rley 0.3.01 → 0.3.04

data/spec/rley/parser/parse_walker_factory_spec.rb

@@ -63,7 +63,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
         parser = Parser::GFGEarleyParser.new(sample_grammar)
         parser.parse(sample_tokens)
       end
-      
+
       let(:accept_entry) { sample_result.accepting_entry }
       let(:accept_index) { sample_result.chart.last_index }
       subject { ParseWalkerFactory.new }
@@ -100,7 +100,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           event3 = walker.next
           expectations = [:visit, 'S. | 0', 4]
           event_expectations(event3, expectations)
-          
+
           # Backtrack created: first alternative selected
           event4 = walker.next
           expectations = [:visit, 'S => a T . | 0', 4]
@@ -112,16 +112,16 @@ module Rley # Open this namespace to avoid module qualifier prefixes
 
           event6 = walker.next
           expectations = [:visit, 'T => b b b . | 1', 4]
-          event_expectations(event6, expectations)        
-          
+          event_expectations(event6, expectations)
+
           event7 = walker.next
           expectations = [:visit, 'T => b b . b | 1', 3]
-          event_expectations(event7, expectations)      
+          event_expectations(event7, expectations)
 
           event8 = walker.next
           expectations = [:visit, 'T => b . b b | 1', 2]
-          event_expectations(event8, expectations)       
-          
+          event_expectations(event8, expectations)
+
           event9 = walker.next
           expectations = [:visit, 'T => . b b b | 1', 1]
           event_expectations(event9, expectations)
@@ -144,7 +144,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
 
           event14 = walker.next
           expectations = [:visit, 'Phi => . S | 0', 0]
-          event_expectations(event14, expectations) 
+          event_expectations(event14, expectations)
 
           event15 = walker.next
           expectations = [:visit, '.Phi | 0', 0]
@@ -153,80 +153,100 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           # Backtracking is occurring
           event16 = walker.next
           expectations = [:backtrack, 'S. | 0', 4]
-          event_expectations(event16, expectations)          
-          
-          event18 = walker.next
+          event_expectations(event16, expectations)
+
+          event17 = walker.next
           expectations = [:visit, 'S => A T . | 0', 4]
+          event_expectations(event17, expectations)
+
+          event18 = walker.next
+          expectations = [:revisit, 'T. | 1', 4]
           event_expectations(event18, expectations)
 
+          # Multiple visit occurred: jump to antecedent of start entry
           event19 = walker.next
-          expectations = [:revisit, 'T. | 1', 4]
+          expectations = [:visit, 'S => A . T | 0', 1]
           event_expectations(event19, expectations)
 
-          # Multiple visit occurred: jump to antecedent of start entry
           event20 = walker.next
-          expectations = [:visit, 'S => A . T | 0', 1]
+          expectations = [:visit, 'A. | 0', 1]
           event_expectations(event20, expectations)
 
+          # Backtrack created: first alternative selected
           event21 = walker.next
-          expectations = [:visit, 'A. | 0', 1]
+          expectations = [:visit, 'A => a . | 0', 1]
           event_expectations(event21, expectations)
-          
-          # Backtrack created: first alternative selected
+
           event22 = walker.next
-          expectations = [:visit, 'A => a . | 0', 1]
-          event_expectations(event22, expectations)                     
-          
-          event23 = walker.next
           expectations = [:visit, 'A => . a | 0', 0]
+          event_expectations(event22, expectations)
+
+          event23 = walker.next
+          expectations = [:visit, '.A | 0', 0]
           event_expectations(event23, expectations)
 
           event24 = walker.next
-          expectations = [:visit, '.A | 0', 0]
+          expectations = [:visit, 'S => . A T | 0', 0]
           event_expectations(event24, expectations)
 
           event25 = walker.next
-          expectations = [:visit, 'S => . A T | 0', 0]
+          expectations = [:revisit, '.S | 0', 0]
           event_expectations(event25, expectations)
 
-          # Backtracking is occurring
           event26 = walker.next
-          expectations = [:backtrack, 'A. | 0', 1]
-          event_expectations(event26, expectations)          
-          
+          expectations = [:revisit, '.Phi | 0', 0]
+          event_expectations(event26, expectations)
+
+          # Backtracking is occurring
           event27 = walker.next
-          expectations = [:visit, 'A => B A . | 0', 1]
+          expectations = [:backtrack, 'A. | 0', 1]
           event_expectations(event27, expectations)
-          
+
           event28 = walker.next
-          expectations = [:revisit, 'A. | 0', 1]
+          expectations = [:visit, 'A => B A . | 0', 1]
           event_expectations(event28, expectations)
 
           event29 = walker.next
-          expectations = [:visit, 'A => B . A | 0', 0]
-          event_expectations(event29, expectations)           
+          expectations = [:revisit, 'A. | 0', 1]
+          event_expectations(event29, expectations)
 
           event30 = walker.next
-          expectations = [:visit, 'B. | 0', 0]
+          expectations = [:visit, 'A => B . A | 0', 0]
           event_expectations(event30, expectations)
 
           event31 = walker.next
-          expectations = [:visit, 'B => . | 0', 0]
+          expectations = [:visit, 'B. | 0', 0]
           event_expectations(event31, expectations)
 
           event32 = walker.next
-          expectations = [:visit, '.B | 0', 0]
-          event_expectations(event32, expectations)  
+          expectations = [:visit, 'B => . | 0', 0]
+          event_expectations(event32, expectations)
 
           event33 = walker.next
+          expectations = [:visit, '.B | 0', 0]
+          event_expectations(event33, expectations)
+
+          event34 = walker.next
           expectations = [:visit, 'A => . B A | 0', 0]
-          event_expectations(event33, expectations)        
+          event_expectations(event34, expectations)
+          
+          event35 = walker.next
+          expectations = [:revisit, '.A | 0', 0]
+          event_expectations(event35, expectations)
+
+          event36 = walker.next
+          expectations = [:revisit, '.S | 0', 0]
+          event_expectations(event36, expectations)
+
+          event37 = walker.next
+          expectations = [:revisit, '.Phi | 0', 0]
+          event_expectations(event37, expectations)          
         end
-        
+
         it 'should raise an exception at end of visit' do
           walker = subject.build_walker(accept_entry, accept_index)
-          32.times { walker.next }
-          
+          37.times { walker.next }
+
           expect{ walker.next }.to raise_error(StopIteration)
         end
 

data/spec/rley/parser/state_set_spec.rb

@@ -54,8 +54,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           # Adding states
           subject.push_state(state1)
           subject.push_state(state2)
-          allow(dotted_rule1).to receive(:production).and_return(:dummy)
-          allow(dotted_rule2).to receive(:production).and_return(a_prod)
+          expect(dotted_rule1).to receive(:production).and_return(:dummy)
+          expect(dotted_rule2).to receive(:production).and_return(a_prod)
           expect(subject.states_for(a_prod)).to eq([state2])
         end
         
@@ -67,11 +67,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           # Adding states
           subject.push_state(state1)
           subject.push_state(state2)
-          allow(dotted_rule1).to receive(:production).and_return(prod1)
-          allow(prod1).to receive(:lhs).and_return(:dummy)          
-          allow(dotted_rule2).to receive(:production).and_return(prod2)
-          allow(dotted_rule2).to receive(:reduce_item?).and_return(true)
-          allow(prod2).to receive(:lhs).and_return(non_term) 
+          expect(dotted_rule1).to receive(:production).and_return(prod1)
+          expect(prod1).to receive(:lhs).and_return(:dummy)          
+          expect(dotted_rule2).to receive(:production).and_return(prod2)
+          expect(dotted_rule2).to receive(:reduce_item?).and_return(true)
+          expect(prod2).to receive(:lhs).and_return(non_term) 
           expect(subject.states_rewriting(non_term)).to eq([state2])
         end
         
@@ -103,7 +103,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
         it 'should complain when impossible predecessor of parse state' do
           subject.push_state(state1)
           subject.push_state(state2)
-          allow(dotted_rule1).to receive(:prev_position).and_return(nil) 
+          expect(dotted_rule1).to receive(:prev_position).and_return(nil) 
           err = StandardError
           expect { subject.predecessor_state(state1) }.to raise_error(err)
         end

data/spec/rley/support/grammar_L0_helper.rb

@@ -0,0 +1,81 @@
+# Load the builder class
+require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/parser/token'
+
+
+module GrammarL0Helper
+  ########################################
+  # Factory method. Define a grammar for a micro English-like language
+  # based on Jurafky & Martin L0 language (chapter 12 of the book).
+  # It defines the syntax of a sentence in a language with a 
+  # very limited syntax and lexicon in the context of airline reservation.
+  def grammar_L0_builder()
+    builder = Rley::Syntax::GrammarBuilder.new
+    builder.add_terminals('Noun', 'Verb', 'Pronoun', 'Proper-Noun')
+    builder.add_terminals('Determiner', 'Preposition', )
+    builder.add_production('S' => %w[NP VP])
+    builder.add_production('NP' => 'Pronoun')
+    builder.add_production('NP' => 'Proper-Noun')
+    builder.add_production('NP' => %w[Determiner Nominal])
+    builder.add_production('Nominal' => %w[Nominal Noun])
+    builder.add_production('Nominal' => 'Noun')
+    builder.add_production('VP' => 'Verb')
+    builder.add_production('VP' => %w[Verb NP])
+    builder.add_production('VP' => %w[Verb NP PP])
+    builder.add_production('VP' => %w[Verb PP])
+    builder.add_production('PP' => %w[Preposition PP])
+    builder
+  end
+  
+  # Return the language lexicon.
+  # A lexicon is just a Hash with pairs of the form:
+  # word => terminal symbol name
+  def lexicon_L0()
+    lexicon = {
+      'flight' => 'Noun',
+      'breeze' => 'Noun',
+      'trip' => 'Noun',
+      'morning' => 'Noun',
+      'is' => 'Verb',
+      'prefer' => 'Verb',
+      'like' => 'Verb',
+      'need' => 'Verb',
+      'want' => 'Verb',
+      'fly' => 'Verb',
+      'me' => 'Pronoun',
+      'I' => 'Pronoun',
+      'you' => 'Pronoun',
+      'it' => 'Pronoun',
+      'Alaska' => 'Proper-Noun',
+      'Baltimore' => 'Proper-Noun',
+      'Chicago' => 'Proper-Noun',
+      'United' => 'Proper-Noun',
+      'American' => 'Proper-Noun',
+      'the' => 'Determiner',
+      'a' => 'Determiner',
+      'an' => 'Determiner',
+      'this' => 'Determiner',
+      'these' => 'Determiner',
+      'that' => 'Determiner',
+      'from' => 'Preposition',
+      'to' => 'Preposition',
+      'on' => 'Preposition',
+      'near' => 'Preposition'
+    }
+  end
+  
+  
+  # Highly simplified tokenizer implementation.
+  def tokenizer_L0(aText, aGrammar)
+    tokens = aText.scan(/\S+/).map do |word|
+      term_name = lexicon_L0[word]
+      if term_name.nil?
+        fail StandardError, "Word '#{word}' not found in lexicon"
+      end
+      terminal = aGrammar.name2symbol[term_name]
+      Rley::Parser::Token.new(word, terminal)
+    end
+    
+    return tokens
+  end
+end # module

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rley
 version: !ruby/object:Gem::Version
-  version: 0.3.01
+  version: 0.3.04
 platform: ruby
 authors:
 - Dimitri Geshef
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-23 00:00:00.000000000 Z
+date: 2016-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -36,40 +36,40 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.5'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 3.5.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.5'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 3.5.0
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -249,12 +249,10 @@ files:
 - spec/rley/ptree/terminal_node_spec.rb
 - spec/rley/ptree/token_range_spec.rb
 - spec/rley/sppf/alternative_node_spec.rb
-- spec/rley/sppf/antecedence_graph.rb
-- spec/rley/sppf/forest_representation.rb
-- spec/rley/sppf/gfg_representation.rb
 - spec/rley/sppf/non_terminal_node_spec.rb
 - spec/rley/support/ambiguous_grammar_helper.rb
 - spec/rley/support/expectation_helper.rb
+- spec/rley/support/grammar_L0_helper.rb
 - spec/rley/support/grammar_abc_helper.rb
 - spec/rley/support/grammar_b_expr_helper.rb
 - spec/rley/support/grammar_helper.rb

data/spec/rley/sppf/antecedence_graph.rb

@@ -1,87 +0,0 @@
-# Mix-in module that generates a Graphviz's DOT file
-# that represents the precedence graph of parse entries.
-module AntecedenceGraph
-
-  def generate_graph(aParsing, aFile)
-    heading = build_heading()
-    aFile.puts(heading)
-    
-    fill_graph(aParsing, aFile)
-    
-    trailing = build_trailing()
-    aFile.puts(trailing)
-  end
-  
-private
-
-  def build_heading()
-    text = <<-END_STRING
-  digraph entries {
-  size = "7,9.5";
-  page = "8.5,11";
-  ratio = fill;
-  rankdir = "BT"; // Draw arrows from bottom to top
-END_STRING
-
-    return text
-  end
-  
-  def build_trailing()
-    return '}'
-  end
-  
-  def fill_graph(aParsing, aFile)
-    # Associate to each parse entry a node id
-    oid2node_id = build_nodes_id(aParsing)
-    aParsing.chart.sets.each_with_index do |entry_set, chart_index|
-      # Create the graph nodes
-      aFile.puts ''
-      aFile.puts(cluster_heading(chart_index))
-      
-      entry_set.entries.each do |entry|
-        aFile.puts %Q(    #{oid2node_id[entry]}[label="#{entry}"];)
-      end
-      aFile.puts '  }'  # Close cluster
-      
-      # Create the edges
-      aFile.puts ''
-      entry_set.entries.each do |entry|
-        antecedents = aParsing.antecedence[entry]
-        antecedents.each do |antec|
-          aFile.puts "  #{oid2node_id[antec]} -> #{oid2node_id[entry]};"
-        end
-      end      
-    end    
-  end
-  
-  # For each parse entry, associate a graph node id
-  def build_nodes_id(aParsing)
-    # Create a Hash with pairs of the form: object id => node id
-    oid2node_id = {} 
-    
-    aParsing.chart.sets.each_with_index do |entry_set, chart_index|
-      entry_set.entries.each_with_index do |entry, entry_index|
-        oid2node_id[entry] = "node_#{chart_index}_#{entry_index}"
-      end
-    end
-
-    return oid2node_id
-  end
-
-  
-  def cluster_heading(anIndex)
-    text = <<-END_STRING
-  subgraph cluster_chart_#{anIndex} {
-    style = rounded;
-    color = blue;
-    fontsize = 24.0;
-    labeljust = "r";    
-    label="chart[#{anIndex}]";
-END_STRING
-
-  return text
-  end
-  
-  
-
-end # module

data/spec/rley/sppf/forest_representation.rb

@@ -1,136 +0,0 @@
-require 'set'
-
-# Mix-in module that generates a Graphviz's DOT file
-# that represents a parse forest.
-class ForestRepresentation
-
-  def generate_graph(aPForest, aFile)
-    heading = build_heading()
-    aFile.puts(heading)
-    
-    fill_graph(aPForest, aFile)
-    
-    trailing = build_trailing()
-    aFile.puts(trailing)
-  end
-  
-private
-
-  def build_heading()
-    text = <<-END_STRING
-  digraph gfg {
-  size="7,9.5";
-  page="8.5,11";
-  ratio = fill;
-
-END_STRING
-
-    return text
-  end
-  
-  def build_trailing()
-    return '}'
-  end
-  
-  def fill_graph(aPForest, aFile)
-    visitees = Set.new
-    visit_node(aPForest.root, aFile, visitees)
-  end
-
-  def visit_node(aNode, aFile, visitees)
-    return if visitees.include?(aNode)
-    visitees << aNode
-    aFile.puts %Q(    node_#{aNode.object_id}[shape=box, fontsize=18.0, label="#{aNode.to_string(0)}"];)
-    
-    if aNode.kind_of?(Rley::SPPF::CompositeNode)
-      aNode.subnodes.each do |snode|
-        # puts snode.to_string(0)
-        next unless snode
-        visit_node(snode, aFile, visitees)
-        aFile.puts %Q(    node_#{aNode.object_id} -> node_#{snode.object_id};)
-      end
-    end
-  end
-
-=begin
-  def fill_graph(aGFGraph, aFile)
-    all_vertices = aGFGraph.vertices.dup
-    (itemized, endpoints) = all_vertices.partition do |vertex| 
-      vertex.is_a?(Rley::GFG::ItemVertex) 
-    end
-    
-    # Group start/end nodes by non-terminal symbol
-    group_endings = endpoints.group_by { |endpoint| endpoint.non_terminal }
-    
-    # Group item vertices by lhs non-terminal symbol
-    group_items = itemized.group_by { |vertex| vertex.lhs }    
-    
-    aFile.puts ''
-    group_endings.each_pair do |nonterm, nodes|
-      text = <<-END_STRING    
-  subgraph cluster_#{nonterm} {
-    color = transparent;
-END_STRING
-      aFile.puts text
-      aFile.puts '    // Define the start and end nodes'      
-      nodes.each do |vertex|
-        # Emit the start/end nodes
-        aFile.puts %Q(    node_#{vertex.object_id}[shape=box, fontsize=18.0, label="#{vertex.label}"];)
-      end
-      
-      # Create sub-clusters by production
-      subnodes = group_items[nonterm]
-      subclusters = subnodes.group_by { |vertex| vertex.dotted_item.production }
-      subclusters.each_pair do |prod, vertices|
-        aFile.puts ''
-        aFile.puts cluster_heading(prod)
-        vertices.each do |vertex|
-          aFile.puts %Q(      node_#{vertex.object_id}[label="#{vertex.label}"];)
-        end
-        aFile.puts cluster_trailing(prod)
-      end      
-      aFile.puts '  }'
-    end
-
-    aFile.puts ''
-    aFile.puts '  // Draw the edges'
-    aGFGraph.vertices.each do |from_vertex|
-      from_vertex.edges.each do |anEdge|
-        if from_vertex.is_a?(Rley::GFG::EndVertex)
-          to_dotted_item = anEdge.successor.dotted_item
-          label = "RET_#{to_dotted_item.production.object_id}_#{to_dotted_item.prev_position}"
-          aFile.puts "  node_#{from_vertex.object_id}->node_#{anEdge.successor.object_id}[color=red, style=dashed, arrowhead=onormal,label=#{label}];"
-        else
-          if anEdge.is_a?(Rley::GFG::ScanEdge)
-            aFile.puts %Q(  node_#{from_vertex.object_id}->node_#{anEdge.successor.object_id}[fontsize=18.0, label="#{anEdge.terminal}"];)
-          else
-            if anEdge.successor.is_a?(Rley::GFG::StartVertex)
-              from_dotted_item = from_vertex.dotted_item
-              label = "CALL_#{from_dotted_item.production.object_id}_#{from_dotted_item.position}" 
-              aFile.puts "  node_#{from_vertex.object_id}->node_#{anEdge.successor.object_id}[color=green, label=#{label}];"              
-            else
-              aFile.puts "  node_#{from_vertex.object_id}->node_#{anEdge.successor.object_id};"
-            end
-          end
-        end
-      end  
-    end    
-  end
-
-
-  def cluster_heading(anObject)
-    text = <<-END_STRING
-    subgraph cluster_#{anObject.object_id} {
-      style = rounded;
-      color = blue;
-END_STRING
-
-  return text
-  end
-  
-  def cluster_trailing(anObject) 
-   return '    }'
-  end
-=end
-  
-end # class