rley 0.4.01 → 0.4.02

data/spec/rley/parser/error_reason_spec.rb

@@ -1,5 +1,5 @@
 require_relative '../../spec_helper'
-require_relative '../../../lib/rley/parser/token'
+require_relative '../../../lib/rley/tokens/token'
 
 # Load the class under test
 require_relative '../../../lib/rley/parser/error_reason'
@@ -61,7 +61,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
     describe UnexpectedToken do
       let(:err_lexeme) { '-'}
       let(:err_terminal) { Syntax::Terminal.new('MINUS') }
-      let(:err_token) { Token.new(err_lexeme, err_terminal) }
+      let(:err_token) { Tokens::Token.new(err_lexeme, err_terminal) }
       let(:terminals) do
         ['PLUS', 'LPAREN'].map { |name| Syntax::Terminal.new(name) }
       end
@@ -90,7 +90,7 @@ MSG_END
     describe PrematureInputEnd do
       let(:err_lexeme) { '+'}
       let(:err_terminal) { Syntax::Terminal.new('PLUS') }
-      let(:err_token) { Token.new(err_lexeme, err_terminal) }
+      let(:err_token) { Tokens::Token.new(err_lexeme, err_terminal) }
       let(:terminals) do
         ['INT', 'LPAREN'].map { |name| Syntax::Terminal.new(name) }
       end

data/spec/rley/parser/gfg_chart_spec.rb

@@ -2,7 +2,7 @@ require_relative '../../spec_helper'
 require 'stringio'
 
 require_relative '../../../lib/rley/syntax/terminal'
-require_relative '../../../lib/rley/parser/token'
+require_relative '../../../lib/rley/tokens/token'
 require_relative '../../../lib/rley/gfg/start_vertex'
 require_relative '../../../lib/rley/parser/parse_entry'
 require_relative '../../../lib/rley/parser/parse_tracer'
@@ -39,7 +39,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
 
       let(:token_seq) do
         literals = %w(a a b c c)
-        literals.map { |lexeme| Token.new(lexeme, nil) }
+        literals.map { |lexeme| Tokens::Token.new(lexeme, nil) }
       end
 
       # Helper method. Create an array of dotted items

data/spec/rley/parser/gfg_earley_parser_spec.rb

@@ -4,7 +4,7 @@ require_relative '../../../lib/rley/syntax/verbatim_symbol'
 require_relative '../../../lib/rley/syntax/non_terminal'
 require_relative '../../../lib/rley/syntax/production'
 require_relative '../../../lib/rley/syntax/grammar_builder'
-require_relative '../../../lib/rley/parser/token'
+require_relative '../../../lib/rley/tokens/token'
 require_relative '../../../lib/rley/parser/dotted_item'
 require_relative '../../../lib/rley/parser/gfg_parsing'
 
@@ -288,7 +288,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           builder.add_terminals(t_x)
           builder.add_production('Ss' => %w(A A x))
           builder.add_production('A' => [])
-          tokens = [ Token.new('x', t_x) ]
+          tokens = [ Tokens::Token.new('x', t_x) ]
 
           instance = GFGEarleyParser.new(builder.grammar)
           expect { instance.parse(tokens) }.not_to raise_error

data/spec/rley/parser/gfg_parsing_spec.rb

@@ -6,7 +6,7 @@ require_relative '../../../lib/rley/syntax/verbatim_symbol'
 require_relative '../../../lib/rley/syntax/production'
 require_relative '../../../lib/rley/syntax/grammar_builder'
 require_relative '../../../lib/rley/parser/dotted_item'
-require_relative '../../../lib/rley/parser/token'
+require_relative '../../../lib/rley/tokens/token'
 require_relative '../../../lib/rley/parser/parse_tracer'
 require_relative '../../../lib/rley/gfg/grm_flow_graph'
 require_relative '../../../lib/rley/parser/grm_items_builder'
@@ -298,7 +298,7 @@ SNIPPET
 
         let(:token_seq1) do
           %w(a a b c c).map do |letter|
-            Token.new(letter, sample_grammar1.name2symbol[letter])
+            Tokens::Token.new(letter, sample_grammar1.name2symbol[letter])
           end
         end
 

data/spec/rley/parser/groucho_spec.rb

@@ -54,7 +54,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
             raise StandardError, "Word '#{word}' not found in lexicon"
           end
           terminal = aGrammar.name2symbol[term_name]
-          Rley::Parser::Token.new(word, terminal)
+          Rley::Tokens::Token.new(word, terminal)
         end
 
         return tokens

data/spec/rley/parser/parse_tracer_spec.rb

@@ -6,7 +6,7 @@ require_relative '../../../lib/rley/syntax/non_terminal'
 require_relative '../../../lib/rley/syntax/production'
 require_relative '../../../lib/rley/parser/dotted_item'
 require_relative '../../../lib/rley/parser/parse_state'
-require_relative '../../../lib/rley/parser/token'
+require_relative '../../../lib/rley/tokens/token'
 
 # Load the class under test
 require_relative '../../../lib/rley/parser/parse_tracer'
@@ -18,7 +18,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
 
       let(:token_seq) do
         literals = %w(I saw John with a dog)
-        literals.map { |lexeme| Token.new(lexeme, nil) }
+        literals.map { |lexeme| Tokens::Token.new(lexeme, nil) }
       end
 
       subject { ParseTracer.new(1, output, token_seq) }

data/spec/rley/parser/parse_tree_builder_spec.rb

@@ -1,178 +1,251 @@
 require_relative '../../spec_helper'
-require_relative '../../../lib/rley/parser/token'
-require_relative '../../../lib/rley/parser/earley_parser'
-require_relative '../../../lib/rley/parser/parsing'
+
+require_relative '../../../lib/rley/parser/gfg_earley_parser'
+require_relative '../../../lib/rley/parser/parse_walker_factory'
+
+require_relative '../support/expectation_helper'
+require_relative '../support/grammar_b_expr_helper'
+
 # Load the class under test
 require_relative '../../../lib/rley/parser/parse_tree_builder'
-require_relative '../support/grammar_abc_helper'
 
 module Rley # Open this namespace to avoid module qualifier prefixes
-  module Parser # Open this namespace to avoid module qualifier prefixes
+  module Parser
     describe ParseTreeBuilder do
-      include GrammarABCHelper # Mix-in module with builder for grammar abc
+      include ExpectationHelper # Mix-in with expectation on parse entry sets
+      include GrammarBExprHelper # Mix-in for basic arithmetic language
 
-      let(:grammar_abc) do
-        builder = grammar_abc_builder
-        builder.grammar
+      let(:sample_grammar) do
+          builder = grammar_expr_builder()
+          builder.grammar
       end
 
-      let(:capital_a) { grammar_abc.name2symbol['A'] }
-      let(:capital_s) { grammar_abc.name2symbol['S'] }
-      let(:small_a) { grammar_abc.name2symbol['a'] }
-      let(:small_b) { grammar_abc.name2symbol['b'] }
-      let(:small_c) { grammar_abc.name2symbol['c'] }
+      let(:sample_tokens) do
+        expr_tokenizer('2 + 3 * 4', sample_grammar)
+      end
 
-      let(:start_prod) { grammar_abc.start_production }
+      let(:sample_result) do
+        parser = Parser::GFGEarleyParser.new(sample_grammar)
+        parser.parse(sample_tokens)
+      end
+
+      subject { ParseTreeBuilder.new(sample_tokens) }
 
-      let(:tokens_abc) do
-        %w(a a b c c).map do |letter|
-          Token.new(letter, grammar_abc.name2symbol[letter])
+      # Emit a text representation of the current path.
+      def path_to_s()
+        text_parts = subject.curr_path.map do |path_element|
+          path_element.to_s
         end
+        return text_parts.join('/')
       end
 
-      let(:sample_parsing) do
-        parser = EarleyParser.new(grammar_abc)
-        parser.parse(tokens_abc)
+      def next_event(eventType, anEntryText)
+        event = @walker.next
+        subject.receive_event(*event)
+        expect(event[0]).to eq(eventType)
+        expect(event[1].to_s).to eq(anEntryText)
       end
 
-      subject { ParseTreeBuilder.new(start_prod, low: 0, high: 5) }
+      def expected_curr_parent(anExpectation)
+        expect(subject.curr_parent.to_string(0)).to eq(anExpectation)
+      end
+
+      def expected_curr_path(anExpectation)
+        expect(path_to_s).to eq(anExpectation)
+      end
+
+      def expected_first_child(anExpectation)
+          child = subject.curr_parent.subnodes.first
+          expect(child.to_string(0)).to eq(anExpectation)
+      end
 
       context 'Initialization:' do
-        it 'should be created with a proposition and a range' do
-          expect { ParseTreeBuilder.new(start_prod, {}) }.not_to raise_error
+        it 'should be created with a sequence of tokens' do
+          expect { ParseTreeBuilder.new(sample_tokens) }.not_to raise_error
         end
 
-        it 'should have a root node at start' do
-            expect(subject.root.symbol).to eq(capital_s)
+        it 'should know the input tokens' do
+          expect(subject.tokens).to eq(sample_tokens)
         end
 
-        it 'should have current path at start' do
-            expect(subject.current_path).not_to be_empty
+        it 'should have an empty path' do
+          expect(subject.curr_path).to be_empty
         end
+      end # context
 
-        it 'should have current node at start' do
-            expect(subject.current_node.symbol).to eq(capital_a)
+      context 'Parse tree construction:' do
+        before(:each) do
+          factory = ParseWalkerFactory.new
+          accept_entry = sample_result.accepting_entry
+          accept_index = sample_result.chart.last_index
+          @walker = factory.build_walker(accept_entry, accept_index)
         end
-      end # context
 
-      context 'Adding nodes to parse tree:' do
-        it 'should process parse state for a non-terminal node' do
-          # Expectation:
-          # S[0, 5]
-          # +- A[0,5]
-          expect(subject.root.symbol).to eq(capital_s)
-          expect(subject.root.children.size).to eq(1)
-          child1 = subject.root.children[0]
-          expect(child1.symbol).to eq(capital_a)
-          expect(child1.range.low).to eq(0)
-          expect(child1.range.high).to eq(5)
-          expect(subject.current_node).to eq(child1)
-
-          # Add children to A
-          other_state = sample_parsing.chart.state_sets.last.states.first
-          subject.use_complete_state(other_state)
-
-          # Tree is:
-          # S[0,5]
-          # +- A[0,5]
-          #    +- a[0, ?]
-          #    +- A[?, ?]
-          #    +- c[?, 5]
-          expect(child1.children.size).to eq(3) # a A c
-          %w(a A c).each_with_index do |letter, i|
-            grm_symbol = grammar_abc.name2symbol[letter]
-            expect(child1.children[i].symbol).to eq(grm_symbol)
-          end
-          expect(child1.children[0].range.low).to eq(0)
-          expect(child1.children[-1].range.high).to eq(5)
-
-          subject.move_down # ... to c
-          subject.range = { low: 4 }
-          expect(child1.children[-1].range.low).to eq(4)
-          expect(child1.children.last).to eq(subject.current_node)
-          subject.move_back # ... to A
-          expect(subject.current_node).to eq(child1.children[1])
-          grand_child_A = subject.current_node
-
-          other_state = sample_parsing.chart.state_sets[4].first
-          subject.use_complete_state(other_state)
-          expect(grand_child_A.children.size).to eq(3) # a A c
-          %w(a A c).each_with_index do |letter, i|
-            grm_symbol = grammar_abc.name2symbol[letter]
-            expect(grand_child_A.children[i].symbol).to eq(grm_symbol)
-          end
+        it 'should initialize the root node' do
+          next_event(:visit, 'P. | 0')
+          tree = subject.tree
+
+          expect(tree.root.to_string(0)).to eq('P[0, 5]')
+          expected_curr_path('P[0, 5]')
         end
-      end # context
 
-      context 'Moving the current node:' do
-        it 'should move down to last child' do
-          # Tree is:
-          # S[0,?]
-          # +- A[0,?]
-
-          # Add children to A
-          parse_state = sample_parsing.chart.state_sets.last.states.first
-          subject.use_complete_state(parse_state)
-
-          # Tree is:
-          # S[0,?]
-          # +- A[0,?]
-          #    +- a[0, ?]
-          #    +- A[?, ?]
-          #    +- c[?, ?]
-          subject.move_down # ...to grand-child c
-          expect(subject.current_node.symbol).to eq(small_c)
-
-
-          subject.move_back # ...to grand-child A
-          expect(subject.current_node.symbol).to eq(capital_a)
-
-          # Add more children
-          other_state = sample_parsing.chart.state_sets[4].states.first
-          subject.use_complete_state(other_state)
-
-          # Tree is:
-          tree_snapshot = <<-SNIPPET
-S[0, 5]
-+- A[0, 5]
-   +- a[0, ?]: '(nil)'
-   +- A[1, ?]
-      +- a[1, ?]: '(nil)'
-      +- A[?, ?]
-      +- c[?, ?]: '(nil)'
-   +- c[?, 5]: '(nil)'
-SNIPPET
-          expect(subject.root.to_string(0)).to eq(tree_snapshot.chomp)
-
-          subject.move_down # ...to grand-grand-child c
-          expect(subject.current_node.to_string(0)).to eq("c[?, ?]: '(nil)'")
-
-          subject.move_back # ...to grand-grand-child A
-          expect(subject.current_node.to_string(0)).to eq('A[?, ?]')
-
-          subject.move_back # ...to grand-grand-child a
-          expect(subject.current_node.to_string(0)).to eq("a[1, ?]: '(nil)'")
-
-          subject.move_back # ...to grand-child a
-          expect(subject.current_node.to_string(0)).to eq("a[0, ?]: '(nil)'")
-
-          subject.move_back # ...to S
-          expect(subject.current_node.symbol).to eq(capital_s)
+        it 'should initialize the first child of the root node' do
+          next_event(:visit, 'P. | 0') # Event 1
+          next_event(:visit, 'P => S . | 0') # Event 2
+          next_event(:visit, 'S. | 0') # Event 3
+          next_event(:visit, 'S => S + M . | 0') # Event 4
+          expected_curr_path('P[0, 5]/S[0, 5]')          
+          next_event(:visit, 'M. | 2') # Event 5
+          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]')
+          next_event(:visit, 'M => M * T . | 2') # Event 6
+          next_event(:visit, 'T. | 4') # Event 7
+          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/T[4, 5]')
+          next_event(:visit, 'T => integer . | 4') # Event 8
         end
 
-        it 'should move through deeply nested structure' do
+        it 'should build token node when scan edge was detected' do
+          8.times do
+            event = @walker.next
+            subject.receive_event(*event)
+          end
+
+          next_event(:visit, 'T => . integer | 4') # Event 9
+          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/T[4, 5]')
+          expected_first_child("integer[4, 5]: '4'")
+          expect(subject.curr_parent.subnodes.size).to eq(1)          
         end
-      end # context
 
-      context 'Parse tree building:' do
-        it 'should build a parse tree' do
-          expect(subject.parse_tree).to be_kind_of(PTree::ParseTree)
-          actual = subject.parse_tree
-          expect(actual.root).to eq(subject.root)
+        it 'should handle the remaining events' do
+          9.times do
+            event = @walker.next
+            subject.receive_event(*event)
+          end
+
+          next_event(:visit, '.T | 4') # Event 10
+          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]')
+
+          next_event(:visit, 'M => M * . T | 2') # Event 11
+          
+          next_event(:visit, 'M => M . * T | 2') # Event 12          
+          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]')
+          expect(subject.curr_parent.subnodes.size).to eq(2)
+          expected_first_child("*[3, 4]: '*'")
+
+          next_event(:visit, 'M. | 2') # Event 13
+          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/M[2, 3]')
+
+          next_event(:visit, 'M => T . | 2') # Event 14
+          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/M[2, 3]')
+
+          next_event(:visit, 'T. | 2') # Event 15
+          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/M[2, 3]/T[2, 3]')
+
+          next_event(:visit, 'T => integer . | 2') # Event 16
+          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/M[2, 3]/T[2, 3]')
+          expect(subject.curr_parent.subnodes.size).to eq(1)
+          expected_first_child("integer[2, 3]: '3'")
+
+          next_event(:visit, 'T => . integer | 2') # Event 17
+          
+          next_event(:visit, '.T | 2') # Event 18
+          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/M[2, 3]')          
+
+          next_event(:visit, 'M => . T | 2') # Event 19
+          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]/M[2, 3]')
+
+          next_event(:visit, '.M | 2') # Event 20
+          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]')
+ 
+          next_event(:visit, 'M => . M * T | 2') # Event 21
+          expected_curr_path('P[0, 5]/S[0, 5]/M[2, 5]') 
+
+          next_event(:revisit, '.M | 2') # Revisit Event 22
+          expected_curr_path('P[0, 5]/S[0, 5]')          
+ 
+          next_event(:visit, 'S => S + . M | 0') # Event 23
+          expected_curr_path('P[0, 5]/S[0, 5]')
+
+          next_event(:visit, 'S => S . + M | 0') # Event 24
+          expected_curr_path('P[0, 5]/S[0, 5]')
+          expect(subject.curr_parent.subnodes.size).to eq(2)
+          expected_first_child("+[1, 2]: '+'")          
+          
+          next_event(:visit, 'S. | 0') # Event 25
+          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]')          
+
+          next_event(:visit, 'S => M . | 0') # Event 26
+          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]')
+          
+          next_event(:visit, 'M. | 0') # Event 27
+          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]')
+
+          next_event(:visit, 'M => T . | 0') # Event 28
+          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]') 
+
+          next_event(:visit, 'T. | 0') # Event 29
+          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]/T[0, 1]')           
+
+          next_event(:visit, 'T => integer . | 0') # Event 30
+          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]/T[0, 1]')
+
+          next_event(:visit, 'T => . integer | 0') # Event 31
+          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]/T[0, 1]')
+          expect(subject.curr_parent.subnodes.size).to eq(1)
+          expected_first_child("integer[0, 1]: '2'")
+
+          next_event(:visit, '.T | 0') # Event 32
+          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]')  
+
+          next_event(:visit, 'M => . T | 0') # Event 33
+          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]/M[0, 1]')           
+
+          next_event(:visit, '.M | 0') # Event 34
+          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]') 
+
+          next_event(:visit, 'S => . M | 0') # Event 35
+          expected_curr_path('P[0, 5]/S[0, 5]/S[0, 1]')
+          
+          next_event(:visit, '.S | 0') # Event 36
+          expected_curr_path('P[0, 5]/S[0, 5]')
+
+          next_event(:visit, 'S => . S + M | 0') # Event 37
+          expected_curr_path('P[0, 5]/S[0, 5]')
+
+          next_event(:revisit, '.S | 0') # Event 38
+          expected_curr_path('P[0, 5]')
+
+          next_event(:visit, 'P => . S | 0') # Event 39
+          expected_curr_path('P[0, 5]')            
+
+          next_event(:visit, '.P | 0') # Event 39         
+          expect(path_to_s).to be_empty
+        end
+        
+        it 'should build parse trees' do
+          loop do
+            event = @walker.next
+            subject.receive_event(*event)
+            break if path_to_s.empty?
+          end
+
+          # Lightweight sanity check
+          expect(subject.tree).not_to be_nil
+          expect(subject.tree).to be_kind_of(PTree::ParseTree)
+          expect(subject.tree.root.to_s).to eq('P[0, 5]')
+          expect(subject.tree.root.subnodes.size).to eq(1)
+          child_node = subject.tree.root.subnodes[0]
+          expect(child_node.to_s).to eq('S[0, 5]')
+          expect(child_node.subnodes.size).to eq(3)
+          first_grandchild = child_node.subnodes[0]
+          expect(first_grandchild.to_s).to eq('S[0, 1]')
+          second_grandchild = child_node.subnodes[1]
+          expect(second_grandchild.to_s).to eq("+[1, 2]: '+'")
+          third_grandchild = child_node.subnodes[2]
+          expect(third_grandchild.to_s).to eq('M[2, 5]')           
         end
       end # context
+
     end # describe
   end # module
 end # module
-
 # End of file