rley 0.2.01 → 0.2.02

data/lib/rley/ptree/non_terminal_node.rb

@@ -16,6 +16,19 @@ module Rley # This module is used as a namespace
         children << aChildNode
       end
       
+      # Emit a (formatted) string representation of the node.
+      # Mainly used for diagnosis/debugging purposes.
+      def to_string(indentation)
+        connector = '+- '
+        selfie = super(indentation)
+        prefix = "\n" + (' ' * connector.size * indentation) + connector
+        children_repr = children.reduce('') do |sub_result, child|
+          sub_result << prefix + child.to_string(indentation + 1) 
+        end
+        
+        return selfie + children_repr
+      end
+      
       # Part of the 'visitee' role in Visitor design pattern.
       # @param aVisitor[ParseTreeVisitor] the visitor
       def accept(aVisitor)

data/lib/rley/ptree/parse_tree_node.rb

@@ -17,6 +17,13 @@ module Rley # This module is used as a namespace
       def range=(aRange)
         range.assign(aRange)
       end
+      
+      # Emit a (formatted) string representation of the node.
+      # Mainly used for diagnosis/debugging purposes.
+      def to_string(indentation)
+        
+        return "#{symbol.name}#{range.to_string(indentation)}"
+      end
     end # class
   end # module
 end # module

data/lib/rley/ptree/terminal_node.rb

@@ -10,6 +10,13 @@ module Rley # This module is used as a namespace
         super(aTerminalSymbol, aRange)
       end
       
+      # Emit a (formatted) string representation of the node.
+      # Mainly used for diagnosis/debugging purposes.
+      def to_string(indentation)
+        value = token.nil? ? '(nil)' : token.lexeme
+        super(indentation) + ": '#{value}'"
+      end
+      
       # Part of the 'visitee' role in Visitor design pattern.
       # @param aVisitor[ParseTreeVisitor] the visitor
       def accept(aVisitor)

data/lib/rley/ptree/token_range.rb

@@ -42,6 +42,24 @@ module Rley # This module is used as a namespace
         assign_high(aRange) if high.nil?
       end
 
+      # Tell whether the given index value lies outside the range
+      def out_of_range?(index)
+        result = false
+        result = true if !low.nil? && index < low
+        result = true if !high.nil? && index > high
+
+        return result
+      end
+
+      # Emit a (formatted) string representation of the range.
+      # Mainly used for diagnosis/debugging purposes.
+      def to_string(_indentation)
+        low_text = low.nil? ? '?' : low.to_s
+        high_text = high.nil? ? '?' : high.to_s
+        
+        return "[#{low_text}, #{high_text}]"
+      end 
+
       private
 
       def assign_low(aRange)

data/spec/rley/formatter/json_spec.rb

@@ -70,8 +70,7 @@ module Rley # Re-open the module to get rid of qualified names
         end
       end # context
       
-
-     
+ 
       context 'Formatting events:' do   
         it 'should render a parse tree in JSON' do
           instance = Json.new(destination)

data/spec/rley/parser/earley_parser_spec.rb

@@ -6,6 +6,7 @@ require_relative '../../../lib/rley/syntax/production'
 require_relative '../../../lib/rley/syntax/grammar_builder'
 require_relative '../../../lib/rley/parser/token'
 require_relative '../../../lib/rley/parser/dotted_item'
+require_relative '../support/ambiguous_grammar_helper'
 # Load the class under test
 require_relative '../../../lib/rley/parser/earley_parser'
 
@@ -38,9 +39,9 @@ module Rley # Open this namespace to avoid module qualifier prefixes
 
       # Grammar 1: A very simple language
       # (based on example in N. Wirth "Compiler Construction" book, p. 6)
-      # S ::= A.
-      # A ::= "a" A "c".
-      # A ::= "b".
+      # S => A.
+      # A => "a" A "c".
+      # A => "b".
       # Let's create the grammar piece by piece
       let(:nt_S) { Syntax::NonTerminal.new('S') }
       let(:nt_A) { Syntax::NonTerminal.new('A') }
@@ -333,7 +334,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           compare_state_texts(parse_result.chart[1], expected)
         end
 
-        it 'should parse an ambiguous grammar' do
+        it 'should parse an ambiguous grammar (I)' do
           # Grammar 3: A ambiguous arithmetic expression language
           # (based on example in article on Earley's algorithm in Wikipedia)
           # P => S.
@@ -442,6 +443,78 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           compare_state_texts(parse_result.chart[5], expected)
         end
 
+        it 'should parse an ambiguous grammar (II)' do
+          self.extend(AmbiguousGrammarHelper)
+          grammar = grammar_builder.grammar
+          instance = EarleyParser.new(grammar)
+          tokens = tokenize('abc + def + ghi', grammar)
+          expect { instance.parse(tokens) }.not_to raise_error
+          parse_result = instance.parse(tokens)
+          expect(parse_result.success?).to eq(true)
+
+          ###################### S(0): . abc + def + ghi
+          # Expectation chart[0]:
+          expected = [
+            'S => . E | 0',       # Start rule
+            'E => . E + E | 0',   # predict from (1)
+            'E => . id | 0'       # predict from (1)
+          ]
+          compare_state_texts(parse_result.chart[0], expected)
+
+          ###################### S(1): abc . + def + ghi
+          # Expectation chart[1]:
+          expected = [
+            'E => id . | 0',      # scan from S(0) 3
+            'S => E . | 0',       # complete from (1) and S(0) 2
+            'E => E . + E | 0'    # complete from (1) and S(0) 3
+          ]
+          compare_state_texts(parse_result.chart[1], expected)
+
+          ###################### S(2): abc + . def + ghi
+          # Expectation chart[2]:
+          expected = [
+            'E => E + . E | 0',   # Scan from S(1) 3
+            'E => . E + E | 2',   # predict from (1)
+            'E => . id | 2'       # predict from (1)
+          ]
+          compare_state_texts(parse_result.chart[2], expected)
+
+          ###################### S(3): abc + def .  + ghi
+          # Expectation chart[3]:
+          expected = [
+            'E => id . | 2',      # Scan from S(2) 3
+            'E => E + E . | 0',   # complete from (1) and S(2) 1
+            'E => E . + E | 2',   # complete from (1) and S(2) 2
+            'S => E . | 0',       # complete from (1) and S(0) 1
+            'E => E . + E | 0'    # complete from (1) and S(0) 2
+          ]
+          compare_state_texts(parse_result.chart[3], expected)
+
+          ###################### S(4): abc + def + . ghi
+          # Expectation chart[4]:
+          expected = [
+            'E => E + . E | 2',   # Scan from S(3) 3
+            'E => E + . E | 0',   # Scan from S(3) 5
+            'E => . E + E | 4',   # predict from (1)
+            'E => . id | 4'       # predict from (1)
+          ]
+          compare_state_texts(parse_result.chart[4], expected)
+
+          ###################### S(5): abc + def + ghi .
+          # Expectation chart[5]:
+          expected = [
+            'E => id . | 4',      # Scan from S(4) 4
+            'E => E + E . | 2',   # complete from (1) and S(4) 1
+            'E => E + E . | 0',   # complete from (1) and S(4) 2
+            'E => E . + E | 4',   # complete from (1) and S(4) 3
+            'E => E . + E | 2',   # complete from (1) and S(2) 2
+            'S => E . | 0',       # complete from (1) and S(0) 1
+            'E => E . + E | 0',   # complete from (1) and S(0) 2
+          ]
+          compare_state_texts(parse_result.chart[5], expected)
+        end
+
+
 
         it 'should parse an invalid simple input' do
           # Parse an erroneous input (b is missing)

data/spec/rley/parser/parse_tree_builder_spec.rb

@@ -21,8 +21,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
       let(:small_a) { grammar_abc.name2symbol['a'] }
       let(:small_b) { grammar_abc.name2symbol['b'] }
       let(:small_c) { grammar_abc.name2symbol['c'] }
-      
-      let(:start_prod) { grammar_abc.start_production } 
+
+      let(:start_prod) { grammar_abc.start_production }
 
       let(:tokens_abc) do
         %w(a a b c c).map do |letter|
@@ -34,7 +34,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
         parser = EarleyParser.new(grammar_abc)
         parser.parse(tokens_abc)
       end
-      
+
       subject { ParseTreeBuilder.new(start_prod, { low: 0, high: 5 }) }
 
       context 'Initialization:' do
@@ -71,7 +71,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           # Add children to A
           other_state = sample_parsing.chart.state_sets.last.states.first
           subject.use_complete_state(other_state)
-          
+
           # Tree is:
           # S[0,5]
           # +- A[0,5]
@@ -85,7 +85,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           end
           expect(child1.children[0].range.low).to eq(0)
           expect(child1.children[-1].range.high).to eq(5)
-          
+
           subject.move_down # ... to c
           subject.range = { low: 4 }
           expect(child1.children[-1].range.low).to eq(4)
@@ -132,46 +132,47 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           subject.use_complete_state(other_state)
 
           # Tree is:
-          # S[0,?]
-          # +- A[0,?]
-          #    +- a[0, ?]
-          #    +- A[?, ?]
-          #       +- a[?, ?]
-          #       +- A[?, ?]
-          #       +- c [?, ?]
-          #    +- c[?, ?]
+          tree_snapshot = <<-SNIPPET
+S[0, 5]
++- A[0, 5]
+   +- a[0, ?]: '(nil)'
+   +- A[1, ?]
+      +- a[1, ?]: '(nil)'
+      +- A[?, ?]
+      +- c[?, ?]: '(nil)'
+   +- c[?, 5]: '(nil)'
+SNIPPET
+          expect(subject.root.to_string(0)).to eq(tree_snapshot.chomp)
 
           subject.move_down # ...to grand-grand-child c
-          expect(subject.current_node.symbol).to eq(small_c)
+          expect(subject.current_node.to_string(0)).to eq("c[?, ?]: '(nil)'")
 
           subject.move_back # ...to grand-grand-child A
-          expect(subject.current_node.symbol).to eq(capital_a)
+          expect(subject.current_node.to_string(0)).to eq('A[?, ?]')
 
           subject.move_back # ...to grand-grand-child a
-          expect(subject.current_node.symbol).to eq(small_a)
+          expect(subject.current_node.to_string(0)).to eq("a[1, ?]: '(nil)'")
 
-          subject.move_back # ...to grand-child A
-          expect(subject.current_node.symbol).to eq(capital_a)
-          
           subject.move_back # ...to grand-child a
-          expect(subject.current_node.symbol).to eq(small_a)
-
-          subject.move_back # ...to child A
-          expect(subject.current_node.symbol).to eq(capital_a)
+          expect(subject.current_node.to_string(0)).to eq("a[0, ?]: '(nil)'")
 
           subject.move_back # ...to S
-          expect(subject.current_node.symbol).to eq(capital_s)           
+          expect(subject.current_node.symbol).to eq(capital_s)
+        end
+        
+        it 'should move through deeply nested structure' do
+        
         end
       end # context
-      
-      context 'Parse tree building:' do       
+
+      context 'Parse tree building:' do
         it 'should build a parse tree' do
           expect(subject.parse_tree).to be_kind_of(PTree::ParseTree)
           actual = subject.parse_tree
           expect(actual.root).to eq(subject.root)
         end
       end # context
-      
+
     end # describe
   end # module
 end # module

data/spec/rley/parser/parsing_spec.rb

@@ -21,9 +21,9 @@ module Rley # Open this namespace to avoid module qualifier prefixes
       include GrammarBExprHelper # Mix-in with builder for simple expressions
 
       # Grammar 1: A very simple language
-      # S ::= A.
-      # A ::= "a" A "c".
-      # A ::= "b".
+      # S => A.
+      # A => "a" A "c".
+      # A => "b".
       let(:nt_S) { Syntax::NonTerminal.new('S') }
       let(:nt_A) { Syntax::NonTerminal.new('A') }
       let(:a_) { Syntax::VerbatimSymbol.new('a') }
@@ -139,6 +139,215 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           b_expr_grammar.name2symbol[aSymbolName]
         end
 
+        subject do
+          parser = EarleyParser.new(b_expr_grammar)
+          tokens = expr_tokenizer('2 + 3 * 4', b_expr_grammar)
+          instance = parser.parse(tokens)
+        end
+
+        # Helper. Build a state tracker and a parse tree builder.
+        def prepare_parse_tree(aParsing)
+          # Accessing private methods by sending message
+          state_tracker = aParsing.send(:new_state_tracker)
+          builder = aParsing.send(:tree_builder, state_tracker.state_set_index)
+          return [state_tracker, builder]
+        end
+
+
+        it 'should create the root of a parse tree' do
+          (state_tracker, builder) = prepare_parse_tree(subject)
+          # The root node should correspond to the start symbol and
+          # its direct children should correspond to rhs of start production
+expected_text = <<-SNIPPET
+P[0, 5]
++- S[0, 5]
+SNIPPET
+          root_text = builder.root.to_string(0)
+          expect(root_text).to eq(expected_text.chomp)
+
+          expect(state_tracker.state_set_index).to eq(subject.tokens.size)
+          expected_state = 'P => S . | 0'
+          expect(state_tracker.parse_state.to_s).to eq(expected_state)
+          expect(builder.current_node.to_string(0)).to eq('S[0, 5]')
+        end
+
+        it 'should use a reduce item for a matched non-terminal' do
+          # Setup
+          (state_tracker, builder) = prepare_parse_tree(subject)
+          # Same state as in previous example
+
+          # Given matched symbol is S[0, 5]
+          # And its reduce item is S => S + M . | 0
+          # Then add child nodes corresponding to the rhs symbols
+          # And make M[?, 5] the current symbol
+          subject.insert_matched_symbol(state_tracker, builder)
+          expected_text = <<-SNIPPET
+P[0, 5]
++- S[0, 5]
+   +- S[0, ?]
+   +- +[?, ?]: '(nil)'
+   +- M[?, 5]
+SNIPPET
+          root_text = builder.root.to_string(0)
+          expect(root_text).to eq(expected_text.chomp)
+          expected_state = 'S => S + M . | 0'
+          expect(state_tracker.parse_state.to_s).to eq(expected_state)
+          expect(state_tracker.state_set_index).to eq(5)
+          expect(builder.current_node.to_string(0)).to eq('M[?, 5]')
+
+          # Second similar test
+
+          # Given matched symbol is M[?, 5]
+          # And its reduce item is M => M * T . | 2
+          # Then add child nodes corresponding to the rhs symbols
+          # And make T[?, 5] the current symbol
+          subject.insert_matched_symbol(state_tracker, builder)
+          expected_text = <<-SNIPPET
+P[0, 5]
++- S[0, 5]
+   +- S[0, ?]
+   +- +[?, ?]: '(nil)'
+   +- M[2, 5]
+      +- M[2, ?]
+      +- *[?, ?]: '(nil)'
+      +- T[?, 5]
+SNIPPET
+          root_text = builder.root.to_string(0)
+          expect(root_text).to eq(expected_text.chomp)
+          expected_state = 'M => M * T . | 2'
+          expect(state_tracker.parse_state.to_s).to eq(expected_state)
+          expect(state_tracker.state_set_index).to eq(5)
+          expect(builder.current_node.to_string(0)).to eq('T[?, 5]')
+        end
+
+
+
+        it 'should use a previous item for a terminal symbol' do
+          # Setup
+          (state_tracker, builder) = prepare_parse_tree(subject)
+          3.times do
+            subject.insert_matched_symbol(state_tracker, builder)
+          end
+
+          # Given matched symbol is T[?, 5]
+          # And its reduce item is T => integer . | 4
+          # Then add child node corresponding to the rhs symbol
+          # And make integer[4, 5]: '(nil)' the current symbol
+          expected_text = <<-SNIPPET
+P[0, 5]
++- S[0, 5]
+   +- S[0, ?]
+   +- +[?, ?]: '(nil)'
+   +- M[2, 5]
+      +- M[2, ?]
+      +- *[?, ?]: '(nil)'
+      +- T[4, 5]
+         +- integer[4, 5]: '(nil)'
+SNIPPET
+          root_text = builder.root.to_string(0)
+          expect(root_text).to eq(expected_text.chomp)
+          expected_state = 'T => integer . | 4'
+          expect(state_tracker.parse_state.to_s).to eq(expected_state)
+          expect(state_tracker.state_set_index).to eq(5)
+          expect(builder.current_node.to_string(0)).to eq("integer[4, 5]: '(nil)'")
+
+          # Given current tree symbol is integer[4, 5]: '(nil)'
+          # And its previous item is T => . integer | 4
+          # Then attach the token to the terminal node
+          # And decrement the state index by one
+          # Make *[?, ?]: '(nil)' the current symbol
+          subject.insert_matched_symbol(state_tracker, builder)
+          expected_text = <<-SNIPPET
+P[0, 5]
++- S[0, 5]
+   +- S[0, ?]
+   +- +[?, ?]: '(nil)'
+   +- M[2, 5]
+      +- M[2, ?]
+      +- *[?, ?]: '(nil)'
+      +- T[4, 5]
+         +- integer[4, 5]: '4'
+SNIPPET
+          root_text = builder.root.to_string(0)
+          expect(root_text).to eq(expected_text.chomp)
+          expected_state = 'T => . integer | 4'
+          expect(state_tracker.parse_state.to_s).to eq(expected_state)
+          expect(state_tracker.state_set_index).to eq(4)
+          next_symbol = "*[?, ?]: '(nil)'"
+          expect(builder.current_node.to_string(0)).to eq(next_symbol)
+        end
+
+        it 'should handle [no symbol before dot, terminal tree node] case' do
+          # Setup
+          (state_tracker, builder) = prepare_parse_tree(subject)
+          4.times do
+            subject.insert_matched_symbol(state_tracker, builder)
+          end
+
+          # Given current tree symbol is *[?, ?]: '(nil)'
+          # And current dotted item is T => . integer | 4
+          # When one retrieves the parse state expecting the T
+          # Then new parse state is changed to: M => M * . T | 2
+          subject.insert_matched_symbol(state_tracker, builder)
+
+          expected_text = <<-SNIPPET
+P[0, 5]
++- S[0, 5]
+   +- S[0, ?]
+   +- +[?, ?]: '(nil)'
+   +- M[2, 5]
+      +- M[2, ?]
+      +- *[?, ?]: '(nil)'
+      +- T[4, 5]
+         +- integer[4, 5]: '4'
+SNIPPET
+          root_text = builder.root.to_string(0)
+          expect(root_text).to eq(expected_text.chomp)
+          expected_state = 'M => M * . T | 2'
+          expect(state_tracker.parse_state.to_s).to eq(expected_state)
+          expect(state_tracker.state_set_index).to eq(4)
+          next_symbol = "*[?, ?]: '(nil)'"
+          expect(builder.current_node.to_string(0)).to eq(next_symbol)
+
+          subject.insert_matched_symbol(state_tracker, builder)
+          next_symbol = 'M[2, ?]'
+          expect(builder.current_node.to_string(0)).to eq(next_symbol)
+        end
+
+        it 'should handle the end of parse tree generation' do
+          # Begin setup
+          is_done = false
+          (state_tracker, builder) = prepare_parse_tree(subject)
+          16.times do
+            is_done = subject.insert_matched_symbol(state_tracker, builder)
+          end
+
+          expected_text = <<-SNIPPET
+P[0, 5]
++- S[0, 5]
+   +- S[0, 1]
+      +- M[0, 1]
+         +- T[0, 1]
+            +- integer[0, 1]: '2'
+   +- +[1, 2]: '+'
+   +- M[2, 5]
+      +- M[2, 3]
+         +- T[2, 3]
+            +- integer[2, 3]: '3'
+      +- *[3, 4]: '*'
+      +- T[4, 5]
+         +- integer[4, 5]: '4'
+SNIPPET
+          root_text = builder.root.to_string(0)
+          expect(root_text).to eq(expected_text.chomp)
+          
+          expected_state = 'T => . integer | 0'
+          expect(state_tracker.parse_state.to_s).to eq(expected_state)
+          expect(state_tracker.state_set_index).to eq(0)
+          expect(is_done).to eq(true)
+        end
+
+
 
         it 'should build the parse tree for a simple non-ambiguous grammar' do
           parser = EarleyParser.new(sample_grammar1)
@@ -155,76 +364,24 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           expect(ptree).to be_kind_of(PTree::ParseTree)
 
           # Expect parse tree:
-          # P[0, 5]
-          # +- S[0, 5]
-          #    +- S[0, 1]
-          #       +- M[0, 1]
-          #          +- T[0, 1]
-          #          +- integer(2)[0, 1]
-          #    +- +[?, ?]
-          #    +- M[2, 5]
-          expect(ptree.root.symbol). to eq(grm_symbol('P'))
-          expect(ptree.root.range). to eq([0, 5])
-          expect(ptree.root.children.size). to eq(1)
-
-          node = ptree.root.children[0] # S
-          expect(node.symbol). to eq(grm_symbol('S'))
-          expect(node.range). to eq([0, 5])
-          expect(node.children.size). to eq(3)
-
-          (node_s, node_plus, node_m) = node.children
-          expect(node_s.symbol).to eq(grm_symbol('S'))
-          expect(node_s.range).to eq(low: 0, high: 1)
-          expect(node_s.children.size).to eq(1)
-          expect(node_plus.symbol).to eq(grm_symbol('+'))
-          expect(node_plus.range).to eq(low: 0, high: 1)  # TODO: fix this
-          expect(node_plus.token.lexeme). to eq('+')
-          expect(node_m.symbol).to eq(grm_symbol('M'))
-          expect(node_m.range).to eq(low: 2, high: 5)
-          expect(node_m.children.size).to eq(3)
-          
-          node = node_s.children[0] # M
-          expect(node.symbol).to eq(grm_symbol('M'))
-          expect(node.range).to eq([0, 1])
-          expect(node.children.size).to eq(1)
-          
-          node = node.children[0] # T
-          expect(node.symbol).to eq(grm_symbol('T'))
-          expect(node.range).to eq([0, 1])
-          expect(node.children.size).to eq(1)
-          
-          node = node.children[0] # integer(2)
-          expect(node.symbol).to eq(grm_symbol('integer'))
-          expect(node.range).to eq([0, 1])
-          expect(node.token.lexeme).to eq('2')
-          
-          (node_m2, node_star, node_t3) = node_m.children
-          expect(node_m2.symbol).to eq(grm_symbol('M'))
-          expect(node_m2.range).to eq([2, 3])
-          expect(node_m2.children.size).to eq(1)
-          
-          node_t2 = node_m2.children[0] # T
-          expect(node_t2.symbol).to eq(grm_symbol('T'))
-          expect(node_t2.range).to eq([2, 3])
-          expect(node_t2.children.size).to eq(1)
-          
-          node = node_t2.children[0] # integer(3)
-          expect(node.symbol).to eq(grm_symbol('integer'))
-          expect(node.range).to eq([2, 3])
-          expect(node.token.lexeme).to eq('3')
-          
-          expect(node_star.symbol).to eq(grm_symbol('*'))
-          expect(node_star.range).to eq([2, 3])  # Fix this
-          expect(node_star.token.lexeme). to eq('*')
-          
-          expect(node_t3.symbol).to eq(grm_symbol('T'))
-          expect(node_t3.range).to eq([4, 5])
-          expect(node_t3.children.size).to eq(1)
-          
-          node = node_t3.children[0] # integer(4)
-          expect(node.symbol).to eq(grm_symbol('integer'))
-          expect(node.range).to eq([4, 5])
-          expect(node.token.lexeme).to eq('4')
+          expected_text = <<-SNIPPET
+P[0, 5]
++- S[0, 5]
+   +- S[0, 1]
+      +- M[0, 1]
+         +- T[0, 1]
+            +- integer[0, 1]: '2'
+   +- +[1, 2]: '+'
+   +- M[2, 5]
+      +- M[2, 3]
+         +- T[2, 3]
+            +- integer[2, 3]: '3'
+      +- *[3, 4]: '*'
+      +- T[4, 5]
+         +- integer[4, 5]: '4'
+SNIPPET
+          actual =  ptree.root.to_string(0)
+          expect(actual).to eq(expected_text.chomp)
         end
       end # context
     end # describe