rley 0.4.08 → 0.5.00

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ce91c79f9088da09449654c9b6e1b6e0581cf1c
-  data.tar.gz: 3b1e3cc1ce21d3b031b7b09cdbb7606e2eb5a747
+  metadata.gz: a0a5683c3a64ffa38c57d916ecbca45fa2f8f0b2
+  data.tar.gz: 77c4d6c99c3836c37e129cce40a9457fe74fbb8e
 SHA512:
-  metadata.gz: 7bd55bc558c857bab67f080169c1f9d72705fb66fda177c63032fdb0e8e6674efd1119f9fafba3681593dbc9b027beb05081bfe3f38fb99f46068f94b2563282
-  data.tar.gz: 54186ced4b193fa133b48ecdb2cddad7df0960cd2499b2ae48bc586a657cc7a2e49466b42570d78e8be4d0d123b6c821f5c9ab75b98d2e3c3d04d4876fd61f0e
+  metadata.gz: 7f4d20ab683694d83fb8f16933bd3cf7dccfa39e1166d0ae2571503f576074c53d13ad26006016ea4518627e3d94596ec91af527394e33d71d825f12c64611dc
+  data.tar.gz: 620870b9f7ddb191fb1899a13ae08dd97feb01ef33e96d6b40e752ac4e01d47b5bfd0f09107a017babe00fec08cb44579d45fae0d8583a4d8d801dd1f76de216

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+### 0.5.00 / 2017-08-20
+* [CHANGE] Concrete Syntax tree generation re-designed in prevision of customized tree generation.
+* [NEW] Class `Parser::CSTBuilder` Builder class that creates Concrete Syntax (parse) Tree.
+
+
 ### 0.4.08 / 2017-08-06
 * [FIX] File `/spec/spec_helper.rb` replaced deprecated syntax for `SimpleCov::Formatter::MultiFormatter` construction
 * [NEW] File `examples/data_formats/JSON/json_minifier.rb` Added a working JSON minifier to the demo app.

data/lib/rley/constants.rb

@@ -3,7 +3,7 @@
 
 module Rley # Module used as a namespace
   # The version number of the gem.
-  Version = '0.4.08'.freeze
+  Version = '0.5.00'.freeze
 
   # Brief description of the gem.
   Description = "Ruby implementation of the Earley's parsing algorithm".freeze

data/lib/rley/parse_tree_visitor.rb

@@ -14,6 +14,7 @@ module Rley # This module is used as a namespace
     # Build a visitor for the given ptree.
     # @param aParseTree [ParseTree] the parse tree to visit.
     def initialize(aParseTree, aTraversalStrategy = :post_order)
+      raise StandardError if aParseTree.nil?
       @ptree = aParseTree
       @subscribers = []
       @traversal = aTraversalStrategy

data/lib/rley/parser/cst_builder.rb

@@ -0,0 +1,271 @@
+require_relative '../tokens/token_range'
+require_relative '../syntax/terminal'
+require_relative '../syntax/non_terminal'
+require_relative '../gfg/end_vertex'
+require_relative '../gfg/item_vertex'
+require_relative '../gfg/start_vertex'
+require_relative '../ptree/non_terminal_node'
+require_relative '../ptree/terminal_node'
+require_relative '../ptree/parse_tree'
+
+module Rley # This module is used as a namespace
+  module Parser # This module is used as a namespace
+    # Structure used internally by CSTBuilder class.
+    CSTRawNode = Struct.new(:range, :symbol, :children) do
+      def initialize(aRange, aSymbol)
+        super
+        self.range = aRange
+        self.symbol = aSymbol
+        self.children = nil
+      end
+    end # Struct
+
+
+    # The purpose of a CSTBuilder is to build piece by piece a CST
+    # (Concrete Syntax Tree) from a sequence of input tokens and
+    # visit events produced by walking over a GFGParsing object.
+    # Uses the Builder GoF pattern.
+    # The Builder pattern creates a complex object
+    # (say, a parse tree) from simpler objects (terminal and non-terminal
+    # nodes) and using a step by step approach.
+    class CSTBuilder
+      # @return [Array<Token>] The sequence of input tokens
+      attr_reader(:tokens)
+
+      # Link to CST object (being) built.
+      attr_reader(:result)
+
+
+      # Create a new builder instance.
+      # @param theTokens [Array<Token>] The sequence of input tokens.
+      def initialize(theTokens)
+        @tokens = theTokens
+        @stack = []
+      end
+
+      # Receive events resulting from a visit of GFGParsing object.
+      # These events are produced by a specialized Enumerator created
+      # with a ParseWalkerFactory instance.
+      # @param anEvent [Symbol] Kind of visit event. Should be: :visit
+      # @param anEntry [ParseEntry] The entry being visited
+      # @param anIndex [anIndex] The token index associated with anEntry
+      def receive_event(anEvent, anEntry, anIndex)
+        # puts "Event: #{anEvent} #{anEntry} #{anIndex}"
+        if anEntry.dotted_entry? # A N => alpha . beta pattern?
+          process_item_entry(anEvent, anEntry, anIndex)
+        elsif anEntry.start_entry? # A .N pattern?
+          process_start_entry(anEvent, anEntry, anIndex)
+        elsif anEntry.end_entry? # A N. pattern?
+          process_end_entry(anEvent, anEntry, anIndex)
+        else
+          raise NotImplementedError
+        end
+
+        @last_visitee = anEntry
+      end
+
+      protected
+
+      # Return the stack
+      def stack()
+        return @stack
+      end
+
+      private
+
+      # Return the top of stack element.
+      def tos()
+        return @stack.last
+      end
+
+      # Handler for visit events for ParseEntry matching N. pattern
+      # @param anEvent [Symbol] Kind of visit event. Should be: :visit
+      # @param anEntry [ParseEntry] The entry being visited
+      # @param anIndex [anIndex] The token index at end of anEntry
+      def process_end_entry(anEvent, anEntry, anIndex)
+        case anEvent
+          when :visit
+            range = { low: anEntry.origin, high: anIndex }
+            non_terminal = entry2nonterm(anEntry)
+            # Create raw node and push onto stack
+            push_raw_node(range, non_terminal)
+          else
+            raise NotImplementedError
+        end
+      end
+
+      # Handler for visit events for ParseEntry matching .N pattern
+      # @param anEvent [Symbol] Kind of visit event. Should be: :visit
+      # @param anEntry [ParseEntry] The entry being visited
+      # @param anIndex [anIndex] The token index at end of anEntry
+      def process_start_entry(anEvent, anEntry, anIndex)
+        raise NotImplementedError unless [:visit, :revisit].include?(anEvent)
+      end
+
+      # Handler for visit events for ParseEntry matching N => alpha* . beta*
+      # @param anEvent [Symbol] Kind of visit event. Should be: :visit
+      # @param anEntry [ParseEntry] The entry being visited
+      # @param anIndex [anIndex] The token index at end of anEntry
+      def process_item_entry(anEvent, anEntry, anIndex)
+        # TODO: what if rhs is empty?
+        case anEvent
+          when :visit, :revisit
+            dot_pos = anEntry.vertex.dotted_item.position
+            if dot_pos.zero? || dot_pos < 0
+              # Check for pattern: N => alpha* .
+              process_exit_entry(anEntry, anIndex) if anEntry.exit_entry?
+
+              # Check for pattern: N => . alpha*
+              process_entry_entry(anEntry, anIndex) if anEntry.entry_entry?
+            else
+              # (pattern: N => alpha+ . beta+)
+              process_middle_entry(anEntry, anIndex)
+            end
+          else
+            $stderr.puts "waiko '#{anEvent}'"
+            raise NotImplementedError
+        end
+      end
+
+      # @param anEntry [ParseEntry] Entry matching (pattern: N => alpha* .)
+      # @param anIndex [anIndex] The token index at end of anEntry
+      def process_exit_entry(anEntry, anIndex)
+        production =  anEntry.vertex.dotted_item.production
+        count_rhs = production.rhs.members.size
+        init_TOS_children(count_rhs) # Create placeholders for children
+        build_terminal(anEntry, anIndex) if terminal_before_dot?(anEntry)
+      end
+
+      # @param anEntry [ParseEntry] Entry matching pattern: N => alpha+ . beta+
+      # @param anIndex [anIndex] The token index at end of anEntry
+      def process_middle_entry(anEntry, anIndex)
+        build_terminal(anEntry, anIndex) if terminal_before_dot?(anEntry)
+      end
+
+
+
+      # @param anEntry [ParseEntry] Entry matching (pattern: N => . alpha)
+      # @param anIndex [anIndex] The token index at end of anEntry
+      def process_entry_entry(anEntry, anIndex)
+        dotted_item = anEntry.vertex.dotted_item
+        rule = dotted_item.production
+        previous_tos = stack.pop
+        non_terminal = entry2nonterm(anEntry)
+        # For debugging purposes
+        raise StandardError if previous_tos.symbol != non_terminal
+        
+        new_node = new_parent_node(rule, previous_tos.range,
+                                     tokens, previous_tos.children)
+        if stack.empty?
+          @result = create_tree(new_node)
+        else
+          place_TOS_child(new_node, nil)
+        end
+      end
+
+      # Create a raw node with given range
+      # and push it on top of stack.
+      def push_raw_node(aRange, aSymbol)
+        raw_node = CSTRawNode.new(Tokens::TokenRange.new(aRange), aSymbol)
+        stack.push(raw_node)
+      end
+
+      # Initialize children array of TOS with nil placeholders.
+      # The number of elements equals the number of symbols at rhs.
+      def init_TOS_children(aCount)
+        tos.children = Array.new(aCount)
+      end
+
+      # Does the position on the left side of the dot correspond
+      # a terminal symbol?
+      # @param anEntry [ParseEntry] The entry being visited
+      def terminal_before_dot?(anEntry)
+        prev_symbol = anEntry.prev_symbol
+        return prev_symbol && prev_symbol.terminal?
+      end
+
+      # A terminal symbol was detected at left of dot.
+      # Build a raw node for that terminal and make it
+      # a child of TOS.
+      # @param anEntry [ParseEntry] The entry being visited
+      # @param anIndex [anIndex] The token index at end of anEntry
+      def build_terminal(anEntry, anIndex)
+        # First, build node for terminal...
+        term_symbol = anEntry.prev_symbol
+        token_position = anIndex - 1
+        token = tokens[token_position]
+        term_node = new_leaf_node(term_symbol, token_position, token)
+
+        # Second make it a child of TOS...
+        pos = anEntry.vertex.dotted_item.prev_position # position in rhs of rule
+        place_TOS_child(term_node, pos)
+      end
+
+      # Method to override
+      # Create a parse tree object with given
+      # node as root node.
+      def create_tree(aRootNode)
+        return Rley::PTree::ParseTree.new(aRootNode)
+      end
+
+      # Method to override
+      # Factory method for creating a node object for the given
+      # input token.
+      # @param _terminal [Terminal] Terminal symbol associated with the token
+      # @param aTokenPosition [Integer] Position of token in the input stream
+      # @param aToken [Token] The input token
+      def new_leaf_node(_terminal, aTokenPosition, aToken)
+        PTree::TerminalNode.new(aToken, aTokenPosition)
+      end
+
+      # Method to override.
+      # Factory method for creating a parent node object.
+      # @param aProduction [Production] Production rule
+      # @param aRange [Range] Range of tokens matched by the rule
+      # @param theTokens [Array] The input tokens
+      # @param theChildren [Array] Children nodes (one per rhs symbol)
+      def new_parent_node(aProduction, aRange, theTokens, theChildren)
+        node = Rley::PTree::NonTerminalNode.new(aProduction.lhs, aRange)
+        theChildren.reverse_each { |child| node.add_subnode(child) }
+        return node
+      end
+
+      # Place the given node object as one of the children of the TOS
+      # (TOS = Top Of Stack).
+      # Each child has a position that is dictated by the position of the
+      # related grammar symbol in the right-handed side (RHS) of the grammar
+      # rule.
+      # @param aNode [TerminalNode, NonTerminalNode] Node object to be placed
+      # @param aRHSPos [Integer, NilClass] Position in RHS of rule.
+      # If the position is provided, then the node will placed in the children
+      # array at that position.
+      # If the position is nil, then the node will be placed at the position of
+      # the rightmost nil element in children array.
+      def place_TOS_child(aNode, aRHSPos)
+        if aRHSPos.nil?
+          # Retrieve index of most rightmost nil child...
+          pos = tos.children.rindex { |child| child.nil? }
+          raise StandardError, 'Internal error' if pos.nil?
+        else
+          pos = aRHSPos
+        end
+
+        tos.children[pos] = aNode
+      end
+      
+      # Retrieve non-terminal symbol of given parse entry
+      def entry2nonterm(anEntry)
+        case anEntry.vertex
+        when GFG::StartVertex, GFG::EndVertex
+          non_terminal = anEntry.vertex.non_terminal
+        when GFG::ItemVertex
+          non_terminal = anEntry.vertex.lhs
+        end
+
+        return non_terminal
+      end
+    end # class
+  end # module
+end # module
+
+# End of file

data/lib/rley/parser/parse_tree_factory.rb

@@ -1,5 +1,6 @@
 require_relative 'parse_rep_creator'
-require_relative 'parse_tree_builder'
+# require_relative 'parse_tree_builder' # TODO remove this line
+require_relative 'cst_builder'
 
 module Rley # This module is used as a namespace
   module Parser # This module is used as a namespace
@@ -12,7 +13,7 @@ module Rley # This module is used as a namespace
       # Create a Builder, that is, an object
       # that will create piece by piece the forest
       def builder(aParseResult)
-        ParseTreeBuilder.new(aParseResult.tokens)
+        CSTBuilder.new(aParseResult.tokens)
       end
     end # class
   end # module

data/lib/rley/parser/parse_walker_factory.rb

@@ -124,9 +124,13 @@ module Rley # This module is used as a namespace
               event = [:revisit, anEntry, index]
 
             when GFG::StartVertex
+              # Even for non-ambiguous parse, can be caused by
+              # left recursive rule e.g. (S => S A)
               event = [:revisit, anEntry, index]
 
             when GFG::ItemVertex
+              # Even for non-ambiguous parse, can be caused by
+              # left recursive rule e.g. (S => S A)            
               # Skip item entries while revisiting
               event = [:revisit, anEntry, index]
             else

data/lib/rley/tokens/token_range.rb

@@ -29,6 +29,8 @@ module Rley # This module is used as a namespace
             result = low == other[:low] && high == other[:high]
           when TokenRange
             result = low == other.low && high == other.high
+          when Range
+            result = low == other.first && high == other.last
           when Array
             result = low == other[0] && high == other[1]
         end

data/spec/rley/parser/cst_builder_spec.rb

@@ -0,0 +1,439 @@
+require_relative '../../spec_helper'
+
+require_relative '../../../lib/rley/parser/gfg_earley_parser'
+require_relative '../../../lib/rley/parser/parse_walker_factory'
+
+require_relative '../support/expectation_helper'
+require_relative '../support/grammar_b_expr_helper'
+require_relative '../support/grammar_arr_int_helper'
+
+# Load the class under test
+require_relative '../../../lib/rley/parser/cst_builder'
+
+module Rley # Open this namespace to avoid module qualifier prefixes
+  module Parser
+    describe CSTBuilder do
+      include ExpectationHelper # Mix-in with expectation on parse entry sets
+      include GrammarBExprHelper # Mix-in for basic arithmetic language
+      include GrammarArrIntHelper # Mix-in for array of integers language
+
+      let(:sample_grammar) do
+          builder = grammar_expr_builder
+          builder.grammar
+      end
+
+      let(:sample_tokens) do
+        expr_tokenizer('2 + 3 * 4', sample_grammar)
+      end
+
+      subject { CSTBuilder.new(sample_tokens) }
+
+      def init_walker(theParser, theTokens)
+        result = theParser.parse(theTokens)
+        factory = ParseWalkerFactory.new
+        accept_entry = result.accepting_entry
+        accept_index = result.chart.last_index
+        @walker = factory.build_walker(accept_entry, accept_index)
+      end
+
+      def skip_events(count)
+        count.times do
+          event = @walker.next
+          subject.receive_event(*event)
+        end
+      end
+
+      def get_stack(aBuilder)
+        return aBuilder.send(:stack)
+      end
+
+      def create_range(low, high)
+        return Tokens::TokenRange.new({low: low, high: high })
+      end
+
+      context 'Initialization:' do
+        it 'should be created with a sequence of tokens' do
+          expect { CSTBuilder.new(sample_tokens) }.not_to raise_error
+        end
+
+        it 'should know the input tokens' do
+          expect(subject.tokens).to eq(sample_tokens)
+        end
+
+        it "shouldn't know the result yet" do
+          expect(subject.result).to be_nil
+        end
+
+        it 'should have an empty stack' do
+          expect(subject.send(:stack)).to be_empty
+        end
+      end # context
+
+      
+      
+      context 'Parse tree construction (no null symbol):' do
+        before(:each) do
+          parser = Parser::GFGEarleyParser.new(sample_grammar)
+          init_walker(parser, sample_tokens)
+        end
+
+        # Event: visit P. | 0 5
+        # Event: visit P => S . | 0 5
+        # Event: visit S. | 0 5
+        # Event: visit S => S + M . | 0 5
+        # Event: visit M. | 2 5
+        # Event: visit M => M * T . | 2 5
+        # Event: visit T. | 4 5
+        # Event: visit T => integer . | 4 5
+        # Event: visit T => . integer | 4 4
+        # Event: visit .T | 4 4
+        # Event: visit M => M * . T | 2 4
+        # Event: visit M => M . * T | 2 3
+        # Event: visit M. | 2 3
+        # Event: visit M => T . | 2 3
+        # Event: visit T. | 2 3
+        # Event: visit T => integer . | 2 3
+        # Event: visit T => . integer | 2 2
+        # Event: visit .T | 2 2
+        # Event: visit M => . T | 2 2
+        # Event: visit .M | 2 2
+        # Event: visit M => . M * T | 2 2
+        # Event: revisit .M | 2 2  <= revisit because of left recursive rule
+        # Event: visit S => S + . M | 0 2
+        # Event: visit S => S . + M | 0 1
+        # Event: visit S. | 0 1
+        # Event: visit S => M . | 0 1
+        # Event: visit M. | 0 1
+        # Event: visit M => T . | 0 1
+        # Event: visit T. | 0 1
+        # Event: visit T => integer . | 0 1
+        # Event: visit T => . integer | 0 0
+        # Event: visit .T | 0 0
+        # Event: visit M => . T | 0 0
+        # Event: visit .M | 0 0
+        # Event: visit S => . M | 0 0
+        # Event: visit .S | 0 0
+        # Event: visit S => . S + M | 0 0
+        # Event: revisit .S | 0 0
+        # Event: visit P => . S | 0 0
+        # Event: visit .P | 0 0
+        
+        it 'should react to a first end event' do
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(1)
+          expect(stack.last.range).to eq(create_range(0, 5))
+          expect(stack.last.children).to be_nil
+        end
+
+        it 'should react to a first exit event' do
+          skip_events(1)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(1)
+          expect(stack.last.children).to eq([nil])
+        end
+
+        it 'should react to a second end event' do
+          skip_events(2)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(2)
+          expect(stack.last.range).to eq(create_range(0, 5))
+          expect(stack.last.children).to be_nil
+        end
+
+        it 'should react to a second exit event' do
+          skip_events(3)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(2)
+          expect(stack.last.children).to eq([nil, nil, nil])
+        end
+
+        it 'should react to an exit event that creates a terminal node' do
+          skip_events(7)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(4)
+          expect(stack.last.children.size).to eq(1)
+          child = stack.last.children[-1]
+          expect(child).to be_kind_of(PTree::TerminalNode)
+          expect(child.to_s).to eq("integer[4, 5]: '4'")
+        end
+
+
+        it 'should react to a first entry event' do
+          skip_events(8)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(3) # Element popped
+          expect(stack.last.children.size).to eq(3)
+          child = stack.last.children[-1]
+          expect(child).to be_kind_of(PTree::NonTerminalNode)
+          expect(child.to_s).to eq('T[4, 5]')
+        end
+
+        it 'should react to a first start event' do
+          skip_events(9)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(3)
+        end
+
+        it 'should react to an middle event that creates a terminal node' do
+          skip_events(10)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(3)
+          expect(stack.last.children.size).to eq(3)
+          child = stack.last.children[1]
+          expect(child).to be_kind_of(PTree::TerminalNode)
+          expect(child.to_s).to eq("*[3, 4]: '*'")
+        end
+
+        it 'should react to an exit event that creates a terminal node' do
+          skip_events(15)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(5)
+          expect(stack.last.children.size).to eq(1)
+          child = stack.last.children[-1]
+          expect(child).to be_kind_of(PTree::TerminalNode)
+          expect(child.to_s).to eq("integer[2, 3]: '3'")
+        end
+
+        it 'should ignore to a revisit event' do
+          skip_events(21)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(2)
+          expect(stack.last.children.size).to eq(3)
+          child = stack.last.children[-1]
+          expect(child).to be_kind_of(PTree::NonTerminalNode)
+          expect(child.to_s).to eq('M[2, 5]')
+        end
+
+        it 'should react to a 2nd middle event that creates a terminal node' do
+          skip_events(22)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(2)
+          expect(stack.last.children.size).to eq(3)
+          child = stack.last.children[1]
+          expect(child).to be_kind_of(PTree::TerminalNode)
+          expect(child.to_s).to eq("+[1, 2]: '+'")
+        end
+
+        it 'should react to a exit event that creates a terminal node' do
+          skip_events(29)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(5)
+          expect(stack.last.children.size).to eq(1)
+          child = stack.last.children[-1]
+          expect(child).to be_kind_of(PTree::TerminalNode)
+          expect(child.to_s).to eq("integer[0, 1]: '2'")
+        end
+
+
+        it 'should react to entry event 31' do
+          skip_events(30)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(4)
+          expect(stack.last.children.size).to eq(1)
+          child = stack.last.children[-1]
+          expect(child).to be_kind_of(PTree::NonTerminalNode)
+          expect(child.to_s).to eq('T[0, 1]')
+        end
+
+        it 'should react to entry event 33' do
+          skip_events(32)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(3)
+          expect(stack.last.children.size).to eq(1)
+          child = stack.last.children[-1]
+          expect(child).to be_kind_of(PTree::NonTerminalNode)
+          expect(child.to_s).to eq('M[0, 1]')
+        end
+
+        it 'should react to entry event 35' do
+          skip_events(34)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(2)
+          expect(stack.last.children.size).to eq(3)
+          child = stack.last.children[0]
+          expect(child).to be_kind_of(PTree::NonTerminalNode)
+          expect(child.to_s).to eq('S[0, 1]')
+        end
+
+        it 'should react to entry event 37' do
+          skip_events(36)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack.size).to eq(1)
+          expect(stack.last.children.size).to eq(1)
+          child = stack.last.children[0]
+          expect(child).to be_kind_of(PTree::NonTerminalNode)
+          expect(child.to_s).to eq('S[0, 5]')
+        end
+
+        it 'should react to entry event that creates the tree' do
+          skip_events(38)
+          event = @walker.next
+          expect { subject.receive_event(*event) }.not_to raise_error
+          stack = get_stack(subject)
+          expect(stack).to be_empty
+          expect(subject.result).to be_kind_of(PTree::ParseTree)
+
+          # Lightweight sanity check
+          expect(subject.result.root.to_s).to eq('P[0, 5]')
+          expect(subject.result.root.subnodes.size).to eq(1)
+          child_node = subject.result.root.subnodes[0]
+          expect(child_node.to_s).to eq('S[0, 5]')
+          expect(child_node.subnodes.size).to eq(3)
+          first_grandchild = child_node.subnodes[0]
+          expect(first_grandchild.to_s).to eq('S[0, 1]')
+          second_grandchild = child_node.subnodes[1]
+          expect(second_grandchild.to_s).to eq("+[1, 2]: '+'")
+          third_grandchild = child_node.subnodes[2]
+          expect(third_grandchild.to_s).to eq('M[2, 5]')
+        end
+      end # context
+
+      context 'Parse tree construction with null symbol:' do
+        def next_event(expectation)
+          event = @walker.next
+          (ev_type, entry, index) = event
+          actual = "#{ev_type} #{entry} #{index}"
+          expect(actual).to eq(expectation)
+          @instance.receive_event(*event)
+        end
+
+        let(:array_grammar) do
+            builder = grammar_arr_int_builder
+            builder.grammar
+        end
+
+        before(:each) do
+          @parser = Parser::GFGEarleyParser.new(array_grammar)
+        end
+
+        # The visit events were generated with the following snippets:
+        # 13.times do
+          # event = @walker.next
+          # subject.receive_event(*event)
+        # end
+        # The events are:
+        # Event: visit P. | 0 2
+        # Event: visit P => arr . | 0 2
+        # Event: visit arr. | 0 2
+        # Event: visit arr => [ sequence ] . | 0 2
+        # Event: visit arr => [ sequence . ] | 0 1
+        # Event: visit sequence. | 1 1
+        # Event: visit sequence => . | 1 1
+        # Event: visit .sequence | 1 1
+        # Event: visit arr => [ . sequence ] | 0 1
+        # Event: visit arr => . [ sequence ] | 0 0
+        # Event: visit .arr | 0 0
+        # Event: visit P => . arr | 0 0
+        #   Event: visit .P | 0 0
+        it 'should build a tree for an empty array' do
+          empty_arr_tokens = arr_int_tokenizer('[ ]', array_grammar)
+          @instance = CSTBuilder.new(empty_arr_tokens)
+          init_walker(@parser, empty_arr_tokens)
+          stack = get_stack(@instance)
+
+          next_event('visit P. | 0 2')
+          expect(stack.size).to eq(1)
+          # stack: [P[0, 2]]
+          expect(stack.last.range).to eq(create_range(0, 2))
+          expect(stack.last.children).to be_nil
+
+          next_event('visit P => arr . | 0 2')
+          expect(stack.last.children).to eq([nil])
+
+          next_event('visit arr. | 0 2')
+          expect(stack.size).to eq(2)
+          # stack: [arr[0, 2], P[0, 2]]
+          expect(stack.last.range).to eq(create_range(0, 2))
+          expect(stack.last.children).to be_nil
+
+          next_event('visit arr => [ sequence ] . | 0 2')
+          expect(stack.size).to eq(2)
+          expect(stack.last.range).to eq(create_range(0, 2))
+          expect(stack.last.children.size).to eq(3)
+          child = stack.last.children.last
+          expect(child.to_s).to eq("][1, 2]: ']'")
+
+          next_event('visit arr => [ sequence . ] | 0 1')
+          expect(stack.size).to eq(2)
+
+          next_event('visit sequence. | 1 1')
+          expect(stack.size).to eq(3)
+          # stack: [sequence[1, 1], arr[0, 2], P[0, 2]]
+          expect(stack.last.range).to eq(create_range(1, 1))
+          expect(stack.last.children).to be_nil
+
+          next_event('visit sequence => . | 1 1')
+          expect(stack.size).to eq(2)
+          # stack: [arr[0, 2], P[0, 2]]
+          expect(stack.last.range).to eq(create_range(0, 2))
+          sequence = stack.last.children[1]
+          expect(sequence).to be_kind_of(PTree::NonTerminalNode)
+          expect(sequence.subnodes).to be_empty
+          expect(sequence.to_s).to eq('sequence[1, 1]')
+
+          next_event('visit .sequence | 1 1')
+          expect(stack.size).to eq(2)
+
+          next_event('visit arr => [ . sequence ] | 0 1')
+          expect(stack.size).to eq(2)
+          expect(stack.last.range).to eq(create_range(0, 2))
+          sequence = stack.last.children[0]
+          expect(sequence).to be_kind_of(PTree::TerminalNode)
+          expect(sequence.to_s).to eq("[[0, 1]: '['")
+
+          next_event('visit arr => . [ sequence ] | 0 0')
+          expect(stack.size).to eq(1)
+          # stack: [P[0, 2]]
+          expect(stack.last.range).to eq(create_range(0, 2))
+          expect(stack.last.children.size).to eq(1)
+          expect(stack.last.children[0]).to be_kind_of(PTree::NonTerminalNode)
+          expect(stack.last.children[0].to_s).to eq('arr[0, 2]')
+
+          next_event('visit .arr | 0 0')
+          expect(stack.size).to eq(1)
+
+          next_event('visit P => . arr | 0 0')
+          expect(stack).to be_empty
+          expect(@instance.result).not_to be_nil
+          
+          next_event('visit .P | 0 0')
+          expect(stack).to be_empty
+          expect(@instance.result).not_to be_nil          
+        end
+      end # context
+    end # describe
+  end # module
+end # module
+# End of file

data/spec/rley/support/grammar_arr_int_helper.rb

@@ -0,0 +1,41 @@
+# Load the builder class
+require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/tokens/token'
+
+
+module GrammarArrIntHelper
+  # Factory method. Creates a grammar builder for a grammar of
+  # array of integers.
+  # (based on the article about Earley's algorithm in Wikipedia)
+  def grammar_arr_int_builder()
+    builder = Rley::Syntax::GrammarBuilder.new do
+      add_terminals('[', ']', ',', 'integer')
+      rule 'P' => 'arr'
+      rule 'arr' => %w( [ sequence ] )
+      rule 'sequence' => ['list']
+      rule 'sequence' => []
+      rule 'list' => %w[list , integer]
+      rule 'list' => 'integer'
+    end
+    builder
+  end
+
+  # Basic tokenizer for array of integers
+  def arr_int_tokenizer(aText, aGrammar)
+    tokens = aText.scan(/\S+/).map do |lexeme|
+      case lexeme
+        when '[', ']', ','
+          terminal = aGrammar.name2symbol[lexeme]
+        when /^[-+]?\d+$/
+          terminal = aGrammar.name2symbol['integer']
+        else
+          msg = "Unknown input text '#{lexeme}'"
+          raise StandardError, msg
+      end
+      Rley::Tokens::Token.new(lexeme, terminal)
+    end
+
+    return tokens
+  end
+end # module
+# End of file

data/spec/rley/tokens/token_range_spec.rb

@@ -55,6 +55,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           expect(subject == me).to eq(true)
           equal = TokenRange.new(low: 0, high: 5)
           expect(subject == equal).to eq(true)
+          # expect(subject == [0..5]).to eq(true)
           expect(subject == [0, 5]).to eq(true)
         end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rley
 version: !ruby/object:Gem::Version
-  version: 0.4.08
+  version: 0.5.00
 platform: ruby
 authors:
 - Dimitri Geshef
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-06 00:00:00.000000000 Z
+date: 2017-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: coveralls
@@ -163,6 +163,7 @@ files:
 - lib/rley/parse_forest_visitor.rb
 - lib/rley/parse_tree_visitor.rb
 - lib/rley/parser/base_parser.rb
+- lib/rley/parser/cst_builder.rb
 - lib/rley/parser/dotted_item.rb
 - lib/rley/parser/error_reason.rb
 - lib/rley/parser/gfg_chart.rb
@@ -225,6 +226,7 @@ files:
 - spec/rley/parse_forest_visitor_spec.rb
 - spec/rley/parse_tree_visitor_spec.rb
 - spec/rley/parser/ambiguous_parse_spec.rb
+- spec/rley/parser/cst_builder_spec.rb
 - spec/rley/parser/dotted_item_spec.rb
 - spec/rley/parser/error_reason_spec.rb
 - spec/rley/parser/gfg_chart_spec.rb
@@ -252,6 +254,7 @@ files:
 - spec/rley/support/expectation_helper.rb
 - spec/rley/support/grammar_abc_helper.rb
 - spec/rley/support/grammar_ambig01_helper.rb
+- spec/rley/support/grammar_arr_int_helper.rb
 - spec/rley/support/grammar_b_expr_helper.rb
 - spec/rley/support/grammar_helper.rb
 - spec/rley/support/grammar_l0_helper.rb
@@ -315,6 +318,7 @@ test_files:
 - spec/rley/gfg/start_vertex_spec.rb
 - spec/rley/gfg/vertex_spec.rb
 - spec/rley/parser/ambiguous_parse_spec.rb
+- spec/rley/parser/cst_builder_spec.rb
 - spec/rley/parser/dotted_item_spec.rb
 - spec/rley/parser/error_reason_spec.rb
 - spec/rley/parser/gfg_chart_spec.rb