rley 0.7.06 → 0.8.01

data/spec/rley/parse_rep/parse_forest_factory_spec.rb

@@ -4,7 +4,7 @@ require_relative '../../spec_helper'
 
 require_relative '../../../lib/rley/parser/gfg_earley_parser'
 
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
 require_relative '../support/grammar_helper'
 require_relative '../support/expectation_helper'
 
@@ -22,15 +22,15 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           # "SPPF-Style Parsing From Earley Recognizers" in
           # Notes in Theoretical Computer Science 203, (2008), pp. 53-67
           # contains a hidden left recursion and a cycle
-          builder = Syntax::GrammarBuilder.new do
+          builder = Syntax::BaseGrammarBuilder.new do
             add_terminals('a', 'b')
             rule 'Phi' => 'S'
-            rule 'S' => %w[A T]
-            rule 'S' => %w[a T]
+            rule 'S' => 'A T'
+            rule 'S' => 'a T'
             rule 'A' => 'a'
-            rule 'A' => %w[B A]
+            rule 'A' => 'B A'
             rule 'B' => []
-            rule 'T' => %w[b b b]
+            rule 'T' => 'b b b'
           end
           builder.grammar
       end
@@ -49,11 +49,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
       end
 
       # Emit a text representation of the current path.
-      def path_to_s()
+      def path_to_s
         text_parts = subject.curr_path.map do |path_element|
           path_element.to_string(0)
         end
-        return text_parts.join('/')
+        text_parts.join('/')
       end
 
       context 'Initialization:' do

data/spec/rley/parse_rep/parse_tree_factory_spec.rb

@@ -3,7 +3,7 @@
 require_relative '../../spec_helper'
 
 require_relative '../../../lib/rley/parser/gfg_earley_parser'
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
 require_relative '../support/grammar_helper'
 require_relative '../support/grammar_abc_helper'
 require_relative '../support/expectation_helper'
@@ -38,11 +38,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
       end
 
       # Emit a text representation of the current path.
-      def path_to_s()
+      def path_to_s
         text_parts = subject.curr_path.map do |path_element|
           path_element.to_string(0)
         end
-        return text_parts.join('/')
+        text_parts.join('/')
       end
 
 

data/spec/rley/parse_tree_visitor_spec.rb

@@ -11,7 +11,7 @@ require_relative './support/grammar_sppf_helper'
 require_relative '../../lib/rley/parse_tree_visitor'
 
 module Rley # Open this namespace to avoid module qualifier prefixes
-  describe ParseTreeVisitor do    
+  describe ParseTreeVisitor do
     let(:grammar_abc) do
       sandbox = Object.new
       sandbox.extend(GrammarABCHelper)
@@ -45,7 +45,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
     # Capital letters represent non-terminal nodes
     let(:grm_abc_ptree1) do
       engine = Rley::Engine.new
-      engine.use_grammar(grammar_abc)     
+      engine.use_grammar(grammar_abc)
       parse_result = engine.parse(grm_abc_tokens1)
       ptree = engine.convert(parse_result)
       ptree
@@ -161,6 +161,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
         subject.end_visit_ptree(grm_abc_ptree1)
       end
 
+      # rubocop: disable Naming/VariableNumber
       it 'should begin the visit when requested' do
         # Reminder: parse tree structure is
         # S[0,5]
@@ -209,11 +210,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
         expectations.each do |(msg, args)|
           expect(listener1).to receive(msg).with(*args).ordered
         end
-        
+
         # Here we go...
         subject.start
       end
-      
+
       it 'should also visit in pre-order' do
         # Reminder: parse tree structure is
         # S[0,5]
@@ -228,7 +229,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
         root = grm_abc_ptree1.root
         # Here we defeat encapsulation for the good cause
         subject.instance_variable_set(:@traversal, :pre_order)
-        
+
         children = root.subnodes
         big_a_1 = children[0]
         big_a_1_children = big_a_1.subnodes
@@ -239,7 +240,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
         expectations = [
           [:before_ptree, [grm_abc_ptree1]]
           # TODO: fix this test
-          # [:before_subnodes, [root, children]],          
+          # [:before_subnodes, [root, children]],
           # [:before_non_terminal, [root]],
 
           # [:before_non_terminal, [big_a_1]],
@@ -267,10 +268,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
         expectations.each do |(msg, args)|
           expect(listener1).to receive(msg).with(*args).ordered
         end
-        
+
         # Here we go...
         subject.start
-      end      
+      end
+      # rubocop: enable Naming/VariableNumber
     end # context
   end # describe
 end # module

data/spec/rley/parser/dangling_else_spec.rb

@@ -0,0 +1,445 @@
+# frozen_string_literal: true
+
+require_relative '../../spec_helper'
+require 'stringio'
+require_relative '../../../lib/rley/syntax/match_closest'
+require_relative '../../../lib/rley/syntax/non_terminal'
+require_relative '../../../lib/rley/syntax/production'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
+require_relative '../../../lib/rley/lexical/token'
+require_relative '../../../lib/rley/base/dotted_item'
+require_relative '../../../lib/rley/parser/gfg_parsing'
+
+require_relative '../support/expectation_helper'
+
+# Load the class under test
+require_relative '../../../lib/rley/parser/gfg_earley_parser'
+
+module Rley # Open this namespace to avoid module qualifier prefixes
+  module Parser # Open this namespace to avoid module qualifier prefixes
+    describe GFGEarleyParser do
+      include ExpectationHelper # Mix-in with expectation on parse entry sets
+    
+      Keyword = {
+        'else' => 'ELSE',
+        'false' => 'FALSE',
+        'if' => 'IF',
+        'then' => 'THEN',
+        'true' => 'TRUE'
+      }.freeze      
+      
+      def tokenizer(aTextToParse)
+        scanner = StringScanner.new(aTextToParse)
+        tokens = []
+
+        loop do
+          scanner.skip(/\s+/)
+          break if scanner.eos?
+          curr_pos = scanner.pos          
+          lexeme = scanner.scan(/\S+/)        
+
+          term_name = Keyword[lexeme]
+          unless term_name
+            if lexeme =~ /\d+/
+              term_name = 'INTEGER'
+            else 
+              err_msg = "Unknown token '#{lexeme}'"
+              raise StandardError,  err_msg
+            end
+          end
+          pos = Rley::Lexical::Position.new(1, curr_pos + 1)
+          tokens << Rley::Lexical::Token.new(lexeme, term_name, pos)            
+        end
+
+        tokens
+      end
+
+      let(:input) { 'if false then if true then 1 else 2' }
+
+      context 'Ambiguous parse: ' do
+        # Factory method. Creates a grammar builder for a simple grammar. 
+        def grammar_if_else_amb
+          builder = Rley::Syntax::BaseGrammarBuilder.new do
+            add_terminals('IF', 'THEN', 'ELSE')
+            add_terminals('FALSE', 'TRUE', 'INTEGER')
+            
+            rule 'program' => 'stmt'
+            rule 'stmt' => 'IF boolean THEN stmt'
+            rule 'stmt' => 'IF boolean THEN stmt ELSE stmt'
+            rule 'stmt' => 'literal'
+            rule 'literal' => 'boolean'
+            rule 'literal' => 'INTEGER'
+            rule 'boolean' => 'FALSE'
+            rule 'boolean' => 'TRUE'
+          end
+          
+          builder.grammar
+        end
+
+        subject { GFGEarleyParser.new(grammar_if_else_amb) }
+
+        it 'should parse a valid simple input' do
+          tokens = tokenizer(input)
+          parse_result = subject.parse(tokens)
+          expect(parse_result.success?).to eq(true)
+          expect(parse_result.ambiguous?).to eq(true)
+          ######################
+          # Expectation chart[0]:
+          expected = [
+            '.program | 0',                                 # initialization
+            'program => . stmt | 0',                        # start rule
+            '.stmt | 0',                                    # call rule
+            'stmt => . IF boolean THEN stmt | 0',           # start rule
+            'stmt => . IF boolean THEN stmt ELSE stmt | 0', # start rule
+            'stmt => . literal | 0',                        # start rule
+            '.literal | 0',                                 # call rule
+            'literal => . boolean | 0',                     # start rule
+            'literal => . INTEGER | 0',                     # start rule
+            '.boolean | 0',                                 # call rule
+            'boolean => . FALSE | 0',                       # start rule
+            'boolean => . TRUE | 0'                         # start rule
+          ]
+          compare_entry_texts(parse_result.chart[0], expected)
+          expected_terminals(parse_result.chart[0], %w[FALSE IF INTEGER TRUE])
+
+          ######################
+          # Expectation chart[1]:
+          expected = [
+            'stmt => IF . boolean THEN stmt | 0',           # start rule
+            'stmt => IF . boolean THEN stmt ELSE stmt | 0', # start rule
+            '.boolean | 1',
+            'boolean => . FALSE | 1',                       # start rule
+            'boolean => . TRUE | 1'                         # start rule
+          ]
+          result1 = parse_result.chart[1]
+          expect(result1.entries.size).to eq(5)
+          compare_entry_texts(result1, expected)
+          expected_terminals(result1, %w[FALSE TRUE])
+
+          ######################
+          # Expectation chart[2]:
+          expected = [
+            'boolean => FALSE . | 1',
+            'boolean. | 1',
+            'stmt => IF boolean . THEN stmt | 0',
+            'stmt => IF boolean . THEN stmt ELSE stmt | 0'
+          ]
+          result2 = parse_result.chart[2]
+          expect(result2.entries.size).to eq(4)
+          compare_entry_texts(result2, expected)
+          expected_terminals(result2, %w[THEN])
+
+          ######################
+          # Expectation chart[3]:
+          expected = [
+            'stmt => IF boolean THEN . stmt | 0',
+            'stmt => IF boolean THEN . stmt ELSE stmt | 0',
+            '.stmt | 3',
+            'stmt => . IF boolean THEN stmt | 3',
+            'stmt => . IF boolean THEN stmt ELSE stmt | 3',
+            'stmt => . literal | 3',
+            '.literal | 3',
+            'literal => . boolean | 3',
+            'literal => . INTEGER | 3',
+            '.boolean | 3',
+            'boolean => . FALSE | 3',
+            'boolean => . TRUE | 3'
+          ]
+          result3 = parse_result.chart[3]
+          expect(result3.entries.size).to eq(12)
+          compare_entry_texts(result3, expected)
+          expected_terminals(result3, %w[FALSE IF INTEGER TRUE])
+
+
+          ######################
+          # Expectation chart[4]:
+          expected = [
+            'stmt => IF . boolean THEN stmt | 3',
+            'stmt => IF . boolean THEN stmt ELSE stmt | 3',
+            '.boolean | 4',
+            'boolean => . FALSE | 4',
+            'boolean => . TRUE | 4'
+          ]
+          result4 = parse_result.chart[4]
+          expect(result4.entries.size).to eq(5)
+          compare_entry_texts(result4, expected)
+          expected_terminals(result4, %w[FALSE TRUE])
+
+          ######################
+          # Expectation chart[5]:
+          expected = [
+            'boolean => TRUE . | 4',
+            'boolean. | 4',
+            'stmt => IF boolean . THEN stmt | 3',
+            'stmt => IF boolean . THEN stmt ELSE stmt | 3'
+          ]
+          result5 = parse_result.chart[5]
+          expect(result5.entries.size).to eq(4)
+          compare_entry_texts(result5, expected)
+          expected_terminals(result5, %w[THEN])
+
+          ######################
+          # Expectation chart[6]:
+          expected = [
+            'stmt => IF boolean THEN . stmt | 3',
+            'stmt => IF boolean THEN . stmt ELSE stmt | 3',
+            '.stmt | 6',
+            'stmt => . IF boolean THEN stmt | 6',
+            'stmt => . IF boolean THEN stmt ELSE stmt | 6',
+            'stmt => . literal | 6',
+            '.literal | 6',
+            'literal => . boolean | 6',
+            'literal => . INTEGER | 6',
+            '.boolean | 6',
+            'boolean => . FALSE | 6',
+            'boolean => . TRUE | 6'
+          ]
+          result6 = parse_result.chart[6]
+          expect(result6.entries.size).to eq(12)
+          compare_entry_texts(result6, expected)
+          expected_terminals(result6, %w[FALSE IF INTEGER TRUE])
+
+          ######################
+          # Expectation chart[7]:
+          expected = [
+            'literal => INTEGER . | 6',
+            'literal. | 6',
+            'stmt => literal . | 6',
+            'stmt. | 6',
+            'stmt => IF boolean THEN stmt . | 3',
+            'stmt => IF boolean THEN stmt . ELSE stmt | 3',
+            'stmt. | 3',
+            'stmt => IF boolean THEN stmt . | 0',
+            'stmt => IF boolean THEN stmt . ELSE stmt | 0',
+            'stmt. | 0',
+            'program => stmt . | 0',
+            'program. | 0'
+          ]
+          result7 = parse_result.chart[7]
+          expect(result7.entries.size).to eq(12)
+          compare_entry_texts(result7, expected)
+          expected_terminals(result7, %w[ELSE])
+
+          # Expectation chart[8]:
+          expected = [
+            'stmt => IF boolean THEN stmt ELSE . stmt | 3',
+            'stmt => IF boolean THEN stmt ELSE . stmt | 0',
+            '.stmt | 8',
+            'stmt => . IF boolean THEN stmt | 8',
+            'stmt => . IF boolean THEN stmt ELSE stmt | 8',
+            'stmt => . literal | 8',
+            '.literal | 8',
+            'literal => . boolean | 8',
+            'literal => . INTEGER | 8',
+            '.boolean | 8',
+            'boolean => . FALSE | 8',
+            'boolean => . TRUE | 8'
+          ]
+          result8 = parse_result.chart[8]
+          expect(result8.entries.size).to eq(12)
+          compare_entry_texts(result8, expected)
+          expected_terminals(result8, %w[FALSE IF INTEGER TRUE])
+
+          ######################
+          # Expectation chart[9]:
+          expected = [
+            'literal => INTEGER . | 8',
+            'literal. | 8',
+            'stmt => literal . | 8',
+            'stmt. | 8',
+            'stmt => IF boolean THEN stmt ELSE stmt . | 3',
+            'stmt => IF boolean THEN stmt ELSE stmt . | 0',
+            'stmt. | 3',
+            'stmt. | 0',
+            'stmt => IF boolean THEN stmt . | 0',
+            'stmt => IF boolean THEN stmt . ELSE stmt | 0',
+            'program => stmt . | 0',
+            'program. | 0'
+          ]
+          result9 = parse_result.chart[9]
+          expect(result9.entries.size).to eq(12)
+          compare_entry_texts(result9, expected)
+          expected_terminals(result9, %w[ELSE])
+
+          ######################
+          # Expectation chart[10]:
+          result10 = parse_result.chart[10]
+          expect(result10).to be_nil
+
+          # The parse is ambiguous since there more than one dotted item
+          # that matches the stmt. | 0 exit node on chart[9]:
+          # stmt => IF boolean THEN stmt ELSE stmt . | 0'
+          # stmt => IF boolean THEN stmt . | 0'
+          #
+          # This is related to the "dangling else problem"
+        end
+      end # context
+
+      context 'Disambiguated parse: ' do
+        def match_else_with_if(grammar)
+          # Brittle code
+          prod = grammar.rules[2]
+          constraint = Syntax::MatchClosest.new(prod.rhs.members, 4, 'IF')
+          prod.constraints << constraint
+        end
+
+        # Factory method. Creates a grammar builder for a simple grammar.
+        def grammar_if_else
+          builder = Rley::Syntax::BaseGrammarBuilder.new do
+            add_terminals('IF', 'THEN', 'ELSE')
+            add_terminals('FALSE', 'TRUE', 'INTEGER')
+
+            rule 'program' => 'stmt'
+            rule 'stmt' => 'IF boolean THEN stmt'
+
+            # To prevent dangling else issue, the ELSE must match the closest preceding IF
+            # rule 'stmt' => 'IF boolean THEN stmt ELSE{closest IF} stmt'
+            rule 'stmt' => 'IF boolean THEN stmt ELSE stmt'
+            rule 'stmt' => 'literal'
+            rule 'literal' => 'boolean'
+            rule 'literal' => 'INTEGER'
+            rule 'boolean' => 'FALSE'
+            rule 'boolean' => 'TRUE'
+          end
+
+          grm = builder.grammar
+          match_else_with_if(grm)
+
+          grm
+        end
+
+        subject { GFGEarleyParser.new(grammar_if_else) }
+
+        it 'should cope with dangling else problem' do
+          tokens = tokenizer(input)
+          parse_result = subject.parse(tokens)
+          expect(parse_result.success?).to eq(true)
+          expect(parse_result.ambiguous?).to eq(true)
+          ######################
+          # Expectation chart[0]:
+          expected = [
+            '.program | 0',                                 # initialization
+            'program => . stmt | 0',                        # start rule
+            '.stmt | 0',                                    # call rule
+            'stmt => . IF boolean THEN stmt | 0',           # start rule
+            'stmt => . IF boolean THEN stmt ELSE stmt | 0', # start rule
+            'stmt => . literal | 0',                        # start rule
+            '.literal | 0',                                 # call rule
+            'literal => . boolean | 0',                     # start rule
+            'literal => . INTEGER | 0',                     # start rule
+            '.boolean | 0',                                 # call rule
+            'boolean => . FALSE | 0',                       # start rule
+            'boolean => . TRUE | 0'                         # start rule
+          ]
+          compare_entry_texts(parse_result.chart[0], expected)
+          expected_terminals(parse_result.chart[0], %w[FALSE IF INTEGER TRUE])
+
+          # The parser should work as the previous version...
+          # we skip chart[2] and chart[3]
+          ######################
+          # Expectation chart[4]:
+          expected = [
+            'stmt => IF . boolean THEN stmt | 3',
+            'stmt => IF . boolean THEN stmt ELSE stmt | 3',
+            '.boolean | 4',
+            'boolean => . FALSE | 4',
+            'boolean => . TRUE | 4'
+          ]
+          result4 = parse_result.chart[4]
+          expect(result4.entries.size).to eq(5)
+          compare_entry_texts(result4, expected)
+          expected_terminals(result4, %w[FALSE TRUE])
+
+          ######################
+          # Before reading ELSE
+          # Expectation chart[7]:
+          expected = [
+            'literal => INTEGER . | 6',
+            'literal. | 6',
+            'stmt => literal . | 6',
+            'stmt. | 6',
+            'stmt => IF boolean THEN stmt . | 3',
+            'stmt => IF boolean THEN stmt . ELSE stmt | 3',
+            'stmt. | 3',
+            'stmt => IF boolean THEN stmt . | 0',
+            'stmt => IF boolean THEN stmt . ELSE stmt | 0',
+            'stmt. | 0',
+            'program => stmt . | 0',
+            'program. | 0'
+          ]
+          result7 = parse_result.chart[7]
+          expect(result7.entries.size).to eq(12)
+          compare_entry_texts(result7, expected)
+          expected_terminals(result7, %w[ELSE])
+
+          ######################
+          # After reading ELSE
+          # Expectation chart[8]:
+          expected = [
+            'stmt => IF boolean THEN stmt ELSE . stmt | 3',
+            # 'stmt => IF boolean THEN stmt ELSE . stmt | 0', # Excluded
+            '.stmt | 8',
+            'stmt => . IF boolean THEN stmt | 8',
+            'stmt => . IF boolean THEN stmt ELSE stmt | 8',
+            'stmt => . literal | 8',
+            '.literal | 8',
+            'literal => . boolean | 8',
+            'literal => . INTEGER | 8',
+            '.boolean | 8',
+            'boolean => . FALSE | 8',
+            'boolean => . TRUE | 8'
+          ]
+          result8 = parse_result.chart[8]
+          found = parse_result.chart.search_entries(4, {before: 'IF'})
+          expect(result8.entries.size).to eq(11)
+          compare_entry_texts(result8, expected)
+          expected_terminals(result8, %w[FALSE IF INTEGER TRUE])
+
+          # How does it work?
+          # ELSE was just read at position 7
+          # We look backwards to nearest IF; there is one at position 3
+          # In chart[8], we should exclude the dotted item:
+          # 'stmt => IF boolean THEN stmt ELSE . stmt | 0'
+          # Reasoning?
+          # On chart[4], we find two entries for the IF .:
+          # 'stmt => IF . boolean THEN stmt | 3',
+          # 'stmt => IF . boolean THEN stmt ELSE stmt | 3'
+          #  Only these productions that still applies at 8 must be retained
+          # 'stmt => IF boolean THEN stmt ELSE . stmt | 3',
+          # 'stmt => IF boolean THEN stmt ELSE . stmt | 0', # To exclude
+          # Where to place the check?
+          # At the dotted item?
+          # call, return scan nodes
+          # So if one has an annotated production rule:
+          # stmt => IF boolean THEN stmt ELSE{ closest: IF }  stmt
+          # then the dotted item:
+          # stmt => IF boolean THEN stmt ELSE . stmt
+          # should bear the constraint
+
+          ######################
+          # Expectation chart[9]:
+          expected = [
+            'literal => INTEGER . | 8',
+            'literal. | 8',
+            'stmt => literal . | 8',
+            'stmt. | 8',
+            'stmt => IF boolean THEN stmt ELSE stmt . | 3',
+            # 'stmt => IF boolean THEN stmt ELSE stmt . | 0', # Excluded
+            'stmt. | 3',
+            'stmt => IF boolean THEN stmt . | 0',
+            'stmt => IF boolean THEN stmt . ELSE stmt | 0',
+            'stmt. | 0',
+            'program => stmt . | 0',
+            'program. | 0'
+          ]
+          result9 = parse_result.chart[9]
+          expect(result9.entries.size).to eq(11)
+          compare_entry_texts(result9, expected)
+          expected_terminals(result9, ['ELSE'])
+        end
+      end # context
+    end # describe
+  end # module
+end # module
+  
+