rley 0.7.08 → 0.8.03

data/spec/rley/parser/gfg_earley_parser_spec.rb

@@ -5,7 +5,7 @@ require 'stringio'
 require_relative '../../../lib/rley/syntax/verbatim_symbol'
 require_relative '../../../lib/rley/syntax/non_terminal'
 require_relative '../../../lib/rley/syntax/production'
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
 require_relative '../../../lib/rley/lexical/token'
 require_relative '../../../lib/rley/base/dotted_item'
 require_relative '../../../lib/rley/parser/gfg_parsing'
@@ -13,6 +13,7 @@ require_relative '../../../lib/rley/parser/gfg_parsing'
 # Load builders and lexers for sample grammars
 require_relative '../support/grammar_abc_helper'
 require_relative '../support/ambiguous_grammar_helper'
+require_relative '../support/grammar_int_seq_helper'
 require_relative '../support/grammar_pb_helper'
 require_relative '../support/grammar_helper'
 require_relative '../support/expectation_helper'
@@ -21,6 +22,7 @@ require_relative '../support/expectation_helper'
 require_relative '../../../lib/rley/parser/gfg_earley_parser'
 
 module Rley # Open this namespace to avoid module qualifier prefixes
+  # rubocop: disable Metrics/BlockLength
   module Parser # Open this namespace to avoid module qualifier prefixes
     describe GFGEarleyParser do
       include GrammarABCHelper # Mix-in module with builder for grammar abc
@@ -281,6 +283,91 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           compare_entry_texts(parse_result.chart[5], expected)
         end
 
+        it 'should support Kleene plus ' do
+          extend(GrammarIntSeqHelper)
+          grammar = grammar_int_seq_builder.grammar
+          instance = GFGEarleyParser.new(grammar)
+          tokens = int_seq_tokenizer('6, 36, 216')
+          parse_result = nil
+          expect { parse_result = instance.parse(tokens) }.not_to raise_error
+          expect(parse_result.success?).to eq(true)
+
+          ###################### S(0): . 6, 36, 216
+          # Expectation chart[0]:
+          expected = [
+            '.S | 0',                                   # Initialization
+            'S => . sequence | 0',                      # start rule
+            'S => . | 0',                               # start rule
+            '.sequence | 0',                            # call rule
+            'S. | 0',                                   # exit rule
+            'sequence => . sequence comma integer | 0', # start rule
+            'sequence => . integer | 0',                # start rule
+            '.integer | 0',                             # call rule
+            'integer => . digit_plus | 0',              # start rule
+            '.digit_plus | 0',                          # call rule
+            'digit_plus => . digit_plus digit | 0',     # start rule (generated)
+            'digit_plus => . digit | 0'                 # start rule (generated)
+          ]
+          compare_entry_texts(parse_result.chart[0], expected)
+
+          ###################### S(1): 6 ., 36, 216
+          # Expectation chart[1]:
+          expected = [
+            'digit_plus => digit . | 0',                # Scan
+            'digit_plus. | 0',                          # exit rule
+            'integer => digit_plus . | 0',              # end rule
+            'digit_plus => digit_plus . digit | 0',     # rule (generated)
+            'integer. | 0',                             # exit rule
+            'sequence => integer . | 0',                # end rule
+            'sequence. | 0',                            # exit rule
+            'S => sequence . | 0',                      # end rule
+            'sequence => sequence . comma integer | 0', #  rule
+            'S. | 0'                                    # exit rule
+          ]
+          compare_entry_texts(parse_result.chart[1], expected)
+
+          ###################### S(2): 6 , . 36, 216
+          # Expectation chart[2]:
+          expected = [
+            'sequence => sequence comma . integer | 0', #  Scan
+            '.integer | 2',                             # call rule
+            'integer => . digit_plus | 2',              # start rule
+            '.digit_plus | 2',                          # call rule
+            'digit_plus => . digit_plus digit | 2',     # start rule (generated)
+            'digit_plus => . digit | 2'                 # start rule (generated)
+          ]
+          compare_entry_texts(parse_result.chart[2], expected)
+
+          ###################### S(3): 6 , 3 . 6. , 216
+          # Expectation chart[3]:
+          expected = [
+            'digit_plus => digit . | 2',                # Scan
+            'digit_plus. | 2',                          # exit rule
+            'integer => digit_plus . | 2',              # end rule
+            'digit_plus => digit_plus . digit | 2',     # rule (generated)
+            'integer. | 2',                             # exit rule
+            'sequence => sequence comma integer . | 0', # rule
+            'sequence. | 0',                            # exit rule
+            'S => sequence . | 0',                      # end rule
+            'sequence => sequence . comma integer | 0'  #  rule
+          ]
+          compare_entry_texts(parse_result.chart[3], expected)
+
+          ###################### S(4): 6 , 36 . . , 216
+          # Expectation chart[4]:
+          expected = [
+            'digit_plus => digit_plus digit . | 2',     # Scan
+            'digit_plus. | 2',                          # exit rule
+            'integer => digit_plus . | 2',              # end rule
+            'digit_plus => digit_plus . digit | 2',     #
+            'integer. | 2',                             # exit rule
+            'sequence => sequence comma integer . | 0', # rule
+            'sequence. | 0',                            # exit rule
+            'S => sequence . | 0'                       # end rule
+          ]
+          compare_entry_texts(parse_result.chart[4], expected)
+        end
+
         it 'should parse a nullable grammar' do
           # Simple but problematic grammar for the original Earley parser
           # (based on example in D. Grune, C. Jacobs "Parsing Techniques" book)
@@ -288,7 +375,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           # A => ;
           t_x = Syntax::VerbatimSymbol.new('x')
 
-          builder = Syntax::GrammarBuilder.new do
+          builder = Notation::GrammarBuilder.new do
             add_terminals(t_x)
             rule 'Ss' => 'A A x'
             rule 'A' => []
@@ -334,11 +421,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           t_plus = Syntax::VerbatimSymbol.new('+')
           t_star = Syntax::VerbatimSymbol.new('*')
 
-          builder = Syntax::GrammarBuilder.new do
+          builder = Syntax::BaseGrammarBuilder.new do
             add_terminals(t_int, t_plus, t_star)
             rule 'P' => 'S'
-            rule 'S' => %w[S + S]
-            rule 'S' => %w[S * S]
+            rule 'S' => 'S + S'
+            rule 'S' => 'S * S'
             rule 'S' => 'L'
             rule 'L' => 'integer'
           end
@@ -733,7 +820,7 @@ MSG
           t_star = Syntax::VerbatimSymbol.new('*')
           t_slash = Syntax::VerbatimSymbol.new('/')
 
-          builder = Syntax::GrammarBuilder.new do
+          builder = Syntax::BaseGrammarBuilder.new do
             add_terminals(t_a, t_star, t_slash)
             rule 'Z' => 'E'
             rule 'E' => %w[E Q F]
@@ -848,10 +935,11 @@ MSG
           # S => ;
           # This grammar requires a time that is quadratic in the number of
           # input tokens
-          builder = Syntax::GrammarBuilder.new
-          builder.add_terminals('a')
-          builder.add_production('S' => %w[a S])
-          builder.add_production('S' => [])
+          builder = Notation::GrammarBuilder.new do
+            add_terminals('a')
+            rule('S' => 'a S')
+            rule('S' => '')
+          end
           grammar = builder.grammar
           tokens = build_token_sequence(%w[a a a a], grammar)
 
@@ -931,8 +1019,28 @@ MSG
           ]
           compare_entry_texts(parse_result.chart[4], expected)
         end
+
+        it 'should support modifier(s) in start rule' do
+          # An implicit EOF marker is a special terminal
+          # that denotes the end of input string but doesn't
+          # appear explicitly as some character or text in the input.
+          builder = Notation::GrammarBuilder.new do
+            add_terminals('a', 'b', 'EOF')
+
+            rule('S' => 'a_or_b* EOF')
+            rule('a_or_b' => 'a')
+            rule('a_or_b' => 'b')
+          end
+          grammar = builder.grammar
+          tokens = build_token_sequence(%w[EOF], grammar)
+          tokens[0].instance_variable_set(:@lexeme, '')
+          instance = GFGEarleyParser.new(grammar)
+          parse_result = instance.parse(tokens)
+          expect(parse_result.success?).to eq(true)
+        end
       end # context
     end # describe
   end # module
+  # rubocop: enable Metrics/BlockLength
 end # module
 # End of file

data/spec/rley/parser/gfg_parsing_spec.rb

@@ -6,7 +6,7 @@ require 'stringio'
 require_relative '../../../lib/rley/syntax/non_terminal'
 require_relative '../../../lib/rley/syntax/verbatim_symbol'
 require_relative '../../../lib/rley/syntax/production'
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
 require_relative '../../../lib/rley/base/dotted_item'
 require_relative '../../../lib/rley/lexical/token'
 require_relative '../../../lib/rley/parser/parse_tracer'
@@ -316,6 +316,7 @@ State[5]
   S => S . + M | 0
   P. | 0
 REPR
+          expect(expectation).to eq(expectation)
         end
       end # context
 

data/spec/rley/parser/parse_walker_factory_spec.rb

@@ -2,7 +2,7 @@
 
 require_relative '../../spec_helper'
 
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
 require_relative '../support/grammar_helper'
 require_relative '../support/expectation_helper'
 
@@ -45,7 +45,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           # "SPPF=Style Parsing From Earley Recognizers" in
           # Notes in Theoretical Computer Science 203, (2008), pp. 53-67
           # contains a hidden left recursion and a cycle
-          builder = Syntax::GrammarBuilder.new do
+          builder = Syntax::BaseGrammarBuilder.new do
             add_terminals('a', 'b')
             rule 'Phi' => 'S'
             rule 'S' => %w[A T]

data/spec/rley/support/ambiguous_grammar_helper.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Load the builder class
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/notation/grammar_builder'
 require_relative '../../../lib/rley/lexical/token'
 
 
@@ -10,7 +10,7 @@ module AmbiguousGrammarHelper
   # expression grammar.
   # (based on an example from Fisher and LeBlanc: "Crafting a Compiler")
   def grammar_builder
-    Rley::Syntax::GrammarBuilder.new do
+    Rley::Notation::GrammarBuilder.new do
       add_terminals('+', 'id')
       rule 'S' => 'E'
       rule 'E' => 'E + E'

data/spec/rley/support/grammar_abc_helper.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
 # Load the builder class
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
 
 module GrammarABCHelper
   # Factory method. Creates a grammar builder for a simple grammar.
   # (based on example in N. Wirth "Compiler Construction" book, p. 6)
   def grammar_abc_builder
-    Rley::Syntax::GrammarBuilder.new do
+    Rley::Syntax::BaseGrammarBuilder.new do
       add_terminals('a', 'b', 'c')
       rule 'S' => 'A'
       rule 'A' => 'a A c'

data/spec/rley/support/grammar_ambig01_helper.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Load the builder class
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
 require_relative '../../../lib/rley/lexical/token'
 
 
@@ -11,7 +11,7 @@ module GrammarAmbig01Helper
   # Grammar 3: An ambiguous arithmetic expression language
   # (based on example in article on Earley's algorithm in Wikipedia)
   def grammar_ambig01_builder
-    Rley::Syntax::GrammarBuilder.new do
+    Rley::Syntax::BaseGrammarBuilder.new do
       add_terminals('integer', '+', '*')
       rule 'P' => 'S'
       rule 'S' => 'S + S'

data/spec/rley/support/grammar_arr_int_helper.rb

@@ -3,7 +3,7 @@
 require 'strscan'
 
 # Load the builder class
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
 require_relative '../../../lib/rley/lexical/token'
 
 
@@ -12,7 +12,7 @@ module GrammarArrIntHelper
   # array of integers.
   # (based on the article about Earley's algorithm in Wikipedia)
   def grammar_arr_int_builder
-    Rley::Syntax::GrammarBuilder.new do
+    Rley::Syntax::BaseGrammarBuilder.new do
       add_terminals('[', ']', ',', 'integer')
       rule 'P' => 'arr'
       rule 'arr' => '[ sequence ]'

data/spec/rley/support/grammar_b_expr_helper.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Load the builder class
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
 require_relative '../../../lib/rley/lexical/token'
 
 
@@ -10,7 +10,7 @@ module GrammarBExprHelper
   # expression grammar.
   # (based on the article about Earley's algorithm in Wikipedia)
   def grammar_expr_builder
-    Rley::Syntax::GrammarBuilder.new do
+    Rley::Syntax::BaseGrammarBuilder.new do
       add_terminals('+', '*', 'integer')
       rule 'P' => 'S'
       rule 'S' => 'S + M'

data/spec/rley/support/grammar_int_seq_helper.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'strscan'
+
+# Load the builder class
+require_relative '../../../lib/rley/notation/grammar_builder'
+require_relative '../../../lib/rley/lexical/token'
+
+
+module GrammarIntSeqHelper
+  # Factory method. Creates a builder for a grammar of sequence
+  # of positive integers.
+  def grammar_int_seq_builder
+    Rley::Notation::GrammarBuilder.new do
+      add_terminals('comma', 'digit')
+      rule 'S' => 'sequence'
+      rule 'S' => ''
+      rule 'sequence' => 'sequence comma integer'
+      rule 'sequence' => 'integer'
+      rule 'integer' => 'digit+'
+    end
+  end
+
+  # Basic tokenizer for sequence positive integers
+  def int_seq_tokenizer(aText)
+    scanner = StringScanner.new(aText)
+    tokens = []
+
+    loop do
+      scanner.skip(/\s+/)
+      break if scanner.eos?
+
+      curr_pos = scanner.pos
+
+      if (lexeme = scanner.scan(/,/))
+        terminal = 'comma'
+      elsif (lexeme = scanner.scan(/\d/))
+        terminal = 'digit'
+      else
+        msg = "Unknown input text '#{scanner.scan(/.*/)}'"
+        raise StandardError, msg
+      end
+
+      pos = Rley::Lexical::Position.new(1, curr_pos + 1)
+      tokens << Rley::Lexical::Token.new(lexeme, terminal, pos)
+    end
+
+    return tokens
+  end
+end # module
+# End of file

data/spec/rley/support/grammar_l0_helper.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Load the builder class
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
 require_relative '../../../lib/rley/lexical/token'
 
 
@@ -12,7 +12,7 @@ module GrammarL0Helper
   # It defines the syntax of a sentence in a language with a
   # very limited syntax and lexicon in the context of airline reservation.
   def grammar_l0_builder
-    Rley::Syntax::GrammarBuilder.new do
+    Rley::Syntax::BaseGrammarBuilder.new do
       add_terminals('Noun', 'Verb', 'Pronoun', 'Proper-Noun')
       add_terminals('Determiner', 'Preposition')
       rule 'S' => 'NP VP'

data/spec/rley/support/grammar_pb_helper.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Load the builder class
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
 require_relative '../../../lib/support/base_tokenizer'
 require_relative '../../../lib/rley/lexical/token'
 
@@ -14,7 +14,7 @@ class GrammarPBHelper
   # "A Graphical Model for Context-Free Grammar Parsing"
   def grammar
     @grammar ||= begin
-      builder = Rley::Syntax::GrammarBuilder.new do
+      builder = Rley::Syntax::BaseGrammarBuilder.new do
         add_terminals('int', '+', '(', ')')
         rule 'S' => 'E'
         rule 'E' => 'int'

data/spec/rley/support/grammar_sppf_helper.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Load the builder class
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
 
 
 module GrammarSPPFHelper
@@ -11,7 +11,7 @@ module GrammarSPPFHelper
   # Notes in Theoretical Computer Science 203, (2008), pp. 53-67
   # contains a hidden left recursion and a cycle
   def grammar_sppf_builder
-    builder = Rley::Syntax::GrammarBuilder.new do
+    builder = Rley::Syntax::BaseGrammarBuilder.new do
       add_terminals('a', 'b')
       rule 'Phi' => 'S'
       rule 'S' => %w[A T]

data/spec/rley/syntax/{grammar_builder_spec.rb → base_grammar_builder_spec.rb}

@@ -3,14 +3,14 @@
 require_relative '../../spec_helper'
 
 # Load the class under test
-require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/rley/syntax/base_grammar_builder'
 
 module Rley # Open this namespace to avoid module qualifier prefixes
   module Syntax # Open this namespace to avoid module qualifier prefixes
-    describe GrammarBuilder do
+    describe BaseGrammarBuilder do
       context 'Initialization without argument:' do
         it 'could be created without argument' do
-          expect { GrammarBuilder.new }.not_to raise_error
+          expect { BaseGrammarBuilder.new }.not_to raise_error
         end
 
         it 'should have no grammar symbols at start' do
@@ -25,12 +25,12 @@ module Rley # Open this namespace to avoid module qualifier prefixes
       context 'Initialization with argument:' do
         it 'could be created with a block argument' do
           expect do
-            GrammarBuilder.new { nil }
+            BaseGrammarBuilder.new { nil }
           end.not_to raise_error
         end
 
         it 'could have grammar symbols from block argument' do
-          instance = GrammarBuilder.new do
+          instance = BaseGrammarBuilder.new do
             add_terminals('a', 'b', 'c')
           end
           expect(instance.symbols.size).to eq(3)
@@ -80,7 +80,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
 
       context 'Adding productions:' do
         subject do
-          instance = GrammarBuilder.new
+          instance = BaseGrammarBuilder.new
           instance.add_terminals('a', 'b', 'c')
           instance
         end
@@ -108,18 +108,36 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           expect_rhs = [subject['a'], subject['A'], subject['c']]
           expect(new_prod.rhs.members).to eq(expect_rhs)
 
-          # GrammarBuilder#rule is an alias of add_production
+          # BaseGrammarBuilder#rule is an alias of add_production
           subject.rule('A' => ['b'])
           expect(subject.productions.size).to eq(4)
           new_prod = subject.productions.last
           expect(new_prod.lhs).to eq(subject['A'])
           expect(new_prod.rhs[0]).to eq(subject['b'])
         end
+
+        it "should support Kleene's plus" do
+          instance = BaseGrammarBuilder.new
+          instance.add_terminals('plus', 'minus', 'digit')
+
+          instance.add_production('integer' => 'value')
+          instance.add_production('integer' => 'sign value')
+          instance.add_production('sign' => 'plus')
+          instance.add_production('sign' => 'minus')
+          expect(instance.productions.size).to eq(4)
+          instance.add_production('value' => 'digit+')
+          expect(instance.productions.size).to eq(7) # Two additional rules generated
+          prod_plus = instance.productions.select { |prod| prod.lhs.name == 'digit_plus' }
+          expect(prod_plus.size).to eq(2)
+          last_prod = instance.productions.last
+          expect(last_prod.lhs.name).to eq('value')
+          expect(last_prod.rhs.members[0].name).to eq('digit_plus')
+        end
       end # context
 
       context 'Building grammar:' do
         subject do
-          instance = GrammarBuilder.new do
+          instance = BaseGrammarBuilder.new do
             add_terminals('a', 'b', 'c')
             add_production('S' => ['A'])
             add_production('A' => %w[a A c])
@@ -141,14 +159,14 @@ module Rley # Open this namespace to avoid module qualifier prefixes
         end
 
         it 'should complain in absence of symbols' do
-          instance = GrammarBuilder.new
+          instance = BaseGrammarBuilder.new
           err = StandardError
           msg = 'No symbol found for grammar'
           expect { instance.grammar }.to raise_error(err, msg)
         end
 
         it 'should complain in absence of productions' do
-          instance = GrammarBuilder.new
+          instance = BaseGrammarBuilder.new
           instance.add_terminals('a', 'b', 'c')
           err = StandardError
           msg = 'No production found for grammar'
@@ -183,7 +201,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           t_star = VerbatimSymbol.new('*')
           t_slash = VerbatimSymbol.new('/')
 
-          builder = GrammarBuilder.new
+          builder = BaseGrammarBuilder.new
           builder.add_terminals(t_a, t_star, t_slash)
           builder.add_production('S' => 'E')
           builder.add_production('E' => %w[E Q F])