rley 0.6.09 → 0.7.00

data/spec/rley/engine_spec.rb

@@ -49,10 +49,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
       end
 
       def each()
+        pos = Rley::Lexical::Position.new(1, 1) # Dummy position
         lexemes = @input.scan(/\S/)
         lexemes.each do |ch|
           if ch =~ /[abc]/
-            yield Rley::Lexical::Token.new(ch, ch)
+            yield Rley::Lexical::Token.new(ch, ch, pos)
           else
              raise StandardError, "Invalid character #{ch}"
           end

data/spec/rley/formatter/asciitree_spec.rb

@@ -30,7 +30,8 @@ module Rley # Re-open the module to get rid of qualified names
       # Helper method that mimicks the output of a tokenizer
       # for the language specified by grammar_abc
       let(:grm_abc_tokens1) do
-        %w[a a b c c].map { |ch| Lexical::Token.new(ch, ch) }
+        pos = Lexical::Position.new(1, 2) # Dummy position
+        %w[a a b c c].map { |ch| Lexical::Token.new(ch, ch, pos) }
       end
 
       # Factory method that builds a sample parse tree.

data/spec/rley/formatter/bracket_notation_spec.rb

@@ -30,7 +30,8 @@ module Rley # Re-open the module to get rid of qualified names
       # Helper method that mimicks the output of a tokenizer
       # for the language specified by grammar_abc
       let(:grm_abc_tokens1) do
-        %w[a a b c c].map { |ch| Lexical::Token.new(ch, ch) }
+        pos = Lexical::Position.new(1, 2) # Dummy position
+        %w[a a b c c].map { |ch| Lexical::Token.new(ch, ch, pos) }
       end
 
       # Factory method that builds a sample parse tree.

data/spec/rley/formatter/debug_spec.rb

@@ -31,7 +31,8 @@ module Rley # Re-open the module to get rid of qualified names
       # Helper method that mimicks the output of a tokenizer
       # for the language specified by grammar_abc
       let(:grm_abc_tokens1) do
-        %w[a a b c c].map { |ch| Lexical::Token.new(ch, ch) }
+        pos = Lexical::Position.new(1, 2) # Dummy position
+        %w[a a b c c].map { |ch| Lexical::Token.new(ch, ch, pos) }
       end
 
       let(:grammar_sppf) do
@@ -40,7 +41,8 @@ module Rley # Re-open the module to get rid of qualified names
       end
 
       let(:sample_tokens) do
-        %w[a b b b].map { |ch| Lexical::Token.new(ch, ch) }
+        pos = Lexical::Position.new(1, 2) # Dummy position
+        %w[a b b b].map { |ch| Lexical::Token.new(ch, ch, pos) }
       end
 
       # Factory method that builds a sample parse tree.

data/spec/rley/formatter/json_spec.rb

@@ -29,7 +29,8 @@ module Rley # Re-open the module to get rid of qualified names
       # Helper method that mimicks the output of a tokenizer
       # for the language specified by grammar_abc
       let(:grm_abc_tokens1) do
-        %w[a a b c c].map { |ch| Lexical::Token.new(ch, ch) }
+        pos = Lexical::Position.new(1, 2) # Dummy position
+        %w[a a b c c].map { |ch| Lexical::Token.new(ch, ch, pos) }
       end
 
       # Factory method that builds a sample parse tree.

data/spec/rley/lexical/token_spec.rb

@@ -9,14 +9,15 @@ module Rley # Open this namespace to avoid module qualifier prefixes
   module Lexical # Open this namespace to avoid module qualifier prefixes
     describe Token do
       let(:lexeme) { '"some text"' }
-      let(:sample_terminal) { Syntax::Terminal.new('if') }
+      let(:a_terminal) { Syntax::Terminal.new('if') }
+      let(:a_pos) { Position.new(3, 4) }
 
       context 'Initialization:' do
         # Default instantiation rule
-        subject { Token.new(lexeme, sample_terminal) }
+        subject { Token.new(lexeme, a_terminal, a_pos) }
 
-        it 'should be created with a lexeme and a terminal argument' do
-          expect { Token.new(lexeme, sample_terminal) }.not_to raise_error
+        it 'should be created with a lexeme, a terminal and position' do
+          expect { Token.new(lexeme, a_terminal, a_pos) }.not_to raise_error
         end
 
         it 'should know its lexeme' do
@@ -24,8 +25,12 @@ module Rley # Open this namespace to avoid module qualifier prefixes
         end
 
         it 'should know its terminal' do
-          expect(subject.terminal).to eq(sample_terminal)
+          expect(subject.terminal).to eq(a_terminal)
         end
+        
+        it 'should know its terminal' do
+          expect(subject.position).to eq(a_pos)
+        end        
       end # context
     end # describe
   end # module

data/spec/rley/parse_rep/ambiguous_parse_spec.rb

@@ -61,7 +61,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
 
         let(:expr_tokens) do
           sentence = '2 + 3 * 4'
-          tokenizer_ambig01(sentence, grammar_ambig01)
+          tokenizer_ambig01(sentence)
         end
 
         let(:sentence_result) do

data/spec/rley/parse_rep/ast_builder_spec.rb

@@ -94,7 +94,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
       end
 
       let(:sample_tokens) do
-        arr_int_tokenizer('[2 , 3, 5 ]', sample_grammar)
+        arr_int_tokenizer('[2 , 3, 5 ]')
       end
 
       subject { ASTBuilder.new(sample_tokens) }

data/spec/rley/parse_rep/cst_builder_spec.rb

@@ -23,7 +23,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
       end
 
       let(:sample_tokens) do
-        expr_tokenizer('2 + 3 * 4', sample_grammar)
+        expr_tokenizer('2 + 3 * 4')
       end
 
       subject { CSTBuilder.new(sample_tokens) }
@@ -357,7 +357,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
         # Event: visit P => . arr | 0 0
         #   Event: visit .P | 0 0
         it 'should build a tree for an empty array' do
-          empty_arr_tokens = arr_int_tokenizer('[ ]', array_grammar)
+          empty_arr_tokens = arr_int_tokenizer('[ ]')
           @instance = CSTBuilder.new(empty_arr_tokens)
           init_walker(@parser, empty_arr_tokens)
           stack = get_stack(@instance)

data/spec/rley/parse_rep/groucho_spec.rb

@@ -48,13 +48,14 @@ module Rley # Open this namespace to avoid module qualifier prefixes
 
       # Highly simplified tokenizer implementation.
       def tokenizer(aText, aGrammar)
+        pos = Rley::Lexical::Position.new(1, 2) # Dummy position
         tokens = aText.scan(/\S+/).map do |word|
           term_name = Groucho_lexicon[word]
           if term_name.nil?
             raise StandardError, "Word '#{word}' not found in lexicon"
           end
           terminal = aGrammar.name2symbol[term_name]
-          Rley::Lexical::Token.new(word, terminal)
+          Rley::Lexical::Token.new(word, terminal, pos)
         end
 
         return tokens

data/spec/rley/parse_rep/parse_forest_builder_spec.rb

@@ -285,7 +285,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
 
         let(:sentence_tokens) do
           sentence = 'I prefer a morning flight'
-          tokenizer_l0(sentence, grammar_l0)
+          tokenizer_l0(sentence)
         end
 
         let(:sentence_result) do

data/spec/rley/parse_tree_visitor_spec.rb

@@ -25,7 +25,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
     # Helper method that mimicks the output of a tokenizer
     # for the language specified by grammar_abc
     let(:grm_abc_tokens1) do
-      %w[a a b c c].map { |ch| Lexical::Token.new(ch, ch) }
+      pos = Lexical::Position.new(1, 2) # Dummy position
+      %w[a a b c c].map { |ch| Lexical::Token.new(ch, ch, pos) }
     end
 
     # Factory method that builds a sample parse tree.

data/spec/rley/parser/error_reason_spec.rb

@@ -62,7 +62,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
     describe UnexpectedToken do
       let(:err_lexeme) { '-' }
       let(:err_terminal) { Syntax::Terminal.new('MINUS') }
-      let(:err_token) { Lexical::Token.new(err_lexeme, err_terminal) }
+      let(:pos) { Lexical::Position.new(3, 4) }
+      let(:err_token) { Lexical::Token.new(err_lexeme, err_terminal, pos) }
       let(:terminals) do
         %w[PLUS LPAREN].map { |name| Syntax::Terminal.new(name) }
       end
@@ -81,7 +82,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
       context 'Provided services:' do
         it 'should emit a message' do
           text = <<MESSAGE_END
-Syntax error at or near token 4 >>>-<<<
+Syntax error at or near token line 3, column 4 >>>-<<<
 Expected one of: ['PLUS', 'LPAREN'], found a 'MINUS' instead.
 MESSAGE_END
           expect(subject.to_s).to eq(text.chomp)
@@ -93,7 +94,8 @@ MESSAGE_END
     describe PrematureInputEnd do
       let(:err_lexeme) { '+' }
       let(:err_terminal) { Syntax::Terminal.new('PLUS') }
-      let(:err_token) { Lexical::Token.new(err_lexeme, err_terminal) }
+      let(:pos) { Lexical::Position.new(3, 4) }
+      let(:err_token) { Lexical::Token.new(err_lexeme, err_terminal, pos) }
       let(:terminals) do
         %w[INT LPAREN].map { |name| Syntax::Terminal.new(name) }
       end
@@ -112,7 +114,7 @@ MESSAGE_END
       context 'Provided services:' do
         it 'should emit a message' do
           text = <<MESSAGE_END
-Premature end of input after '+' at position 4
+Premature end of input after '+' at position line 3, column 4
 Expected one of: ['INT', 'LPAREN'].
 MESSAGE_END
           expect(subject.to_s).to eq(text.chomp)

data/spec/rley/parser/gfg_earley_parser_spec.rb

@@ -288,7 +288,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           builder.add_terminals(t_x)
           builder.add_production('Ss' => %w[A A x])
           builder.add_production('A' => [])
-          tokens = [Lexical::Token.new('x', t_x)]
+          pos = Lexical::Position.new(1, 1)
+          tokens = [Lexical::Token.new('x', t_x, pos)]
 
           instance = GFGEarleyParser.new(builder.grammar)
           expect { instance.parse(tokens) }.not_to raise_error
@@ -299,8 +300,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           expected = [
             '.Ss | 0',              # Initialization
             "Ss => . A A 'x' | 0",  # start rule
-            '.A | 0',               # call rule         
-            'A => . | 0',           # start rule             
+            '.A | 0',               # call rule
+            'A => . | 0',           # start rule
             'A. | 0',               # exit rule
             "Ss => A . A 'x' | 0",  # end rule
             "Ss => A A . 'x' | 0"   # end rule
@@ -559,7 +560,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           parse_result = subject.parse(wrong)
           expect(parse_result.success?).to eq(false)
           err_msg = <<-MSG
-Syntax error at or near token 3 >>>c<<<
+Syntax error at or near token line 1, column 5 >>>c<<<
 Expected one of: ['a', 'b'], found a 'c' instead.
 MSG
           expect(parse_result.failure_reason.message).to eq(err_msg.chomp)
@@ -590,34 +591,35 @@ MSG
             'S => . E | 0',               # start rule
             '.E | 0',                     # call rule
             'E => . int | 0',             # start rule
-            "E => . '(' E '+' E ')' | 0", # start rule
-            "E => . E '+' E | 0"          # start rule
+            "E => . ( E + E ) | 0", # start rule
+            "E => . E + E | 0"          # start rule
           ]
           compare_entry_texts(parse_result.chart[0], expected)
 
           ###################### S(1) == 1 . +
           # Expectation chart[1]:
           expected = [
-            'E => int . | 0',             # scan '1'
-            'E. | 0',                     # exit rule
-            'S => E . | 0',               # end rule
-            "E => E . '+' E | 0",         # end rule
-            'S. | 0'                      # exit rule
+            'E => int . | 0',         # scan '1'
+            'E. | 0',                 # exit rule
+            'S => E . | 0',           # end rule
+            'E => E . + E | 0',       # end rule
+            'S. | 0'                  # exit rule
           ]
           compare_entry_texts(parse_result.chart[1], expected)
 
           ###################### S(2) == 1 + .
           # Expectation chart[2]:
           expected = [
-            "E => E '+' . E | 0",         # scan '+'
-            '.E | 2',                     # exit rule
-            'E => . int | 2',             # start rule
-            "E => . '(' E '+' E ')' | 2", # start rule
-            "E => . E '+' E | 2"          # start rule
+            'E => E + . E | 0',       # scan '+'
+            '.E | 2',                 # exit rule
+            'E => . int | 2',         # start rule
+            'E => . ( E + E ) | 2',   # start rule
+            'E => . E + E | 2'        # start rule
           ]
           compare_entry_texts(parse_result.chart[2], expected)
 
-          err_msg = "Premature end of input after '+' at position 2"
+          err_msg = "Premature end of input after '+' at position line 1, "
+          err_msg << "column 3"
           err_msg << "\nExpected one of: ['int', '(']."
           expect(parse_result.failure_reason.message).to eq(err_msg)
         end
@@ -636,76 +638,76 @@ MSG
           ###################### S(0) == . 7 + 8 + 9
           # Expectation chart[0]:
           expected = [
-            '.S | 0',                     # initialization
-            'S => . E | 0',               # start rule
-            '.E | 0',                     # call rule
-            'E => . int | 0',             # start rule
-            "E => . '(' E '+' E ')' | 0", # start rule
-            "E => . E '+' E | 0"          # start rule
+            '.S | 0',               # initialization
+            'S => . E | 0',         # start rule
+            '.E | 0',               # call rule
+            'E => . int | 0',       # start rule
+            'E => . ( E + E ) | 0', # start rule
+            'E => . E + E | 0'      # start rule
           ]
           compare_entry_texts(parse_result.chart[0], expected)
 
           ###################### S(1) == 7 . + 8 + 9
           # Expectation chart[1]:
           expected = [
-            'E => int . | 0',             # scan '7'
-            'E. | 0',                     # exit rule
-            'S => E . | 0',               # end rule
-            "E => E . '+' E | 0",         # end rule
-            'S. | 0'                      # exit rule
+            'E => int . | 0',       # scan '7'
+            'E. | 0',               # exit rule
+            'S => E . | 0',         # end rule
+            'E => E . + E | 0',     # end rule
+            'S. | 0'                # exit rule
           ]
           compare_entry_texts(parse_result.chart[1], expected)
 
           ###################### S(2) == 7 + . 8 + 9
           # Expectation chart[2]:
           expected = [
-            "E => E '+' . E | 0",         # scan '+'
-            '.E | 2',                     # exit rule
-            'E => . int | 2',             # start rule
-            "E => . '(' E '+' E ')' | 2", # start rule
-            "E => . E '+' E | 2"          # start rule
+            'E => E + . E | 0',       # scan '+'
+            '.E | 2',                 # exit rule
+            'E => . int | 2',         # start rule
+            'E => . ( E + E ) | 2',   # start rule
+            'E => . E + E | 2'        # start rule
           ]
           compare_entry_texts(parse_result.chart[2], expected)
 
           ###################### S(3) == 7 + 8 . + 9
           # Expectation chart[3]:
           expected = [
-            'E => int . | 2',             # scan '8'
-            'E. | 2',                     # exit rule
-            "E => E '+' E . | 0",         # end rule
-            "E => E . '+' E | 2",         # end rule
-            'E. | 0',                     # exit rule
-            'S => E . | 0',               # end rule
-            "E => E . '+' E | 0",         # end rule
-            'S. | 0'                      # exit rule
+            'E => int . | 2',         # scan '8'
+            'E. | 2',                 # exit rule
+            'E => E + E . | 0',       # end rule
+            'E => E . + E | 2',       # end rule
+            'E. | 0',                 # exit rule
+            'S => E . | 0',           # end rule
+            'E => E . + E | 0',       # end rule
+            'S. | 0'                  # exit rule
           ]
           compare_entry_texts(parse_result.chart[3], expected)
 
           ###################### S(4) == 7 + 8 + . 9
           # Expectation chart[4]:
           expected = [
-            "E => E '+' . E | 2",         # scan '+'
-            "E => E '+' . E | 0",         # scan '+'
-            '.E | 4',                     # exit rule
-            'E => . int | 4',             # start rule
-            "E => . '(' E '+' E ')' | 4", # start rule
-            "E => . E '+' E | 4"          # start rule
+            'E => E + . E | 2',       # scan '+'
+            'E => E + . E | 0',       # scan '+'
+            '.E | 4',                 # exit rule
+            'E => . int | 4',         # start rule
+            'E => . ( E + E ) | 4',   # start rule
+            'E => . E + E | 4'        # start rule
           ]
           compare_entry_texts(parse_result.chart[4], expected)
 
           ###################### S(5) == 7 + 8 + 9 .
           # Expectation chart[5]:
           expected = [
-            'E => int . | 4',             # scan '9'
-            'E. | 4',                     # exit rule
-            "E => E '+' E . | 2",         # end rule
-            "E => E '+' E . | 0",         # end rule
-            "E => E . '+' E | 4",         # exit rule (not shown in paper)
-            'E. | 2',                     # exit rule
-            'E. | 0',                     # exit rule
-            "E => E . '+' E | 2",         # end rule
-            'S => E . | 0',               # end rule
-            "E => E . '+' E | 0",         # end rule
+            'E => int . | 4',         # scan '9'
+            'E. | 4',                 # exit rule
+            'E => E + E . | 2',       # end rule
+            'E => E + E . | 0',       # end rule
+            'E => E . + E | 4',       # exit rule (not shown in paper)
+            'E. | 2',                 # exit rule
+            'E. | 0',                 # exit rule
+            'E => E . + E | 2',       # end rule
+            'S => E . | 0',           # end rule
+            'E => E . + E | 0',       # end rule
             'S. | 0'
           ]
           compare_entry_texts(parse_result.chart[5], expected)

data/spec/rley/parser/gfg_parsing_spec.rb

@@ -305,7 +305,7 @@ SNIPPET
 
         subject do
           parser = GFGEarleyParser.new(b_expr_grammar)
-          tokens = expr_tokenizer('2 + 3 * 4', b_expr_grammar)
+          tokens = expr_tokenizer('2 + 3 * 4')
           parser.parse(tokens)
         end
         

data/spec/rley/parser/parse_tracer_spec.rb

@@ -15,11 +15,12 @@ module Rley # Open this namespace to avoid module qualifier prefixes
   module Parser # Open this namespace to avoid module qualifier prefixes
     describe ParseTracer do
       let(:output) { StringIO.new('', 'w') }
+      let(:tpos) { Lexical::Position.new(3, 4) }
 
       let(:token_seq) do
         literals = %w[I saw John with a dog]
-        literals.map do |lexeme| 
-          Lexical::Token.new(lexeme, double('fake-terminal')) 
+        literals.map do |lexeme|
+          Lexical::Token.new(lexeme, double('fake-terminal'), tpos)
         end
       end
 

data/spec/rley/sppf/token_node_spec.rb

@@ -12,10 +12,13 @@ module Rley # Open this namespace to avoid module qualifier prefixes
   module SPPF # Open this namespace to avoid module qualifier prefixes
     describe TokenNode do
       let(:sample_symbol) { Syntax::Terminal.new('Noun') }
-      let(:sample_token) { Lexical::Token.new('language', sample_symbol) }
-      let(:sample_position) { 3 } 
+      let(:sample_position) { Lexical::Position.new(3, 4) }
+      let(:sample_token) do 
+        Lexical::Token.new('language', sample_symbol, sample_position) 
+      end
+      let(:sample_rank) { 3 }
 
-      subject { TokenNode.new(sample_token, sample_position) }
+      subject { TokenNode.new(sample_token, sample_rank) }
 
       context 'Initialization:' do
         it 'should know its token' do
@@ -23,9 +26,9 @@ module Rley # Open this namespace to avoid module qualifier prefixes
         end
 
         it 'should know its token range' do
-          expect(subject.origin).to eq(sample_position)
-          expect(subject.range.low).to eq(sample_position)
-          expect(subject.range.high).to eq(sample_position + 1)
+          expect(subject.origin).to eq(sample_rank)
+          expect(subject.range.low).to eq(sample_rank)
+          expect(subject.range.high).to eq(sample_rank + 1)
         end
       end # context