rley 0.6.09 → 0.7.00

data/spec/rley/support/ambiguous_grammar_helper.rb

@@ -29,7 +29,8 @@ module AmbiguousGrammarHelper
           msg = "Unknown input text '#{lexeme}'"
           raise StandardError, msg
       end
-      Rley::Lexical::Token.new(lexeme, terminal)
+      pos = Rley::Lexical::Position.new(3, 4) # dummy pos
+      Rley::Lexical::Token.new(lexeme, terminal, pos)
     end
 
     return tokens

data/spec/rley/support/expectation_helper.rb

@@ -7,6 +7,7 @@ module ExpectationHelper
   # Helper method. Compare the data from all the parse entries
   # of a given ParseEntrySet with an array of expectation strings.
   def compare_entry_texts(anEntrySet, expectations)
+    raise StandardError, "Nil entry set" if anEntrySet.nil?
     (0...expectations.size).each do |i|
       expect(anEntrySet.entries[i].to_s).to eq(expectations[i])
     end

data/spec/rley/support/grammar_ambig01_helper.rb

@@ -21,19 +21,28 @@ module GrammarAmbig01Helper
   end
 
   # Highly simplified tokenizer implementation.
-  def tokenizer_ambig01(aText, aGrammar)
-    tokens = aText.scan(/\S+/).map do |lexeme|
+  def tokenizer_ambig01(aText)
+    scanner = StringScanner.new(aText)
+    tokens = []
+    
+    loop do
+      scanner.skip(/\s+/)
+      curr_pos = scanner.pos
+      lexeme = scanner.scan(/\S+/)
+      break unless lexeme
       case lexeme
         when '+', '*'
-          terminal = aGrammar.name2symbol[lexeme]
+          terminal = lexeme
         when /^[-+]?\d+$/
-          terminal = aGrammar.name2symbol['integer']
+          terminal = 'integer'
         else
           msg = "Unknown input text '#{lexeme}'"
           raise StandardError, msg
       end
-      Rley::Lexical::Token.new(lexeme, terminal)
-    end
+
+      pos = Rley::Lexical::Position.new(1, curr_pos + 1)
+      tokens << Rley::Lexical::Token.new(lexeme, terminal, pos)      
+    end    
 
     return tokens
   end

data/spec/rley/support/grammar_arr_int_helper.rb

@@ -23,29 +23,30 @@ module GrammarArrIntHelper
   end
 
   # Basic tokenizer for array of integers
-  def arr_int_tokenizer(aText, aGrammar)
-    tokens = []
+  def arr_int_tokenizer(aText)
     scanner = StringScanner.new(aText)
-
-    until scanner.eos?
+    tokens = []
+    
+    loop do
       scanner.skip(/\s+/)
-      lexeme = scanner.scan(/[\[,\]]/)
-      if lexeme
-        terminal = aGrammar.name2symbol[lexeme]
-        tokens << Rley::Lexical::Token.new(lexeme, terminal)
-        next
-      end
-      lexeme = scanner.scan(/^[-+]?\d+/)
-      if lexeme
-        terminal = aGrammar.name2symbol['integer']
-        tokens << Rley::Lexical::Token.new(lexeme, terminal)
+      curr_ch = scanner.peek(1)
+      break if curr_ch.nil? || curr_ch.empty?
+      curr_pos = scanner.pos
+      
+      if (lexeme = scanner.scan(/[\[\],]/))
+        terminal = lexeme
+      elsif (lexeme = scanner.scan(/[-+]?\d+/))
+        terminal = 'integer'
       else
         msg = "Unknown input text '#{lexeme}'"
         raise StandardError, msg
       end
+
+      pos = Rley::Lexical::Position.new(1, curr_pos + 1)
+      tokens << Rley::Lexical::Token.new(lexeme, terminal, pos)      
     end
 
-    return tokens
+    return tokens    
   end
 end # module
 # End of file

data/spec/rley/support/grammar_b_expr_helper.rb

@@ -21,21 +21,30 @@ module GrammarBExprHelper
   end
 
   # Basic expression tokenizer
-  def expr_tokenizer(aText, aGrammar)
-    tokens = aText.scan(/\S+/).map do |lexeme|
+  def expr_tokenizer(aText)
+    scanner = StringScanner.new(aText)
+    tokens = []
+    
+    loop do
+      scanner.skip(/\s+/)
+      curr_pos = scanner.pos
+      lexeme = scanner.scan(/\S+/)
+      break unless lexeme
       case lexeme
         when '+', '*'
-          terminal = aGrammar.name2symbol[lexeme]
+          terminal = lexeme
         when /^[-+]?\d+$/
-          terminal = aGrammar.name2symbol['integer']
+          terminal = 'integer'
         else
           msg = "Unknown input text '#{lexeme}'"
           raise StandardError, msg
       end
-      Rley::Lexical::Token.new(lexeme, terminal)
-    end
 
-    return tokens
+      pos = Rley::Lexical::Position.new(1, curr_pos + 1)
+      tokens << Rley::Lexical::Token.new(lexeme, terminal, pos)      
+    end    
+
+    return tokens 
   end
 end # module
 # End of file

data/spec/rley/support/grammar_helper.rb

@@ -8,19 +8,23 @@ module GrammarHelper
   # Synopsis:
   #   build_token_sequence(%w(a a b c c), grm1)
   def build_token_sequence(literals, aGrammar)
+    col = 1
     tokens = literals.map do |lexeme|
+      pos = Rley::Lexical::Position.new(1, col)
       case lexeme
         when String
           terminal = aGrammar.name2symbol[lexeme]
-          Rley::Lexical::Token.new(lexeme, terminal)
+          token = Rley::Lexical::Token.new(lexeme, terminal, pos)
 
         when Hash # lexeme is reality a Hash: literal => terminal name
           sub_array = lexeme.to_a
           sub_array.map do |(literal, name)|
             terminal = aGrammar.name2symbol[name]
-            Rley::Lexical::Token.new(literal, terminal)
+            token = Rley::Lexical::Token.new(literal, terminal, pos)
           end
       end
+        col += lexeme.length + 1
+        token
     end
 
     return tokens.flatten

data/spec/rley/support/grammar_l0_helper.rb

@@ -66,14 +66,22 @@ module GrammarL0Helper
   end
 
   # Highly simplified tokenizer implementation.
-  def tokenizer_l0(aText, aGrammar)
-    tokens = aText.scan(/\S+/).map do |word|
+  def tokenizer_l0(aText)
+    scanner = StringScanner.new(aText)
+    tokens = []
+    
+    loop do
+      scanner.skip(/\s+/)
+      curr_pos = scanner.pos
+      word = scanner.scan(/\S+/)
+      break unless word
+
       term_name = lexicon_l0[word]
       if term_name.nil?
         raise StandardError, "Word '#{word}' not found in lexicon"
       end
-      terminal = aGrammar.name2symbol[term_name]
-      Rley::Lexical::Token.new(word, terminal)
+      pos = Rley::Lexical::Position.new(1, curr_pos + 1)
+      tokens << Rley::Lexical::Token.new(word, term_name, pos)      
     end
 
     return tokens

data/spec/rley/support/grammar_pb_helper.rb

@@ -1,5 +1,6 @@
 # Load the builder class
 require_relative '../../../lib/rley/syntax/grammar_builder'
+require_relative '../../../lib/support/base_tokenizer'
 require_relative '../../../lib/rley/lexical/token'
 
 
@@ -12,36 +13,60 @@ class GrammarPBHelper
   def grammar()
     @grammar ||= begin
       builder = Rley::Syntax::GrammarBuilder.new do
-        t_int = Rley::Syntax::Literal.new('int', /[-+]?\d+/)
-        t_plus = Rley::Syntax::VerbatimSymbol.new('+')
-        t_lparen = Rley::Syntax::VerbatimSymbol.new('(')
-        t_rparen = Rley::Syntax::VerbatimSymbol.new(')')
-        add_terminals(t_int, t_plus, t_lparen, t_rparen)
+        add_terminals('int', '+', '(', ')')
         rule 'S' => 'E'
         rule 'E' => 'int'
-        rule 'E' => %w[( E + E )]
-        rule 'E' => %w[E + E]
+        rule 'E' => '( E + E )'
+        rule 'E' => 'E + E'
       end
       builder.grammar
     end
   end
 
-  # Basic expression tokenizer
-  def tokenize(aText)
-    tokens = aText.scan(/\S+/).map do |lexeme|
-      case lexeme
-        when '+', '(', ')'
-          terminal = @grammar.name2symbol[lexeme]
-        when /^[-+]?\d+$/
-          terminal = @grammar.name2symbol['int']
-        else
-          msg = "Unknown input text '#{lexeme}'"
-          raise StandardError, msg
+  # # Basic expression tokenizer
+  # def tokenize(aText)
+    # tokens = aText.scan(/\S+/).map do |lexeme|
+      # case lexeme
+        # when '+', '(', ')'
+          # terminal = @grammar.name2symbol[lexeme]
+        # when /^[-+]?\d+$/
+          # terminal = @grammar.name2symbol['int']
+        # else
+          # msg = "Unknown input text '#{lexeme}'"
+          # raise StandardError, msg
+      # end
+      # pos = Rley::Lexical::Position.new(1, 4) # Dummy position
+      # Rley::Lexical::Token.new(lexeme, terminal, pos)
+    # end
+
+    # return tokens
+  # end
+  
+  
+  class PB_Tokenizer < BaseTokenizer
+
+    protected
+
+    def recognize_token()
+      token = nil
+
+      if (lexeme = scanner.scan(/[\(\)]/)) # Single characters
+        # Delimiters, separators => single character token
+        token = build_token(lexeme, lexeme)
+      elsif (lexeme = scanner.scan(/(?:\+)(?=\s|$)/)) # Single char occurring alone
+        token = build_token(lexeme, lexeme)
+       elsif (lexeme = scanner.scan(/[+-]?[0-9]+/))
+        token = build_token('int', lexeme)
       end
-      Rley::Lexical::Token.new(lexeme, terminal)
     end
+  end # class
 
-    return tokens
+  # Basic tokenizer
+  # @return [Array<Rley::Lexical::Token>]
+  def tokenize(aText)
+    tokenizer = PB_Tokenizer.new(aText)
+    tokenizer.tokens
   end
-end # module
+  
+end # class
 # End of file

data/spec/support/base_tokenizer_spec.rb

@@ -0,0 +1,77 @@
+require_relative '../spec_helper'
+
+# Load the class under test
+require_relative '../../lib/support/base_tokenizer'
+
+describe BaseTokenizer do
+  let(:sample_input) { '7 + (8 + 9)' }
+  context 'Standard creation & initialization:' do
+    subject { BaseTokenizer.new(sample_input) }
+
+    it 'should be initialized with a text argument' do
+      expect { BaseTokenizer.new(sample_input) }.not_to raise_error
+    end
+
+    it 'should have a scanner initialized' do
+      expect(subject.scanner).to be_kind_of(StringScanner)
+    end
+
+    it 'should have line number initialized' do
+      expect(subject.lineno).to eq(1)
+    end
+  end # context
+
+
+  context 'Provided services:' do
+    class PB_Tokenizer < BaseTokenizer
+      @@lexeme2name = {
+        '(' => 'LPAREN',
+        ')' => 'RPAREN',
+        '+' => 'PLUS',
+      }.freeze
+
+      protected
+
+      def recognize_token()
+        token = nil
+
+        if (lexeme = scanner.scan(/[\(\)]/)) # Single characters
+          # Delimiters, separators => single character token
+          token = build_token(@@lexeme2name[lexeme], lexeme)
+        elsif (lexeme = scanner.scan(/(?:\+)(?=\s)/)) # Single char occurring alone
+          token = build_token(@@lexeme2name[lexeme], lexeme)
+         elsif (lexeme = scanner.scan(/[+-]?[0-9]+/))
+          token = build_token('int', lexeme)
+        end
+      end
+    end # class
+
+    # Basic tokenizer
+    # @return [Array<Rley::Lexical::Token>]
+    def tokenize(aText)
+      tokenizer = PB_Tokenizer.new(aText)
+      tokenizer.tokens
+    end
+
+    it 'should return a sequence of tokens' do
+      sequence = tokenize(sample_input)
+      checks = [
+        ['int', 7, [1, 1]],
+        ['PLUS', '+', [1, 3]],
+        ['LPAREN', '(', [1, 5]],
+        ['int', 8, [1, 6]],
+        ['PLUS', '+', [1, 8]],
+        ['int', 9, [1, 10]],
+        ['RPAREN', ')', [1, 11]]
+      ]
+      sequence.each_with_index do |token, i|
+        (tok_type, tok_value, tok_pos) = checks[i]
+        (line, col) = tok_pos
+        expect(token.terminal).to eq(tok_type)
+        expect(token.lexeme).to eq(tok_value.to_s)
+        expect(token.position.line).to eq(line)
+        expect(token.position.column).to eq(col)
+      end
+    end
+  end
+end # describe

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rley
 version: !ruby/object:Gem::Version
-  version: 0.6.09
+  version: 0.7.00
 platform: ruby
 authors:
 - Dimitri Geshef
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-20 00:00:00.000000000 Z
+date: 2018-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: coveralls
@@ -224,6 +224,7 @@ files:
 - lib/rley/syntax/symbol_seq.rb
 - lib/rley/syntax/terminal.rb
 - lib/rley/syntax/verbatim_symbol.rb
+- lib/support/base_tokenizer.rb
 - spec/rley/base/dotted_item_spec.rb
 - spec/rley/base/grm_items_builder_spec.rb
 - spec/rley/engine_spec.rb
@@ -291,6 +292,7 @@ files:
 - spec/rley/syntax/terminal_spec.rb
 - spec/rley/syntax/verbatim_symbol_spec.rb
 - spec/spec_helper.rb
+- spec/support/base_tokenizer_spec.rb
 homepage: https://github.com/famished-tiger/Rley
 licenses:
 - MIT
@@ -376,3 +378,4 @@ test_files:
 - spec/rley/syntax/symbol_seq_spec.rb
 - spec/rley/syntax/terminal_spec.rb
 - spec/rley/syntax/verbatim_symbol_spec.rb
+- spec/support/base_tokenizer_spec.rb