rley 0.6.09 → 0.7.00

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a924baa9568e3076c5c8ebb0d3f1e9ff162ab09
-  data.tar.gz: f01496851e2679a598a34f6635caf27e90aeda19
+  metadata.gz: 2b462d4c492ffb698715478492a962d65e41834c
+  data.tar.gz: 282ab2ed83d7b1ead2646c8dd98176d1753a142e
 SHA512:
-  metadata.gz: 80b5ac648702c5ab11e2e84e6748b0be336f8ff39c61d6f05b49fda2a50ed7213cd82f4711590d8173434feaa83e4236a0b7ed56c74242eac53f2608d6ab1d4c
-  data.tar.gz: b8cf7f4a64c2526a784b0f6396e031040e57244a15ae760d5b050900fb26ad38c98e5fb66316fa29381384482fa2b40722502e11b7ede794fb1f65e31508a2f8
+  metadata.gz: 514e4a9429b4fd1231001269cd18e96fa70d4b9145c60115f042f338e0a0063871f979ba9c04e971ee589c8e2d3919fece6b3af4499af50f03899f44318a0598
+  data.tar.gz: 870e01cb9e693c126b9fa13915dadcec72559553147e0bf220f6e39a6431c6b59c7fb1b67aa1a19d8eadf0076fc6378948fbf1f56850c88d9f99584fefb7f259

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+### 0.7.00 / 2018-11-24  
+- Version bump. Core class `Token` is changed.
+
+* [NEW] Structure `Lexical::Position` to hold the line, column position of a token.
+* [NEW] Class `BaseTokenizer`: Provides basic tokenizer operations to customized through subclassing.
+* [CHANGE] Class `Lexical::Token`: Attribute `position` added.
+* [CHANGE] Method `Lexical::Token#initialize`: Add a third argument for specifying the position of the token.
+* [CHANGE] Many classes and examples updated to conform to `Token` class change.   
+* [FIX] Missing methods in class `JSONPair` added.
+
 ### 0.6.09 / 2018-10-20
 * [FIXED] Method `GrmFlowGraph#traverse_df` now returns a meaningful message when the grammar uses a terminal symbol without declaring it first.
 

data/README.md

@@ -148,14 +148,25 @@ The subset of English grammar is based on an example from the NLTK book.
 
 ### Creating a tokenizer
 ```ruby
+  require 'strscan'
+
     # A tokenizer reads the input string and converts it into a sequence of tokens.
     # Remark: Rley doesn't provide tokenizer functionality.
     # Highly simplified tokenizer implementation
     def tokenizer(aTextToParse)
-      tokens = aTextToParse.scan(/\S+/).map do |word|
+      scanner = StringScanner.new(aTextToParse)
+      tokens = []
+
+      loop do
+        scanner.skip(/\s+/)
+        curr_pos = scanner.pos
+        word = scanner.scan(/\S+/)
+        break unless word
+
         term_name = Lexicon[word]
         raise StandardError, "Word '#{word}' not found in lexicon" if term_name.nil?
-        Rley::Lexical::Token.new(word, term_name)
+        pos = Rley::Lexical::Position.new(1, curr_pos + 1)
+        tokens << Rley::Lexical::Token.new(word, term_name, pos)
       end
 
       return tokens

data/examples/NLP/benchmark_pico_en.rb

@@ -63,10 +63,13 @@ Lexicon = {
 # Rley doesn't provide tokenizer functionality.
 # (Highly simplified tokenizer implementation).
 def tokenizer(aTextToParse)
+  offset = -1
   tokens = aTextToParse.scan(/\S+/).map do |word|
     term_name = Lexicon[word]
     raise StandardError, "Word '#{word}' not found in lexicon" if term_name.nil?
-    Rley::Lexical::Token.new(word, term_name)
+    pos = Rley::Lexical::Position.new(1, offset + 1)
+    offset += word.length 
+    Rley::Lexical::Token.new(word, term_name, pos)
   end
 
   return tokens

data/examples/NLP/engtagger.rb

@@ -147,10 +147,13 @@ lexicon = clean_text(text)
 tokens = tagged.scan(GET_TAG).map { |tag, word| [word, tag.upcase] }
 
 def tokenizer(lexicon, tokens)
+  pos = -1
   rley_tokens = []
   lexicon.each_with_index do |word, i|
     term_name = tokens[i].last
-    rley_tokens << Rley::Lexical::Token.new(word, term_name)
+    rank = Rley::Lexical::Position.new(1, pos + 1)
+    pos += word.length + 1 # Assuming one space between words.
+    rley_tokens << Rley::Lexical::Token.new(word, term_name, pos)
   end
   return rley_tokens
 end

data/examples/NLP/nano_eng/nano_en_demo.rb

@@ -1,3 +1,4 @@
+require 'strscan'
 require 'rley' # Load Rley library
 
 ########################################
@@ -67,16 +68,26 @@ Lexicon = {
 # Step 4. Creating a tokenizer
 # A tokenizer reads the input string and converts it into a sequence of tokens
 # Highly simplified tokenizer implementation.
-def tokenizer(aTextToParse)
-  tokens = aTextToParse.scan(/\S+/).map do |word|
+def tokenizer(aTextToParse) 
+  scanner = StringScanner.new(aTextToParse)
+  tokens = []
+  
+  loop do
+    scanner.skip(/\s+/)
+    curr_pos = scanner.pos
+    word = scanner.scan(/\S+/)
+    break unless word
+
     term_name = Lexicon[word]
     raise StandardError, "Word '#{word}' not found in lexicon" if term_name.nil?
-    Rley::Lexical::Token.new(word, term_name)
+    pos = Rley::Lexical::Position.new(1, curr_pos + 1)
+    tokens << Rley::Lexical::Token.new(word, term_name, pos)
   end
 
-  return tokens
+  return tokens  
 end
 
+
 ########################################
 # Step 5. Parsing the input
 input_to_parse = 'John saw Mary'

data/examples/NLP/pico_en_demo.rb

@@ -56,21 +56,6 @@ Lexicon = {
   'with' => 'Preposition'
 }.freeze
 
-Position = Struct.new(:line, :column) do
-  def to_s()
-    "line #{line}, column #{column}"
-  end
-end
-
-class NLPToken < Rley::Lexical::Token
-  attr_reader(:position)
-  
-  def initialize(theLexeme, aTerminal, aPosition)
-    super(theLexeme, aTerminal)
-    @position = aPosition
-  end
-end
-
 ########################################
 # Step 4. Create a tokenizer
 # A tokenizer reads the input string and converts it into a sequence of tokens.
@@ -88,8 +73,8 @@ def tokenizer(aTextToParse)
 
     term_name = Lexicon[word]
     raise StandardError, "Word '#{word}' not found in lexicon" if term_name.nil?
-    pos = Position.new(1, curr_pos + 1)
-    tokens << NLPToken.new(word, term_name, pos)
+    pos = Rley::Lexical::Position.new(1, curr_pos + 1)
+    tokens << Rley::Lexical::Token.new(word, term_name, pos)
   end
 
   return tokens

data/examples/data_formats/JSON/json_ast_builder.rb

@@ -7,7 +7,7 @@ require_relative 'json_ast_nodes'
 # The Builder pattern creates a complex object
 # (say, a parse tree) from simpler objects (terminal and non-terminal
 # nodes) and using a step by step approach.
-class JSONASTBuilder < Rley::ParseRep::ParseTreeBuilder 
+class JSONASTBuilder < Rley::ParseRep::ASTBaseBuilder 
   Terminal2NodeClass = {
     'false' => JSONBooleanNode,
     'true' => JSONBooleanNode,
@@ -63,7 +63,7 @@ class JSONASTBuilder < Rley::ParseRep::ParseTreeBuilder
     return JSONPair.new(theChildren[0], theChildren[2], aProduction.lhs)
   end
 
-  # rule 'object' => %w[begin-object member-list end-object]
+  # rule 'array' => %w[begin-array array-items end-array]
   def reduce_array_0(aProduction, _range, _tokens, theChildren)
     second_child = theChildren[1]
     second_child.symbol = aProduction.lhs

data/examples/data_formats/JSON/json_ast_nodes.rb

@@ -27,6 +27,9 @@ JSONTerminalNode = Struct.new(:token, :value, :position) do
   def accept(aVisitor)
     aVisitor.visit_terminal(self)
   end
+  
+  def done!
+  end
 end
 
 
@@ -71,6 +74,9 @@ class JSONCompositeNode
   def accept(aVisitor)
     aVisitor.visit_nonterminal(self)
   end
+  
+  def done!
+  end
 
   alias subnodes children
 end # class
@@ -96,7 +102,7 @@ end # class
 class JSONPair
   attr_reader(:name)
   attr_reader(:value)
-  attr_reader(:symbol)
+  attr_accessor(:symbol)
 
   def initialize(aName, aValue, aSymbol)
     @name = aName
@@ -115,6 +121,16 @@ class JSONPair
   def accept(aVisitor)
     aVisitor.visit_nonterminal(self)
   end
+  
+  def done!
+  end
+  
+  def to_ruby
+    rep = {}
+    rep[name.to_ruby] = value.to_ruby
+
+    return rep    
+  end
 end # class
 
 class JSONObjectNode < JSONCompositeNode
@@ -123,7 +139,7 @@ class JSONObjectNode < JSONCompositeNode
   end
 
   # Convert this tree node in a simpler Ruby representation.
-  # Basically a JSON object corresponds to a Ruhy Hash
+  # Basically a JSON object corresponds to a Ruby Hash
   def to_ruby()
     rep = {}
     members.each do |pair|

data/examples/data_formats/JSON/json_lexer.rb

@@ -23,6 +23,7 @@ class JSONLexer
   def initialize(source)
     @scanner = StringScanner.new(source)
     @lineno = 1
+    @line_start = 0
   end
 
   def tokens()
@@ -48,7 +49,7 @@ class JSONLexer
       case curr_ch
         when '{', '}', '[', ']', ',', ':'
           token_type = @@lexeme2name[curr_ch]
-          token = Rley::Lexical::Token.new(curr_ch, token_type)
+          token = build_token(curr_ch, token_type)
 
         when /[ftn]/ # First letter of keywords
           @scanner.pos = scanner.pos - 1 # Simulate putback
@@ -57,7 +58,7 @@ class JSONLexer
             invalid_keyw = scanner.scan(/\w+/)
             raise ScanError.new("Invalid keyword: #{invalid_keyw}")
           else
-            token = Rley::Lexical::Token.new(keyw, keyw)
+            token = build_token(keyw, keyw)
           end
 
         # LITERALS
@@ -66,12 +67,12 @@ class JSONLexer
           end_delimiter = scanner.getch
           err_msg = 'No closing quotes (") found'
           raise ScanError.new(err_msg) if end_delimiter.nil?
-          token = Rley::Lexical::Token.new(value, 'string')
+          token = build_token(value, 'string')
 
         when /[-0-9]/ # Start character of number literal found
           @scanner.pos = scanner.pos - 1 # Simulate putback
           value = scanner.scan(/-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9])?/)
-          token = Rley::Lexical::Token.new(value, 'number')
+          token = build_token(value, 'number')
 
         else # Unknown token
           erroneous = curr_ch.nil? ? '' : curr_ch
@@ -84,6 +85,11 @@ class JSONLexer
 
     return token
   end
+  
+  def build_token(lexeme, token)
+    pos = Rley::Lexical::Position.new(lineno, scanner.pos - line_start)
+    Rley::Lexical::Token.new(lexeme, token, pos)
+  end
 
   def skip_whitespaces()
     matched = scanner.scan(/[ \t\f\n\r]+/)

data/examples/general/calc_iter1/calc_lexer.rb

@@ -42,13 +42,13 @@ class CalcLexer
     skip_whitespaces
     curr_ch = scanner.peek(1)
     return nil if curr_ch.nil?
-    
+
     token = nil
 
     if '()+/'.include? curr_ch
       # Single character token
       token = build_token(@@lexeme2name[curr_ch], scanner.getch)
-      
+
     elsif (lexeme = scanner.scan(/\*\*/))
       token = build_token(@@lexeme2name[lexeme], lexeme)
     elsif (lexeme = scanner.scan(/\*/))
@@ -66,9 +66,10 @@ class CalcLexer
 
     return token
   end
-  
+
   def build_token(aSymbolName, aLexeme)
-    return Rley::Lexical::Token.new(aLexeme, aSymbolName)    
+    pos = Rley::Lexical::Position.new(1, scanner.pos)
+    return Rley::Lexical::Token.new(aLexeme, aSymbolName, pos)
   end
 
   def skip_whitespaces()

data/examples/general/calc_iter2/calc_lexer.rb

@@ -75,7 +75,8 @@ class CalcLexer
   end
   
   def build_token(aSymbolName, aLexeme)
-    return Rley::Lexical::Token.new(aLexeme, aSymbolName)    
+    pos = Rley::Lexical::Position.new(1, scanner.pos)
+    return Rley::Lexical::Token.new(aLexeme, aSymbolName, pos)    
   end
 
   def skip_whitespaces()

data/examples/general/left.rb

@@ -17,9 +17,12 @@ grammar = builder.grammar
 
 # Highly simplified tokenizer implementation.
 def tokenizer(aText, aGrammar)
+  index = 0
   tokens = aText.scan(/\./).map do |dot|
     terminal = aGrammar.name2symbol['DOT']
-    Rley::Lexical::Token.new(dot, terminal)
+    index += 1
+    pos = Rley::Lexical::Position.new(1, index)
+    Rley::Lexical::Token.new(dot, terminal, pos)
   end
 
   return tokens

data/examples/general/right.rb

@@ -17,9 +17,12 @@ grammar = builder.grammar
 
 # Highly simplified tokenizer implementation.
 def tokenizer(aText, aGrammar)
+  index = 0
   tokens = aText.scan(/\./).map do |dot|
     terminal = aGrammar.name2symbol['DOT']
-    Rley::Lexical::Token.new(dot, terminal)
+    index += 1
+    pos = Rley::Lexical::Position.new(1, index)
+    Rley::Lexical::Token.new(dot, terminal, pos)
   end
 
   return tokens

data/lib/rley/constants.rb

@@ -3,7 +3,7 @@
 
 module Rley # Module used as a namespace
   # The version number of the gem.
-  Version = '0.6.09'.freeze
+  Version = '0.7.00'.freeze
 
   # Brief description of the gem.
   Description = "Ruby implementation of the Earley's parsing algorithm".freeze

data/lib/rley/lexical/token.rb

@@ -1,5 +1,13 @@
 module Rley # This module is used as a namespace
   module Lexical # This module is used as a namespace
+    # A Position is the location of a lexeme within a source file.
+    Position = Struct.new(:line, :column) do
+      def to_s
+        "line #{line}, column #{column}"
+      end
+    end
+  
+  
     # In Rley, a (lexical) token is an object created by a lexer (tokenizer)
     # and passed to the parser. Such token an object is created when a lexer
     # detects that a sequence of characters(a lexeme) from the input stream
@@ -17,15 +25,19 @@ module Rley # This module is used as a namespace
 
       # @return [Syntax::Terminal] Terminal symbol corresponding to the lexeme.
       attr_reader(:terminal)
+      
+      # @return [Position] The position of the lexeme in the source file.      
+      attr_reader(:position)
 
       # Constructor.
       # @param theLexeme [String] the lexeme (= piece of text from input)
-      # @param aTerminal [Syntax::Terminal]
+      # @param aTerminal [Syntax::Terminal, String]
       #   The terminal symbol corresponding to the lexeme.
-      def initialize(theLexeme, aTerminal)
+      def initialize(theLexeme, aTerminal, aPosition)
         raise 'Internal error: nil terminal symbol detected' if aTerminal.nil?
         @lexeme = theLexeme
         @terminal = aTerminal
+        @position = aPosition
       end
     end # class
   end # module

data/lib/rley/parser/error_reason.rb

@@ -84,7 +84,7 @@ module Rley # Module used as a namespace
         err_msg = "Syntax error at or near token #{position} "
         err_msg << ">>>#{last_token.lexeme}<<<\n"
         err_msg << expectations
-        err_msg << ", found a '#{last_token.terminal.name}' instead."
+        err_msg << ", found a '#{last_token.terminal}' instead."
 
         return err_msg
       end

data/lib/rley/parser/gfg_earley_parser.rb

@@ -30,6 +30,10 @@ module Rley # This module is used as a namespace
 
         aTokenSequence.each_with_index do |token, i|
           parse_for_token(result, i)
+          if token.terminal.kind_of?(String)
+            symb = grammar.name2symbol[token.terminal]
+            token.instance_variable_set(:@terminal, symb)
+          end
           scan_success = scan_rule(result, i, token)
           break unless scan_success
         end

data/lib/rley/syntax/terminal.rb

@@ -14,16 +14,20 @@ module Rley # This module is used as a namespace
       end
       
       # Return true iff the symbol is a terminal
-      def terminal?()
+      def terminal?
         return true
       end
       
       # @return [false] Return true if the symbol derives
       # the empty string. As terminal symbol corresponds to a input token
       # it is by definition non-nullable.
-      def nullable?() 
+      def nullable?
         false
       end
+      
+      def to_s
+        name
+      end
     end # class
   end # module
 end # module

data/lib/support/base_tokenizer.rb

@@ -0,0 +1,197 @@
+require 'strscan'
+require_relative '../rley/lexical/token'
+
+class BaseTokenizer
+  attr_reader(:scanner)
+  attr_reader(:lineno)
+  attr_reader(:line_start)
+  
+  class ScanError < StandardError; end
+
+  # Constructor. Initialize a tokenizer for Skeem.
+  # @param source [String] Skeem text to tokenize.
+  def initialize(source)
+    @scanner = StringScanner.new('')
+    restart(source)
+  end
+
+  # @param source [String] Skeem text to tokenize.
+  def restart(source)
+    @scanner.string = source
+    @lineno = 1
+    @line_start = 0
+  end
+
+  # @return [Array<SkmToken>] | Returns a sequence of tokens
+  def tokens
+    tok_sequence = []
+    until @scanner.eos?
+      token = _next_token
+      tok_sequence << token unless token.nil?
+    end
+
+    return tok_sequence
+  end
+  
+  protected
+  
+  # Patterns:
+  # Unambiguous single character
+  # Conditional single character (e.g. '+' operator, '+' prefix for positive numbers) 
+  def _next_token
+    skip_whitespaces
+    curr_ch = scanner.peek(1)
+    return nil if curr_ch.nil? || curr_ch.empty?
+
+    token = recognize_token()
+    if token.nil? # Unknown token
+      curr_ch = scanner.peek(1)
+      erroneous = curr_ch.nil? ? '' : scanner.scan(/./)
+      sequel = scanner.scan(/.{1,20}/)
+      erroneous += sequel unless sequel.nil?
+      raise ScanError, "Unknown token #{erroneous} on line #{lineno}"
+    end
+
+    return token
+  end
+  
+  def recognize_token()
+=begin
+    if "()'`".include? curr_ch # Single characters
+      # Delimiters, separators => single character token
+      token = build_token(@@lexeme2name[curr_ch], scanner.getch)
+    elsif (lexeme = scanner.scan(/(?:\.)(?=\s)/)) # Single char occurring alone
+      token = build_token('PERIOD', lexeme)
+    elsif (lexeme = scanner.scan(/,@?/))
+      token = build_token(@@lexeme2name[lexeme], lexeme)
+    elsif (lexeme = scanner.scan(/#(?:(?:true)|(?:false)|(?:u8)|[\\\(tfeiodx]|(?:\d+[=#]))/))
+      token = cardinal_token(lexeme)        
+    elsif (lexeme = scanner.scan(/[+-]?[0-9]+(?=\s|[|()";]|$)/))
+      token = build_token('INTEGER', lexeme) # Decimal radix
+    elsif (lexeme = scanner.scan(/[+-]?[0-9]+(?:\.[0-9]+)?(?:(?:e|E)[+-]?[0-9]+)?/))
+      # Order dependency: must be tested after INTEGER case
+      token = build_token('REAL', lexeme)
+    elsif (lexeme = scanner.scan(/"(?:\\"|[^"])*"/)) # Double quotes literal?
+      token = build_token('STRING_LIT', lexeme)
+    elsif (lexeme = scanner.scan(/[a-zA-Z!$%&*\/:<=>?@^_~][a-zA-Z0-9!$%&*+-.\/:<=>?@^_~+-]*/))
+      keyw = @@keywords[lexeme.upcase]
+      tok_type = keyw ? keyw : 'IDENTIFIER'
+      token = build_token(tok_type, lexeme)
+    elsif (lexeme = scanner.scan(/\|(?:[^|])*\|/)) # Vertical bar delimited
+      token = build_token('IDENTIFIER', lexeme)
+    elsif (lexeme = scanner.scan(/([\+\-])((?=\s|[|()";])|$)/))
+      #  # R7RS peculiar identifiers case 1: isolated plus and minus as identifiers
+      token = build_token('IDENTIFIER', lexeme)
+    elsif (lexeme = scanner.scan(/[+-][a-zA-Z!$%&*\/:<=>?@^_~+-@][a-zA-Z0-9!$%&*+-.\/:<=>?@^_~+-]*/))
+      # R7RS peculiar identifiers case 2
+      token = build_token('IDENTIFIER', lexeme)
+    elsif (lexeme = scanner.scan(/\.[a-zA-Z!$%&*\/:<=>?@^_~+-@.][a-zA-Z0-9!$%&*+-.\/:<=>?@^_~+-]*/))
+      # R7RS peculiar identifiers case 4
+      token = build_token('IDENTIFIER', lexeme)
+=end  
+  end
+  
+  def build_token(aSymbolName, aLexeme, aFormat = :default)
+    begin
+      value = convert_to(aLexeme, aSymbolName, aFormat)
+      col = scanner.pos - aLexeme.size - @line_start + 1
+      pos = Rley::Lexical::Position.new(@lineno, col)
+      token = Rley::Lexical::Token.new(value, aSymbolName, pos)
+    rescue StandardError => exc
+      puts "Failing with '#{aSymbolName}' and '#{aLexeme}'"
+      raise exc
+    end
+
+    return token
+  end
+  
+  def convert_to(aLexeme, aSymbolName, aFormat)
+    return aLexeme
+  end
+ 
+  def skip_whitespaces
+    pre_pos = scanner.pos
+
+    loop do
+      ws_found = false
+      cmt_found = false
+      found = scanner.skip(/[ \t\f]+/)
+      ws_found = true if found
+      found = scanner.skip(/(?:\r\n)|\r|\n/)
+      if found
+        ws_found = true
+        next_line
+      end
+      # next_ch = scanner.peek(1)
+      # if next_ch == ';'
+        # cmt_found = true
+        # scanner.skip(/;[^\r\n]*(?:(?:\r\n)|\r|\n)?/)
+        # next_line
+      # end
+      break unless ws_found or cmt_found
+    end
+
+    curr_pos = scanner.pos
+    return if curr_pos == pre_pos
+  end
+  
+  def next_line
+    @lineno += 1
+    @line_start = scanner.pos
+  end  
+end # class
+=begin  
+require 'base_tokenizer'
+
+class PB_Tokenizer < BaseTokenizer
+  @@lexeme2name = {
+    '(' => 'LPAREN',
+    ')' => 'RPAREN',
+    '+' => 'PLUS',
+  }.freeze
+
+  protected
+  
+  def recognize_token()
+    token = nil
+    curr_ch = scanner.peek(1)
+    
+    if '()'.include? curr_ch # Single characters
+      # Delimiters, separators => single character token
+      token = build_token(@@lexeme2name[curr_ch], scanner.getch)
+    elsif (lexeme = scanner.scan(/(?:\+)(?=\s)/)) # Single char occurring alone
+      token = build_token(@@lexeme2name[lexeme], lexeme)
+     elsif (lexeme = scanner.scan(/[+-]?[0-9]+/))
+      token = build_token('INTEGER', lexeme)
+    end
+  end
+end # class
+
+  # Basic tokenizer
+  # @return [Array<Rley::Lexical::Token>]
+  def tokenize(aText)
+    tokenizer = PB_Tokenizer.new(aText)
+    tokenizer.token
+  end
+
+=end 
+=begin  
+  # Basic expression tokenizer
+  def tokenize(aText)
+    tokens = aText.scan(/\S+/).map do |lexeme|
+      case lexeme
+        when '+', '(', ')'
+          terminal = @grammar.name2symbol[lexeme]
+        when /^[-+]?\d+$/
+          terminal = @grammar.name2symbol['int']
+        else
+          msg = "Unknown input text '#{lexeme}'"
+          raise StandardError, msg
+      end
+      pos = Rley::Lexical::Position.new(1, 4) # Dummy position
+      Rley::Lexical::Token.new(lexeme, terminal, pos)
+    end
+
+    return tokens
+  end
+=end