fbtxt-lexer 0.0.1

data/lib/fbtxt/lexer/lexer-tokenize_line.rb

@@ -0,0 +1,381 @@
+module Fbtxt
+class Lexer
+
+
+
+def _tokenize_line( line, lineno )
+  tokens = []
+  errors = []   ## keep a list of errors - why? why not?
+
+
+  pos = 0        ## note - usually same as offset[1] aka offset[end] after match
+  ## track last offset (begin/end) - to report error on no match
+  ##   or no match in end of string
+  offset = [0,0]
+  m = nil
+
+  ## track number of geo text seen
+  ##    (use for - do NOT break on two spaces if no geo text seen yet!!)
+  @geo_count = 0
+
+  ####
+  ## quick hack - keep re state/mode between tokenize calls!!!
+  @re  ||= RE     ## note - switch between RE & INSIDE_RE
+
+
+  if @re == RE  ## top-level
+    ### check for modes once (per line) here to speed-up parsing
+    ###   for now goals only possible for start of line!!
+    ###        fix - remove optional [] - why? why not?
+
+    ####
+    ## note - ord e.g. (45) for match number can only start a (match) line
+    ##                "inline" use NOT possible
+    ## note -  ord (for ordinal number!!!) e.g match number (1), (42), etc.
+    if (m = START_WITH_ORD.match(line))
+       ## note -  strip enclosing () and convert to integer
+       tokens << Token.new(:ORD, m[:ord],
+                                lineno: lineno, offset: m.offset(:ord),
+                                value: m[:value].to_i(10)  )
+
+       offset = m.offset(0)
+       pos    = offset[1]      ## update pos
+
+    elsif (m = START_WITH_GROUP_DEF_LINE_RE.match( line ))
+      _trace( "ENTER GROUP_DEF_RE MODE" )
+      @re = GROUP_DEF_RE
+
+      tokens << Token.new( :GROUP_DEF, m[:group_def],
+                               lineno: lineno, offset: m.offset(:group_def) )
+
+
+      offset = m.offset(0)
+      pos = offset[1]    ## update pos
+
+    elsif (m = START_WITH_PROP_KEY_RE.match( line ))
+      ##  start with prop key (match will switch into prop mode!!!)
+      ##   - fix - remove leading spaces in regex (upstream) - why? why not?
+      ##
+      ###  switch into new mode
+      ##  switch context  to PROP_RE
+        _trace("ENTER PROP_RE MODE" )
+
+        ##  check for (well-known) property (e.g. yellow,red,ref,attn, etc.)
+        ## find prop spec (token_sym, mode , ..)
+        prop = KNOWN_PROP_KEYS[ m[:key].downcase ]
+
+        if prop
+          ## e.g. :PROP_YELLOWCARD, PROP_CARDS_RE
+          prop_token, prop_re, _ = prop
+
+          @re = prop_re
+          tokens << Token.new( prop_token, m[:key],
+                                   lineno: lineno, offset: m.offset(:key))
+        else   ## assume (team) line-up
+          @re = PROP_LINEUP_RE
+          ## fix-fix-fix - rename to PROP_LINEUP !!
+          tokens << Token.new(:PROP, m[:key],
+                                 lineno: lineno, offset: m.offset(:key))
+        end
+
+        offset = m.offset(0)
+        pos    = offset[1]     ## update pos
+
+    elsif (m = START_WITH_YEAR.match(line))
+       tokens << Token.new(:YEAR, m[:year],
+                                 lineno: lineno, offset: m.offset(:year),
+                                 value:  m[:year].to_i(10) )
+
+       offset = m.offset(0)
+       pos    = offset[1]    ## update pos
+
+    ###
+    ### todo/fix
+    ###   rename to START_WITH_ROUND_DEF_OUTLINE_RE !!!!
+    elsif (m = ROUND_DEF_OUTLINE_RE.match( line ))
+      _trace( "ENTER ROUND_DEF_RE MODE" )
+      @re = ROUND_DEF_RE
+
+      ## note - return ROUND_DEF NOT  ROUND_OUTLINE token
+      ##   fix - add leading ▪ too!!
+      tokens << Token.new( :ROUND_DEF, m[:round_outline],
+                            lineno: lineno, offset: m.offset(:round_outline))
+
+      offset = m.offset(0)
+      pos    = offset[1]    ## update pos
+    elsif (m = ROUND_OUTLINE_RE.match( line ))
+      _trace( "ROUND_OUTLINE" )
+      ## note - derive round level from no of (leading) markers
+      ##             e.g. ▪/:: is 1, ▪▪/::: is 2, ▪▪▪/:::: is 3, etc.
+      ##       note  - ascii-style starts with double ::, thus, autodecrement by one!
+      round_level = m[:round_marker].size
+      round_level -= 1  if m[:round_marker].start_with?( '::' )
+
+      tokens << Token.new( :ROUND_OUTLINE, m[:round_outline],
+                           lineno: lineno, offset: m.offset(:round_outline),
+                           value: { outline: m[:round_outline],
+                                    level: round_level})
+
+      ## note - eats-up line for now (change later to only eat-up marker e.g. »|>>)
+      offset = m.offset(0)
+      pos    = offset[1]       ## update pos
+    elsif (m = START_GOAL_LINE_RE.match( line ))   ## line starting with ( - assume
+      ##  switch context to GOAL_RE (goalline(s))
+      ####
+      ##  note - check for alternate goal line styles / formats
+      if START_GOAL_LINE_COMPAT_RE.match(line )
+        ## "legacy" style starting with minute e.g.
+        ##  (6 Puskás 0-1, 9 Czibor 0-2, 11 Morlock 1-2, 18 Rahn 2-2,
+        ##    84 Rahn 3-2)
+        @re = GOAL_COMPAT_RE
+        _trace( "ENTER GOAL_COMPAT_RE MODE" )
+
+        tokens << Token.virtual( :GOALS_COMPAT, lineno: lineno )
+      elsif START_GOAL_LINE_ALT_RE.match( line )
+        ##  goals with scores e.g.
+        ##    (1-0 Franck Ribéry, 2-0 Ivica Olić, 2-1 Wayne Rooney)
+        ##         -or-
+        ##      (Dion Beljo  1-0
+        ##                   1-1  Andreas Gruber
+        ##   Matthias Seidl  2-1)
+        @re = GOAL_ALT_RE
+        _trace( "ENTER GOAL_ALT_RE MODE" )
+
+        tokens << Token.virtual( :GOALS_ALT, lineno: lineno )
+      else
+        ## "standard" / default style
+        @re = GOAL_RE
+        _trace( "ENTER GOAL_RE MODE" )
+
+        tokens << Token.virtual( :GOALS, lineno: lineno )
+      end
+
+      ## note - eat-up ( for now
+      ##   pass along "virtual" GOALS or GOALS_ALT token
+      ##      (see INLINE_GOALS for the starting goal line inline)
+      ##
+      ## fix-fix-fix
+      ##  keep offset at [0,0] - why? why not?
+      ##    do NOT eat-up
+      ##   or better
+      ##    add tokens << Token.literal( '(', lineno: lineno, offset: ...) !!!
+      offset = m.offset(0)
+      pos    = offset[1]      ## update pos
+    end
+  end
+
+
+
+  old_pos = -1   ## allows to backtrack to old pos (used in geo)
+
+
+
+
+  ctx = Context.new( self,
+                     line:   line,
+                     lineno: lineno,
+                     errors: errors )
+
+
+  while m = @re.match( line, pos )
+    # if debug?
+    #  pp m
+    #  puts "pos: #{pos}"
+    # end
+    offset = m.offset(0)
+    ctx.offset = offset
+
+
+
+    if offset[0] != pos
+      ## match NOT starting at start/begin position!!!
+      ##  report parse error!!!
+      msg =  "parse error (tokenize) - skipping >#{line[pos..(offset[0]-1)]}< in line #{lineno}@#{offset[0]},#{offset[1]} >#{line}<"
+      errors << msg
+
+      log( msg )
+      puts "!! WARN - #{msg}"
+    end
+
+
+    ##
+    ## todo/fix - also check if possible
+    ##   if no match but not yet end off string!!!!
+    ##    report skipped text run too!!!
+
+    old_pos = pos
+    pos     = offset[1]
+
+#    pp offset  if debug?
+
+    ##
+    ## note: racc requires pairs e.g. [:TOKEN, VAL]
+    ##         for VAL use "text" or ["text", { opts }]  array
+
+
+
+  t = if    @re == ROUND_DEF_RE      then   _on_round_def( m, ctx: ctx )
+      elsif @re == GROUP_DEF_RE      then   _on_group_def( m, ctx: ctx )
+      elsif @re == GEO_RE
+           ### note - possibly end inline geo on [ (and others?? in the future
+           ## note: break on double spaces e.g.
+           ## e.g. Jul/16 @ Arena Auf Schalke, Gelsenkirchen  Serbia 0-1 England
+           if m[:spaces]
+                 ### note - do NOT break out
+                 ##           if not text seen yet!!!
+                 if @geo_count > 0
+                    ## get out-off geo mode and backtrack (w/ next)
+                    ##
+                    ## todo/fix
+                    ##   add virtual geo_end token!!!
+                    _trace( "LEAVE GEO_RE MODE, BACK TO TOP_LEVEL/RE" )
+                    @re = RE
+                    pos = old_pos
+                    next   ## backtrack (resume new loop step)
+                 else
+                     nil   ## skip spaces
+                 end
+           elsif m[:space]
+               nil    ## skip (single) space
+           elsif m[:text]
+               @geo_count += 1
+                ## keep pos - why? why not?
+               Token.new(:GEO, m[:text],
+                                lineno: lineno, offset: m.offset(:text))
+           elsif m[:geo_end]   ## "hacky" special comma; always ends geo mode!!!
+                 ## get out-off geo mode and backtrack (w/ next)
+                    ## todo/fix
+                    ##   add (semi-) virtual geo_end token!!!
+                 _trace( "LEAVE GEO_RE MODE, BACK TO TOP_LEVEL/RE" )
+                 @re = RE
+                 pos = old_pos
+                 next   ## backtrack (resume new loop step)
+           elsif m[:sym]
+              case m[:sym]
+                ## note - reset geo_count to 0 (avoids break on two spaces)
+                ##                     if separator seen!!
+              when ',' then @geo_count = 0
+                            Token.literal( m[:sym], lineno: lineno, offset: m.offset(:sym))
+              when '›' then @geo_count = 0;
+                            Token.literal( ',', lineno: lineno, offset: m.offset(:sym))
+                                ## note - treat geo sep › (unicode) like comma for now!!!
+              when '>' then @geo_count = 0;
+                            Token.literal( ',', lineno: lineno, offset: m.offset(:sym))
+                               ## note - treat geo sep > (ascii) like comma for now!!!
+
+
+              when '[','▪'
+                    ##
+                    ## todo/fix
+                    ##   add virtual geo_end token!!!
+                 ## get out-off geo mode and backtrack (w/ next)
+                 _trace( "LEAVE GEO_RE MODE, BACK TO TOP_LEVEL/RE" )
+                 @re = RE
+                 pos = old_pos
+                 next   ## backtrack (resume new loop step)
+               ## fix-fix-fix  merge  '▪' with '['
+              else
+                 Token.literal( m[:sym], lineno: lineno, offset: m.offset(:sym))
+              end
+           else
+             ctx.warn_on_else( m, mode: 'GEO' )
+             nil
+           end
+      elsif @re == PROP_CARDS_RE       then  _on_prop_cards( m, ctx: ctx )
+      elsif @re == PROP_LINEUP_RE      then  _on_prop_lineup( m, ctx: ctx )
+      elsif @re == PROP_ATTENDANCE_RE  then  _on_prop_attendance( m, ctx: ctx )
+      elsif @re == PROP_REFEREE_RE     then  _on_prop_referee( m, ctx: ctx )
+      elsif @re == PROP_PENALTIES_RE   then  _on_prop_penalties( m, ctx: ctx )
+      elsif @re == PROP_COACH_RE       then  _on_prop_coach( m, ctx: ctx )
+
+      elsif @re == GOAL_COMPAT_RE      then  _on_goal_compat( m, ctx: ctx )
+      elsif @re == GOAL_ALT_RE         then  _on_goal_alt( m, ctx: ctx )
+      elsif @re == GOAL_RE             then  _on_goal( m, ctx: ctx )
+      ###################################################
+      ## assume TOP_LEVEL (a.k.a. RE) machinery
+      else
+          _on_top( m, ctx: ctx )
+      end
+
+
+    tokens << t    if t
+
+#    if debug?
+#      print ">"
+#      print "*" * pos
+#      puts "#{line[pos..-1]}<"
+#    end
+  end
+
+
+
+  ## check if no match in end of string
+  if offset[1] != line.size
+    msg =  "parse error (tokenize) - skipping >#{line[offset[1]..-1]}< in line #{lineno}@#{offset[1]},#{line.size} >#{line}<"
+    errors << msg
+
+    log( msg )
+    puts "!! WARN - #{msg}"
+  end
+
+
+  # if @re == GOAL_RE   ### ALWAYS switch back to top level mode
+  #   puts "  LEAVE GOAL_RE MODE, BACK TO TOP_LEVEL/RE"  if debug?
+  #   @re = RE
+  # end
+
+   if @re == GEO_RE   ### ALWAYS switch back to top level mode
+     _trace( "LEAVE GEO_RE MODE, BACK TO TOP_LEVEL/RE" )
+     @re = RE
+   end
+
+   ### ALWAYS switch back to top level mode
+   @re = RE  if @re == GROUP_DEF_RE ||
+                @re == ROUND_DEF_RE
+
+=begin
+   ##
+   ## if in prop mode continue if   last token is [,-]
+   ##        otherwise change back to "standard" mode
+   if @re == PROP_LINEUP_RE     ||
+      @re == PROP_CARDS_RE      ||
+      @re == PROP_PENALTIES_RE  ||
+      @re == PROP_ATTENDANCE_RE ||
+      @re == PROP_REFEREE_RE
+
+     ## note - add :CARDS_SEP_ALT (aka  -)!!! too
+     ##   maybe later  CARDS_NONE_LEFT too - why? why not?
+     if [',', '-', ';', :CARDS_SEP_ALT].include?( tokens[-1].type)
+        ## continue/stay in PROP_RE mode
+        ##  todo/check - auto-add PROP_CONT token or such
+        ##                to help parser with possible NEWLINE
+        ##                  conflicts  - why? why not?
+     else
+        ## switch back to top-level mode!!
+        _trace( "LEAVE PROP_RE MODE, BACK TO TOP_LEVEL/RE" )
+        @re = RE
+        ## note - auto-add PROP_END (<PROP_END>)
+        tokens << Token.virtual(:PROP_END, lineno: lineno)
+     end
+   end
+=end
+
+
+
+  ##########
+  ## note - auto-add end token
+  ##     inside multi-line token format that is,  prop_cont or goal_cont
+  if is_prop_cont? || is_goal_cont?
+      ## do NOT add end token; continue
+      _trace( "auto-continue - is_prop_cont? #{is_prop_cont?}, is_goal_cont? #{is_goal_cont?}" )
+  else
+      tokens << Token.virtual(:END, lineno: lineno)
+  end
+
+
+  [tokens,errors]
+end
+
+
+end ## class Lexer
+end ## module Fbtxt

data/lib/fbtxt/lexer/lexer-tokenize_norm.rb

@@ -0,0 +1,93 @@
+###
+##  tokenize pipeline (step 2) - normalize tokens
+##
+##    transform (normalize) tokens (using simple patterns)
+##      to help along the (racc look ahead 1 - LA1) parser
+
+module Fbtxt
+class Lexer
+
+
+def normalize_tokens( tokens_by_line )
+
+    tokens_by_line = tokens_by_line.map do |tokens|
+
+        nodes = []
+
+        buf = Tokens.new( tokens )
+        ## pp buf
+
+
+    loop do
+          break if buf.eos?
+
+          if buf.match?( :DATE, :TIME )   ## merge DATE TIME into DATETIME
+               date = buf.next
+               time = buf.next
+               ## puts "DATETIME:"
+               ## pp date, time
+
+               ##  note:  time value is { time: {} } or
+               ##                       { time: {}, time_local {} }
+               text  = date.text + ' ' + time.text,  ## concat string of two tokens
+               value = { date: date.value }.merge( time.value )
+
+               nodes << Token.new(:DATETIME, text,
+                                      lineno: date.lineno,
+                                      offset: [date.offset[0],
+                                               time.offset[1]],
+                                      value: value )
+          ### support  date time with comma too - why? why not?
+          elsif buf.match?( :DATE, ',', :TIME )
+               date = buf.next
+               _    = buf.next  ## ignore comma
+               time = buf.next
+               ## puts "DATETIME:"
+               ## pp date, time
+               text  = date.text + ', ' + time.text  ## concat string of two tokens
+               value =  { date: date.value }.merge( time.value )
+
+               nodes << Token.new(:DATETIME, text,
+                                      lineno: date.lineno,
+                                      offset: [date.offset[0],
+                                               time.offset[1]],
+                                     value: value )
+          elsif buf.match?( :GOAL_MINUTE, ',', :GOAL_MINUTE )
+             ## note - only advance by two tokens!
+             ##     allows more :GOAL_MINUTE sequences!! e.g. 12,13,14 etc!!!
+             ##
+             ## help parser with comma shift/reduce conflict
+             ##   change ',' to GOAL_MINUTE_SEP !!!
+             nodes << buf.next   ## pass through goal_minute
+             comma = buf.next  ## eat-up goal_minute_sep a.k.a. comma (,)
+                           ##   and replace with dedicated sep(arator)
+             nodes << Token.new( :GOAL_MINUTE_SEP,
+                                      comma.text,
+                                      lineno: comma.lineno,
+                                      offset: comma.offset,
+                                      value:  comma.value)
+          elsif buf.match?( ',', :INLINE_ATTENDANCE )
+             ## note  - allow optional comma before inline attendance
+             ## help parser with comma shift/reduce conflict
+             ##   change ',' to INLINE_ATTENDANCE_SEP !!!
+             comma = buf.next  ## eat-up inline_attendance_sep a.k.a. comma (,)
+                           ##   and replace with dedicated sep(arator)
+             nodes << Token.new(:INLINE_ATTENDANCE_SEP,
+                                    comma.text,
+                                    lineno: comma.lineno,
+                                    offset: comma.offset,
+                                    value:  comma.value)
+             nodes << buf.next   ## pass through inline_attendance
+          else
+             ## pass through
+             nodes << buf.next
+          end
+    end  # loop
+    nodes
+  end  # map tokens_by_line
+
+ tokens_by_line
+end
+
+end ## class Lexer
+end ## module Fbtxt

data/lib/fbtxt/lexer/lexer.rb

@@ -0,0 +1,192 @@
+
+module Fbtxt
+
+
+
+class Lexer
+  include Debuggable     ## auto-adds debug?, _trace, _info, etc.
+
+
+def initialize( txt )
+   raise ArgumentError, "text as string expected for lexer; got #{txt.class}"  unless txt.is_a?(String)
+
+   @txt   = txt
+end
+
+
+
+##
+##  check if lexer is in prop(erty) mode
+##      note - property lines auto-continue
+##     and break on blank or another property line
+##     note - follow-up line/match  MUST
+##        add  PROP_END token to last line!!!
+def is_prop_cont?    ## use prop_mode? or such - why? why not?
+   @re == PROP_LINEUP_RE     ||
+   @re == PROP_CARDS_RE      ||
+   @re == PROP_PENALTIES_RE  ||
+   @re == PROP_ATTENDANCE_RE ||
+   @re == PROP_REFEREE_RE    ||
+   @re == PROP_COACH_RE
+end
+
+##
+##  auto-continue
+##    ends on closing-parenthesis `)`
+def is_goal_cont?
+   @re == GOAL_RE          ||
+   @re == GOAL_ALT_RE      ||
+   @re == GOAL_COMPAT_RE
+end
+
+
+
+
+def tokenize_with_errors( flatten: true )
+
+    tokens_by_line = []   ## note: add tokens line-by-line (flatten later)
+    errors         = []   ## keep a list of errors - why? why not?
+
+
+    txt = _prep_doc( @txt )
+
+
+
+    ####
+    ## quick hack - keep re state/mode between tokenize calls!!!
+    @re  ||= RE     ## note - switch between RE & INSIDE_RE
+
+    lineno = 0
+    txt.each_line do |line|
+        lineno += 1
+
+        ## todo - "inlined virtual/collapsed/folded newlines"
+        ##   check for "↵" !!!
+        ##   and add to lineno
+
+
+        ## note - KEEP leading spaces for indent
+        ##         use rstrip (NOT left/leading & right/trainling strip) only!!
+        ## note -   remove/strip trailing newline (and optional spaces)!!!
+        ##          trailing whitespace may incl. \n or \r\n!!!
+        line = line.rstrip
+
+
+        ###  skip comments
+        ##      todo/check - change to blank line
+        ##                     to keep lineno (closer to orginal) - why? why not?
+        next  if line.match?(/\A  [ ]* ## optional leading space(s)
+                                   \#
+                                    /x )
+
+        ##  strip (inline) end-of-line comments (from line)
+        ##    check/discuss: make - inline comment require trailing space
+        ##                      e.g.   #1 vs # 1   - why? why not?
+        line = line.sub( /   [ ]*      ## (eat-up) optional leading space(s) too - why? why not?
+                              \#{1,}.*?
+                             \z
+                            /x, '' )
+
+
+        #  support __END__ marker to cut-off input
+        break if line.match?( /\A [ ]*   ## optional leading space(s)
+                                   __END__
+                                 \z
+                               /x )
+
+
+
+        ## auto-fixes line-by-line (e.g. check for tabs, smart quotes, etc.)
+        line = _prep_line( line )
+
+
+        _trace( "line #{lineno}: >#{line}<" )
+
+
+        ######
+        ### special case for empty line (aka BLANK)
+        if line.empty?
+            ## finish prop (if in prop mode)
+            tokens_by_line[-1] << Token.virtual(:PROP_END)   if is_prop_cont?
+
+           ## note - blank always resets parser mode to std/top-level!!!
+           @re = RE
+           tokens_by_line << [Token.virtual(:BLANK, lineno: lineno)]
+        elsif (m = HEADING_RE.match(line))
+            ## finish prop (if in prop mode)
+            tokens_by_line[-1] << Token.virtual(:PROP_END)   if is_prop_cont?
+
+           ## note - heading always resets parser mode to std/top-level!!!
+           @re = RE
+           _trace( 'HEADING' )
+           ## note - derive heading level from no of (leading) markers
+           ##             e.g. = is 1, == is 2, == is 3, etc.
+           heading_level = m[:heading_marker].size
+           tokens_by_line << [Token.new(:"H#{heading_level}", m[:heading], lineno: lineno)]
+        elsif (m = NOTA_BENE_RE.match(line))
+            ## finish prop (if in prop mode)
+            tokens_by_line[-1] << Token.virtual(:PROP_END)   if is_prop_cont?
+
+           ## note - nota bene always resets parser mode to std/top-level!!!
+           @re = RE
+           tokens_by_line << [Token.new(:NOTA_BENE, m[:nota_bene], lineno: lineno)]
+        else
+
+         ## finish prop (if in prop mode) and new prop upcoming!!
+         ##
+         ##   todo/fix-fix-fix -  add check for and track identation (left-side)
+         ##      (i) break if identation is same or less!!!
+         ##      (ii) handle "sub" properties too
+         if is_prop_cont? && (m = START_WITH_PROP_KEY_RE.match( line ))
+            _trace( "LEAVE PROP_RE MODE, BACK TO TOP_LEVEL/RE" )
+            @re = RE
+            tokens_by_line[-1] << Token.virtual(:PROP_END)
+         end
+
+          more_tokens, more_errors = _tokenize_line( line, lineno )
+
+          tokens_by_line  << more_tokens
+          errors          += more_errors
+        end
+
+
+        ## output last line from tokens by line in debug mode
+        _trace( "  #{tokens_by_line[-1].size} token(s): " + tokens_by_line[-1].pretty_inspect )
+
+    end # each line
+
+    ## finish prop (if in prop mode)
+    tokens_by_line[-1] << Token.virtual(:PROP_END)  if is_prop_cont?
+
+
+    ## note - always switch back to top-level at the end
+    @re = RE
+
+
+   #################
+   ##    transform (normalize) tokens (using simple patterns)
+   ##      to help along the (racc look ahead 1 - LA1) parser
+   tokens_by_line = normalize_tokens( tokens_by_line )
+    ## puts "tokens_by_line:"
+    ## pp tokens_by_line
+
+
+
+    ## flatten tokens
+    ##    check - simple use tokens_by_line.flatten - why? why not?
+    ##      tip: use flatten(1) !! - flattens exactly one level deep (only)
+    if flatten
+      tokens = []
+      tokens_by_line.each do |tok_line|
+         tokens += tok_line
+      end
+
+      [tokens,errors]
+   else
+      [tokens_by_line, errors]
+   end
+end   # method tokenize
+
+
+end  # class Lexer
+end # module Fbtxt