fbtxt-parser 0.9.0

data/lib/fbtxt/parser/lexer-tokenize.rb

@@ -0,0 +1,468 @@
+module Fbtxt
+class Lexer
+
+
+###
+## use nested class for context - why? why not?
+##   note: first arg passed in MUST be ref to lexer (instance)
+class Context
+   ## passed along to on_round_def etc. handlers in tokenize_line
+   ##   note - for now only offset (in line begin/end) gets updated !!!
+     attr_writer :offset
+     attr_reader :lineno
+
+     def initialize( lexer,
+                     line:,
+                     lineno:,
+                     errors: )
+        @lexer   = lexer
+        @line    = line
+        @lineno  = lineno
+        @errors  = errors
+
+        @offset = [0,0]   ## or use [] aka [nil,nil] for not defined??? why? why not?
+        ## @offset = offset    ## MatchData offset e.g. [m.begin(0),m.end(0)]
+     end
+
+
+
+     def warn_on_else( match, mode: 'TOP' )
+         if match[:any]
+           _add_warn( "unexpected char >#{match[:any]}< (#{mode})" )
+         else
+         ##  internal error - shouldn't really happen
+           _add_warn( "internal error - unknown match (#{mode}): #{match.inspect}")
+         end
+     end
+
+
+     def _add_warn( msg )
+        ## note - warns gets logged as error for now too
+        ##          maybe add @warns later - why? why not?
+        ##
+        ##  note - add +1 to offset (start at one - not zero-based)
+        ##           will match with (external) text editors
+        msg =  "parse error (tokenize) - " +
+                          msg +
+                " in line @#{@lineno}:#{@offset[0]+1},#{@offset[1]+1} >#{@line}<  "
+
+        @errors << msg
+        @lexer.log( "!! WARN - #{msg}" )
+
+        @lexer._warn( msg )
+     end
+
+=begin
+     ##  use report/log/??_parses_error
+     def _add_error( msg )
+         msg = "parse error (tokenize) -" +
+                          msg +
+                " in line #{@lineno}@#{@offset[0]},#{@offse[1]} >#{@line}<  "
+
+        @errors << msg
+     end
+=end
+
+end  # class Context
+
+
+
+
+
+def _tokenize_line( line, lineno )
+  tokens = []
+  errors = []   ## keep a list of errors - why? why not?
+
+
+  pos = 0        ## note - usually same as offset[1] aka offset[end] after match
+  ## track last offset (begin/end) - to report error on no match
+  ##   or no match in end of string
+  offset = [0,0]
+  m = nil
+
+  ## track number of geo text seen
+  ##    (use for - do NOT break on two spaces if no geo text seen yet!!)
+  @geo_count = 0
+
+  ####
+  ## quick hack - keep re state/mode between tokenize calls!!!
+  @re  ||= RE     ## note - switch between RE & INSIDE_RE
+
+
+  if @re == RE  ## top-level
+    ### check for modes once (per line) here to speed-up parsing
+    ###   for now goals only possible for start of line!!
+    ###        fix - remove optional [] - why? why not?
+
+    ####
+    ## note - ord e.g. (45) for match number can only start a (match) line
+    ##                "inline" use NOT possible
+    ## note -  ord (for ordinal number!!!) e.g match number (1), (42), etc.
+    if (m = START_WITH_ORD.match(line))
+       ## note -  strip enclosing () and convert to integer
+       tokens << Token.new(:ORD, m[:ord],
+                                lineno: lineno, offset: m.offset(:ord),
+                                value: m[:value].to_i(10)  )
+
+       offset = m.offset(0)
+       pos    = offset[1]      ## update pos
+    elsif (m = START_WITH_YEAR.match(line))
+       tokens << Token.new(:YEAR, m[:year],
+                                 lineno: lineno, offset: m.offset(:year),
+                                 value:  m[:year].to_i(10) )
+
+       offset = m.offset(0)
+       pos    = offset[1]    ## update pos
+
+    elsif (m = START_WITH_GROUP_DEF_LINE_RE.match( line ))
+      _trace( "ENTER GROUP_DEF_RE MODE" )
+      @re = GROUP_DEF_RE
+
+      tokens << Token.new( :GROUP_DEF, m[:group_def],
+                               lineno: lineno, offset: m.offset(:group_def) )
+
+
+      offset = m.offset(0)
+      pos = offset[1]    ## update pos
+
+    elsif (m = START_WITH_PROP_KEY_RE.match( line ))
+      ##  start with prop key (match will switch into prop mode!!!)
+      ##   - fix - remove leading spaces in regex (upstream) - why? why not?
+      ##
+      ###  switch into new mode
+      ##  switch context  to PROP_RE
+        _trace("ENTER PROP_RE MODE" )
+        key = m[:key]
+
+
+        ### todo/fix - add prop yellow/red cards too - why? why not?
+        ##  todo/fix - separate sent off and red card
+        ##     sent-off - incl. red card, yellow/red card and the era before red cards!!
+        if ['sent-off',
+            'sent off'].include?( key.downcase)
+          @re = PROP_CARDS_RE    ## use CARDS_RE ???
+          tokens << Token.new(:PROP_SENTOFF, m[:key],
+                                   lineno: lineno, offset: m.offset(:key))
+        elsif ['red cards'].include?( key.downcase )
+          @re = PROP_CARDS_RE    ## use CARDS_RE ???
+          tokens << Token.new(:PROP_REDCARDS, m[:key],
+                                   lineno: lineno, offset: m.offset(:key))
+        elsif ['yellow cards'].include?( key.downcase )
+          @re = PROP_CARDS_RE
+          tokens << Token.new(:PROP_YELLOWCARDS, m[:key],
+                                   lineno: lineno, offset: m.offset(:key))
+
+        ## todo: check allow yellow/red cards: or such as key!!!
+        ##   check for alternate spellings
+        ##   'yellow/red cards' - note - meaning closes to yellow or red !!
+        ##    thus use yellow-red cards as default/standard - why? why not?
+        elsif ['yellow-red cards',
+               'yellowred cards'].include?( key.downcase )
+          @re = PROP_CARDS_RE
+          tokens << Token.new(:PROP_YELLOWREDCARDS, m[:key],
+                                   lineno: lineno, offset: m.offset(:key))
+
+
+        elsif ['ref', 'referee',
+               'refs', 'referees'   ## note - allow/support assistant refs
+              ].include?( key.downcase )
+          @re = PROP_REFEREE_RE
+          tokens << Token.new(:PROP_REFEREE, m[:key],
+                                   lineno: lineno, offset: m.offset(:key))
+        elsif ['att', 'attn', 'attendance'].include?( key.downcase )
+          @re = PROP_ATTENDANCE_RE
+          tokens << Token.new(:PROP_ATTENDANCE, m[:key],
+                                   lineno: lineno, offset: m.offset(:key))
+
+     #   elsif ['goals'].include?( key.downcase )
+     #     @re = PROP_GOAL_RE
+     #     tokens << [:PROP_GOALS, m[:key]]
+
+        elsif ['penalties',
+               'penalty shootout',
+               'penalty shoot-out',
+               'penalty kicks'].include?( key.downcase )
+          @re = PROP_PENALTIES_RE
+          tokens << Token.new(:PROP_PENALTIES, m[:key],
+                                  lineno: lineno, offset: m.offset(:key))
+        else   ## assume (team) line-up
+          @re = PROP_LINEUP_RE
+          ## fix-fix-fix - rename to PROP_LINEUP !!
+          tokens << Token.new(:PROP, m[:key],
+                                 lineno: lineno, offset: m.offset(:key))
+        end
+
+        offset = m.offset(0)
+        pos    = offset[1]     ## update pos
+    ###
+    ### todo/fix
+    ###   rename to START_WITH_ROUND_DEF_OUTLINE_RE !!!!
+    elsif (m = ROUND_DEF_OUTLINE_RE.match( line ))
+      _trace( "ENTER ROUND_DEF_RE MODE" )
+      @re = ROUND_DEF_RE
+
+      ## note - return ROUND_DEF NOT  ROUND_OUTLINE token
+      ##   fix - add leading ▪ too!!
+      tokens << Token.new( :ROUND_DEF, m[:round_outline],
+                            lineno: lineno, offset: m.offset(:round_outline))
+
+      offset = m.offset(0)
+      pos    = offset[1]    ## update pos
+    elsif (m = ROUND_OUTLINE_RE.match( line ))
+      _trace( "ROUND_OUTLINE" )
+      ## note - derive round level from no of (leading) markers
+      ##             e.g. ▪/:: is 1, ▪▪/::: is 2, ▪▪▪/:::: is 3, etc.
+      ##       note  - ascii-style starts with double ::, thus, autodecrement by one!
+      round_level = m[:round_marker].size
+      round_level -= 1  if m[:round_marker].start_with?( '::' )
+
+      tokens << Token.new( :ROUND_OUTLINE, m[:round_outline],
+                           lineno: lineno, offset: m.offset(:round_outline),
+                           value: { outline: m[:round_outline],
+                                    level: round_level})
+
+      ## note - eats-up line for now (change later to only eat-up marker e.g. »|>>)
+      offset = m.offset(0)
+      pos    = offset[1]       ## update pos
+    elsif (m = START_GOAL_LINE_RE.match( line ))   ## line starting with ( - assume
+      ##  switch context to GOAL_RE (goalline(s))
+      ####
+      ##  note - check for alternate goal line styles / formats
+      if START_GOAL_LINE_COMPAT_RE.match(line )
+        ## "legacy" style starting with minute e.g.
+        ##  (6 Puskás 0-1, 9 Czibor 0-2, 11 Morlock 1-2, 18 Rahn 2-2,
+        ##    84 Rahn 3-2)
+        @re = GOAL_COMPAT_RE
+        _trace( "ENTER GOAL_COMPAT_RE MODE" )
+
+        tokens << Token.virtual( :GOALS_COMPAT, lineno: lineno )
+      elsif START_GOAL_LINE_ALT_RE.match( line )
+        ##  goals with scores e.g.
+        ##    (1-0 Franck Ribéry, 2-0 Ivica Olić, 2-1 Wayne Rooney)
+        ##         -or-
+        ##      (Dion Beljo  1-0
+        ##                   1-1  Andreas Gruber
+        ##   Matthias Seidl  2-1)
+        @re = GOAL_ALT_RE
+        _trace( "ENTER GOAL_ALT_RE MODE" )
+
+        tokens << Token.virtual( :GOALS_ALT, lineno: lineno )
+      else
+        ## "standard" / default style
+        @re = GOAL_RE
+        _trace( "ENTER GOAL_RE MODE" )
+
+        tokens << Token.virtual( :GOALS, lineno: lineno )
+      end
+
+      ## note - eat-up ( for now
+      ##   pass along "virtual" GOALS or GOALS_ALT token
+      ##      (see INLINE_GOALS for the starting goal line inline)
+      ##
+      ## fix-fix-fix
+      ##  keep offset at [0,0] - why? why not?
+      ##    do NOT eat-up
+      ##   or better
+      ##    add tokens << Token.literal( '(', lineno: lineno, offset: ...) !!!
+      offset = m.offset(0)
+      pos    = offset[1]      ## update pos
+    end
+  end
+
+
+
+  old_pos = -1   ## allows to backtrack to old pos (used in geo)
+
+
+
+
+  ctx = Context.new( self,
+                     line:   line,
+                     lineno: lineno,
+                     errors: errors )
+
+
+  while m = @re.match( line, pos )
+    # if debug?
+    #  pp m
+    #  puts "pos: #{pos}"
+    # end
+    offset = m.offset(0)
+    ctx.offset = offset
+
+
+
+    if offset[0] != pos
+      ## match NOT starting at start/begin position!!!
+      ##  report parse error!!!
+      msg =  "parse error (tokenize) - skipping >#{line[pos..(offset[0]-1)]}< in line #{lineno}@#{offset[0]},#{offset[1]} >#{line}<"
+      errors << msg
+
+      log( msg )
+      puts "!! WARN - #{msg}"
+    end
+
+
+    ##
+    ## todo/fix - also check if possible
+    ##   if no match but not yet end off string!!!!
+    ##    report skipped text run too!!!
+
+    old_pos = pos
+    pos     = offset[1]
+
+#    pp offset  if debug?
+
+    ##
+    ## note: racc requires pairs e.g. [:TOKEN, VAL]
+    ##         for VAL use "text" or ["text", { opts }]  array
+
+
+
+  t = if    @re == ROUND_DEF_RE      then   _on_round_def( m, ctx: ctx )
+      elsif @re == GROUP_DEF_RE      then   _on_group_def( m, ctx: ctx )
+      elsif @re == GEO_RE
+           ### note - possibly end inline geo on [ (and others?? in the future
+           ## note: break on double spaces e.g.
+           ## e.g. Jul/16 @ Arena Auf Schalke, Gelsenkirchen  Serbia 0-1 England
+           if m[:spaces]
+                 ### note - do NOT break out
+                 ##           if not text seen yet!!!
+                 if @geo_count > 0
+                    ## get out-off geo mode and backtrack (w/ next)
+                    ##
+                    ## todo/fix
+                    ##   add virtual geo_end token!!!
+                    _trace( "LEAVE GEO_RE MODE, BACK TO TOP_LEVEL/RE" )
+                    @re = RE
+                    pos = old_pos
+                    next   ## backtrack (resume new loop step)
+                 else
+                     nil   ## skip spaces
+                 end
+           elsif m[:space]
+               nil    ## skip (single) space
+           elsif m[:text]
+               @geo_count += 1
+                ## keep pos - why? why not?
+               Token.new(:GEO, m[:text],
+                                lineno: lineno, offset: m.offset(:text))
+           elsif m[:geo_end]   ## "hacky" special comma; always ends geo mode!!!
+                 ## get out-off geo mode and backtrack (w/ next)
+                    ## todo/fix
+                    ##   add (semi-) virtual geo_end token!!!
+                 _trace( "LEAVE GEO_RE MODE, BACK TO TOP_LEVEL/RE" )
+                 @re = RE
+                 pos = old_pos
+                 next   ## backtrack (resume new loop step)
+           elsif m[:sym]
+              case m[:sym]
+                ## note - reset geo_count to 0 (avoids break on two spaces)
+                ##                     if separator seen!!
+              when ',' then @geo_count = 0
+                            Token.literal( m[:sym], lineno: lineno, offset: m.offset(:sym))
+              when '›' then @geo_count = 0;
+                            Token.literal( ',', lineno: lineno, offset: m.offset(:sym))
+                                ## note - treat geo sep › (unicode) like comma for now!!!
+              when '>' then @geo_count = 0;
+                            Token.literal( ',', lineno: lineno, offset: m.offset(:sym))
+                               ## note - treat geo sep > (ascii) like comma for now!!!
+
+
+              when '[','▪'
+                    ##
+                    ## todo/fix
+                    ##   add virtual geo_end token!!!
+                 ## get out-off geo mode and backtrack (w/ next)
+                 _trace( "LEAVE GEO_RE MODE, BACK TO TOP_LEVEL/RE" )
+                 @re = RE
+                 pos = old_pos
+                 next   ## backtrack (resume new loop step)
+               ## fix-fix-fix  merge  '▪' with '['
+              else
+                 Token.literal( m[:sym], lineno: lineno, offset: m.offset(:sym))
+              end
+           else
+             ctx.warn_on_else( m, mode: 'GEO' )
+             nil
+           end
+      elsif @re == PROP_CARDS_RE       then  _on_prop_cards( m, ctx: ctx )
+      elsif @re == PROP_LINEUP_RE      then  _on_prop_lineup( m, ctx: ctx )
+      elsif @re == PROP_ATTENDANCE_RE  then  _on_prop_attendance( m, ctx: ctx )
+      elsif @re == PROP_REFEREE_RE     then  _on_prop_referee( m, ctx: ctx )
+      elsif @re == PROP_PENALTIES_RE   then  _on_prop_penalties( m, ctx: ctx )
+      elsif @re == GOAL_COMPAT_RE      then  _on_goal_compat( m, ctx: ctx )
+      elsif @re == GOAL_ALT_RE         then  _on_goal_alt( m, ctx: ctx )
+      elsif @re == GOAL_RE             then  _on_goal( m, ctx: ctx )
+      ###################################################
+      ## assume TOP_LEVEL (a.k.a. RE) machinery
+      else
+          _on_top( m, ctx: ctx )
+      end
+
+
+    tokens << t    if t
+
+#    if debug?
+#      print ">"
+#      print "*" * pos
+#      puts "#{line[pos..-1]}<"
+#    end
+  end
+
+  ## check if no match in end of string
+  if offset[1] != line.size
+    msg =  "parse error (tokenize) - skipping >#{line[offset[1]..-1]}< in line #{lineno}@#{offset[1]},#{line.size} >#{line}<"
+    errors << msg
+
+    log( msg )
+    puts "!! WARN - #{msg}"
+  end
+
+
+  # if @re == GOAL_RE   ### ALWAYS switch back to top level mode
+  #   puts "  LEAVE GOAL_RE MODE, BACK TO TOP_LEVEL/RE"  if debug?
+  #   @re = RE
+  # end
+
+   if @re == GEO_RE   ### ALWAYS switch back to top level mode
+     _trace( "LEAVE GEO_RE MODE, BACK TO TOP_LEVEL/RE" )
+     @re = RE
+   end
+
+   ### ALWAYS switch back to top level mode
+   @re = RE  if @re == GROUP_DEF_RE ||
+                @re == ROUND_DEF_RE
+
+   ##
+   ## if in prop mode continue if   last token is [,-]
+   ##        otherwise change back to "standard" mode
+   if @re == PROP_LINEUP_RE     ||
+      @re == PROP_CARDS_RE      ||
+      @re == PROP_PENALTIES_RE  ||
+      @re == PROP_ATTENDANCE_RE ||
+      @re == PROP_REFEREE_RE
+
+     ## note - add :CARDS_SEP_ALT (aka  -)!!! too
+     ##   maybe later  CARDS_NONE_LEFT too - why? why not?
+     if [',', '-', ';', :CARDS_SEP_ALT].include?( tokens[-1].type)
+        ## continue/stay in PROP_RE mode
+        ##  todo/check - auto-add PROP_CONT token or such
+        ##                to help parser with possible NEWLINE
+        ##                  conflicts  - why? why not?
+     else
+        ## switch back to top-level mode!!
+        _trace( "LEAVE PROP_RE MODE, BACK TO TOP_LEVEL/RE" )
+        @re = RE
+        ## note - auto-add PROP_END (<PROP_END>)
+        tokens << Token.virtual(:PROP_END, lineno: lineno)
+     end
+   end
+
+
+  [tokens,errors]
+end
+
+
+end ## class Lexer
+end ## module Fbtxt

data/lib/fbtxt/parser/lexer.rb

@@ -0,0 +1,231 @@
+
+module Fbtxt
+
+class LexerResult
+   attr_reader :tokens, :errors
+   def initialize( tokens, errors=[] )
+       @tokens, @errors = tokens, errors
+   end
+
+   def ok?()  @errors.size == 0; end
+   def nok?() !ok?; end
+end  # class LexerResult
+
+
+
+class Lexer
+  include Debuggable     ## auto-adds debug?, _trace, _info, etc.
+
+
+def initialize( txt )
+   raise ArgumentError, "text as string expected for lexer; got #{txt.class}"  unless txt.is_a?(String)
+
+   @txt   = txt
+end
+
+
+def tokenize_with_errors( flatten: true )
+
+    tokens_by_line = []   ## note: add tokens line-by-line (flatten later)
+    errors         = []   ## keep a list of errors - why? why not?
+
+
+    txt = _prep_doc( @txt )
+
+
+
+    ####
+    ## quick hack - keep re state/mode between tokenize calls!!!
+    @re  ||= RE     ## note - switch between RE & INSIDE_RE
+
+    lineno = 0
+    txt.each_line do |line|
+        lineno += 1
+
+        ## todo - "inlined virtual/collapsed/folded newlines"
+        ##   check for "↵" !!!
+        ##   and add to lineno
+
+
+        ## note - KEEP leading spaces for indent
+        ##         use rstrip (NOT left/leading & right/trainling strip) only!!
+        ## note -   remove/strip trailing newline (and optional spaces)!!!
+        ##          trailing whitespace may incl. \n or \r\n!!!
+        line = line.rstrip
+
+
+        ###  skip comments
+        ##      todo/check - change to blank line
+        ##                     to keep lineno (closer to orginal) - why? why not?
+        next  if line.match?(/\A  [ ]* ## optional leading space(s)
+                                   \#
+                                    /x )
+
+        ##  strip (inline) end-of-line comments (from line)
+        ##    check/discuss: make - inline comment require trailing space
+        ##                      e.g.   #1 vs # 1   - why? why not?
+        line = line.sub( /   [ ]*      ## (eat-up) optional leading space(s) too - why? why not?
+                              \#{1,}.*?
+                             \z
+                            /x, '' )
+
+
+        #  support __END__ marker to cut-off input
+        break if line.match?( /\A [ ]*   ## optional leading space(s)
+                                   __END__
+                                 \z
+                               /x )
+
+
+
+        ## auto-fixes line-by-line (e.g. check for tabs, smart quotes, etc.)
+        line = _prep_line( line )
+
+
+        _trace( "line #{lineno}: >#{line}<" )
+
+
+        ######
+        ### special case for empty line (aka BLANK)
+        if line.empty?
+           ## note - blank always resets parser mode to std/top-level!!!
+           @re = RE
+           tokens_by_line << [Token.virtual(:BLANK, lineno: lineno)]
+        elsif (m = HEADING_RE.match(line))
+           ## note - heading always resets parser mode to std/top-level!!!
+           @re = RE
+           _trace( 'HEADING' )
+           ## note - derive heading level from no of (leading) markers
+           ##             e.g. = is 1, == is 2, == is 3, etc.
+           heading_level = m[:heading_marker].size
+           tokens_by_line << [Token.new(:"H#{heading_level}", m[:heading], lineno: lineno)]
+        elsif (m = NOTA_BENE_RE.match(line))
+           ## note - nota bene always resets parser mode to std/top-level!!!
+           @re = RE
+           tokens_by_line << [Token.new(:NOTA_BENE, m[:nota_bene], lineno: lineno)]
+        else
+
+          more_tokens, more_errors = _tokenize_line( line, lineno )
+
+          tokens_by_line  << more_tokens
+          errors          += more_errors
+        end
+
+        ## output last line from tokens by line in debug mode
+        _trace( "  #{tokens_by_line[-1].size} token(s): " + tokens_by_line[-1].pretty_inspect )
+
+    end # each line
+
+
+
+
+    tokens_by_line = tokens_by_line.map do |tokens|
+
+        #################
+        ##    transform tokens (using simple patterns)
+        ##      to help along the (racc look ahead 1 - LA1) parser
+        nodes = []
+
+        buf = Tokens.new( tokens )
+        ## pp buf
+
+
+    loop do
+          break if buf.eos?
+
+          if buf.match?( :DATE, :TIME )   ## merge DATE TIME into DATETIME
+               date = buf.next
+               time = buf.next
+               ## puts "DATETIME:"
+               ## pp date, time
+
+               ##  note:  time value is { time: {} } or
+               ##                       { time: {}, time_local {} }
+               text  = date.text + ' ' + time.text,  ## concat string of two tokens
+               value = { date: date.value }.merge( time.value )
+
+               nodes << Token.new(:DATETIME, text,
+                                      lineno: date.lineno,
+                                      offset: [date.offset[0],
+                                               time.offset[1]],
+                                      value: value )
+          ### support  date time with comma too - why? why not?
+          elsif buf.match?( :DATE, ',', :TIME )
+               date = buf.next
+               _    = buf.next  ## ignore comma
+               time = buf.next
+               ## puts "DATETIME:"
+               ## pp date, time
+               text  = date.text + ', ' + time.text  ## concat string of two tokens
+               value =  { date: date.value }.merge( time.value )
+
+               nodes << Token.new(:DATETIME, text,
+                                      lineno: date.lineno,
+                                      offset: [date.offset[0],
+                                               time.offset[1]],
+                                     value: value )
+          elsif buf.match?( :GOAL_MINUTE, ',', :GOAL_MINUTE )
+             ## note - only advance by two tokens!
+             ##     allows more :GOAL_MINUTE sequences!! e.g. 12,13,14 etc!!!
+             ##
+             ## help parser with comma shift/reduce conflict
+             ##   change ',' to GOAL_MINUTE_SEP !!!
+             nodes << buf.next   ## pass through goal_minute
+             comma = buf.next  ## eat-up goal_minute_sep a.k.a. comma (,)
+                           ##   and replace with dedicated sep(arator)
+             nodes << Token.new( :GOAL_MINUTE_SEP,
+                                      comma.text,
+                                      lineno: comma.lineno,
+                                      offset: comma.offset,
+                                      value:  comma.value)
+          elsif buf.match?( ',', :INLINE_ATTENDANCE )
+             ## note  - allow optional comma before inline attendance
+             ## help parser with comma shift/reduce conflict
+             ##   change ',' to INLINE_ATTENDANCE_SEP !!!
+             comma = buf.next  ## eat-up inline_attendance_sep a.k.a. comma (,)
+                           ##   and replace with dedicated sep(arator)
+             nodes << Token.new(:INLINE_ATTENDANCE_SEP,
+                                    comma.text,
+                                    lineno: comma.lineno,
+                                    offset: comma.offset,
+                                    value:  comma.value)
+             nodes << buf.next   ## pass through inline_attendance
+          else
+             ## pass through
+             nodes << buf.next
+          end
+    end  # loop
+    nodes
+  end  # map tokens_by_line
+
+
+    ## puts "tokens_by_line:"
+    ## pp tokens_by_line
+
+    if flatten
+      ## flatten tokens
+      tokens = []
+      tokens_by_line.each do |tok_line|
+
+        ## if debug?
+        ##   pp tok_line
+        ## end
+
+         tokens  += tok_line
+
+         ## auto-add newlines  (unless BLANK!!)
+         unless tok_line[0] && tok_line[0].type == :BLANK
+            ## note - reuse lineno from first token in line
+            ##                  use last - why? why not?
+            tokens  << Token.newline( lineno: tok_line[0].lineno )
+         end
+      end
+      [tokens,errors]
+   else
+      [tokens_by_line, errors]
+   end
+end   # method tokenize
+
+
+end  # class Lexer
+end # module Fbtxt