sportdb-parser 0.7.1 → 0.7.2

data/lib/sportdb/parser/lexer-prep_doc.rb

@@ -0,0 +1,131 @@
+module SportDb
+class Lexer
+
+
+
+HTML_COMMENT_RE = %r{  <!--
+                            .*?   ## note - use non-greedy/lazy *? match
+                         -->
+                       }xm      ## note - turn on multi-line (newline) match (for dot (.))
+
+
+##
+## check for "literal"  (multi-line) note blocks
+##   eg.  nb:  or note:
+##   space required after double colon - why? why not?
+PREPROC_NOTA_BENE_RE = %r{
+         ^
+    [ ]* (?: nb | note) [ ]* : [ ]+
+       .+?  ## non-greedy
+
+    ## positive lookahead
+    ##    note - must end with blank line or end-of-file/document
+      (?=          \n[ ]*\n
+                 | \z
+        )
+}xim
+
+
+
+##
+##  note - [] block may NOT incl. square brackets
+##       what about comments (e.g. #)?
+##    todo/check - rename to NOTE_BLOCK or TEXT_BLOCK or ???
+PREPROC_BLOCK_RE = %r{  \[
+                      [^\[\]\#]*?  ## note - use non-greedy/lazy *? match
+                  \]
+}xm  ## note - turn on multi-line match (for dot(.))
+
+
+
+
+
+
+
+def _prep_doc( txt )
+    ##  preprocess automagically
+    ##   strip html comments
+    ##      keep empty lines?            - yes  (turn in BLANK tokens)
+    ##      keep leading spaces (indent) - yes  (maybe used later in upstream parser!!)
+    ##
+    ##  note - KEEP empty lines (get turned into BLANK token!!!!)
+
+
+    ### normalize unicode (decomposed chars to composed chars)
+    ##
+    ##  note:  é is decomposed (in two chars e.g.)
+    ##   e (101)
+    ##   ́  (769)
+    ##   vs
+    ##     é (233)
+    txt = txt.unicode_normalize(:nfc)
+
+
+    ##  "universal" newlines
+    ##      replace all windows-style  cr+lf (\r\n) to lf (\n) only
+    txt = txt.gsub( "\r\n", "\n" )
+
+
+
+    ###
+    ## quick hack for now
+    ##   remove  html-style comments <!-- -->
+    ##           (incl. multi-line)  with two spaces
+    ##       will mess-up lineno tracking!!!
+    ##    fix later to have function lineno & colno!!!
+    ##
+    ##  todo/fix - why? why not?
+    ##   to keep lineno intact
+    ##     replace with  space and newline
+
+    ###
+    ## add more "native" multi-line comment-styles
+    ##  e.g.    #[[ ... ]]  or  #<<< .. >>> or #<< .. >>
+    ##                 or such - why? why not?
+
+    txt = txt.gsub( HTML_COMMENT_RE ) do |m|
+                     _trace('preproc html comment:', m )
+                        '  '
+                   end
+
+
+
+   txt = txt.gsub( PREPROC_NOTA_BENE_RE ) do |m|
+       if m.include?( "\n" )   ## check for newlines (\n) and replace
+            _trace('preproc (multi-line) note/nota bene block:', m )
+           m.gsub( "\n", '↵' )
+       else
+         m
+       end
+    end
+
+
+    #####
+    ## (another) quick hack for now
+    ##   turn multi-line note blocks into
+    ##             single-line note blocks
+    ##             by changing newline (\n) to ⏎ (unicode U+23CE)
+    ##              or why not  to ___ ?
+    ##
+    ##  unicode options for return/arrows:
+    ##   -  ↵ (U+21B5): Downwards Arrow With Corner Leftwards.
+    ##                This is the most common "carriage return" symbol.
+    ##   -  ⏎ (U+23CE): Return Symbol.
+    ##               Specifically designated as the keyboard's "Return" key symbol,
+    ##                often used in user interfaces.
+
+    txt = txt.gsub( PREPROC_BLOCK_RE ) do |m|
+       if m.include?( "\n" )   ## check for newlines (\n) and replace
+         _trace( 'preproc (multi-line) block:', m )
+         m.gsub( "\n", '↵' )
+       else
+         m
+       end
+    end
+
+
+    txt
+end
+
+end  # class Lexer
+end  # module SportDb

data/lib/sportdb/parser/lexer-prep_line.rb

@@ -0,0 +1,63 @@
+module SportDb
+class Lexer
+
+
+######
+## auto-fix checks line-by-line
+
+def _prep_line( line )
+
+       ##
+       ##  first check for tabs
+       ##    add error/warn
+       ##    for auto-fix - replace tabs with two spaces
+
+        line = line.gsub( "\t" ) do |_|
+                  ## report error here
+                  ## todo/add error here
+                  _warn( "auto-fix; replacing tab (\\t) with two spaces in line #{line.inspect}" )
+                   '  '   ## replace with two spaces
+                 end
+
+
+        ## U+00A0 (160)  -- non-breaking space (unicode)
+        line = line.gsub( "\u00A0" ) do |uni|
+                  ## report error here
+                  ## todo/add error here
+                  _warn( "auto-fix; replacing non-breaking unicode space (#{uni}/#{uni.ord}) w/ ascii space ( /#{" ".ord}) in line #{line.inspect}" )
+                   ' '   ## replace with space
+                 end
+
+        ###
+        ## todo/fix - print unicode numbers for [–−]
+        ##                different candidates to differentiate and document!!!
+        ##   – => U+2013 (8211)     -- En Dash     (unicode)
+        ##   − => U+2212 (8722)     -- Minus Sign  (unicode)
+        line = line.gsub( /[–−]/ ) do |uni|
+                  ## report error here
+                  ## todo/add error here
+                  _warn( "auto-fix; replacing unicode dash (#{uni}/#{uni.ord}) w/ ascii dash (-/#{"-".ord}) in line #{line.inspect}" )
+                   '-'   ## replace with ascii dash (-)
+                  end
+
+        ####   add more unsmart quotes
+        ## smart quotes
+        line = line.gsub( /[‘’]/ ) do |uni|
+                  ## report error here
+                  ## todo/add error here
+                  _warn( "auto-fix; replacing unicode (smart) quote (#{uni}/#{uni.ord}) w/ ascii quote ('/#{"'".ord}) in line #{line.inspect}" )
+                   "'"
+                  end
+
+        line = line.gsub( /[“”]/ ) do |uni|
+                  ## report error here
+                  ## todo/add error here
+                  _warn( %Q{auto-fix; replacing unicode (smart) double quote (#{uni}/#{uni.ord}) w/ ascii double quote ("/#{'"'.ord}) in line #{line.inspect}} )
+                   '"'
+                  end
+
+   line
+end
+
+end  # class Lexer
+end  # module SportDb

data/lib/sportdb/parser/lexer-tokenize.rb

@@ -0,0 +1,449 @@
+module SportDb
+class Lexer
+
+
+###
+## use nested class for context - why? why not?
+##   note: first arg passed in MUST be ref to lexer (instance)
+class Context
+   ## passed along to on_round_def etc. handlers in tokenize_line
+   ##   note - for now only offset (in line begin/end) gets updated !!!
+     attr_writer :offset
+     attr_reader :lineno
+
+     def initialize( lexer,
+                     line:,
+                     lineno:,
+                     errors: )
+        @lexer   = lexer
+        @line    = line
+        @lineno  = lineno
+        @errors  = errors
+
+        @offset = [0,0]   ## or use [] aka [nil,nil] for not defined??? why? why not?
+        ## @offset = offset    ## MatchData offset e.g. [m.begin(0),m.end(0)]
+     end
+
+
+
+     def warn_on_else( match, mode: 'TOP' )
+         if match[:any]
+           _add_warn( "unexpected char >#{match[:any]}< (#{mode})" )
+         else
+         ##  internal error - shouldn't really happen
+           _add_warn( "internal error - unknown match (#{mode}): #{match.inspect}")
+         end
+     end
+
+
+     def _add_warn( msg )
+        ## note - warns gets logged as error for now too
+        ##          maybe add @warns later - why? why not?
+        ##
+        ##  note - add +1 to offset (start at one - not zero-based)
+        ##           will match with (external) text editors
+        msg =  "parse error (tokenize) - " +
+                          msg +
+                " in line @#{@lineno}:#{@offset[0]+1},#{@offset[1]+1} >#{@line}<  "
+
+        @errors << msg
+        @lexer.log( "!! WARN - #{msg}" )
+
+        @lexer._warn( msg )
+     end
+
+=begin
+     ##  use report/log/??_parses_error
+     def _add_error( msg )
+         msg = "parse error (tokenize) -" +
+                          msg +
+                " in line #{@lineno}@#{@offset[0]},#{@offse[1]} >#{@line}<  "
+
+        @errors << msg
+     end
+=end
+
+end  # class Context
+
+
+
+
+
+def _tokenize_line( line, lineno )
+  tokens = []
+  errors = []   ## keep a list of errors - why? why not?
+
+
+  pos = 0        ## note - usually same as offset[1] aka offset[end] after match
+  ## track last offset (begin/end) - to report error on no match
+  ##   or no match in end of string
+  offset = [0,0]
+  m = nil
+
+  ## track number of geo text seen
+  ##    (use for - do NOT break on two spaces if no geo text seen yet!!)
+  @geo_count = 0
+
+  ####
+  ## quick hack - keep re state/mode between tokenize calls!!!
+  @re  ||= RE     ## note - switch between RE & INSIDE_RE
+
+
+  if @re == RE  ## top-level
+    ### check for modes once (per line) here to speed-up parsing
+    ###   for now goals only possible for start of line!!
+    ###        fix - remove optional [] - why? why not?
+
+    ####
+    ## note - ord e.g. (45) for match number can only start a (match) line
+    ##                "inline" use NOT possible
+    ## note -  ord (for ordinal number!!!) e.g match number (1), (42), etc.
+    if (m = START_WITH_ORD.match(line))
+       ## note -  strip enclosing () and convert to integer
+       tokens << Token.new(:ORD, m[:ord],
+                                lineno: lineno, offset: m.offset(:ord),
+                                value: m[:value].to_i(10)  )
+
+       offset = m.offset(0)
+       pos    = offset[1]      ## update pos
+    elsif (m = START_WITH_YEAR.match(line))
+       tokens << Token.new(:YEAR, m[:year],
+                                 lineno: lineno, offset: m.offset(:year),
+                                 value:  m[:year].to_i(10) )
+
+       offset = m.offset(0)
+       pos    = offset[1]    ## update pos
+
+    elsif (m = START_WITH_GROUP_DEF_LINE_RE.match( line ))
+      _trace( "ENTER GROUP_DEF_RE MODE" )
+      @re = GROUP_DEF_RE
+
+      tokens << Token.new( :GROUP_DEF, m[:group_def],
+                               lineno: lineno, offset: m.offset(:group_def) )
+
+
+      offset = m.offset(0)
+      pos = offset[1]    ## update pos
+
+    elsif (m = START_WITH_PROP_KEY_RE.match( line ))
+      ##  start with prop key (match will switch into prop mode!!!)
+      ##   - fix - remove leading spaces in regex (upstream) - why? why not?
+      ##
+      ###  switch into new mode
+      ##  switch context  to PROP_RE
+        _trace("ENTER PROP_RE MODE" )
+        key = m[:key]
+
+
+        ### todo/fix - add prop yellow/red cards too - why? why not?
+        ##  todo/fix - separate sent off and red card
+        ##     sent-off - incl. red card, yellow/red card and the era before red cards!!
+        if ['sent off'].include?( key.downcase)
+          @re = PROP_CARDS_RE    ## use CARDS_RE ???
+          tokens << Token.new(:PROP_SENTOFF, m[:key],
+                                   lineno: lineno, offset: m.offset(:key))
+        elsif ['red cards'].include?( key.downcase )
+          @re = PROP_CARDS_RE    ## use CARDS_RE ???
+          tokens << Token.new(:PROP_REDCARDS, m[:key],
+                                   lineno: lineno, offset: m.offset(:key))
+        elsif ['yellow cards'].include?( key.downcase )
+          @re = PROP_CARDS_RE
+          tokens << Token.new(:PROP_YELLOWCARDS, m[:key],
+                                   lineno: lineno, offset: m.offset(:key))
+        elsif ['ref', 'referee',
+               'refs', 'referees'   ## note - allow/support assistant refs
+              ].include?( key.downcase )
+          @re = PROP_REFEREE_RE
+          tokens << Token.new(:PROP_REFEREE, m[:key],
+                                   lineno: lineno, offset: m.offset(:key))
+        elsif ['att', 'attn', 'attendance'].include?( key.downcase )
+          @re = PROP_ATTENDANCE_RE
+          tokens << Token.new(:PROP_ATTENDANCE, m[:key],
+                                   lineno: lineno, offset: m.offset(:key))
+
+     #   elsif ['goals'].include?( key.downcase )
+     #     @re = PROP_GOAL_RE
+     #     tokens << [:PROP_GOALS, m[:key]]
+
+        elsif ['penalties',
+               'penalty shootout',
+               'penalty shoot-out',
+               'penalty kicks'].include?( key.downcase )
+          @re = PROP_PENALTIES_RE
+          tokens << Token.new(:PROP_PENALTIES, m[:key],
+                                  lineno: lineno, offset: m.offset(:key))
+        else   ## assume (team) line-up
+          @re = PROP_LINEUP_RE
+          ## fix-fix-fix - rename to PROP_LINEUP !!
+          tokens << Token.new(:PROP, m[:key],
+                                 lineno: lineno, offset: m.offset(:key))
+        end
+
+        offset = m.offset(0)
+        pos    = offset[1]     ## update pos
+    ###
+    ### todo/fix
+    ###   rename to START_WITH_ROUND_DEF_OUTLINE_RE !!!!
+    elsif (m = ROUND_DEF_OUTLINE_RE.match( line ))
+      _trace( "ENTER ROUND_DEF_RE MODE" )
+      @re = ROUND_DEF_RE
+
+      ## note - return ROUND_DEF NOT  ROUND_OUTLINE token
+      ##   fix - add leading ▪ too!!
+      tokens << Token.new( :ROUND_DEF, m[:round_outline],
+                            lineno: lineno, offset: m.offset(:round_outline))
+
+      offset = m.offset(0)
+      pos    = offset[1]    ## update pos
+    elsif (m = ROUND_OUTLINE_RE.match( line ))
+      _trace( "ROUND_OUTLINE" )
+      ## note - derive round level from no of (leading) markers
+      ##             e.g. ▪/:: is 1, ▪▪/::: is 2, ▪▪▪/:::: is 3, etc.
+      ##       note  - ascii-style starts with double ::, thus, autodecrement by one!
+      round_level = m[:round_marker].size
+      round_level -= 1  if m[:round_marker].start_with?( '::' )
+
+      tokens << Token.new( :ROUND_OUTLINE, m[:round_outline],
+                           lineno: lineno, offset: m.offset(:round_outline),
+                           value: { outline: m[:round_outline],
+                                    level: round_level})
+
+      ## note - eats-up line for now (change later to only eat-up marker e.g. »|>>)
+      offset = m.offset(0)
+      pos    = offset[1]       ## update pos
+    elsif (m = START_GOAL_LINE_RE.match( line ))   ## line starting with ( - assume
+      ##  switch context to GOAL_RE (goalline(s))
+      ####
+      ##  note - check for alternate goal line styles / formats
+      if START_GOAL_LINE_COMPAT_RE.match(line )
+        ## "legacy" style starting with minute e.g.
+        ##  (6 Puskás 0-1, 9 Czibor 0-2, 11 Morlock 1-2, 18 Rahn 2-2,
+        ##    84 Rahn 3-2)
+        @re = GOAL_COMPAT_RE
+        _trace( "ENTER GOAL_COMPAT_RE MODE" )
+
+        tokens << Token.virtual( :GOALS_COMPAT, lineno: lineno )
+      elsif START_GOAL_LINE_ALT_RE.match( line )
+        ##  goals with scores e.g.
+        ##    (1-0 Franck Ribéry, 2-0 Ivica Olić, 2-1 Wayne Rooney)
+        ##         -or-
+        ##      (Dion Beljo  1-0
+        ##                   1-1  Andreas Gruber
+        ##   Matthias Seidl  2-1)
+        @re = GOAL_ALT_RE
+        _trace( "ENTER GOAL_ALT_RE MODE" )
+
+        tokens << Token.virtual( :GOALS_ALT, lineno: lineno )
+      else
+        ## "standard" / default style
+        @re = GOAL_RE
+        _trace( "ENTER GOAL_RE MODE" )
+
+        tokens << Token.virtual( :GOALS, lineno: lineno )
+      end
+
+      ## note - eat-up ( for now
+      ##   pass along "virtual" GOALS or GOALS_ALT token
+      ##      (see INLINE_GOALS for the starting goal line inline)
+      ##
+      ## fix-fix-fix
+      ##  keep offset at [0,0] - why? why not?
+      ##    do NOT eat-up
+      ##   or better
+      ##    add tokens << Token.literal( '(', lineno: lineno, offset: ...) !!!
+      offset = m.offset(0)
+      pos    = offset[1]      ## update pos
+    end
+  end
+
+
+
+  old_pos = -1   ## allows to backtrack to old pos (used in geo)
+
+
+
+
+  ctx = Context.new( self,
+                     line:   line,
+                     lineno: lineno,
+                     errors: errors )
+
+
+  while m = @re.match( line, pos )
+    # if debug?
+    #  pp m
+    #  puts "pos: #{pos}"
+    # end
+    offset = m.offset(0)
+    ctx.offset = offset
+
+
+
+    if offset[0] != pos
+      ## match NOT starting at start/begin position!!!
+      ##  report parse error!!!
+      msg =  "parse error (tokenize) - skipping >#{line[pos..(offset[0]-1)]}< in line #{lineno}@#{offset[0]},#{offset[1]} >#{line}<"
+      errors << msg
+
+      log( msg )
+      puts "!! WARN - #{msg}"
+    end
+
+
+    ##
+    ## todo/fix - also check if possible
+    ##   if no match but not yet end off string!!!!
+    ##    report skipped text run too!!!
+
+    old_pos = pos
+    pos     = offset[1]
+
+#    pp offset  if debug?
+
+    ##
+    ## note: racc requires pairs e.g. [:TOKEN, VAL]
+    ##         for VAL use "text" or ["text", { opts }]  array
+
+
+
+  t = if    @re == ROUND_DEF_RE      then   _on_round_def( m, ctx: ctx )
+      elsif @re == GROUP_DEF_RE      then   _on_group_def( m, ctx: ctx )
+      elsif @re == GEO_RE
+           ### note - possibly end inline geo on [ (and others?? in the future
+           ## note: break on double spaces e.g.
+           ## e.g. Jul/16 @ Arena Auf Schalke, Gelsenkirchen  Serbia 0-1 England
+           if m[:spaces]
+                 ### note - do NOT break out
+                 ##           if not text seen yet!!!
+                 if @geo_count > 0
+                    ## get out-off geo mode and backtrack (w/ next)
+                    ##
+                    ## todo/fix
+                    ##   add virtual geo_end token!!!
+                    _trace( "LEAVE GEO_RE MODE, BACK TO TOP_LEVEL/RE" )
+                    @re = RE
+                    pos = old_pos
+                    next   ## backtrack (resume new loop step)
+                 else
+                     nil   ## skip spaces
+                 end
+           elsif m[:space]
+               nil    ## skip (single) space
+           elsif m[:text]
+               @geo_count += 1
+                ## keep pos - why? why not?
+               Token.new(:GEO, m[:text],
+                                lineno: lineno, offset: m.offset(:text))
+           elsif m[:geo_end]   ## "hacky" special comma; always ends geo mode!!!
+                 ## get out-off geo mode and backtrack (w/ next)
+                    ## todo/fix
+                    ##   add (semi-) virtual geo_end token!!!
+                 _trace( "LEAVE GEO_RE MODE, BACK TO TOP_LEVEL/RE" )
+                 @re = RE
+                 pos = old_pos
+                 next   ## backtrack (resume new loop step)
+           elsif m[:sym]
+              case m[:sym]
+                ## note - reset geo_count to 0 (avoids break on two spaces)
+                ##                     if separator seen!!
+              when ',' then @geo_count = 0
+                            Token.literal( m[:sym], lineno: lineno, offset: m.offset(:sym))
+              when '›' then @geo_count = 0;
+                            Token.literal( ',', lineno: lineno, offset: m.offset(:sym))
+                                ## note - treat geo sep › (unicode) like comma for now!!!
+              when '>' then @geo_count = 0;
+                            Token.literal( ',', lineno: lineno, offset: m.offset(:sym))
+                               ## note - treat geo sep > (ascii) like comma for now!!!
+              when '[' then
+                    ##
+                    ## todo/fix
+                    ##   add virtual geo_end token!!!
+                 ## get out-off geo mode and backtrack (w/ next)
+                 _trace( "LEAVE GEO_RE MODE, BACK TO TOP_LEVEL/RE" )
+                 @re = RE
+                 pos = old_pos
+                 next   ## backtrack (resume new loop step)
+              else
+                 Token.literal( m[:sym], lineno: lineno, offset: m.offset(:sym))
+              end
+           else
+             ctx.warn_on_else( m, mode: 'GEO' )
+             nil
+           end
+      elsif @re == PROP_CARDS_RE       then  _on_prop_cards( m, ctx: ctx )
+      elsif @re == PROP_LINEUP_RE      then  _on_prop_lineup( m, ctx: ctx )
+      elsif @re == PROP_ATTENDANCE_RE  then  _on_prop_attendance( m, ctx: ctx )
+      elsif @re == PROP_REFEREE_RE     then  _on_prop_referee( m, ctx: ctx )
+      elsif @re == PROP_PENALTIES_RE   then  _on_prop_penalties( m, ctx: ctx )
+      elsif @re == GOAL_COMPAT_RE      then  _on_goal_compat( m, ctx: ctx )
+      elsif @re == GOAL_ALT_RE         then  _on_goal_alt( m, ctx: ctx )
+      elsif @re == GOAL_RE             then  _on_goal( m, ctx: ctx )
+      ###################################################
+      ## assume TOP_LEVEL (a.k.a. RE) machinery
+      else
+          _on_top( m, ctx: ctx )
+      end
+
+
+    tokens << t    if t
+
+#    if debug?
+#      print ">"
+#      print "*" * pos
+#      puts "#{line[pos..-1]}<"
+#    end
+  end
+
+  ## check if no match in end of string
+  if offset[1] != line.size
+    msg =  "parse error (tokenize) - skipping >#{line[offset[1]..-1]}< in line #{lineno}@#{offset[1]},#{line.size} >#{line}<"
+    errors << msg
+
+    log( msg )
+    puts "!! WARN - #{msg}"
+  end
+
+
+  # if @re == GOAL_RE   ### ALWAYS switch back to top level mode
+  #   puts "  LEAVE GOAL_RE MODE, BACK TO TOP_LEVEL/RE"  if debug?
+  #   @re = RE
+  # end
+
+   if @re == GEO_RE   ### ALWAYS switch back to top level mode
+     _trace( "LEAVE GEO_RE MODE, BACK TO TOP_LEVEL/RE" )
+     @re = RE
+   end
+
+   ### ALWAYS switch back to top level mode
+   @re = RE  if @re == GROUP_DEF_RE ||
+                @re == ROUND_DEF_RE
+
+   ##
+   ## if in prop mode continue if   last token is [,-]
+   ##        otherwise change back to "standard" mode
+   if @re == PROP_LINEUP_RE     ||
+      @re == PROP_CARDS_RE      ||
+      @re == PROP_PENALTIES_RE  ||
+      @re == PROP_ATTENDANCE_RE ||
+      @re == PROP_REFEREE_RE
+     if [',', '-', ';'].include?( tokens[-1].type)
+        ## continue/stay in PROP_RE mode
+        ##  todo/check - auto-add PROP_CONT token or such
+        ##                to help parser with possible NEWLINE
+        ##                  conflicts  - why? why not?
+     else
+        ## switch back to top-level mode!!
+        _trace( "LEAVE PROP_RE MODE, BACK TO TOP_LEVEL/RE" )
+        @re = RE
+        ## note - auto-add PROP_END (<PROP_END>)
+        tokens << Token.virtual(:PROP_END, lineno: lineno)
+     end
+   end
+
+
+  [tokens,errors]
+end
+
+
+end ## class Lexer
+end ## module SportDb