fbtxt-parser 0.9.0

data/lib/fbtxt/parser/lexer-on_round_def.rb

@@ -0,0 +1,37 @@
+module Fbtxt
+class Lexer
+
+
+
+### note - add comma (,) as optional separator
+ROUND_DEF_RE = Regexp.union(  SPACES_RE,
+                              DURATION_RE,  # note - duration MUST match before date
+                              DATE_RE,  ## note - date must go before time (e.g. 12.12. vs 12.12)
+                              / (?<sym> [:|,] ) /x,
+                              ANY_RE
+                           )
+
+
+def _on_round_def( m, ctx: )      ## note - m is MatchData object
+
+
+           if m[:spaces] || m[:space]
+               nil    ## skip spaces
+           elsif m[:date]
+              Token.new(:DATE, m[:date],
+                           lineno: ctx.lineno, offset: m.offset(:date),
+                           value: _build_date(m))
+           elsif m[:duration]
+              Token.new(:DURATION, m[:duration],
+                            lineno: ctx.lineno, offset: m.offset(:duration),
+                            value: _build_duration( m ))
+           elsif m[:sym]
+              Token.literal( m[:sym], lineno: ctx.lineno, offset: m.offset(:sym))
+           else
+              ctx.warn_on_else( m, mode: 'ROUND_DEF' )
+              nil
+           end
+end
+
+end ## class Lexer
+end ## module Fbtxt

data/lib/fbtxt/parser/lexer-on_top.rb

@@ -0,0 +1,133 @@
+module Fbtxt
+class Lexer
+
+
+def _on_top( m, ctx: )      ## note - m is MatchData object
+
+        ##  note - top-level (for now always) assumes TEAM for TEXT match!!
+        ##           fix/fix/fix change TEXT_RE/:text to  TEAM_RE/:team !!!
+
+        if m[:space] || m[:spaces]
+           nil   ## skip space(s)
+        elsif m[:text]         then Token.new(:TEAM,  m[:text],
+                                                      lineno: ctx.lineno, offset: m.offset(:text))
+        elsif m[:team_home]    then Token.new(:TEAM_HOME,  m[:team_home],
+                                                      lineno: ctx.lineno, offset: m.offset(:team_home))
+        elsif m[:team_away]    then Token.new(:TEAM_AWAY,  m[:team_away],
+                                                      lineno: ctx.lineno, offset: m.offset(:team_away))
+        elsif m[:team_neutral] then Token.new(:TEAM_NEUTRAL, m[:team_neutral],
+                                                      lineno: ctx.lineno, offset: m.offset(:team_neutral))
+
+        ## (match) status e.g. cancelled, awarded, etc.
+        ##  inline:  w/o - walkover
+        ##           n/p - not played
+        ##           bye
+        ##           abd/abd. - abandoned
+        ##           void
+        ##           susp/susp. - suspended
+        ##           ppd/ppd. or postp/postp. - postponed
+        ##           awd/awd. - awarded
+        ##           canc/canc. - cancelled/canceled
+        elsif m[:inline_wo]   then Token.new(:INLINE_WO, m[:inline_wo],
+                                                  lineno: ctx.lineno, offset: m.offset(:inline_wo))
+        elsif m[:inline_np]   then Token.new(:INLINE_NP, m[:inline_np],
+                                                  lineno: ctx.lineno, offset: m.offset(:inline_np))
+        elsif m[:inline_bye]  then Token.new(:INLINE_BYE, m[:inline_bye],
+                                                  lineno: ctx.lineno, offset: m.offset(:inline_bye))
+        elsif m[:inline_abd]  then Token.new(:INLINE_ABD, m[:inline_abd],
+                                                  lineno: ctx.lineno, offset: m.offset(:inline_abd))
+        elsif m[:inline_void] then Token.new(:INLINE_VOID, m[:inline_void],
+                                                  lineno: ctx.lineno, offset: m.offset(:inline_void))
+        elsif m[:inline_susp] then Token.new(:INLINE_SUSP, m[:inline_susp],
+                                                  lineno: ctx.lineno, offset: m.offset(:inline_susp))
+        elsif m[:inline_ppd]  then Token.new(:INLINE_PPD, m[:inline_ppd],
+                                                  lineno: ctx.lineno, offset: m.offset(:inline_ppd))
+        elsif m[:inline_awd]  then Token.new(:INLINE_AWD, m[:inline_awd],
+                                                  lineno: ctx.lineno, offset: m.offset(:inline_awd))
+        elsif m[:inline_canc] then Token.new(:INLINE_CANC, m[:inline_canc],
+                                                  lineno: ctx.lineno, offset: m.offset(:inline_canc))
+
+        elsif m[:inline_round_short] then Token.new(:INLINE_ROUND_SHORT, m[0],
+                                                  lineno: ctx.lineno, offset: m.offset(0),
+                                                     value: m[:inline_round_text])
+        elsif m[:inline_round_big] then Token.new(:INLINE_ROUND_BIG, m[0],
+                                                  lineno: ctx.lineno, offset: m.offset(0),
+                                                     value: m[:inline_round_text])
+
+        elsif m[:status]      then Token.new(:STATUS, m[:status],
+                                                  lineno: ctx.lineno, offset: m.offset(:status),
+                                                  value: _build_status( m ))
+        elsif m[:note]
+            ###  todo/check:
+            ##      use value hash - why? why not? or simplify to:
+            ## [:NOTE, [m[:note], {note: m[:note] } ]]
+             Token.new(:NOTE, m[:note],
+                               lineno: ctx.lineno, offset: m.offset(:note))
+
+        elsif m[:attendance]
+             att = {}
+             att[:value] = m[:value].gsub( '_', '' ).to_i(10)
+             ## note - for token id use INLINE_ATTENDANCE  (ATTENDANCE in use for prop!!!)
+            Token.new(:INLINE_ATTENDANCE, m[:attendance],
+                                   lineno: ctx.lineno, offset: m.offset(:attendance),
+                                          value: att)
+
+        elsif m[:time]         then Token.new(:TIME, m[:time],
+                                                lineno: ctx.lineno, offset: m.offset(:time),
+                                                value: _build_time(m))
+        elsif m[:date]         then Token.new(:DATE, m[:date],
+                                                lineno: ctx.lineno, offset: m.offset(:date),
+                                                value: _build_date(m))
+        elsif m[:date_legs]    then Token.new(:DATE_LEGS, m[:date_legs],
+                                                 lineno: ctx.lineno, offset: m.offset(:date_legs),
+                                                 value: _build_date_legs(m))
+
+        elsif m[:score_legs]   then Token.new(:SCORE_LEGS, m[:score_legs],
+                                                  lineno: ctx.lineno, offset: m.offset(:score_legs),
+                                                  value: _build_score_legs( m ))
+        elsif m[:score_full]   then Token.new(:SCORE_FULL, m[:score_full],
+                                                  lineno: ctx.lineno, offset: m.offset(:score_full),
+                                                  value: _build_score_full( m ))
+        elsif m[:score_fuller] then Token.new(:SCORE_FULLER, m[:score_fuller],
+                                                  lineno: ctx.lineno, offset: m.offset(:score_fuller),
+                                                  value: _build_score_fuller( m ))
+        elsif m[:score_fuller_more] then Token.new(:SCORE_FULLER_MORE, m[:score_fuller_more],
+                                                      lineno: ctx.lineno, offset: m.offset(:score_fuller_more),
+                                                      value: _build_score_fuller_more( m ))
+        elsif m[:score]      then Token.new(:SCORE,  m[:score],
+                                                lineno: ctx.lineno, offset: m.offset(:score),
+                                                value: _build_score( m ))
+        elsif m[:score_awd]  then Token.new(:SCORE_AWD, m[:score_awd],
+                                                lineno: ctx.lineno, offset: m.offset(:score_awd),
+                                                value: _build_score_awd( m ))
+        elsif m[:score_abd]  then Token.new(:SCORE_ABD, m[:score_abd],
+                                                lineno: ctx.lineno, offset: m.offset(:score_abd),
+                                                value: _build_score_abd( m ))
+
+        elsif m[:vs]         then Token.new(:VS, m[:vs],
+                                              lineno: ctx.lineno, offset: m.offset(:vs))
+        elsif m[:sym]
+          case m[:sym]  ## return symbols "inline" as is - why? why not?
+          when '@'    ##  enter geo mode
+            _trace( 'ENTER GEO_RE MODE' )
+            @re = GEO_RE
+            @geo_count = 0
+            Token.literal( m[:sym], lineno: ctx.lineno, offset: m.offset(:sym))
+          when '('    ## enter goal scorer mode on "free-floating" open paranthesis!!!
+             _trace( 'ENTER GOAL_RE MODE' )
+             @re = GOAL_RE
+              ## note - eat-up ( for now; do NOT pass along as token
+              ##       pass along "virutal" INLINE GOALS - why? why not?
+              Token.virtual( :INLINE_GOALS, lineno: ctx.lineno )
+          else
+            Token.literal( m[:sym], lineno: ctx.lineno, offset: m.offset(:sym))
+          end
+        else
+           ctx.warn_on_else( m )
+           nil
+        end
+end
+
+
+end ## class Lexer
+end ## module Fbtxt

data/lib/fbtxt/parser/lexer-prep_doc.rb

@@ -0,0 +1,131 @@
+module Fbtxt
+class Lexer
+
+
+
+HTML_COMMENT_RE = %r{  <!--
+                            .*?   ## note - use non-greedy/lazy *? match
+                         -->
+                       }xm      ## note - turn on multi-line (newline) match (for dot (.))
+
+
+##
+## check for "literal"  (multi-line) note blocks
+##   eg.  nb:  or note:
+##   space required after double colon - why? why not?
+PREPROC_NOTA_BENE_RE = %r{
+         ^
+    [ ]* (?: nb | note) [ ]* : [ ]+
+       .+?  ## non-greedy
+
+    ## positive lookahead
+    ##    note - must end with blank line or end-of-file/document
+      (?=          \n[ ]*\n
+                 | \z
+        )
+}xim
+
+
+
+##
+##  note - [] block may NOT incl. square brackets
+##       what about comments (e.g. #)?
+##    todo/check - rename to NOTE_BLOCK or TEXT_BLOCK or ???
+PREPROC_BLOCK_RE = %r{  \[
+                      [^\[\]\#]*?  ## note - use non-greedy/lazy *? match
+                  \]
+}xm  ## note - turn on multi-line match (for dot(.))
+
+
+
+
+
+
+
+def _prep_doc( txt )
+    ##  preprocess automagically
+    ##   strip html comments
+    ##      keep empty lines?            - yes  (turn in BLANK tokens)
+    ##      keep leading spaces (indent) - yes  (maybe used later in upstream parser!!)
+    ##
+    ##  note - KEEP empty lines (get turned into BLANK token!!!!)
+
+
+    ### normalize unicode (decomposed chars to composed chars)
+    ##
+    ##  note:  é is decomposed (in two chars e.g.)
+    ##   e (101)
+    ##   ́  (769)
+    ##   vs
+    ##     é (233)
+    txt = txt.unicode_normalize(:nfc)
+
+
+    ##  "universal" newlines
+    ##      replace all windows-style  cr+lf (\r\n) to lf (\n) only
+    txt = txt.gsub( "\r\n", "\n" )
+
+
+
+    ###
+    ## quick hack for now
+    ##   remove  html-style comments <!-- -->
+    ##           (incl. multi-line)  with two spaces
+    ##       will mess-up lineno tracking!!!
+    ##    fix later to have function lineno & colno!!!
+    ##
+    ##  todo/fix - why? why not?
+    ##   to keep lineno intact
+    ##     replace with  space and newline
+
+    ###
+    ## add more "native" multi-line comment-styles
+    ##  e.g.    #[[ ... ]]  or  #<<< .. >>> or #<< .. >>
+    ##                 or such - why? why not?
+
+    txt = txt.gsub( HTML_COMMENT_RE ) do |m|
+                     _trace('preproc html comment:', m )
+                        '  '
+                   end
+
+
+
+   txt = txt.gsub( PREPROC_NOTA_BENE_RE ) do |m|
+       if m.include?( "\n" )   ## check for newlines (\n) and replace
+            _trace('preproc (multi-line) note/nota bene block:', m )
+           m.gsub( "\n", '↵' )
+       else
+         m
+       end
+    end
+
+
+    #####
+    ## (another) quick hack for now
+    ##   turn multi-line note blocks into
+    ##             single-line note blocks
+    ##             by changing newline (\n) to ⏎ (unicode U+23CE)
+    ##              or why not  to ___ ?
+    ##
+    ##  unicode options for return/arrows:
+    ##   -  ↵ (U+21B5): Downwards Arrow With Corner Leftwards.
+    ##                This is the most common "carriage return" symbol.
+    ##   -  ⏎ (U+23CE): Return Symbol.
+    ##               Specifically designated as the keyboard's "Return" key symbol,
+    ##                often used in user interfaces.
+
+    txt = txt.gsub( PREPROC_BLOCK_RE ) do |m|
+       if m.include?( "\n" )   ## check for newlines (\n) and replace
+         _trace( 'preproc (multi-line) block:', m )
+         m.gsub( "\n", '↵' )
+       else
+         m
+       end
+    end
+
+
+    txt
+end
+
+end  # class Lexer
+end  # module Fbtxt

data/lib/fbtxt/parser/lexer-prep_line.rb

@@ -0,0 +1,63 @@
+module Fbtxt
+class Lexer
+
+
+######
+## auto-fix checks line-by-line
+
+def _prep_line( line )
+
+       ##
+       ##  first check for tabs
+       ##    add error/warn
+       ##    for auto-fix - replace tabs with two spaces
+
+        line = line.gsub( "\t" ) do |_|
+                  ## report error here
+                  ## todo/add error here
+                  _warn( "auto-fix; replacing tab (\\t) with two spaces in line #{line.inspect}" )
+                   '  '   ## replace with two spaces
+                 end
+
+
+        ## U+00A0 (160)  -- non-breaking space (unicode)
+        line = line.gsub( "\u00A0" ) do |uni|
+                  ## report error here
+                  ## todo/add error here
+                  _warn( "auto-fix; replacing non-breaking unicode space (#{uni}/#{uni.ord}) w/ ascii space ( /#{" ".ord}) in line #{line.inspect}" )
+                   ' '   ## replace with space
+                 end
+
+        ###
+        ## todo/fix - print unicode numbers for [–−]
+        ##                different candidates to differentiate and document!!!
+        ##   – => U+2013 (8211)     -- En Dash     (unicode)
+        ##   − => U+2212 (8722)     -- Minus Sign  (unicode)
+        line = line.gsub( /[–−]/ ) do |uni|
+                  ## report error here
+                  ## todo/add error here
+                  _warn( "auto-fix; replacing unicode dash (#{uni}/#{uni.ord}) w/ ascii dash (-/#{"-".ord}) in line #{line.inspect}" )
+                   '-'   ## replace with ascii dash (-)
+                  end
+
+        ####   add more unsmart quotes
+        ## smart quotes
+        line = line.gsub( /[‘’]/ ) do |uni|
+                  ## report error here
+                  ## todo/add error here
+                  _warn( "auto-fix; replacing unicode (smart) quote (#{uni}/#{uni.ord}) w/ ascii quote ('/#{"'".ord}) in line #{line.inspect}" )
+                   "'"
+                  end
+
+        line = line.gsub( /[“”]/ ) do |uni|
+                  ## report error here
+                  ## todo/add error here
+                  _warn( %Q{auto-fix; replacing unicode (smart) double quote (#{uni}/#{uni.ord}) w/ ascii double quote ("/#{'"'.ord}) in line #{line.inspect}} )
+                   '"'
+                  end
+
+   line
+end
+
+end  # class Lexer
+end  # module Fbtxt