fbtxt-lexer 0.0.1

data/lib/fbtxt/lexer/lexer_buffer.rb

@@ -0,0 +1,68 @@
+
+module Fbtxt
+
+## note - Tokens was placed inside Lexer - keep "top-level" for now inside Fbtxt
+##            for easier reuse with (new) lexer variants!!
+
+## transforms
+##
+##  Netherlands  1-2 (1-1)   England
+##   =>  text => team
+##       score|vs
+##       text => team
+
+
+
+## token iter/find better name
+##  e.g. TokenBuffer/Scanner or such ??
+class Tokens
+  def initialize( tokens )
+      @tokens = tokens
+      @pos    = 0
+  end
+
+  def pos()  @pos; end
+  def eos?() @pos >= @tokens.size; end
+
+
+
+
+  ## pattern e.g. [:TEXT, [:VS,:SCORE], :TEXT]
+  def match?( *pattern )
+      ## puts "  starting match? #{pattern.inspect} @ #{@pos}"
+      pattern.each_with_index do |types,offset|
+          tok = peek(offset)
+          return false if tok.nil?   ## no more tokens (cannot match)
+
+          ## if single symbol wrap in array
+          types = types.is_a?(Array) ? types : [types]
+          return false  unless types.include?( tok.type )
+      end
+      true
+  end
+
+
+
+  def cur()           peek(0); end
+
+  def peek(offset=1)
+      ## return nil if eos
+      if @pos+offset >= @tokens.size
+          nil
+      else
+         @tokens[@pos+offset]
+      end
+  end
+
+  def next
+     # if @pos >= @tokens.size
+     #     raise ArgumentError, "end of array - #{@pos} >= #{@tokens.size}"
+     # end
+     #   throw (standard) end of iteration here why? why not?
+
+      t = @tokens[@pos]
+      @pos += 1
+      t
+  end
+end  # class Tokens
+end # module Fbtxt

data/lib/fbtxt/lexer/lexer_context.rb

@@ -0,0 +1,70 @@
+module Fbtxt
+class Lexer
+
+
+###
+## use nested class for context - why? why not?
+##   note: first arg passed in MUST be ref to lexer (instance)
+class Context
+   ## passed along to on_round_def etc. handlers in tokenize_line
+   ##   note - for now only offset (in line begin/end) gets updated !!!
+     attr_writer :offset
+     attr_reader :lineno
+
+     def initialize( lexer,
+                     line:,
+                     lineno:,
+                     errors: )
+        @lexer   = lexer
+        @line    = line
+        @lineno  = lineno
+        @errors  = errors
+
+        @offset = [0,0]   ## or use [] aka [nil,nil] for not defined??? why? why not?
+        ## @offset = offset    ## MatchData offset e.g. [m.begin(0),m.end(0)]
+     end
+
+
+
+     def warn_on_else( match, mode: 'TOP' )
+         if match[:any]
+           _add_warn( "unexpected char >#{match[:any]}< (#{mode})" )
+         else
+         ##  internal error - shouldn't really happen
+           _add_warn( "internal error - unknown match (#{mode}): #{match.inspect}")
+         end
+     end
+
+
+     def _add_warn( msg )
+        ## note - warns gets logged as error for now too
+        ##          maybe add @warns later - why? why not?
+        ##
+        ##  note - add +1 to offset (start at one - not zero-based)
+        ##           will match with (external) text editors
+        msg =  "parse error (tokenize) - " +
+                          msg +
+                " in line @#{@lineno}:#{@offset[0]+1},#{@offset[1]+1} >#{@line}<  "
+
+        @errors << msg
+        @lexer.log( "!! WARN - #{msg}" )
+
+        @lexer._warn( msg )
+     end
+
+=begin
+     ##  use report/log/??_parses_error
+     def _add_error( msg )
+         msg = "parse error (tokenize) -" +
+                          msg +
+                " in line #{@lineno}@#{@offset[0]},#{@offse[1]} >#{@line}<  "
+
+        @errors << msg
+     end
+=end
+
+end  # class Context
+
+
+end ## class Lexer
+end ## module Fbtxt

data/lib/fbtxt/lexer/lexer_token.rb

@@ -0,0 +1,127 @@
+module Fbtxt
+class Lexer
+
+
+
+class Token
+
+  ##   Token.newline( lineno: 1, offset: [1,2] )
+  ##     maps to =>
+  ##   Token.new( :NEWLINE, "\n", lineno: 1, offset: [1,2])
+  ##
+  ## use self.nl ?
+  def self.newline( lineno:, offset: [])
+     new( :NEWLINE, "\n", lineno: lineno, offset: offset )
+  end
+
+  ##   Token.literal( ",", lineno: 4, offset: [5,6])
+  ##    # maps to =>
+  ##   Token.new( ",", ",", lineno: 4, offset: [5,6])
+  ##
+  ## use self.lit?
+  def self.literal( literal, lineno:, offset: [])
+     new( literal, literal, lineno: lineno, offset: offset )
+  end
+
+  ## or use virt or pseudo - why? why not?
+  def self.virtual( type, lineno: -1, offset: [])
+     ## note - offset (start/end) should be same number (zero-width assertions!!)
+     ## e.g.    :GOALS_COMPAT, "<|GOALS_COMPAT|>"
+     new( type, '', lineno: lineno, offset: offset )
+  end
+
+
+
+  attr_reader :type,   :text,
+              :lineno, :offset
+
+  def initialize( type, text='',
+                   lineno:, offset: [],
+                   value: nil  )
+     @type    = type
+     @text    = text       # note - lexeme (string from source)
+     @lineno  = lineno     # note - lineno (integer number - not line as string) !!!
+
+     raise TypeError, "type Array required for offset; got #{offset.inspect}"  unless offset.is_a?( Array )
+     @offset  = offset  # note - for now char offset [start,end] in line (NOT absolute!!)
+                         #     maybe latter add MatchData#byteoffset instead - why? why not?
+     @value   = value   # might be (union of) string/array/hash
+  end
+
+  def value
+       ## note - if value is not set (nil) return text (lexeme)
+       ##             no need to duplicate text as value
+       @value.nil?  ? @text : @value
+  end
+
+
+##  note: do NOT use as_text/text to avoid confusion with (raw) text (lexeme)
+##
+##  use
+##    as_str                -- value (as String)
+##    as_int                -- value (as Integer)
+##    as_hash               -- value (as Hash)
+##    as_ary                -- value (as Array)
+
+  def as_str
+      raise TypeError, "token value #{@value.inspect} is #{@value.class.name} NOT string; sorry"   if @value && !@value.is_a?(String)
+       ## note - if value is not set (nil) return text (lexeme)
+       ##             no need to duplicate text as value
+      @value.nil?  ? @text : @value
+  end
+
+  def as_int
+     raise TypeError, "token value #{@value.inspect} is #{@value.class.name} NOT integer; sorry"    if !@value.is_a?(Integer)
+     @value
+  end
+
+  def as_hash
+     raise TypeError, "token value #{@value.inspect} is #{@value.class.name} NOT hash; sorry"    if !@value.is_a?(Hash)
+     @value
+  end
+
+  def as_ary
+     raise TypeError, "token value #{@value.inspect} is #{@value.class.name} NOT array; sorry"    if !@value.is_a?(Array)
+     @value
+  end
+
+
+  def to_legacy
+    ## return old "legacy" array format
+      if @value.nil?
+         [@type, @text]
+      else
+         [@type, [@text, @value]]
+      end
+  end
+
+
+## pretty print
+  def pretty_print( printer )
+    ## check for literal e.g. "," etc.
+    if @type.is_a?( String ) && @type == @text && @value.nil?
+       printer.text( "[#{@type.inspect}" )
+    elsif @type.is_a?( Symbol ) && @text == ''  && @value.nil?
+      ## assume virtual token  (zero-width)
+      ##   use <!...!> style
+      printer.text( "[<|#{@type}|>" )
+    else
+      printer.text( "[#{@type.inspect} #{@text.inspect}" )
+      printer.text( ", #{value.inspect}")            if @value
+    end
+
+
+    printer.text( " @#{@lineno}" )
+    ## note - for now print only start_offset (offset[0])
+    ##           to keep dump/output shorter
+    ##   note - start counting columns at one (NOT zero), thus, add +1 !!
+    printer.text( ":#{@offset[0]+1}" )         if @offset.is_a?(Array) && @offset.size == 2
+    printer.text( "]" )
+  end
+
+end # class Token
+
+
+
+end  # class Lexer
+end  # module Fbtxt

data/lib/fbtxt/lexer/token-date--helpers.rb

@@ -0,0 +1,130 @@
+module Fbtxt
+class Lexer
+
+
+##  "internal" date helpers
+def self._build_date( m )
+            date = {}
+         ## map month names
+         ## note - allow any/upcase JULY/JUL etc. thus ALWAYS downcase for lookup
+            date[:y]  = m[:year].to_i(10)  if m[:year]
+            ## check - use y too for two-digit year or keep separate - why? why not?
+            date[:yy] = m[:yy].to_i(10)    if m[:yy]    ## two digit year (e.g. 25 or 78 etc.)
+            date[:m] = m[:month].to_i(10)  if m[:month]
+            date[:m] = MONTH_MAP[ m[:month_name].downcase ]   if m[:month_name]
+            date[:d]  = m[:day].to_i(10)   if m[:day]
+            date[:wday] = DAY_MAP[ m[:day_name].downcase ]   if m[:day_name]
+
+            date
+end
+
+def self._build_date_legs( m )
+           legs = {}
+        ## map month names
+         ## note - allow any/upcase JULY/JUL etc. thus ALWAYS downcase for lookup
+            date = {}
+            date[:m] = MONTH_MAP[ m[:month_name1].downcase ]
+            date[:d]  = m[:day1].to_i(10)
+            legs[:date1] = date
+
+            date = {}
+            date[:m] = MONTH_MAP[ m[:month_name2].downcase ]   if m[:month_name2]
+            date[:d]  = m[:day2].to_i(10)
+            legs[:date2] = date
+
+            legs
+end
+
+
+def self._build_duration( m )
+            ## todo/check/fix - if end: works for kwargs!!!!!
+            duration = { start: {}, end: {}}
+
+            duration[:start][:y] = m[:year1].to_i(10)  if m[:year1]
+            duration[:start][:m] = MONTH_MAP[ m[:month_name1].downcase ]   if m[:month_name1]
+            duration[:start][:d]  = m[:day1].to_i(10)   if m[:day1]
+            duration[:start][:wday] = DAY_MAP[ m[:day_name1].downcase ]   if m[:day_name1]
+
+            duration[:end][:y] = m[:year2].to_i(10)  if m[:year2]
+            duration[:end][:m] = MONTH_MAP[ m[:month_name2].downcase ]   if m[:month_name2]
+            duration[:end][:d]  = m[:day2].to_i(10)   if m[:day2]
+            duration[:end][:wday] = DAY_MAP[ m[:day_name2].downcase ]   if m[:day_name2]
+
+            duration
+end
+
+
+
+
+def _build_date( m )      self.class._build_date( m ); end
+def _build_date_legs( m ) self.class._build_date_legs( m ); end
+def _build_duration( m )  self.class._build_duration( m ); end
+
+
+
+
+#############
+## "top-level" add a date parser helper
+
+##  note:  parse_date  - returns Date object
+##        _parse_date (with underscore) - return  hash of "parsed" regex match data!!
+
+def self.parse_date( str, start: nil )
+    if m = _parse_date( str )
+       year  = m[:y]
+       yy    = m[:yy]
+
+       ####
+       ## support two digit shortcut for year
+       if yy && year.nil?
+          ###
+          ## for now assume 00,01 to 30 is 2000,2001 to 2030
+          ##   and          31 to 99   is  1931 to 1999
+          year =   yy <= 30 ?  2000+yy : 1900+yy
+       end
+
+       month = m[:m]
+       day   = m[:d]
+       wday  = m[:wday]
+
+
+      if year.nil?     ## try to calculate year
+        raise ArgumentError, "year required in date >#{str}< or pass along start date"   if start.nil?
+
+        year =  if  month > start.month ||
+                   (month == start.month && day >= start.day)
+                  # assume same year as start_at event (e.g. 2013 for 2013/14 season)
+                  start.year
+                else
+                  # assume year+1 as start_at event (e.g. 2014 for 2013/14 season)
+                  start.year+1
+                end
+      end
+      Date.new( year,month,day )
+    else
+      raise ArgumentError, "unexpected date format; cannot parse >#{str}<"
+    end
+end
+
+
+
+def self._parse_date( str )
+    ## note - strip - leading/trailing spaces automatic - why? why not?
+    m = DATE_RE.match( str.strip )
+
+    if m && m.pre_match == '' && m.post_match == ''
+      ## return hash table with captured components
+      date = _build_date( m )
+      date
+    elsif  m
+        ## note - match BUT not anchored to start and end-of-string!!!
+        ##  report, error somehow??
+      nil
+    else
+      nil  ## no match - return nil
+    end
+end
+
+
+end  # class Lexer
+end  # module Fbtxt

data/lib/fbtxt/lexer/token-date--names.rb

@@ -0,0 +1,108 @@
+module Fbtxt
+class Lexer
+
+
+def self.parse_names( txt )
+  lines = [] # array of lines (with words)
+
+  txt.each_line do |line|
+    line = line.strip
+
+    next if line.empty?
+    next if line.start_with?( '#' )   ## skip comments too
+
+    ## strip inline (until end-of-line) comments too
+    ##   e.g. Janvier  Janv  Jan  ## check janv in use??
+    ##   =>   Janvier  Janv  Jan
+
+    line = line.sub( /#.*/, '' ).strip
+    ## pp line
+
+    values = line.split( /[ \t]+/ )
+    ## pp values
+
+    ## todo/fix -- add check for duplicates
+    lines << values
+  end
+  lines
+
+end # method parse
+
+
+def self.build_names( lines )
+  ## join all words together into a single string e.g.
+  ##   January|Jan|February|Feb|March|Mar|April|Apr|May|June|Jun|...
+  lines.map { |line| line.join('|') }.join('|')
+end
+
+
+def self.build_map( lines, downcase: false )
+   ## note: downcase name!!!
+  ## build a lookup map that maps the word to the index (line no) plus 1 e.g.
+  ##  {"january" => 1,  "jan" => 1,
+  ##   "february" => 2, "feb" => 2,
+  ##   "march" => 3,    "mar" => 3,
+  ##   "april" => 4,    "apr" => 4,
+  ##   "may" => 5,
+  ##   "june" => 6,     "jun" => 6, ...
+  lines.each_with_index.reduce( {} ) do |h,(line,i)|
+    line.each do |name|
+       h[ downcase ? name.downcase : name ] = i+1
+    end  ## note: start mapping with 1 (and NOT zero-based, that is, 0)
+    h
+  end
+end
+
+
+
+
+MONTH_LINES = parse_names( <<TXT )
+January    Jan
+February   Feb
+March      Mar
+April      Apr
+May
+June       Jun
+July       Jul
+August     Aug
+September  Sept  Sep
+October    Oct
+November   Nov
+December   Dec
+TXT
+
+MONTH_NAMES = build_names( MONTH_LINES )
+# pp MONTH_NAMES
+MONTH_MAP   = build_map( MONTH_LINES, downcase: true )
+# pp MONTH_MAP
+
+
+
+DAY_LINES = parse_names( <<TXT )
+Monday                   Mon  Mo
+Tuesday            Tues  Tue  Tu
+Wednesday                Wed  We
+Thursday    Thurs  Thur  Thu  Th
+Friday                   Fri  Fr
+Saturday                 Sat  Sa
+Sunday                   Sun  Su
+TXT
+
+DAY_NAMES = build_names( DAY_LINES )
+# pp DAY_NAMES
+DAY_MAP   = build_map( DAY_LINES, downcase: true )
+# pp DAY_MAP
+
+
+#=>
+# "January|Jan|February|Feb|March|Mar|April|Apr|May|June|Jun|
+#  July|Jul|August|Aug|September|Sept|Sep|October|Oct|
+#  November|Nov|December|Dec"
+#
+# "Monday|Mon|Mo|Tuesday|Tues|Tue|Tu|Wednesday|Wed|We|
+#  Thursday|Thurs|Thur|Thu|Th|Friday|Fri|Fr|
+#  Saturday|Sat|Sa|Sunday|Sun|Su"
+
+
+end   # class Lexer
+end   # module Fbtxt

data/lib/fbtxt/lexer/token-date.rb

@@ -0,0 +1,200 @@
+module Fbtxt
+class Lexer
+
+
+
+# e.g.  Fri Aug 9
+#       Fri  Aug 9
+##      Fri, Aug 9
+##      Fri, Aug 9 2024
+##      Fri, Aug 9, 2024
+##           Aug 9, 2024
+##           Aug 9, 2024
+##   note - eat-up optional comma after DAY_NAMES!!
+##
+##      note - Fri Aug/9  no longer supported!!!
+DATE_I_RE = %r{
+(?<date>
+  \b
+     ## optional day name
+     ((?<day_name>#{DAY_NAMES})
+           (?: ,?[ ]+)
+     )?
+     (?<month_name>#{MONTH_NAMES})
+          [ ]
+     (?<day>\d{1,2})
+          \b
+     ## optional year
+     (      ,? [ ]       ## note - comma optinal with single space required for now
+            (?<year>\d{4})        ## optional year 2025 (yyyy)
+              \b
+     )?
+)}ix
+
+
+### todo/fix - add (opt) day_name later
+##             add (opt) year later
+# e.g.  Aug 9 & Aug 10
+### note - allow shortcut e.g. Aug 9 & 10
+DATE_LEGS_I_RE = %r{
+(?<date_legs>
+ \b
+     (?<month_name1>#{MONTH_NAMES})
+          [ ]
+     (?<day1>\d{1,2})
+    [ ] & [ ]
+     (?:
+        (?<month_name2>#{MONTH_NAMES})
+          [ ]
+      )?  ## note - make 2nd month_name optional
+     (?<day2>\d{1,2})
+  \b
+)}ix
+
+
+###
+# e.g. 3 June  or 10 June
+##   note - allow more spaces between  DAY_NAMES and DAY e.g.
+##    Sun  1 Mar
+##    Wed  4 Mar
+##    Sat 14 Mar
+##    Sat 11 Apr
+##    Sat 11 Apr 2021
+##    Sat 11 Apr 21
+##
+##    Sat, 11 Apr
+##   note - eat-up optional comma after DAY_NAMES!!
+##
+##  note - Sat 14 Mar 17:30
+##          check two-digit year (with NEGATIVE lookahead for time!!!)
+
+DATE_II_RE = %r{
+(?<date>
+  \b
+     ## optional day name
+     ((?<day_name>#{DAY_NAMES})
+           (?: ,?[ ]+)
+     )?
+     (?<day>\d{1,2})
+         [ ]
+     (?<month_name>#{MONTH_NAMES})
+          \b
+     ## optional year
+     (  [ ]
+        (?:
+           (?<year>\d{4})        ## optional year 2025 (yyyy)
+               |
+            (?:
+               (?<yy>\d{2})           ## optional year 25 (yy)
+                ## check NEGATIVE lookahead
+               (?! :|[:h]\d{2})
+            )
+        )
+        \b
+     )?
+)}ix
+
+
+# e.g. iso-date  -  2011-08-25
+##   note - allow/support ("shortcuts") e.g 2011-8-25  or 2011-8-3 / 2011-08-03 etc.
+DATE_III_A_RE = %r{
+(?<date>
+  \b
+   (?<year>\d{4})
+       -
+   (?<month>\d{1,2})
+       -
+   (?<day>\d{1,2})
+  \b
+)}ix
+
+##  starting w/  day/month/year e.g.  25-08-2011
+DATE_III_B_RE = %r{
+(?<date>
+  \b
+     ## optional day name
+     ((?<day_name>#{DAY_NAMES})
+          (?: ,?[ ]+)
+     )?
+   (?<day>\d{1,2})
+       -
+   (?<month>\d{1,2})
+       -
+   (?<year>\d{4})
+  \b
+)}ix
+
+
+
+## allow (short)"european" style  8.8.
+##   note - assume day/month!!!
+DATE_IIII_RE = %r{
+(?<date>
+  \b
+     ## optional day name
+     ((?<day_name>#{DAY_NAMES})
+           (?: ,?[ ]+)
+     )?
+   (?<day>\d{1,2})
+       \.
+   (?<month>\d{1,2})
+       \.
+   (?: (?:
+          (?<year>\d{4})        ## optional year 2025 (yyyy)
+              |
+          (?<yy>\d{2})           ## optional year 25 (yy)
+       )
+        \b
+   )?
+)
+}ix
+
+
+####################
+### 04/03/2026  or 4/3/2026
+##   04/03/26   or 4/3/26
+##   04/03      or 4/3
+DATE_IIIII_RE = %r{
+(?<date>
+  \b
+     ## optional day name
+     ((?<day_name>#{DAY_NAMES})
+          (?: ,?[ ]+)
+     )?
+   (?<day>\d{1,2})
+       /
+   (?<month>\d{1,2})
+    \b
+   (?:
+        /
+       (?:
+          (?<year>\d{4})         ## optional year 2025 (yyyy)
+              |
+          (?<yy>\d{2})           ## optional year 25 (yy)
+       )
+      \b
+   )?
+)
+}ix
+
+
+
+#############################################
+# map tables
+#  note: order matters; first come-first matched/served
+DATE_RE = Regexp.union(
+   DATE_I_RE,
+   DATE_II_RE,
+   DATE_III_A_RE,    ## e.g. 1973-08-14
+   DATE_III_B_RE,
+   DATE_IIII_RE,    ## e.g. 8.8. or 8.13.79 or 08.14.1973
+   DATE_IIIII_RE,   ## e.g.  08/14/1973
+)
+
+## todo - add more format style here; change to Regexp.union later!!!
+DATE_LEGS_RE  =    DATE_LEGS_I_RE
+
+
+
+end  #   class Lexer
+end  # module Fbtxt