sportdb-parser 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1466b82654b4a4f0f823a96709488dedb595d08731a55abc128691e0ffe2a80b
+  data.tar.gz: 14995e94dc079ab61e77d056d15c9a5830dc573129661ca453b2892d087c2061
+SHA512:
+  metadata.gz: 75c2b4f455e8bb1b5e471c39f8fa3b5069bd0bb2a808ad8b246c0f2b060c5416f9f56a3619ad7db7ac5f21a6177c762aa28ae8e9c939b03a2569cf27d34f9b81
+  data.tar.gz: 9c4f9095a61410499ae7628b1eb3295d8f456e62feae45a4c254d9157904326abf6571f3c4a04c078551b6364cd09252509f709bfeef46a569dbe202f4058460

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+### 0.0.1 / 2024-07-12
+
+* Everything is new. First release.

data/Manifest.txt

@@ -0,0 +1,14 @@
+CHANGELOG.md
+Manifest.txt
+README.md
+Rakefile
+bin/fbt
+lib/sportdb/parser.rb
+lib/sportdb/parser/lang.rb
+lib/sportdb/parser/linter.rb
+lib/sportdb/parser/outline_reader.rb
+lib/sportdb/parser/parser.rb
+lib/sportdb/parser/token-date.rb
+lib/sportdb/parser/token-score.rb
+lib/sportdb/parser/token-text.rb
+lib/sportdb/parser/token.rb

data/README.md

@@ -0,0 +1,8 @@
+#  sportdb-parser - football.txt match parser (& tokenizer)
+
+
+
+
+
+
+

data/Rakefile

@@ -0,0 +1,27 @@
+require 'hoe'
+
+
+Hoe.spec 'sportdb-parser' do
+
+  self.version = '0.0.1'
+
+  self.summary = "sportdb-parser - football.txt match parser (& tokenizer)"
+  self.description = summary
+
+  self.urls = { home: 'https://github.com/sportdb/sport.db' }
+
+  self.author = 'Gerald Bauer'
+  self.email = 'gerald.bauer@gmail.com'
+
+  # switch extension to .markdown for gihub formatting
+  self.readme_file  = 'README.md'
+  self.history_file = 'CHANGELOG.md'
+
+  self.licenses = ['Public Domain']
+
+  self.extra_deps = []
+
+  self.spec_extras = {
+    required_ruby_version: '>= 2.2.2'
+  }
+end

data/bin/fbt

@@ -0,0 +1,144 @@
+#!/usr/bin/env ruby
+
+## tip: to test run:
+##   ruby -I ./lib bin/fbt
+
+require 'sportdb/parser'
+
+
+require 'optparse'
+
+##
+## read textfile 
+##   and dump tokens
+##
+##   fbt  ../openfootball/.../euro.txt
+
+
+  SEASON_RE = %r{ (?:
+                       \d{4}-\d{2}
+                     | \d{4}(--[a-z0-9_-]+)?
+                    )
+                  }x
+    SEASON = SEASON_RE.source    ## "inline" helper for embedding in other regexes - keep? why? why not?
+
+
+    ## note: if pattern includes directory add here
+    ##     (otherwise move to more "generic" datafile) - why? why not?
+    MATCH_RE = %r{ (?: ^|/ )      # beginning (^) or beginning of path (/)
+                       #{SEASON}
+                     /[a-z0-9_-]+\.txt$  ## txt e.g /1-premierleague.txt
+                }x
+
+
+def find( path, pattern=MATCH_RE )
+    datafiles = []
+
+    ## check all txt files
+    ## note: incl. files starting with dot (.)) as candidates (normally excluded with just *)
+    candidates = Dir.glob( "#{path}/**/{*,.*}.txt" )
+    ## pp candidates
+    candidates.each do |candidate|
+      datafiles << candidate    if pattern.match( candidate )
+    end
+
+    ## pp datafiles
+    datafiles
+ end
+
+
+
+
+
+
+ args = ARGV
+ opts = { debug: false,
+          metal: false }
+
+ parser = OptionParser.new do |parser|
+  parser.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+##
+## check if git has a offline option?? (use same)
+##             check for other tools - why? why not?
+
+
+  parser.on( "--verbose", "--debug",
+               "turn on verbose / debug output (default: #{opts[:debug]} )" ) do |debug|
+    opts[:debug] = debug
+  end
+
+  parser.on( "--metal",
+                 "turn off typed parse tree; show to the metal tokens"+
+                   " (default: #{opts[:metal]})" ) do |metal|
+    opts[:metal] = metal
+  end
+end
+parser.parse!( args )
+
+puts "OPTS:"
+p opts
+puts "ARGV:"
+p args
+
+
+
+
+
+
+def expand_args( args )
+    paths = []
+
+    args.each do |arg|
+        ## check if directory
+        if Dir.exist?( arg )
+            datafiles = find( arg )
+            puts
+            puts "  found #{datafiles.size} match txt datafiles in #{arg}"
+            pp datafiles
+            paths += datafiles
+        else
+              ## assume it's a file
+            paths << arg
+        end
+    end
+
+    paths
+end
+
+
+paths =  if args.empty?
+            [
+              '../../../openfootball/euro/2020--europe/euro.txt',
+              '../../../openfootball/euro/2024--germany/euro.txt',
+            ]
+         else
+            ## check for directories
+            ##   and auto-expand
+             
+            expand_args( args )
+         end
+
+
+
+SportDb::Parser::Linter.debug = true    if opts[:debug]
+
+linter = SportDb::Parser::Linter.new
+
+
+
+paths.each_with_index do |path,i|
+    puts "==> [#{i+1}/#{paths.size}] reading >#{path}<..."
+    linter.read( path, parse: !opts[:metal] )
+end
+
+if linter.errors?
+    puts
+    pp linter.errors
+    puts "!!   #{linter.errors.size} parse error(s) in #{paths.size} datafiles(s)"
+else
+    puts "OK   no parse errors found in #{paths.size} datafile(s)"
+end
+
+puts "bye"
+

data/lib/sportdb/parser/lang.rb

@@ -0,0 +1,111 @@
+
+## use Sports (not SportDb) for module - why? why not?
+
+
+
+module SportDb
+class Parser
+
+## Group A-Z
+## Group 1-99
+## Group HEX  # used in concaf world cup quali
+## Group 1A or A1, B1  - used anywhere
+##
+##  use "key" of group - why? why not?
+
+GROUP_RE = %r{^
+                Group [ ]
+                   (?<key>[a-z0-9]+)                   
+              $}ix
+def is_group?( text )
+   ## use regex for match
+   GROUP_RE.match?( text )
+end
+
+
+
+
+ROUND_RE = %r{^(
+
+   # round  - note - requiers number e.g. round 1,2, etc.
+      (?: (?: Round |
+              Matchday |
+              Week
+           )
+           [ ] [0-9]+
+      )
+       |
+   # more (kockout) rounds
+   # playoffs  - playoff, play-off, play-offs
+        (?: Play-?offs? 
+           (?: [ ]for[ ]quarter-?finals )?
+        )
+        |    
+   # round32
+        (?: Round[ ]of[ ]32 | 
+            Last[ ]32 )
+          |
+   # round16   
+        (?: Round[ ]of[ ]16 |
+            Last[ ]16 | 
+            8th[ ]finals )
+           |
+   # fifthplace
+         (?:
+             (?: (Fifth|5th)[ -]place 
+                  (?: [ ] (?: match|play-?off|final ))?
+              ) |
+             (?: Match[ ]for[ ](?: fifth|5th )[ -]place )
+         )
+          |
+   # thirdplace
+          (?: 
+              (?: (Third|3rd)[ -]place 
+                     (?: [ ] (?: match|play-?off|final ))? 
+               ) |
+              (?: Match[ ]for[ ](?: third|3rd )[ -]place ) 
+           )
+           |
+   # quarterfinals
+         (?:
+              Quarter-?finals? |
+              Quarters |
+              Last[ ]8
+          )
+          |     
+   # semifinals
+        (?:   
+             Semi-?finals? |
+             Semis |
+             Last[ ]4
+        )
+        |
+   # final
+         Finals? 
+       
+        )$}ix
+
+
+def is_round?( text )
+    ROUND_RE.match?( text )
+end
+
+##
+## keep leg separate (from round) - why? why not?
+##
+LEG_RE = %r{^
+  # leg1
+     (?: 1st|First)[ ]leg 
+     |
+  # leg2 
+     (?: 2nd|Second)[ ]leg
+$}ix
+
+### Pair matches/games if marked with leg1 n leg2
+def is_leg?( text )
+   LEG_RE.match?( text )
+end
+
+
+end  # class Parser
+end  # module SportDb

data/lib/sportdb/parser/linter.rb

@@ -0,0 +1,153 @@
+
+module SportDb
+class Parser
+
+###
+## note - Linter for now nested inside Parser - keep? why? why not?
+class Linter
+   
+def self.debug=(value) @@debug = value; end
+def self.debug?() @@debug ||= false; end  ## note: default is FALSE
+def debug?()  self.class.debug?; end   
+    
+## keep typed - why? why not? 
+## - used anywhere? 
+def self.typed=(value) @@typed = value; end
+def self.typed?() @@typed ||= true; end   ## note: default is TRUE
+def typed?()  self.class.typed?; end   
+
+
+
+attr_reader :errors
+
+def initialize
+  @errors = []
+  @parser = Parser.new   ## use own parser instance (not shared) - why? why not?
+end
+
+
+def errors?() @errors.size > 0; end
+
+
+
+  ## note:  colon (:) MUST be followed by one (or more) spaces
+  ##      make sure mon feb 12 18:10 will not match
+  ##        allow 1. FC Köln etc.
+  ##               Mainz 05:
+  ##           limit to 30 chars max
+  ##          only allow  chars incl. intl buut (NOT ()[]/;)
+  ##
+  ##   Group A:
+  ##   Group B:   - remove colon
+  ##    or lookup first
+
+  ATTRIB_RE = %r{^  
+                   [ ]*?     # slurp leading spaces
+                (?<key>[^:|\]\[()\/; -]
+                       [^:|\]\[()\/;]{0,30}
+                 )
+                   [ ]*?     # slurp trailing spaces
+                   :[ ]+
+                (?<value>.+)
+                    [ ]*?   # slurp trailing spaces
+                   $
+                }ix
+
+
+#########
+## parse - false (default) - tokenize (only)
+##       - true            - tokenize & parse                
+def read( path, parse: false )
+  nodes = OutlineReader.read( path ) 
+
+  ##  process nodes
+  h1 = nil
+  orphans = 0    ## track paragraphs's with no heading
+
+  attrib_found = false
+
+
+  nodes.each do |node|
+    type = node[0]
+    
+    if type == :h1
+        h1 = node[1]  ## get heading text
+        puts
+        puts "  = Heading 1 >#{node[1]}<"
+    elsif type == :p
+
+       if h1.nil?
+         orphans += 1    ## only warn once 
+         puts "!! WARN - no heading for #{orphans} text paragraph(s); skipping parse"
+         next
+       end
+
+       lines = node[1]
+
+       tree = [] 
+       lines.each_with_index do |line,i|
+
+        if debug?
+         puts
+         puts "line >#{line}<"
+        end
+
+
+        ## skip new (experimental attrib syntax)
+        m = nil
+        if attrib_found == false && 
+            m=ATTRIB_RE.match( line )
+          ## note: check attrib regex AFTER group def e.g.:
+          ##         Group A: 
+          ##         Group B:  etc.
+          ##     todo/fix - change Group A: to Group A etc.
+          ##                       Group B: to Group B
+           attrib_found = true
+           ## logger.debug "skipping key/value line - >#{line}<"
+           next
+        end
+
+        if attrib_found
+          ## check if line ends with dot
+          ##  if not slurp up lines to the next do!!!
+          ## logger.debug "skipping key/value line - >#{line}<"
+          attrib_found = false   if line.end_with?( '.' ) 
+              # logger.debug "skipping key/value line (cont.) - >#{line}<"
+              next
+        end    
+    
+        t, error_messages  =  if parse
+                                  @parser.parse_with_errors( line )
+                              else
+                                  @parser.tokenize_with_errors( line )                   
+                              end
+         
+
+         if error_messages.size > 0
+            ## add to "global" error list
+            ##   make a triplet tuple (file / msg / line text)
+            error_messages.each do |msg|
+                @errors << [ path,
+                             msg,
+                             line
+                           ]
+            end
+         end
+
+         pp t   if debug?
+
+         tree << t
+       end
+       
+       ## pp tree
+    else
+        pp node
+        raise ArgumentError, "unsupported (node) type >#{type}<"
+    end
+  end  # each node
+end  # read
+end  # class Linter
+
+
+end   # class Parser
+end   # module SportDb

data/lib/sportdb/parser/outline_reader.rb

@@ -0,0 +1,101 @@
+
+###
+## todo/fix -  move to sportdb-parser - why? why not? !!!!!!
+##
+
+
+module SportDb
+
+class OutlineReader
+
+  def self.debug=(value) @@debug = value; end
+  def self.debug?() @@debug ||= false; end
+  def debug?()  self.class.debug?; end   
+
+
+
+  def self.read( path )   ## use - rename to read_file or from_file etc. - why? why not?
+    txt = File.open( path, 'r:utf-8' ) {|f| f.read }
+    parse( txt )
+  end
+
+  def self.parse( txt )
+    new( txt ).parse
+  end
+
+  def initialize( txt )
+    @txt = txt
+  end
+
+  ## note: skip "decorative" only heading e.g. ========
+  ##  todo/check:  find a better name e.g. HEADING_EMPTY_RE or HEADING_LINE_RE or ???
+  HEADING_BLANK_RE = %r{\A
+                        ={1,}
+                        \z}x
+
+  ## note: like in wikimedia markup (and markdown) all optional trailing ==== too
+  HEADING_RE = %r{\A
+                  (?<marker>={1,})       ## 1. leading ======
+                    [ ]*
+                  (?<text>[^=]+)         ## 2. text   (note: for now no "inline" = allowed)
+                    [ ]*
+                    =*                   ## 3. (optional) trailing ====
+                  \z}x
+
+  def parse
+    outline=[]   ## outline structure
+    start_para = true      ## start new para(graph) on new text line?
+
+    @txt.each_line do |line|
+        line = line.strip      ## todo/fix: keep leading and trailing spaces - why? why not?
+
+        if line.empty?    ## todo/fix: keep blank line nodes?? and just remove comments and process headings?! - why? why not?
+          start_para = true
+          next
+        end
+
+        break if line == '__END__'
+
+        next if line.start_with?( '#' )   ## skip comments too
+        ## strip inline (until end-of-line) comments too
+        ##  e.g Eupen | KAS Eupen ## [de]
+        ##   => Eupen | KAS Eupen
+        ##  e.g bq   Bonaire,  BOE        # CONCACAF
+        ##   => bq   Bonaire,  BOE
+        line = line.sub( /#.*/, '' ).strip
+        pp line    if debug?
+
+        ## todo/check: also use heading blank as paragraph "breaker" or treat it like a comment ?? - why? why not?
+        next if HEADING_BLANK_RE.match( line )  # skip "decorative" only heading e.g. ========
+
+         ## note: like in wikimedia markup (and markdown) all optional trailing ==== too
+        if m=HEADING_RE.match( line )
+           start_para = true
+
+           heading_marker = m[:marker]
+           heading_level  = m[:marker].length   ## count number of = for heading level
+           heading        = m[:text].strip
+
+           puts "heading #{heading_level} >#{heading}<"   if debug?
+           outline << [:"h#{heading_level}", heading]
+        else    ## assume it's a (plain/regular) text line
+           if start_para
+             outline << [:p, [line]]
+             start_para = false
+           else
+             node = outline[-1]    ## get last entry
+             if node[0] == :p      ##  assert it's a p(aragraph) node!!!
+                node[1] << line    ## add line to p(aragraph)
+             else
+               puts "!! ERROR - invalid outline state / format - expected p(aragraph) node; got:"
+               pp node
+               exit 1
+             end
+           end
+        end
+    end
+    outline
+  end # method read
+end # class OutlineReader
+
+end # module SportDb