fbtok 0.4.2 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0bd9a1ff623cc7049c87db9b8344799d9ee49294bf1cd8f2a48cd57886fd0b89
-  data.tar.gz: 35acfef6ccd67319d218e8dd849e3772042d8190afe786f73ba98fa19ca2418b
+  metadata.gz: 6bb0384d86033771326222bc9284264b9bfc44c55210261e6b598fffb3ddf925
+  data.tar.gz: a7692e7b0f394373c364234ec7b7c57933d6bd1344b58d4139beb41381a59921
 SHA512:
-  metadata.gz: 20054a913668ac69f58bc17ebbc4bdc6de6528639f64a52b60925e87129f05fbd9bfe4a87c9ea8be55bf0d5010280e0fdf2f56b8b1b892b9f4744d99aa66239b
-  data.tar.gz: c7980b69a62273c6fffd4c7025ccb086a96070ee8dff51248fab1ddf08ef2625de467e646c4496570f163519d88985e06698c1452e639419c629521320bc26c5
+  metadata.gz: 854430fe426e69eb27651930db928cb24bf7f4b489503a222f7ff67178fb087b465718e71cc4a9421a591ab87c784e35f2a26514c99de9151ddb9a1a13c4ec53
+  data.tar.gz: e602fe075cfbd20eeed1e744f4bdfa35314f2e42db69537cb8ff14ccc87b8fed50400d5f3aa53974e342a0b09c00b7110e0614e0d06ede5595c19e6e652de63d

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-### 0.4.2
+### 0.5.1
 ### 0.0.1 / 2025-01-02
 
 * Everything is new. First release.

data/Manifest.txt

@@ -12,4 +12,6 @@ lib/fbtok/fbquick.rb
 lib/fbtok/fbtok.rb
 lib/fbtok/fbtree.rb
 lib/fbtok/fbx.rb
+lib/fbtok/filepack.rb
 lib/fbtok/pathspec.rb
+lib/fbtok/pathspec_report.rb

data/Rakefile

@@ -2,7 +2,7 @@ require 'hoe'
 
 
 Hoe.spec 'fbtok' do
-  self.version = '0.4.2'
+  self.version = '0.5.1'
 
   self.summary = "fbtok - football.txt lint tools incl. tokenizer, parser & more"
   self.description = summary
@@ -19,7 +19,7 @@ Hoe.spec 'fbtok' do
   self.licenses = ['Public Domain']
 
   self.extra_deps = [
-     ['sportdb-parser', '>= 0.7.1'],
+     ['sportdb-parser', '>= 0.7.2'],
      ['sportdb-quick',  '>= 0.7.0'],
   ]
 

data/lib/fbtok/fbquick.rb

@@ -54,10 +54,27 @@ end
 specs =  if opts[:file]
             read_pathspecs( opts[:file] )
          else
-           args =     ['/sports/openfootball/euro/2021--europe/euro.txt',
-                       '/sports/openfootball/euro/2024--germany/euro.txt']  if args.empty?
-
-            build_pathspecs( args )
+            ## check for filepack
+            ##
+            ## fix-fix-fix
+            ##  add tool specific filepack
+            ##       e.g. filepack.quik|quick.txt or such
+            ##   to match first??
+            ##   check a list of names!!
+            filepack =  if File.file?( './filepack.txt')
+                            read_filepack( './filepack.txt' )
+                        else
+                             nil
+                        end
+
+            path = SportDb::Pathspec.path(
+                          ['/sports/sportdb/sport.db.v2/parser/fbtxt-specs',
+                           '/sports/sportdb/sport.db.v2/parser/fbtxt-samples',
+                           '/sports/openfootball'])
+
+
+            build_pathspecs( args, path: path,
+                                   filepack: filepack )
          end
 
 pp specs
@@ -91,6 +108,7 @@ pp specs
       pp errors
     else
       puts
+      pp datafiles
       puts "  OK - no parse errors in #{datafiles.size} datafile(s)"
     end
 

data/lib/fbtok/fbtok.rb

@@ -6,8 +6,8 @@ def self.main( args=ARGV )
 
 opts = {
     debug: true,
-    warn:  false,
     file:  nil,
+#    warn:  false,
 }
 
 
@@ -23,10 +23,10 @@ parser = OptionParser.new do |parser|
                "turn on verbose / debug output (default: #{opts[:debug]})" ) do |debug|
     opts[:debug] = true
   end
-  parser.on( "-w", "--warn",
-              "turn warnings into errors (default: #{opts[:warn]})" ) do |warn|
-    opts[:warn] = true
-  end
+#  parser.on( "-w", "--warn",
+#              "turn warnings into errors (default: #{opts[:warn]})" ) do |warn|
+#    opts[:warn] = true
+#  end
 
 
   parser.on( "-f FILE", "--file FILE",
@@ -58,10 +58,20 @@ end
 specs =  if opts[:file]
             read_pathspecs( opts[:file] )
          else
-           args = ['/sports/openfootball/euro/2021--europe/euro.txt',
-                   '/sports/openfootball/euro/2024--germany/euro.txt']  if args.empty?
-
-            build_pathspecs( args )
+            ## check for filepack
+            filepack =  if File.file?( './filepack.txt')
+                            read_filepack( './filepack.txt' )
+                        else
+                             nil
+                        end
+
+            path = SportDb::Pathspec.path(
+                          ['/sports/sportdb/sport.db.v2/parser/fbtxt-specs',
+                           '/sports/sportdb/sport.db.v2/parser/fbtxt-samples',
+                           '/sports/openfootball'])
+
+            build_pathspecs( args, path: path,
+                                   filepack: filepack )
          end
 
 
@@ -82,11 +92,20 @@ specs.each_with_index do |rec,i|
       tokens, more_errors = lexer.tokenize_with_errors
 
       ####
-      ## todo - report error on empty file (no tokens!!!)
+      ## todo - report error on empty file (no tokens!!!) - why? why not?
 
       puts "   #{tokens.size} token(s)"
 
-      errors += more_errors    if more_errors.size > 0
+      if more_errors.size > 0
+         ## note - auto-add filename to errors
+         more_errors.each do |msg|
+          ###
+          ###   errors << [ path, *msg ] # note: use splat (*) to add extra values (starting with msg)
+              errors << [path, msg]
+         end
+      end
+
+
    end
 
    if errors.size > 0
@@ -96,6 +115,7 @@ specs.each_with_index do |rec,i|
       puts "!!   #{errors.size} tokenize error(s) in #{datafiles.size} datafiles(s)"
    else
       puts
+      pp datafiles
       puts "OK   no tokenize errors found in #{datafiles.size} datafile(s)"
    end
 

data/lib/fbtok/fbtree.rb

@@ -6,8 +6,8 @@ def self.main( args=ARGV )
 
 opts = {
     debug: true,
-    warn:  false,
     file:  nil,
+##    warn:  false,
 }
 
 parser = OptionParser.new do |parser|
@@ -22,10 +22,10 @@ parser = OptionParser.new do |parser|
                "turn on verbose / debug output (default: #{opts[:debug]})" ) do |debug|
     opts[:debug] = true
   end
-  parser.on( "-w", "--warn",
-              "turn warnings into errors (default: #{opts[:warn]})" ) do |warn|
-    opts[:warn] = true
-  end
+#  parser.on( "-w", "--warn",
+#              "turn warnings into errors (default: #{opts[:warn]})" ) do |warn|
+#    opts[:warn] = true
+#  end
 
 
   parser.on( "-f FILE", "--file FILE",
@@ -38,7 +38,6 @@ parser = OptionParser.new do |parser|
 end
 parser.parse!( args )
 
-
 if opts[:debug]
   puts "OPTS:"
   p opts
@@ -47,8 +46,6 @@ if opts[:debug]
 end
 
 
-# SportDb::Parser::Linter.debug = opts[:debug]
-# SportDb::Parser::Linter.warn  = opts[:warn]
 
 
 
@@ -58,13 +55,37 @@ end
 specs =  if opts[:file]
             read_pathspecs( opts[:file] )
          else
-           args = ['/sports/openfootball/euro/2021--europe/euro.txt',
-                   '/sports/openfootball/euro/2024--germany/euro.txt']  if args.empty?
-
-            build_pathspecs( args )
+            ## check for filepack
+            filepack =  if File.file?( './filepack.txt')
+                            read_filepack( './filepack.txt' )
+                        else
+                             nil
+                        end
+
+            path = SportDb::Pathspec.path(
+                          ['/sports/sportdb/sport.db.v2/parser/fbtxt-specs',
+                           '/sports/sportdb/sport.db.v2/parser/fbtxt-samples',
+                           '/sports/openfootball'])
+
+            build_pathspecs( args, path: path,
+                                   filepack: filepack )
          end
 
 
+pp specs
+
+## if more than single datafile
+##     auto-switch into quite mode for now
+## if specs.size > 0  && specs[0]['datafiles'].size > 1
+##   opts[:debug] = false
+## end
+
+# SportDb::Parser::Linter.debug = opts[:debug]
+# SportDb::Parser::Linter.warn  = opts[:warn]
+
+
+
+
 
 
 specs.each_with_index do |rec,i|
@@ -81,7 +102,14 @@ specs.each_with_index do |rec,i|
 
       dump_tree_stats( tree )
 
-      errors += parser.errors    if parser.errors?
+      if parser.errors?
+         ## note - auto-add filename to errors
+         parser.errors.each do |msg|
+          ###
+          ###   errors << [ path, *msg ] # note: use splat (*) to add extra values (starting with msg)
+              errors << [path, msg]
+         end
+      end
    end
 
 
@@ -92,6 +120,7 @@ specs.each_with_index do |rec,i|
       puts "!!   #{errors.size} parse error(s) in #{datafiles.size} datafiles(s)"
    else
       puts
+      pp datafiles
       puts "OK   no parse errors found in #{datafiles.size} datafile(s)"
    end
 

data/lib/fbtok/fbx.rb

@@ -54,8 +54,8 @@ end
 
 
 args =  [
-              '../../../openfootball/euro/2021--europe/euro.txt',
-              '../../../openfootball/euro/2024--germany/euro.txt',
+              '/sports/openfootball/euro/2021--europe/euro.txt',
+              '/sports/openfootball/euro/2024--germany/euro.txt',
         ]   if args.empty?
 
 pp args

data/lib/fbtok/filepack.rb

@@ -0,0 +1,63 @@
+###################
+##
+##  quick and dirty format for filepacks
+##      that is, named list of files
+##
+##   note -  space indent(ation) does NOT matter
+##
+
+
+
+##  PACK eng|en
+##  add NAME eng|en   - why? why not?
+##
+## change to FILEPACK_NAME - why? why not?
+FILEPACK_HEADER_RE = %r{\A
+                              (?: PACK ) [ ]+
+                              (?<header> .+?)   ## note - use non-greedy
+                          \z}x
+
+##   DIR  /sports/openfootball
+##   CD  /sports/openfootball     -- keep - why? why not?
+FILEPACK_DIR_RE = %r{\A
+                              (?: DIR|CD ) [ ]+
+                              (?<dir> .+? )      ## note - use non-greedy
+
+                      \z}x
+
+
+
+def parse_filepack( txt )
+    h={}
+    recs    = nil
+    basedir = nil
+
+    txt.each_line do |line|
+      line = line.strip
+
+      next  if line.start_with?('#') || line.empty?
+
+      break if line == '__END__'
+
+
+      if m=FILEPACK_HEADER_RE.match(line)
+        keys    = m[:header].strip.split(  /[ ]*\|[ ]*/ )
+        basedir = nil
+        recs = []
+        ## note - normalize keys for now (always downcase)
+        keys.each {|key| h[key.downcase] = recs }
+      elsif m=FILEPACK_DIR_RE.match(line)
+        basedir = m[:dir].strip
+      else
+        file =  basedir ? File.join( basedir, line ) : line
+        recs << file
+      end
+    end
+
+    h
+end
+
+
+def read_filepack( path )
+  parse_filepack( read_text( path ))
+end

data/lib/fbtok/pathspec.rb

@@ -1,13 +1,8 @@
 
 
 module SportDb
-
 class Pathspec
 
-  def self.debug=(value) @@debug = value; end
-  def self.debug?()      @@debug ||= false; end  ## note: default is FALSE
-
-
     SEASON_RE = %r{ (?:
                           (?<season>\d{4}-\d{2})
                         | (?<season>\d{4})
@@ -62,9 +57,6 @@ class Pathspec
     ##      but starting filename e.g. 2024_friendlies.txt or 2024-25_bundesliga.txt
 
 
-def self._norm_seasons( seasons )
-    seasons.map {|season| Season(season) }
-end
 
 
 ## todo/check - rename to glob or expand or such - why? why not?
@@ -74,23 +66,31 @@ end
 ##                    that will use generic **/*.txt and only use ignore filter!!!
 
 
-def self._find( path, seasons: nil )
+def self.find( path, seasons: nil )
+    ##
+    ## note -  only if seasons filter is turn on
+    ##                     MATCH_RE gets used!!!
+    ##            otherwise generic **/*.txt
+    ##
+    ## note -   the ignore/exlude filter always gets used/applied for now
+
+
     ## check - rename dir
     ##          use root_dir or work_dir or cd or such - why? why not?
 
-
     ## note: normalize path - use File.expand_path ??
     ##    change all backslash to slash for now
-    ## path = path.gsub( "\\", '/' )
-    path =  File.expand_path( path )
+    fullpath =  File.expand_path( path )
 
     ####
     ## note - make sure path exists; raise error if not
-    raise Errno::ENOENT, "No such directory - #{path})"  unless Dir.exist?( path )
+    ##   ENOENT => Error No Entity
+    raise Errno::ENOENT, "No such directory - #{path})"  unless Dir.exist?( fullpath )
 
 
     if seasons && seasons.size > 0
-       seasons = _norm_seasons( seasons )    ## norm seasons (string, integer => Season obj)
+       ## norm seasons (string, integer => Season obj)
+       seasons =  seasons.map {|season| Season(season) }
     end
 
 
@@ -99,41 +99,45 @@ def self._find( path, seasons: nil )
     ##     (normally excluded with just *)
     ##  was:  Dir.glob( "#{path}/**/{*,.*}.txt" )
 
-    candidates = Dir.glob( "#{path}/**/*.txt" )
+    candidates = Dir.glob( "#{fullpath}/**/*.txt" )
     ## pp candidates
 
 
     datafiles = []
     candidates.each do |candidate|
-       if m = MATCH_RE.match( candidate )
 
-          ## check for seasons filter
-          next   if seasons && seasons.size > 0 &&
-                    !seasons.include?( Season.parse( m[:season] ))
+         ## (i) check for (optional) seasons filter
+         if seasons && seasons.size > 0
+            if m=MATCH_RE.match( candidate )
+               next   unless seasons.include?( Season.parse( m[:season] ))
+            else
+              next    ## note - no season found in filename; skip too
+            end
+         end
+
 
+         ## (ii) check for (default/built-in) ignore/excludes
+          basename = File.basename( candidate, File.extname( candidate ))
+          dirname  = File.dirname( candidate )
 
-          #########
-          ### exclude squad
-          ##   and .v2 or .v2603
+          ### exclude basenames with:
+          ##   - squad
+          ##   - .v2 or .v2603
           ##
           ##    - worldcup/more/1930_squads.txt   =>   squads
-          ##    - 2014--brazil/cup.v2.txt",
-          ##    - 2014--brazil/cup.v260318_164934.txt",
-
-          basename = File.basename( candidate, File.extname( candidate ))
+          ##    - 2014--brazil/cup.v2.txt
+          ##    - 2014--brazil/cup.v260318_164934.txt
 
-          next   if /squad/i.match( basename )
-          next   if /\.v[0-9][0-9_]*/i.match( basename )
+          next   if /squad/i.match?( basename )
+          next   if /\.v[0-9][0-9_]*/i.match?( basename )
 
           #####
-          ### exclude dirs  w/ squad or wiki
-          dirname  = File.dirname( candidate )
-
-          next   if /squad|wiki/i.match( dirname )
+          ### exclude dirs  with:
+          ##    - squad or wiki
+          next   if /squad|wiki/i.match?( dirname )
 
 
           datafiles << candidate
-       end
     end
 
     ## pp datafiles
@@ -141,22 +145,48 @@ def self._find( path, seasons: nil )
 end
 
 
+def self.path( default=[])
+ ## check for  FBPATH
+ ##         or FBTXT_PATH
+      path = ENV['FBPATH'] || ENV['FBTXT_PATH']
+      if path
+          path.split( /[:;]/)
+      else
+         default
+      end
+end
+end # class Pathspec
+
+
 
 
 
+class Pathspecs    ## change to a module - why? why not?
+
+
+def self.debug=(value) @@debug = value; end
+def self.debug?()      @@debug ||= false; end     ## note: default is FALSE
+
+
 def self.read( src )
-    recs = read_csv( src )
+    ## note: normalize src - use File.expand_path ??
+    ##    change all backslash to slash for now (\ to /)
+    fullsrc =  File.expand_path( src )
+
+    recs = read_csv( fullsrc )
     pp recs     if debug?
 
     ##  note - make pathspecs relative to passed in file arg!!!
-    basedir = File.dirname( src )
+    basedir = File.dirname( fullsrc )
 
     recs.each do |rec|
+        ##
+        ## todo/check - check for season/seasons column - why? why not?
         path = rec['path']
         fullpath = File.expand_path( path, basedir )
-        datafiles =   _find( fullpath )
+        datafiles =   Pathspec.find( fullpath )
 
-        ## add (new) datafiles column (from expanded pathspec)
+        ## auto-add (new) datafiles column (from expanded pathspec)
         rec['datafiles'] = datafiles
     end
 
@@ -165,90 +195,82 @@ end
 
 
 
+def self.build( args, path: [],
+                      filepack: nil )
+  recs = []
 
-end  # class Pathspec
-
-
-
-##
-#  PathspecReport (aka/formerly BatchReport)
-
-class PathspecReport
-  def initialize( specs, title: )
-     @specs = specs
-     @title = title
-  end
-
-  def build
-     buf = String.new
-     buf << "# #{@title} - #{@specs.size} dataset(s)\n\n"
-
-     @specs.each_with_index do |rec,i|
-       datafiles  = rec['datafiles']
-       errors     = rec['errors']
-
-       if errors.size > 0
-         buf << "!! #{errors.size} ERROR(S)  "
-       else
-         buf << "   OK          "
-       end
-       buf << "%-20s" % rec['path']
-       buf << " - #{datafiles.size} datafile(s)"
-       buf << "\n"
-
-       if errors.size > 0
-         buf << errors.pretty_inspect
-         buf << "\n"
+  ## check fo no args case  (and filepack present with default)
+  if args.empty?
+    if filepack && filepack.has_key?('default')
+               recs << { 'path'      => '<default>',
+                         'datafiles' => filepack['default'] }
+    end
+  else
+
+    ## note - collect all "loose/standalone" files (NOT directories)
+    ##             in single default pathspec node
+    more = []
+
+    args.each do |arg|
+       ## note - ALWAYS give priority to existing directory matches (over filepack)
+       ##             e.g. euro or such -  why? why not?
+       ##
+       ##  todo/check  if euro/ works too?
+       ## check if directory
+       if Dir.exist?( arg )
+          recs << { 'path'       => arg,
+                    'datafiles'  => Pathspec.find( arg ) }
+       elsif filepack && filepack.has_key?( arg.downcase )
+           recs << { 'path'      => "<#{arg.downcase}>",
+                     'datafiles' => filepack[arg.downcase] }
+       else   ## assume it's a file
+           file = find_file( arg, path: path )
+            ## check if file (exists) in any path lookup
+            if file
+              ## todo/fix:
+              ##   add File.expand_path upstream in find_file !!!
+              ##           and remove later here
+              ##       also fix upstream Errorno::ENOENT to Errno::ENOENT !!!
+              ##
+              ## make sure path exists; raise error if not
+               ## (auto-)expand path to normalize - yes why? why not?
+              more << File.expand_path(file)
+            else
+              raise Errno::ENOENT, "No such file or directory - #{arg}"
+            end
        end
-     end
+    end
 
-     buf
-  end   # method build
-end  # class BatchReport
+    if more.size > 0
+      recs << { 'path'      => '<input>',
+                'datafiles' =>  more }
+    end
+  end
 
-BatchReport = PathspecReport
+  recs
+end
 
+end  # class Pathspecs
 end  # module Sportdb
 
 
 
+
 ##
 ##  keep helpers as global functions - why? why not?
 
 
-
-def build_pathspecs( args )    ### note:   was expand_args
-   specs = []
-
-   ## note - collect all "loose/standalone" files (NOT directories)
-   ##             in single default pathspec node
-   more = []
-
-   args.each do |arg|
-    ## check if directory
-    if Dir.exist?( arg )
-          datafiles = SportDb::Pathspec._find( arg )
-          specs << { 'path'       => arg,
-                     'datafiles'  => datafiles }
-    elsif File.file?( arg )  ## assume it's a file
-        ## make sure path exists; raise error if not
-        ##   (auto-)expand path to normalize - why? why not?
-          more << arg
-    else
-        raise Errno::ENOENT, "No such file or directory - #{arg}"
-    end
-  end
-
-   if more.size > 0
-      specs << { 'path'      => '<input>',
-                 'datafiles' =>  more }
-   end
-
-  specs
+##  build pathspecs via arguments
+##   - (i) every dir is a pathspec entry/record
+##   - (ii) all files get bundled together into <input> pathspec entry/record
+##
+##   note: was formerly known as expand_args
+def build_pathspecs( args, path: [], filepack: nil )
+   SportDb::Pathspecs.build( args, path: path, filepack: filepack )
 end
 
-
-
+####
+## read pathspecs via csv file (using path column)
 def read_pathspecs( src )
-    SportDb::Pathspec.read( src )
+    SportDb::Pathspecs.read( src )
 end

data/lib/fbtok/pathspec_report.rb

@@ -0,0 +1,42 @@
+module SportDb
+
+##
+#  PathspecReport (aka/formerly BatchReport)
+
+class PathspecReport
+  def initialize( specs, title: )
+     @specs = specs
+     @title = title
+  end
+
+  def build
+     buf = String.new
+     buf << "# #{@title} - #{@specs.size} dataset(s)\n\n"
+
+     @specs.each_with_index do |rec,i|
+       datafiles  = rec['datafiles']
+       errors     = rec['errors']
+
+       if errors.size > 0
+         buf << "!! #{errors.size} ERROR(S)  "
+       else
+         buf << "   OK          "
+       end
+       buf << "%-20s" % rec['path']
+       buf << " - #{datafiles.size} datafile(s)"
+       buf << "\n"
+
+       if errors.size > 0
+         buf << errors.pretty_inspect
+         buf << "\n"
+       end
+     end
+
+     buf
+  end   # method build
+end  # class BatchReport
+
+BatchReport = PathspecReport
+
+
+end  # module Sportdb

data/lib/fbtok.rb

@@ -3,8 +3,9 @@ require 'sportdb/quick'
 
 
 ## our own code
+require_relative 'fbtok/filepack'
 require_relative 'fbtok/pathspec'
-
+require_relative 'fbtok/pathspec_report'
 
 require_relative 'fbtok/fbtok'
 require_relative 'fbtok/fbtree'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fbtok
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.1
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-05-31 00:00:00.000000000 Z
+date: 2026-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sportdb-parser
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.7.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.7.2
 - !ruby/object:Gem::Dependency
   name: sportdb-quick
   requirement: !ruby/object:Gem::Requirement
@@ -100,7 +100,9 @@ files:
 - lib/fbtok/fbtok.rb
 - lib/fbtok/fbtree.rb
 - lib/fbtok/fbx.rb
+- lib/fbtok/filepack.rb
 - lib/fbtok/pathspec.rb
+- lib/fbtok/pathspec_report.rb
 homepage: https://github.com/sportdb/footty
 licenses:
 - Public Domain