sportdb-formats 0.1.7 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 421d04b4b20f49a37bb1a31161bb4ab7b5df3113
-  data.tar.gz: e533ef2bb8dbb3c1053416c98ede4c9d20c42546
+  metadata.gz: 7888347efffbc31760344406232e8f930d8777a0
+  data.tar.gz: 8cba378a58d47f5b4ddc34ee933df6bb19e74cc7
 SHA512:
-  metadata.gz: 6256ca94b85f87865a44b4f892273cb4c2c9f17e7847e765cd3c73f408a005907181e202ec201498fd3214a603aed6af33b4ecbcf4e2af43cca851f7b1ac9a09
-  data.tar.gz: 750122be629a4b902356d845658cfcccbb0403e75e3f713e1ff4bd0da4aefb6b2355bc828aecb8e21d20addb5bfb9c1924df58830eb85e4743aa340c560924df
+  metadata.gz: 54cf5468032c75452e8a595fcf2efb204edec7796478aeef21fdd571c0503eaf1e412c0b8e51e927cb5fab0e7ea363d286885bdd6e7c62f5b422f5f2a7c806b4
+  data.tar.gz: c56db501666da9225f930ed16d9bbdd16f81d968b5d25eea5f04b764bf5c3bed62a33ad6843595c2316102356d89071c324b13db7a3bb8a8e9171f3abf77a534

data/Manifest.txt

@@ -6,6 +6,7 @@ lib/sportdb/formats.rb
 lib/sportdb/formats/datafile.rb
 lib/sportdb/formats/goals.rb
 lib/sportdb/formats/outline_reader.rb
+lib/sportdb/formats/package.rb
 lib/sportdb/formats/scores.rb
 lib/sportdb/formats/season_utils.rb
 lib/sportdb/formats/version.rb
@@ -15,5 +16,6 @@ test/test_datafile.rb
 test/test_datafile_match.rb
 test/test_goals.rb
 test/test_outline_reader.rb
+test/test_package.rb
 test/test_scores.rb
 test/test_season_utils.rb

data/Rakefile

@@ -24,9 +24,11 @@ Hoe.spec 'sportdb-formats' do
     ['date-formats',  '>= 0.2.4'],
     ['csvreader',     '>= 1.2.4'],
     ['sportdb-langs', '>= 0.0.1'],
+
+    ['rubyzip',       '>= 1.2.4' ],
   ]
 
   self.spec_extras = {
-   required_ruby_version: '>= 2.2.2'
+    required_ruby_version: '>= 2.2.2'
   }
 end

data/lib/sportdb/formats.rb

@@ -6,6 +6,10 @@ require 'alphabets'      # downcase_i18n, unaccent, variants, ...
 require 'date/formats'   # DateFormats.parse, find!, ...
 require 'csvreader'
 
+require 'zip'     ## todo/check: if zip is alreay included in a required module
+
+
+
 def read_csv( path )
   CsvHash.read( path, :header_converters => :symbol )
 end
@@ -24,6 +28,7 @@ require 'sportdb/langs'
 require 'sportdb/formats/version' # let version always go first
 require 'sportdb/formats/outline_reader'
 require 'sportdb/formats/datafile'
+require 'sportdb/formats/package'
 require 'sportdb/formats/season_utils'
 
 

data/lib/sportdb/formats/datafile.rb

@@ -19,40 +19,44 @@ module Datafile      # note: keep Datafile in its own top-level module/namespace
   end
 
 
-
-  CLUBS_REGEX = %r{  (?:^|/)               # beginning (^) or beginning of path (/)
-                       (?:[a-z]{1,4}\.)?   # optional country code/key e.g. eng.clubs.txt
+  CLUBS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+                       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.txt
                        clubs\.txt$
                    }x
 
-  CLUBS_WIKI_REGEX = %r{  (?:^|/)               # beginning (^) or beginning of path (/)
+  CLUBS_WIKI_RE = %r{  (?:^|/)               # beginning (^) or beginning of path (/)
                            (?:[a-z]{1,4}\.)?   # optional country code/key e.g. eng.clubs.wiki.txt
                           clubs\.wiki\.txt$
                        }x
 
-  def self.find_clubs( path, pattern: CLUBS_REGEX )            find( path, pattern ); end
-  def self.find_clubs_wiki( path, pattern: CLUBS_WIKI_REGEX )  find( path, pattern ); end
+  CLUB_PROPS_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+                       (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.props.txt
+                      clubs\.props\.txt$
+                   }x
 
-  def self.match_clubs( path )       CLUBS_REGEX.match( path ); end
-  def self.match_clubs_wiki( path )  CLUBS_WIKI_REGEX.match( path ); end
+  def self.find_clubs( path, pattern: CLUBS_RE )            find( path, pattern ); end
+  def self.find_clubs_wiki( path, pattern: CLUBS_WIKI_RE )  find( path, pattern ); end
 
+  def self.match_clubs( path )       CLUBS_RE.match( path ); end
+  def self.match_clubs_wiki( path )  CLUBS_WIKI_RE.match( path ); end
+  def self.match_club_props( path, pattern: CLUB_PROPS_RE ) pattern.match( path ); end
 
 
-  LEAGUES_REGEX = %r{  (?:^|/)               # beginning (^) or beginning of path (/)
-                        (?:[a-z]{1,4}\.)?   # optional country code/key e.g. eng.clubs.wiki.txt
+  LEAGUES_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+                        (?: [a-z]{1,4}\. )?   # optional country code/key e.g. eng.clubs.wiki.txt
                      leagues\.txt$
                     }x
 
-  def self.find_leagues( path, pattern: LEAGUES_REGEX )  find( path, pattern ); end
-  def self.match_leagues( path )  LEAGUES_REGEX.match( path ); end
+  def self.find_leagues( path, pattern: LEAGUES_RE )  find( path, pattern ); end
+  def self.match_leagues( path )  LEAGUES_RE.match( path ); end
 
 
-  CONF_REGEX = %r{  (?:^|/)               # beginning (^) or beginning of path (/)
+  CONF_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
                      \.conf\.txt$
                  }x
 
-  def self.find_conf( path, pattern: CONF_REGEX )  find( path, pattern ); end
-  def self.match_conf( path )  CONF_REGEX.match( path ); end
+  def self.find_conf( path, pattern: CONF_RE )  find( path, pattern ); end
+  def self.match_conf( path )  CONF_RE.match( path ); end
 
 
 

data/lib/sportdb/formats/package.rb

@@ -0,0 +1,160 @@
+
+module Datafile      # note: keep Datafile in its own top-level module/namespace for now - why? why not?
+
+
+  ZIP_RE = %r{ \.zip$
+            }x
+  def self.match_zip( path, pattern: ZIP_RE ) pattern.match( path ); end
+
+
+
+  ## exclude pattern
+  ##  for now exclude all files in directories starting with a dot (e.g. .git/ or .github/ or .build/ etc.)
+  ##  todo/check: rename to EXCLUDE_DOT_DIRS_RE - why? why not?
+  EXCLUDE_RE = %r{  (?: ^|/ )               # beginning (^) or beginning of path (/)
+                        \.[a-zA-Z0-9_-]+  ## (almost) any name BUT must start with dot e.g.  .git, .build, etc.
+                        /
+                     }x
+  def self.match_exclude( path, pattern: EXCLUDE_RE ) pattern.match( path ); end
+
+
+
+class Package; end    ## use a shared base class for DirPackage, ZipPackage, etc.
+
+class DirPackage < Package    ## todo/check: find a better name e.g. UnzippedPackage, FilesystemPackage, etc. - why? why not?
+class Entry
+  def initialize( pack, path )
+    @pack = pack  ## parent package
+    @path = path
+    ## todo/fix!!!!: calculate @name (cut-off pack.path!!!)
+    @name =  path
+  end
+  def name()  @name; end
+  def read()  File.open( @path, 'r:utf-8' ).read; end
+end  # class DirPackage::Entry
+
+
+  attr_reader :name, :path
+
+  def initialize( path )
+    ## todo/fix:  expand_path ?! - why? why not? if you pass in ./ basename will be . and NOT directory name, for example!!!
+    @path = path   ## rename to root_path or base_path or somehting - why? why not?
+
+    basename = File.basename( path )   ## note: ALWAYS keeps "extension"-like name if present (e.g. ./austria.zip => austria.zip)
+    @name = basename
+  end
+
+  def each( pattern:,  extension: 'txt' )    ## todo/check: rename to glob or something - why? why not?
+    ##   use just .* for extension or remove and check if File.file? and skip File.directory? - why? why not?
+    ## note: incl. files starting with dot (.)) as candidates (normally excluded with just *)
+    Dir.glob( "#{@path}/**/{*,.*}.#{extension}" ).each do |path|
+      ## todo/fix: (auto) skip and check for directories
+      if EXCLUDE_RE.match( path )
+        ## note: skip dot dirs (e.g. .build/, .git/, etc.)
+      elsif pattern.match( path )
+        yield( Entry.new( self, path ))
+      else
+        ## puts "  skipping >#{path}<"
+      end
+    end
+  end
+
+  def find( name )
+    Entry.new( self, "#{@path}/#{name}" )
+  end
+end  # class DirPackage
+
+
+## helper wrapper for datafiles in zips
+class ZipPackage < Package
+class Entry
+  def initialize( pack, entry )
+    @pack  = pack
+    @entry = entry
+  end
+
+  def name()  @entry.name; end
+  def read
+    txt = @entry.get_input_stream.read
+    ##  puts "** encoding: #{txt.encoding}"  #=> encoding: ASCII-8BIT
+    txt = txt.force_encoding( Encoding::UTF_8 )
+    txt
+  end
+end # class ZipPackage::Entry
+
+  attr_reader :name, :path
+
+  def initialize( path )
+    @path = path
+
+    extname  = File.extname( path )    ## todo/check: double check if extension is .zip - why? why not?
+    basename = File.basename( path, extname )
+    @name = basename
+  end
+
+  def each( pattern: )
+    Zip::File.open( @path ) do |zipfile|
+      zipfile.each do |entry|
+        if entry.directory?
+          next ## skip
+        elsif entry.file?
+          if EXCLUDE_RE.match( entry.name )
+            ## note: skip dot dirs (e.g. .build/, .git/, etc.)
+          elsif pattern.match( entry.name )
+            yield( Entry.new( self, entry ) )   # wrap entry in uniform access interface / api
+          else
+            ## puts "  skipping >#{entry.name}<"
+          end
+        else
+          puts "** !!! ERROR !!! #{entry.name} is unknown zip file type in >#{@path}<, sorry"
+          exit 1
+        end
+      end
+    end
+  end
+
+  def find( name )
+     entries = match_entry( name )
+     if entries.empty?
+       puts "** !!! ERROR !!! zip entry >#{name}< not found in >#{@path}<; sorry"
+       exit 1
+     elsif entries.size > 1
+       puts "** !!! ERROR !!! ambigious zip entry >#{name}<; found #{entries.size} entries in >#{@path}<:"
+       pp entries
+       exit 1
+     else
+       Entry.new( self, entries[0] )    # wrap entry in uniform access interface / api
+     end
+  end
+
+private
+  def match_entry( name )
+    ## todo/fix:  use Zip::File.glob or find_entry or something better/faster?  why? why not?
+
+    pattern = %r{ #{Regexp.escape( name )}    ## match string if ends with name
+                   $
+                }x
+
+    entries = []
+    Zip::File.open( @path ) do |zipfile|
+      zipfile.each do |entry|
+        if entry.directory?
+          next ## skip
+        elsif entry.file?
+          if EXCLUDE_RE.match( entry.name )
+            ## note: skip dot dirs (e.g. .build/, .git/, etc.)
+          elsif pattern.match( entry.name )
+            entries << entry
+          else
+            ## no match; skip too
+          end
+        else
+          puts "** !!! ERROR !!! #{entry.name} is unknown zip file type in >#{@path}<, sorry"
+          exit 1
+        end
+      end
+    end
+    entries
+  end
+end  # class ZipPackage
+end  # module Datafile

data/lib/sportdb/formats/version.rb

@@ -5,8 +5,8 @@ module SportDb
 module Formats
 
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
-  MINOR = 1
-  PATCH = 7
+  MINOR = 2
+  PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/test/test_datafile.rb

@@ -10,11 +10,26 @@ require 'helper'
 
 class TestDatafile < MiniTest::Test
 
+  def test_exclude
+    assert Datafile.match_exclude( '.build/' )
+    assert Datafile.match_exclude( '.git/' )
+
+    assert Datafile.match_exclude( '/.build/' )
+    assert Datafile.match_exclude( '/.git/' )
+
+    assert Datafile.match_exclude( '.build/leagues.txt' )
+    assert Datafile.match_exclude( '.git/leagues.txt' )
+
+    assert Datafile.match_exclude( '/.build/leagues.txt' )
+    assert Datafile.match_exclude( '/.git/leagues.txt' )
+  end
+
+
+
   CLUBS_DIR   = '../../../openfootball/clubs'    ## source repo directory path
   LEAGUES_DIR = '../../../openfootball/leagues'
   AUSTRIA_DIR = '../../../openfootball/austria'
 
-
   def test_find
     datafiles = Datafile.find_clubs( CLUBS_DIR )
     pp datafiles
@@ -29,6 +44,7 @@ class TestDatafile < MiniTest::Test
     pp datafiles
   end
 
+
   def test_bundle
     datafiles = Datafile.find_clubs( CLUBS_DIR )
     pp datafiles

data/test/test_package.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_package.rb
+
+
+require 'helper'
+
+
+class TestPackage < MiniTest::Test
+
+  def test_read
+    [Datafile::DirPackage.new( '../../../openfootball/england' ),
+     Datafile::ZipPackage.new( 'tmp/england-master.zip' )].each do |eng|
+       assert eng.find( '2015-16/.conf.txt' ).read.start_with?( '= English Premier League 2015/16' )
+       assert eng.find( '2017-18/.conf.txt' ).read.start_with?( '= English Premier League 2017/18' )
+       assert eng.find( '2015-16/1-premierleague-i.txt' ).read.start_with?( '= English Premier League 2015/16' )
+    end
+
+    [Datafile::DirPackage.new( '../../../openfootball/austria' ),
+     Datafile::ZipPackage.new( 'tmp/austria-master.zip' )].each do |at|
+       assert at.find( '2018-19/.conf.txt' ).read.start_with?( '= Österr. Bundesliga 2018/19' )
+    end
+  end  # method test_read
+
+end  # class TestPackage

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sportdb-formats
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.0
 platform: ruby
 authors:
 - Gerald Bauer
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -111,6 +125,7 @@ files:
 - lib/sportdb/formats/datafile.rb
 - lib/sportdb/formats/goals.rb
 - lib/sportdb/formats/outline_reader.rb
+- lib/sportdb/formats/package.rb
 - lib/sportdb/formats/scores.rb
 - lib/sportdb/formats/season_utils.rb
 - lib/sportdb/formats/version.rb
@@ -120,6 +135,7 @@ files:
 - test/test_datafile_match.rb
 - test/test_goals.rb
 - test/test_outline_reader.rb
+- test/test_package.rb
 - test/test_scores.rb
 - test/test_season_utils.rb
 homepage: https://github.com/sportdb/sport.db