sportdb-importers 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 03d3875605ad42d0fd7876f036c2bbbe2dd8999d
-  data.tar.gz: 92435a82e496ef1ca09a9e44395038a81c7b5b21
+  metadata.gz: 6a504f73f34a5e73bb311ee460402b068056d68d
+  data.tar.gz: b8680b47ec3c06958ac8f6718261873521314998
 SHA512:
-  metadata.gz: ac4fea2f6fc7b2fc7d5d981e792977bd1bb44ae8de3bcd72bacae4583828f02f76a1785f9e91f7d68036e10b4743f1cd561e610b1e41612b058c634f5f1b1537
-  data.tar.gz: 2ff8b5dc6f53994431cba566e2031834aa61fd2d6f668a8d4a61af177783791b4981946b58e587ddd2a199f8fa6469ce006925b607111529359503afc627479e
+  metadata.gz: 5c0ea550c7450f524d0314e55365b221841e95c6c5794681ec1b34c5d27cdb721bd0a4408b69ade5e751a76d88a0fddbcdc58bbe1644f5d644ae925311bdec3f
+  data.tar.gz: 358bdd2d3872ff3b80d9916293c9bf372f581ebaf2e0c14de370a80983bb74148acba97b6baa1b0072b59da796c1ba3849d5bc5279c519111a011a46f4fd007f

data/Manifest.txt

@@ -3,7 +3,8 @@ Manifest.txt
 README.md
 Rakefile
 lib/sportdb/importers.rb
-lib/sportdb/importers/import.rb
+lib/sportdb/importers/event.rb
+lib/sportdb/importers/match.rb
 lib/sportdb/importers/version.rb
 test/helper.rb
 test/test_club.rb

data/lib/sportdb/importers.rb

@@ -8,37 +8,20 @@ require 'sportdb/sync'
 ###
 # our own code
 require 'sportdb/importers/version' # let version always go first
-require 'sportdb/importers/import'
+require 'sportdb/importers/event'
+require 'sportdb/importers/match'
 
 
 module SportDb
 class Package
   ## (re)open class - note: adds more machinery; see sportdb-text for first/original/base definition
 
-def read_csv( start: nil )    ### todo/fix - rename to read_csv !!!!!!
+def read_csv( start: nil )
   ## start - season e.g. 1993/94 to start (skip older seasons)
   ## note: assume package holds country/national (club) league
   #  use for importing german bundesliga, english premier league, etc.
 
-  match_by_season( format: 'csv', start: start ).each_with_index do |(season_key, entries),i|
-    puts "season [#{i+1}] >#{season_key}<:"
-
-    entries.each do |entry,j|
-      ## note: assume datafile basename (without extension) is the league key
-      ##  e.g. eng.1, eng.3a, eng.3b, at.1, champs, world, etc.
-      league_key = File.basename( entry.name, File.extname( entry.name ) )  ## get basename WITHOUT extension
-
-      pp [entry.name, season_key, league_key]
-
-      event = CsvEventImporter.parse( entry.read, league:  league_key,
-                                                  season:  season_key )
-
-      puts "added #{event.name} - from source >#{entry.name}<"
-      puts "  #{event.teams.size} teams"
-      puts "  #{event.matches.size} matches"
-      puts "  #{event.rounds.size} rounds"
-    end  # each datafile
-  end # each season
+   each_csv { |entry| SportDb.handle_csv( entry, start: start ) }
 end # method import
 
 end  # class Package
@@ -50,25 +33,101 @@ end  # class Package
 def self.read_csv( path )
   if File.directory?( path )          ## if directory assume "unzipped" package
     DirPackage.new( path ).read_csv
-  elsif File.file?( path ) && File.extname( path ) == '.zip'   ## check if file is a .zip (archive) file
+  elsif File.file?( path ) && File.extname( path ).downcase == '.zip'   ## check if file is a .zip (archive) file
     ZipPackage.new( path ).read_csv
   else                                ## no package; assume single (standalone) datafile
     ## assume single (free-standing) file
+    handle_csv( path )
+  end
+end
+
+##################
+### helper
+##    move handle_csv somewhere else - why? why not?
+def self.handle_csv( source, start: nil )
+  ## todo/fix:  (re)use a more generic filter instead of start for start of season only
+
+  ##  todo/fix: use a "generic" filter_season helper for easy reuse
+  ##     filter_season( clause, season_key )
+  ##   or better filter = SeasonFilter.new( clause )
+  ##             filter.skip? filter.include? ( season_sason_key )?
+  ##             fiteer.before?( season_key )  etc.
+  ##              find some good method names!!!!
+  season_start = start ? Import::Season.new( start ) : nil
+
+  if source.is_a?( Datafile::DirPackage::Entry) ||
+     source.is_a?( Datafile::ZipPackage::Entry)
+    entry = source
+
+    basename = File.basename( entry.name, File.extname( entry.name ) )  ## get basename WITHOUT extension
+
+    ## check if basename is all numbers (and _-) e.g. 2020.csv or 20.csv etc.
+    ##   if yes, assume "mixed" match datafiles (with many/mixed leagues)
+    if basename =~ /^[0-9_-]+$/
+      pp [entry.name, basename]
+
+      CsvMatchImporter.parse( entry.read )
+    else  ## assume "classic" with season
+      league_key = basename
+
+      ## todo/fix: check if season_key is proper season - e.g. matches pattern !!!!
+      season_q   = File.basename( File.dirname( entry.name ))
+      season     = Import::Season.new( season_q )  ## normalize season
+      season_key = season.key
+
+      if season_start && season_start.start_year > season.start_year
+        ## skip if start season before this season
+      else
+        pp [entry.name, season_key, league_key]
+
+        event = CsvEventImporter.parse( entry.read, league:  league_key,
+                                                    season:  season_key )
+
+        puts "added #{event.name} - from source >#{entry.name}<"
+        puts "  #{event.teams.size} teams"
+        puts "  #{event.matches.size} matches"
+        puts "  #{event.rounds.size} rounds"
+      end
+    end
+  else  ## assume (string) filepath for now  - add more options later on!!!!
+    ## assume single (free-standing) file
+    path = source
     full_path = File.expand_path( path )   ## resolve/make path absolute
     ## 1) assume basename is the league key
     ## 2) assume last directory is the season key
-    league_key = File.basename( full_path, File.extname( full_path ) )  ## get basename WITHOUT extension
-    season_key = File.basename( File.dirname( full_path ) )
-
-    event = CsvEventImporter.read( full_path, league: league_key,
-                                              season: season_key )
 
-    puts "added #{event.name} - from source >#{path}<"
-    puts "  #{event.teams.size} teams"
-    puts "  #{event.matches.size} matches"
-    puts "  #{event.rounds.size} rounds"
+    basename = File.basename( full_path, File.extname( full_path ) )  ## get basename WITHOUT extension
+    if basename =~ /^[0-9_-]+$/
+      pp [path, basename]
+      CsvMatchImporter.read( full_path )
+    else ## assume "classic" with season
+      ## 1) assume basename is the league key
+      ## 2) assume last directory is the season key
+      league_key = basename
+
+      season_q   = File.basename( File.dirname( full_path ) )
+      season     = Import::Season.new( season_q )  ## normalize season
+      season_key = season.key
+
+      if season_start && season_start.start_year > season.start_year
+        ## skip if start season before this season
+      else
+        ## todo/fix: check if season_key is proper season - e.g. matches pattern !!!!
+        pp [path, season_key, league_key]
+
+        event = CsvEventImporter.read( full_path, league: league_key,
+                                                  season: season_key )
+
+        puts "added #{event.name} - from source >#{path}<"
+        puts "  #{event.teams.size} teams"
+        puts "  #{event.matches.size} matches"
+        puts "  #{event.rounds.size} rounds"
+      end
+    end
   end
-end
+end  # method self.handle_csv
+
+
 end  # module SportDb
 
 

data/lib/sportdb/importers/{import.rb → event.rb}

@@ -1,6 +1,4 @@
 
-## todo/fix: rename to CsvEventImporter or EventImporter  !!! returns Event!!
-## todo/fix/check: rename to CsvMatchReader and CsvMatchReader to CsvMatchParser - why? why not?
 
 module SportDb
 class CsvEventImporter

data/lib/sportdb/importers/match.rb

@@ -0,0 +1,96 @@
+
+
+module SportDb
+class CsvMatchImporter
+
+  def self.read( path, headers: nil )
+    txt = File.open( path, 'r:utf-8' ) {|f| f.read }
+    parse( txt, headers: headers )
+  end
+
+  def self.parse( txt, headers: nil )
+    new( txt, headers: headers ).parse
+  end
+
+
+  def initialize( txt, headers: nil )
+    @txt     = txt
+    @headers = headers
+  end
+
+
+  def parse
+    ## todo/fix: add headers options (pass throughto CsvMatchReader)
+    ##    add filters too why? why not?
+
+    ##  todo/fix:
+    ##     add normalize: false/mapping: false  flag for NOT mapping club/team names
+    ##       make normalize: false the default, anyways - why? why not?
+    opts = {}
+    opts[:headers] = @headers  if @headers
+
+    matches = CsvMatchParser.parse( @txt, **opts )
+
+    matches.each do |match|
+      league = Import.catalog.leagues.find!( match.league )
+      # pp league
+
+      team1  = Import.catalog.teams.find_by!( name: match.team1, league: league )
+      team2  = Import.catalog.teams.find_by!( name: match.team2, league: league )
+
+      date =  Date.strptime( match.date, '%Y-%m-%d' )
+      ## quick hack - for now always use 2019/20 style season
+      ##   fix!!! - use league to find proper season e.g. 2019 or 2019/20 etc.
+
+      start_year = if date.month >= 7
+                     date.year
+                   else
+                     date.year-1
+                   end
+
+      ## note: for now always assume 2019/20 season
+      season = Import::Season.new( '%d/%d' % [start_year, (start_year+1)%100] )
+      # pp season
+
+
+      event_rec = Sync::Event.find_or_create_by( league: league,
+                                                 season: season )
+
+      team1_rec = Sync::Team.find_or_create( team1 )
+      team2_rec = Sync::Team.find_or_create( team2 )
+
+      ## warn about duplicates?
+      ##  note: for now only allow one (unique) match pair per team
+      match_recs = Model::Match.where( event_id: event_rec.id,
+                                       team1_id: team1_rec.id,
+                                       team2_id: team2_rec.id ).to_a
+      if match_recs.size > 0
+        puts "!! #{match_recs.size} duplicate match record(s) found:"
+        pp match_recs
+        exit 1
+      end
+
+      ## find last pos - check if it can be nil?  yes, is nil if no records found
+      max_pos = Model::Match.where( event_id: event_rec.id ).maximum( 'pos' )
+      max_pos = max_pos ? max_pos+1 : 1
+
+      rec = Model::Match.create!(
+              event_id: event_rec.id,
+              team1_id: team1_rec.id,
+              team2_id: team2_rec.id,
+              ## round_id: round_rec.id,  -- note: now optional
+              pos:      max_pos,
+              date:     date.to_date,
+              score1:   match.score1,
+              score2:   match.score2,
+              score1i:  match.score1i,
+              score2i:  match.score2i )
+
+      ## todo/fix:
+      ##   check if event includes teams?
+      ##    if not (auto-)add teams to event.teams !!!!!!!
+    end
+  end # method parse
+
+end # class CsvEventImporter
+end # module SportDb

data/lib/sportdb/importers/version.rb

@@ -7,7 +7,7 @@ module Importers
 
   MAJOR = 1    ## todo: namespace inside version or something - why? why not??
   MINOR = 1
-  PATCH = 0
+  PATCH = 1
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sportdb-importers
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Gerald Bauer
@@ -67,7 +67,8 @@ files:
 - README.md
 - Rakefile
 - lib/sportdb/importers.rb
-- lib/sportdb/importers/import.rb
+- lib/sportdb/importers/event.rb
+- lib/sportdb/importers/match.rb
 - lib/sportdb/importers/version.rb
 - test/helper.rb
 - test/test_club.rb