sportdb-importers 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bf896cfb7ec92171a5238fcb3b9b434dca7fa651
+  data.tar.gz: ed14f81d52b1487f869d7a0a2574d5b6104cefcc
+SHA512:
+  metadata.gz: 1420fc54d8b03f8ba7467fa3fc6cac2e18c3f2529316012e4f1400288470cca691c9abb2459d5e07fc097b0ffedc6b8c48e6e2e47d2dc31564604b836b4c2214
+  data.tar.gz: 16132b877f95ca4f41e78d153dc5de9633f2d766ba60e0b1a50708bd59819c237be1b1372d2d053e39cb2a7202ed46cc1f356553c55c45c0ae2b543b87b4318b

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+### 0.0.1 / 2019-06-20
+
+* Everything is new. First release.

data/Manifest.txt

@@ -0,0 +1,11 @@
+CHANGELOG.md
+Manifest.txt
+README.md
+Rakefile
+lib/sportdb/importers.rb
+lib/sportdb/importers/import.rb
+lib/sportdb/importers/version.rb
+test/helper.rb
+test/test_club.rb
+test/test_import.rb
+test/test_version.rb

data/README.md

@@ -0,0 +1,50 @@
+# sportdb-importers - tools 'n' scripts for importing sports (football) data in alternate (text) formats incl. comma-separated values (csv) format"
+
+
+* home  :: [github.com/sportdb/sport.db](https://github.com/sportdb/sport.db)
+* bugs  :: [github.com/sportdb/sport.db/issues](https://github.com/sportdb/sport.db/issues)
+* gem   :: [rubygems.org/gems/sportdb-importers](https://rubygems.org/gems/sportdb-importers)
+* rdoc  :: [rubydoc.info/gems/sportdb-importers](http://rubydoc.info/gems/sportdb-importers)
+* forum :: [opensport](http://groups.google.com/group/opensport)
+
+
+
+## Usage
+
+Let's import all datafiles for all seasons (from 1888-89 to today)
+for [England](https://github.com/footballcsv/england), use:
+
+``` ruby
+require 'sportdb/importers'
+
+## note: requires a local copy of the football.db clubs datasets
+##          see https://github.com/openfootball/clubs
+SportDb::Import.config.clubs_dir = './clubs'
+
+
+SportDb.connect( adapter:  'sqlite3',
+                 database: './eng.db' )
+
+SportDb.create_all   ## build database schema / tables
+
+
+## note: requires a local copy of the football.csv england datasets
+##          see https://github.com/footballcsv/england
+pack = CsvPackage.new( './england' )
+pack.import
+```
+
+That's it.
+
+
+## License
+
+The `sportdb-importers` scripts are dedicated to the public domain.
+Use it as you please with no restrictions whatsoever.
+
+
+## Questions? Comments?
+
+Send them along to the
+[Open Sports & Friends Forum/Mailing List](http://groups.google.com/group/opensport).
+Thanks!

data/Rakefile

@@ -0,0 +1,31 @@
+require 'hoe'
+require './lib/sportdb/importers/version.rb'
+
+Hoe.spec 'sportdb-importers' do
+
+  self.version = SportDb::Importers::VERSION
+
+  self.summary = "sportdb-importers - tools 'n' scripts for importing sports (football) data in alternate (text) formats incl. comma-separated values (csv) format"
+  self.description = summary
+
+  self.urls = ['https://github.com/sportdb/sport.db']
+
+  self.author = 'Gerald Bauer'
+  self.email  = 'opensport@googlegroups.com'
+
+  # switch extension to .markdown for gihub formatting
+  self.readme_file  = 'README.md'
+  self.history_file = 'CHANGELOG.md'
+
+  self.licenses = ['Public Domain']
+
+  self.extra_deps = [
+    ['sportdb-text', '>= 0.4.0'],
+    ['sportdb-sync', '>= 1.0.0'],
+  ]
+
+  self.spec_extras = {
+    required_ruby_version: '>= 2.2.2'
+  }
+
+end

data/lib/sportdb/importers/import.rb

@@ -0,0 +1,126 @@
+
+## todo/fix: rename to CsvEventImporter or EventImporter  !!! returns Event!!
+class CsvEventImporter    ## todo/fix/check: rename to CsvMatchReader and CsvMatchReader to CsvMatchParser - why? why not?
+
+  def self.read( path, league:, season:,
+                       headers: nil )
+    txt = File.open( path, 'r:utf-8' ) {|f| f.read }
+    parse( txt, league: league, season: season,
+                headers: headers )
+  end
+
+  def self.parse( txt, league:, season:,
+                       headers: nil  )
+    new( txt, league: league, season: season,
+              headers: headers ).parse
+  end
+
+
+  def initialize( txt, league:, season:, headers: nil )
+    @txt     = txt
+    @headers = headers
+
+    raise ArgumentError("string expected for league; got #{league.class.name}")  unless league.is_a? String
+    raise ArgumentError("string expected for season; got #{season.class.name}")  unless season.is_a? String
+
+    ## try mapping of league here - why? why not?
+    @league  = SportDb::Import.catalog.leagues.find!( league )
+    @season  = SportDb::Import::Season.new( season )
+  end
+
+
+  def parse
+    ## todo/fix: add headers options (pass throughto CsvMatchReader)
+    ##    add filters too why? why not?
+
+    ##  todo/fix:
+    ##     add normalize: false/mapping: false  flag for NOT mapping club/team names
+    ##       make normalize: false the default, anyways - why? why not?
+    opts = {}
+    opts[:headers] = @headers  if @headers
+
+    matches = CsvMatchReader.parse( @txt, **opts )
+
+    matchlist = SportDb::Import::Matchlist.new( matches )
+
+    team_names = matchlist.teams           ## was: find_teams_in_matches_txt( matches_txt )
+    puts "#{team_names.size} teams:"
+    pp team_names
+
+    ## note: allows duplicates - will return uniq struct recs in teams
+    teams = SportDb::Import.catalog.teams.find_by!( name: team_names,
+                                                    league: @league )
+    ## build mapping - name => team struct record
+    team_mappings =  team_names.zip( teams ).to_h
+
+    pp team_mappings
+
+
+    #######
+    # start with database updates / sync here
+
+    event_rec = SportDb::Sync::Event.find_or_create_by( league: @league,
+                                                        season: @season )
+
+    ## todo/fix:
+    ##   add check if event has teams
+    ##   if yes - only double check and do NOT create / add teams
+    ##    number of teams must match (use teams only for lookup/alt name matches)
+
+    ## note: allows duplicates - will return uniq db recs in teams
+    ##                            and mappings from names to uniq db recs
+
+    ## todo/fix: rename to team_recs_cache or team_cache - why? why not?
+
+    # maps struct record "canonical" team name to active record db record!!
+    ## note: use "canonical" team name as hash key for now (and NOT the object itself) - why? why not?
+    team_recs = SportDb::Sync::Team.find_or_create( team_mappings.values.uniq )
+
+    ## todo/fix/check:
+    ##   add check if event has teams
+    ##   if yes - only double check and do NOT create / add teams
+    ##    number of teams must match (use teams only for lookup/alt name matches)
+
+    ## add teams to event
+    ##   todo/fix: check if team is alreay included?
+    ##    or clear/destroy_all first!!!
+    event_rec.teams = team_recs   ## todo/check/fix: use team_ids instead - why? why not?
+
+
+
+    ## add catch-all/unclassified "dummy" round
+    round_rec = SportDb::Model::Round.create!(
+      event_id: event_rec.id,
+      title:    'Matchday ??? / Missing / Catch-All',   ## find a better name?
+      pos:      999,
+      start_at: event_rec.start_at.to_date
+    )
+
+    ## add matches
+    matches.each do |match|
+      team1_rec = SportDb::Sync::Team.cache[ team_mappings[match.team1].name ]
+      team2_rec = SportDb::Sync::Team.cache[ team_mappings[match.team2].name ]
+
+      if match.date.nil?
+        puts "!!! WARN: skipping match - play date missing!!!!!"
+        pp match
+      else
+        rec = SportDb::Model::Game.create!(
+                team1_id: team1_rec.id,
+                team2_id: team2_rec.id,
+                round_id: round_rec.id,
+                pos:      999,    ## make optional why? why not? - change to num?
+                play_at:  Date.strptime( match.date, '%Y-%m-%d' ),
+                score1:   match.score1,
+                score2:   match.score2,
+                score1i:  match.score1i,
+                score2i:  match.score2i,
+              )
+      end
+    end # each match
+
+    event_rec  # note: return event database record
+  end # method parse
+
+end # class CsvEventImporter
+

data/lib/sportdb/importers/version.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+
+
+
+module SportDb
+module Importers
+
+  MAJOR = 1    ## todo: namespace inside version or something - why? why not??
+  MINOR = 0
+  PATCH = 0
+  VERSION = [MAJOR,MINOR,PATCH].join('.')
+
+  def self.version
+    VERSION
+  end
+
+  def self.banner
+    "sportdb-importers/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
+  end
+
+  def self.root
+    File.expand_path( File.dirname(File.dirname(File.dirname(File.dirname(__FILE__)))) )
+  end
+
+end # module Importers
+end # module SportDb

data/lib/sportdb/importers.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+
+
+## 3rd party gemss
+require 'sportdb/text'
+require 'sportdb/sync'
+
+
+###
+# our own code
+require 'sportdb/importers/version' # let version always go first
+require 'sportdb/importers/import'
+
+
+class CsvPackage
+  ## (re)open class - note: adds more machinery; see sportdb-text for first/original/base definition
+
+def import( start: nil )
+  ## start - season e.g. 1993/94 to start (skip older seasons)
+  ## note: assume package holds country/national (club) league
+  #  use for importing german bundesliga, english premier league, etc.
+
+  entries = find_entries_by_season
+  pp entries
+
+  entries.each_with_index do |(season_key, datafiles),i|
+
+    puts "season [#{i+1}] >#{season_key}<:"
+
+    ##  todo/fix: use a "generic" filter_season helper for easy reuse
+    ##     filter_season( clause, season_key )
+    ##   or better filter = SeasonFilter.new( clause )
+    ##             filter.skip? filter.include? ( season_sason_key )?
+    ##             fiteer.before?( season_key )  etc.
+    ##              find some good method names!!!!
+    if start
+      start_year        = start[0..3].to_i
+      season_start_year = season_key[0..3].to_i
+      if season_start_year < start_year
+        puts "skip #{season_start_year} before #{start_year}"
+        next
+      end
+    end
+
+    datafiles.each do |datafile,j|
+      path = expand_path( datafile )
+      ## note: assume datafile basename (without extension) is the league key
+      ##  e.g. eng.1, eng.3a, eng.3b, at.1, champs, world, etc.
+      league_key = File.basename( datafile, File.extname( datafile ) )  ## get basename WITHOUT extension
+
+      pp [path, season_key, league_key]
+
+      event = CsvEventImporter.read( path, league:  league_key,
+                                           season:  season_key )
+
+      puts "added #{event.title}"
+      puts "  #{event.teams.size} teams"
+      puts "  #{event.rounds.size} rounds"
+      puts "  #{event.games.size} games"
+    end  # each datafile
+  end # each season
+end # method import
+
+end  # class CsvPackage
+
+
+
+puts SportDb::Importers.banner   # say hello

data/test/helper.rb

@@ -0,0 +1,27 @@
+## note: use the local version of sportdb gems
+$LOAD_PATH.unshift( File.expand_path( '../sportdb-formats/lib' ))
+$LOAD_PATH.unshift( File.expand_path( '../sportdb-config/lib' ))
+$LOAD_PATH.unshift( File.expand_path( '../sportdb-text/lib' ))
+$LOAD_PATH.unshift( File.expand_path( '../sportdb-models/lib' ))
+$LOAD_PATH.unshift( File.expand_path( '../sportdb-sync/lib' ))
+
+
+## minitest setup
+
+require 'minitest/autorun'
+
+
+## our own code
+require 'sportdb/importers'
+
+
+## use (switch to) "external" datasets
+SportDb::Import.config.leagues_dir = "../../../openfootball/leagues"
+SportDb::Import.config.clubs_dir   = "../../../openfootball/clubs"
+
+
+COUNTRIES = SportDb::Import.catalog.countries
+LEAGUES   = SportDb::Import.catalog.leagues
+CLUBS     = SportDb::Import.catalog.clubs
+TEAMS     = SportDb::Import.catalog.teams
+

data/test/test_club.rb

@@ -0,0 +1,162 @@
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_club.rb
+
+
+require 'helper'
+
+class TestClub < MiniTest::Test
+
+  def test_eng_i
+    ## todo/fix:
+    ##    add guest1_country_id (optional) to league (e.g. wales for english premier leaguage)
+    ##                                               (e.g. canada for mls)
+    ##                                               (e.g. liechtenstein for swiss superleague) etc.
+    ##                                               why? why not?
+
+    ## fetch English Premiere League
+    league = LEAGUES.find!( 'ENG' )
+
+    team_names = [
+      'Manchester City',
+      'Arsenal',
+      'Liverpool',
+      'Cardiff',
+    ]
+
+    recs = TEAMS.find_by!( name:   team_names,
+                           league: league )
+
+    assert_equal 4, recs.size
+
+    assert_equal 'Manchester City FC', recs[0].name
+    ## assert_equal 'Manchester',    recs[0].city.name
+    assert_equal 'England',            recs[0].country.name
+
+    assert_equal 'Arsenal FC', recs[1].name
+    ## assert_equal 'London',     recs[1].city.name
+    assert_equal 'England',    recs[1].country.name
+
+    assert_equal 'Cardiff City FC', recs[3].name
+    ## assert_equal 'Cardiff',    recs[3].city.name
+    assert_equal 'Wales',           recs[3].country.name
+  end
+
+
+  def test_eng_ii
+    headers = {
+      date:    'Date',
+      team1:   'HomeTeam',
+      team2:   'AwayTeam',
+      score1:  'FTHG',
+      score2:  'FTAG',
+      score1i: 'HTHG',
+      score2i: 'HTAG'
+    }
+
+    matches = CsvMatchReader.read( "#{SportDb::Test.data_dir}/england/2017-18/E0.csv",
+                                        headers: headers
+                                 )
+
+    ## pp matches[0..2]
+
+    matchlist = SportDb::Import::Matchlist.new( matches )
+    team_names = matchlist.teams
+    puts "#{team_names.size} team names:"
+    pp team_names
+=begin
+20 team names:
+["Arsenal",
+ "Bournemouth",
+ "Brighton",
+ "Burnley",
+ "Chelsea",
+ "Crystal Palace",
+ "Everton",
+ "Huddersfield",
+ "Leicester",
+ "Liverpool",
+ "Man City",
+ "Man United",
+ "Newcastle",
+ "Southampton",
+ "Stoke",
+ "Swansea",
+ "Tottenham",
+ "Watford",
+ "West Brom",
+ "West Ham"]
+=end
+
+
+    ## fetch English Premiere League
+    league = LEAGUES.find!( 'ENG' )
+
+    recs   = TEAMS.find_by!( name:   team_names,
+                             league: league )
+
+    assert_equal 20, recs.size
+
+    assert_equal 'Arsenal FC',      recs[0].name
+    ## assert_equal '?',    recs[0].city.name
+    assert_equal 'England',         recs[0].country.name
+
+    assert_equal 'AFC Bournemouth', recs[1].name
+    ## assert_equal '?',     recs[1].city.name
+    assert_equal 'England',         recs[1].country.name
+  end
+
+
+  def test_at
+    headers = {
+      ## season: 'Season',  ## check if season required / needed???
+      date:   'Date',
+      team1:  'Home',
+      team2:  'Away',
+      score1: 'HG',
+      score2: 'AG',
+    }
+
+    matches = CsvMatchReader.read( "#{SportDb::Test.data_dir}/austria/AUT.csv",
+                                        headers: headers,
+                                        filters: { 'Season' => '2016/2017' }
+                                  )
+
+    ## pp matches[0..2]
+
+    matchlist = SportDb::Import::Matchlist.new( matches )
+    team_names = matchlist.teams
+    puts "#{team_names.size} team names:"
+    pp team_names
+=begin
+10 team names:
+["AC Wolfsberger",
+ "Admira",
+ "Altach",
+ "Austria Vienna",
+ "Mattersburg",
+ "Rapid Vienna",
+ "Ried",
+ "Salzburg",
+ "St. Polten",
+ "Sturm Graz"]
+=end
+
+    ## fetch Österr. Bundesliga
+    league = LEAGUES.find!( 'AT' )
+
+    recs   = TEAMS.find_by!( name:   team_names,
+                             league: league )
+
+    assert_equal 10, recs.size
+
+    assert_equal 'Wolfsberger AC',    recs[0].name
+    ## assert_equal '?',    recs[0].city.name
+    assert_equal 'Austria',           recs[0].country.name
+
+    assert_equal 'FC Admira Wacker Mödling', recs[1].name
+    ## assert_equal '?',     recs[1].city.name
+    assert_equal 'Austria',                  recs[1].country.name
+  end
+
+end # class TestClub

data/test/test_import.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_import.rb
+
+
+require 'helper'
+
+class TestImport < MiniTest::Test
+
+  def setup
+    SportDb.connect( adapter:  'sqlite3',
+                     database: ':memory:' )
+    SportDb.create_all   ## build schema
+
+    ## turn on logging to console
+    ## ActiveRecord::Base.logger = Logger.new(STDOUT)
+  end
+
+
+  def test_eng
+    ## fix/todo:
+    ##  use Event.read_csv   - why? why not?
+    ##      Event.parse_csv  - why? why not?
+
+    headers = {
+      date:    'Date',
+      team1:   'HomeTeam',
+      team2:   'AwayTeam',
+      score1:  'FTHG',
+      score2:  'FTAG',
+      score1i: 'HTHG',
+      score2i: 'HTAG'
+    }
+
+    event_rec = CsvEventImporter.read( "#{SportDb::Test.data_dir}/england/2017-18/E0.csv",
+                                        headers: headers,
+                                        league: 'ENG',  ## fetch English Premiere League
+                                        season: '2017/18'
+                                     )
+
+    assert_equal 20,   event_rec.teams.count
+    assert_equal 380,  event_rec.games.count
+
+    rec = event_rec.games.first
+    assert_equal 'Arsenal FC',          rec.team1.name
+    assert_equal 'Leicester City FC',   rec.team2.name
+    assert_equal 4,   rec.score1
+    assert_equal 3,   rec.score2
+    assert_equal 2,   rec.score1i
+    assert_equal 2,   rec.score2i
+  end
+
+end # class TestImport

data/test/test_version.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_version.rb
+
+
+require 'helper'
+
+class TestVersion < MiniTest::Test
+
+  def test_version
+    pp SportDb::Importers::VERSION
+    pp SportDb::Importers.banner
+    pp SportDb::Importers.root
+
+    assert true
+  end
+
+end # class TestVersion

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: sportdb-importers
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Gerald Bauer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-05-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sportdb-text
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+- !ruby/object:Gem::Dependency
+  name: sportdb-sync
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.16'
+description: sportdb-importers - tools 'n' scripts for importing sports (football)
+  data in alternate (text) formats incl. comma-separated values (csv) format
+email: opensport@googlegroups.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- CHANGELOG.md
+- Manifest.txt
+- README.md
+files:
+- CHANGELOG.md
+- Manifest.txt
+- README.md
+- Rakefile
+- lib/sportdb/importers.rb
+- lib/sportdb/importers/import.rb
+- lib/sportdb/importers/version.rb
+- test/helper.rb
+- test/test_club.rb
+- test/test_import.rb
+- test/test_version.rb
+homepage: https://github.com/sportdb/sport.db
+licenses:
+- Public Domain
+metadata: {}
+post_install_message: 
+rdoc_options:
+- "--main"
+- README.md
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: sportdb-importers - tools 'n' scripts for importing sports (football) data
+  in alternate (text) formats incl. comma-separated values (csv) format
+test_files: []