sportdb-readers 0.0.1

data/lib/sportdb/readers/outline_reader.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+
+
+module SportDb
+
+## shared "higher-level" outline reader
+##  todo: add CountryOutlineReader - why? why not?
+class LeagueOutlineReader
+  ## split into league + season
+  ##  e.g. Österr. Bundesliga 2015/16   ## or 2015-16
+  ##       World Cup 2018
+  LEAGUE_SEASON_HEADING_REGEX =  /^
+         (?<league>.+?)     ## non-greedy
+            \s+
+         (?<season>\d{4}
+            (?:[\/-]\d{2})?     ## optional 2nd year in season
+         )
+            $/x
+
+  def self.parse( txt )
+    recs=[]
+    OutlineReader.parse( txt ).each do |node|
+      if node[0] == :h1
+        ## check for league and season
+        heading = node[1]
+        if m=heading.match( LEAGUE_SEASON_HEADING_REGEX )
+          puts "league >#{m[:league]}<, season >#{m[:season]}<"
+
+           recs << { league: m[:league],
+                     season: m[:season],
+                     lines:  []
+                   }
+        else
+          puts "** !!! ERROR !!! - CANNOT match league and season in heading; season missing?"
+          pp heading
+          exit 1
+        end
+      elsif node[0] == :l   ## regular (text) line
+        line = node[1]
+        ## note: skip lines if no heading seen
+        if recs.empty?
+          puts "** !! WARN !! - skipping line (no heading) >#{line}<"
+        else
+          recs[-1][:lines] << line
+        end
+      else
+        puts "** !!! ERROR !!! unknown line type; for now only heading 1 for leagues supported; sorry:"
+        pp node
+        exit 1
+      end
+    end
+
+    ## pass 2 - check & map; replace inline (string with data record)
+    recs.each do |rec|
+      league = find_league( rec[:league] )
+      rec[:league] = league
+    end
+
+    recs
+  end # method parse
+
+
+  def self.find_league( name )
+    league = nil
+    m = LEAGUES.match( name )
+    # pp m
+
+    if m.nil?
+      puts "** !!! ERROR !!! no league match found for >#{name}<, add to leagues table; sorry"
+      exit 1
+    elsif m.size > 1
+      puts "** !!! ERROR !!! ambigious league name; too many leagues (#{m.size}) found:"
+      pp m
+      exit 1
+    else
+      league = m[0]
+    end
+
+    league
+  end
+end # class LeagueOutlineReader
+
+end # module SportDb

data/lib/sportdb/readers/sync.rb

@@ -0,0 +1,208 @@
+# encoding: utf-8
+
+
+module SportDb
+
+module Sync
+  class Country
+    def self.find_or_create( country )
+       rec = WorldDb::Model::Country.find_by( key: country.key )
+       if rec.nil?
+         attribs = {
+           key:  country.key,
+           name: country.name,
+           code: country.fifa,  ## fix:  uses fifa code now (should be iso-alpha3 if available)
+           fifa: country.fifa,
+           area: 1,
+           pop:  1
+         }
+         rec = WorldDb::Model::Country.create!( attribs )
+       end
+       rec
+    end
+  end # class Country
+
+
+  class League
+    def self.find( league )
+      SportDb::Model::League.find_by( key: league.key )
+    end
+    def self.find!( league )
+      rec = find( league )
+      if rec.nil?
+        puts "** !!!ERROR!!! db sync - no league match found for:"
+        pp league
+        exit 1
+      end
+      rec
+    end
+
+    def self.find_or_create( league )
+       rec = find( league )
+       if rec.nil?
+         ## use title and not name - why? why not?
+         ##  quick fix:  change name to title
+         attribs = { key:   league.key,
+                     title: league.name }
+         if league.country
+           attribs[ :country_id ] = Country.find_or_create( league.country ).id
+         end
+
+         rec = SportDb::Model::League.create!( attribs )
+       end
+       rec
+    end
+  end # class League
+
+  class Season
+    def self.normalize_key( key )    ## helper for season key (rename to norm_key ???)
+      ## note:  "normalize" season key
+      ##   always use 2017/18  (and not 2017-18 or 2017-2018 or 2017/2018)
+      ## 1) change 2017-18 to 2017/18
+      key = key.tr( '-', '/' )
+      ## 2) check for 2017/2018 - change to 2017/18
+      if key.length == 9
+        key = "#{key[0..3]}/#{key[7..8]}"
+      end
+      key
+    end
+
+    def self.find( key )
+      key = normalize_key( key )
+      SportDb::Model::Season.find_by( key: key )
+    end
+    def self.find!( key )
+      rec = find( key )
+      if rec.nil?
+        puts "** !!!ERROR!!! db sync - no season match found for >#{normalize_key(key)}<:"
+        pp key
+        exit 1
+      end
+      rec
+    end
+
+    def self.find_or_create( key )  ## e.g. key = '2017/18'
+      rec = find( key )
+      if rec.nil?
+         key = normalize_key( key )  ## note: do NOT forget to normalize key e.g. always use slash (2019/20) etc.
+         attribs = { key:   key,
+                     title: key  }
+         rec = SportDb::Model::Season.create!( attribs )
+      end
+      rec
+    end
+  end # class Season
+
+  class Club
+    def self.find_or_create( club )
+      rec = SportDb::Model::Team.find_by( title: club.name )
+      if rec.nil?
+        ## remove all non-ascii a-z chars
+        key  = club.name.downcase.gsub( /[^a-z]/, '' )
+        puts "add club: #{key}, #{club.name}, #{club.country.name} (#{club.country.key})"
+
+        attribs = {
+          key:       key,
+          title:     club.name,
+          country_id: Country.find_or_create( club.country ).id,
+          club:       true,
+          national:   false  ## check -is default anyway - use - why? why not?
+          ## todo/fix: add city if present - why? why not?
+        }
+        if club.alt_names.empty? == false
+          attribs[:synonyms] = club.alt_names.join('|')
+        end
+
+        rec = SportDb::Model::Team.create!( attribs )
+      end
+      rec
+    end
+  end # class Club
+
+  class Event
+    def self.find( league:, season: )
+      SportDb::Model::Event.find_by( league_id: league.id, season_id: season.id  )
+    end
+    def self.find!( league:, season: )
+      rec = find( league: league, season: season )
+      if rec.nil?
+        puts "** !!!ERROR!!! db sync - no event match found for:"
+        pp league
+        pp season
+        exit 1
+      end
+      rec
+    end
+
+    def self.find_or_create( league:, season: )
+      rec = find( league: league, season: season )
+      if rec.nil?
+        ## quick hack/change later !!
+        ##  todo/fix: check season  - if is length 4 (single year) use 2017, 1, 1
+        ##                               otherwise use 2017, 7, 1
+        ##  start_at use year and 7,1 e.g. Date.new( 2017, 7, 1 )
+        ## hack:  fix/todo1!!
+        ##   add "fake" start_at date for now
+        if season.key.size == '4'   ## e.g. assume 2018 etc.
+          year = season.key.to_i
+          start_at = Date.new( year, 1, 1 )
+        else  ## assume 2014/15 etc.
+          year = season.key[0..3].to_i
+          start_at = Date.new( year, 7, 1 )
+        end
+
+        attribs = {
+          league_id: league.id,
+          season_id: season.id,
+          start_at:  start_at  }
+
+        rec = SportDb::Model::Event.create!( attribs )
+      end
+      rec
+    end
+  end  # class Event
+
+  class Round
+    def self.find_or_create( round, event: )
+       rec = SportDb::Model::Round.find_by( title: round.title, event_id: event.id )
+       if rec.nil?
+         attribs = { event_id: event.id,
+                     title:    round.title,
+                     pos:      round.pos,
+                     start_at: event.start_at.to_date
+                   }
+         rec = SportDb::Model::Round.create!( attribs )
+       end
+       rec
+    end
+  end # class Round
+
+  class Match   ## todo/check:  add alias for Game class - why? why not?
+    def self.create_or_update( match, event: )
+       ## note: MUST find round, thus, use bang (!)
+       round_rec = SportDb::Model::Round.find_by!( event_id: event.id,
+                                                   title:    match.round.title )
+
+       rec = SportDb::Model::Game.find_by( round_id: round_rec.id,
+                                           team1_id: match.team1.id,
+                                           team2_id: match.team2.id )
+       if rec.nil?
+         attribs = { round_id: round_rec.id,
+                     team1_id: match.team1.id,
+                     team2_id: match.team2.id,
+                     pos:      999,    ## make optional why? why not? - change to num?
+                     play_at:  match.date.to_date,
+                     score1:   match.score1,
+                     score2:   match.score2,
+                     score1i:  match.score1i,
+                     score2i:  match.score2i }
+         rec = SportDb::Model::Game.create!( attribs )
+       else
+         # update - todo
+       end
+       rec
+    end
+  end # class Match
+
+end # module Sync
+end # module SportDb

data/lib/sportdb/readers/version.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+
+
+module SportDb
+module Readers
+
+  MAJOR = 0    ## todo: namespace inside version or something - why? why not??
+  MINOR = 0
+  PATCH = 1
+  VERSION = [MAJOR,MINOR,PATCH].join('.')
+
+  def self.version
+    VERSION
+  end
+
+  def self.banner
+    "sportdb-readers/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
+  end
+
+  def self.root
+    File.expand_path( File.dirname(File.dirname(File.dirname(File.dirname(__FILE__)))) )
+  end
+
+end # module Readers
+end # module SportDb

data/test/helper.rb

@@ -0,0 +1,21 @@
+## $:.unshift(File.dirname(__FILE__))
+
+## minitest setup
+require 'minitest/autorun'
+
+
+## our own code
+require 'sportdb/readers'
+
+
+
+
+## use (switch to) "external" datasets
+SportDb::Import.config.clubs_dir   = "../../../openfootball/clubs"
+SportDb::Import.config.leagues_dir = "../../../openfootball/leagues"
+
+
+
+LEAGUES   = SportDb::Import.config.leagues
+CLUBS     = SportDb::Import.config.clubs
+COUNTRIES = SportDb::Import.config.countries

data/test/test_match_parser.rb

@@ -0,0 +1,101 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_match_parser.rb
+
+
+require 'helper'
+
+
+
+class TestMatchParser < MiniTest::Test
+
+    ## build ActiveRecord-like club records/structs
+    Club = Struct.new( :key, :title, :synonyms )
+    def Club.read( txt )
+      recs = []
+      txt.each_line do |line|
+        values = line.split( ',' )
+        values = values.map {|value| value.strip }
+        recs << Club.new( values[0], values[1], values.size > 2 ? values[2..-1].join('|') : nil )
+      end
+      recs
+    end
+
+
+  def test_parse
+    txt = <<TXT
+Matchday 1
+
+[Fri Aug/11]
+  Arsenal FC               4-3  Leicester City
+[Sat Aug/12]
+  Watford FC               3-3  Liverpool FC
+  Chelsea FC               2-3  Burnley FC
+  Crystal Palace           0-3  Huddersfield Town
+  Everton FC               1-0  Stoke City
+  Southampton FC           0-0  Swansea City
+  West Bromwich Albion     1-0  AFC Bournemouth
+  Brighton & Hove Albion   0-2  Manchester City
+[Sun Aug/13]
+  Newcastle United         0-2  Tottenham Hotspur
+  Manchester United        4-0  West Ham United
+
+
+Matchday 2
+
+[Sat Aug/19]
+  Swansea City             0-4  Manchester United
+  AFC Bournemouth          0-2  Watford FC
+  Burnley FC               0-1  West Bromwich Albion
+  Leicester City           2-0  Brighton & Hove Albion
+  Liverpool FC             1-0  Crystal Palace
+  Southampton FC           3-2  West Ham United
+  Stoke City               1-0  Arsenal FC
+[Sun Aug/20]
+  Huddersfield Town        1-0  Newcastle United
+  Tottenham Hotspur        1-2  Chelsea FC
+[Mon Aug/21]
+  Manchester City          1-1  Everton FC
+TXT
+
+    clubs_txt = <<TXT
+arsenalfc, Arsenal FC, Arsenal, FC Arsenal
+leicestercityfc, Leicester City FC, Leicester, Leicester City
+watfordfc, Watford FC, Watford, FC Watford
+liverpoolfc, Liverpool FC, Liverpool, FC Liverpool
+chelseafc, Chelsea FC, Chelsea, FC Chelsea
+burnleyfc, Burnley FC, Burnley, FC Burnley
+crystalpalacefc, Crystal Palace FC, Crystal Palace, C Palace, Palace, Crystal P
+huddersfieldtownafc, Huddersfield Town AFC, Huddersfield, Huddersfield Town
+evertonfc, Everton FC, Everton, FC Everton
+stokecityfc, Stoke City FC, Stoke, Stoke City
+southamptonfc, Southampton FC, Southampton, FC Southampton
+swanseacityfc, Swansea City FC, Swansea, Swansea City, Swansea City AFC
+westbromwichalbionfc, West Bromwich Albion FC, West Brom, West Bromwich Albion, West Bromwich, Albion
+afcbournemouth, AFC Bournemouth, Bournemouth, A.F.C. Bournemouth, Bournemouth FC
+brightonhovealbionfc, Brighton & Hove Albion FC, Brighton, Brighton & Hove, Brighton & Hove Albion
+manchestercityfc, Manchester City FC, Man City, Manchester City, Man. City, Manchester C
+newcastleunitedfc, Newcastle United FC, Newcastle, Newcastle Utd, Newcastle United
+tottenhamhotspurfc, Tottenham Hotspur FC, Tottenham, Tottenham Hotspur, Spurs
+manchesterunitedfc, Manchester United FC, Man Utd, Man. United, Manchester U., Manchester Utd, Manchester United
+westhamunitedfc, West Ham United FC, West Ham, West Ham United
+TXT
+
+
+    clubs = Club.read( clubs_txt )
+    pp clubs
+
+    lines = txt.split( /\n+/ )    # note: removes/strips empty lines
+    pp lines
+
+    start_at= Date.new( 2017, 7, 1 )
+
+
+    parser = SportDb::MatchParserSimpleV2.new( lines, clubs, start_at )
+    rounds, matches = parser.parse
+    pp rounds
+    pp matches
+  end   # method test_parse
+end   # class TestMatchParser

data/test/test_reader.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_reader.rb
+
+
+require 'helper'
+
+
+class TestReader < MiniTest::Test
+
+  def test_read
+
+    SportDb.connect( adapter: 'sqlite3', database: ':memory:' )
+    SportDb.create_all   ## build schema
+
+    ## turn on logging to console
+    ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+
+    path = "../../../openfootball/england/2015-16/.conf.txt"
+    # path = "../../../openfootball/england/2017-18/.conf.txt"
+    # path = "../../../openfootball/england/2018-19/.conf.txt"
+    # path = "../../../openfootball/england/2019-20/.conf.txt"
+    recs = SportDb::EventReaderV2.read( path )
+    path = "../../../openfootball/england/2015-16/1-premierleague-i.txt"
+    # path = "../../../openfootball/england/2017-18/1-premierleague-i.txt"
+    # path = "../../../openfootball/england/2018-19/1-premierleague.txt"
+    # path = "../../../openfootball/england/2019-20/1-premierleague.txt"
+    recs = SportDb::MatchReaderV2.read( path )
+    # path = "../../../openfootball/england/2017-18/1-premierleague-ii.txt"
+    #recs = SportDb::MatchReaderV2.read( path )
+  end  # method test_read
+end  # class TestReader