sportdb 1.7.1 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7821792672d5e971444ef9f51adb41f7cb2059b1
-  data.tar.gz: 256d782e620f2cd78adba12da7b207053a9d0c6c
+  metadata.gz: 4a7f836cdab30262bf0a6065c1b1f68c6a54f817
+  data.tar.gz: 5aea61263d22babceecc7f5dc9b5d8abfc593b87
 SHA512:
-  metadata.gz: 571e77542ea420347b555d023846f9c5fb374452c0bbcf3663c7fb543a2fdb877ebf505699c84589d28c13911b387820e98ae7385ba21c475acefb9c21673dcb
-  data.tar.gz: b1fcafb80a003779928c2a2c2c7c610414993f2cecf287507b8807a901c75c81ec68accf4d1cceae2cc371860b7640d674b18246ea428ad3d57419cea9e11bf3
+  metadata.gz: 17ef76f90a0bef64d4f5a38366aef3f019adc144aa1f370ce994c64220fe9d37fac0d9e12ea48556e1cc1b6b4460cd3e0c74cb71ae96b43d980e08c1c08bb4dc
+  data.tar.gz: 162811a0ed6d0ca322addeb12103f0baccdf57a1f51bb8f6ea73359104a064b0b8a77959360debe8ca21d0287c47e5e2dc6b33710f41ff949e8a73f14f5c7da7

data/Manifest.txt

@@ -3,10 +3,10 @@ Manifest.txt
 README.md
 Rakefile
 bin/sportdb
-config/fixtures.de.yml
-config/fixtures.en.yml
-config/fixtures.es.yml
-config/fixtures.pt.yml
+config/fixtures/de.yml
+config/fixtures/en.yml
+config/fixtures/es.yml
+config/fixtures/pt.yml
 data/seasons.txt
 data/setups/all.yml
 lib/sportdb.rb
@@ -45,6 +45,7 @@ lib/sportdb/reader.rb
 lib/sportdb/schema.rb
 lib/sportdb/stats.rb
 lib/sportdb/title.rb
+lib/sportdb/updater.rb
 lib/sportdb/utils.rb
 lib/sportdb/version.rb
 tasks/test.rb

data/Rakefile

@@ -23,11 +23,12 @@ Hoe.spec 'sportdb' do
   # self.history_file = 'History.md'
   
   self.extra_deps = [
-    ['worlddb', '~> 1.7'],  # NB: worlddb already includes
+    ['worlddb', '>= 1.7'],  # NB: worlddb already includes
                                #         - commander
                                #         - logutils
                                #         - textutils
-    
+    ['fetcher', '>= 0.3'],
+
     ## 3rd party
     ['gli', '>= 2.5.6']
 

data/lib/sportdb.rb

@@ -23,6 +23,7 @@ require 'textutils'
 
 require 'worlddb'
 
+require 'fetcher'   # for fetching/downloading fixtures via HTTP/HTTPS etc.
 
 # our own code
 
@@ -57,6 +58,7 @@ require 'sportdb/utils'
 require 'sportdb/reader'
 require 'sportdb/lang'
 
+require 'sportdb/updater'
 require 'sportdb/deleter'
 require 'sportdb/stats'
 
@@ -145,6 +147,11 @@ module SportDb
     Deleter.new.run
   end # method delete!
 
+  def self.update!
+    puts '*** update event fixtures...'
+    Updater.new.run
+  end
+
 
   def self.stats
     stats = Stats.new

data/lib/sportdb/cli/main.rb

@@ -195,6 +195,20 @@ command [:load, :l] do |c|
   end
 end # command load
 
+
+desc 'Pull - Auto-update event fixtures from upstream online sources'
+command :pull do |c|
+  c.action do |g,o,args|
+
+    connect_to_db( opts )
+
+    SportDb.update!
+
+    puts 'Done.'
+  end # action
+end # command pull
+
+
 desc 'Show logs'
 command :logs do |c|
   c.action do |g,o,args|

data/lib/sportdb/lang.rb

@@ -6,56 +6,63 @@ class Lang
 
   include LogUtils::Logging
 
-  def initialize
-
-    @lang = 'en'   # make default lang english/en
+  attr_reader :lang
 
-    # load word lists
+  def initialize
+    # fix/todo: load on demand; only if no fixtures loaded/configured use builtin
+    load_builtin_words
+  end
 
-    @fixtures_en = YAML.load( File.read_utf8( "#{SportDb.config_path}/fixtures.en.yml" ))
-    @fixtures_de = YAML.load( File.read_utf8( "#{SportDb.config_path}/fixtures.de.yml" ))
-    @fixtures_es = YAML.load( File.read_utf8( "#{SportDb.config_path}/fixtures.es.yml" ))
 
-    @fixtures = {
-      'en' => @fixtures_en,
-      'de' => @fixtures_de,
-      'es' => @fixtures_es,
+  def load_builtin_words
+    builtin_words = {
+      'en' => 'fixtures/en',
+      'de' => 'fixtures/de',
+      'es' => 'fixtures/es'
     }
 
+    load_words( builtin_words, SportDb.config_path )
+  end
+
+  def load_words( h, include_path )
+    @lang = 'en'   # make default lang english/en
+    @words = {}  # resets fixtures
+    @cache = {}  # reset cached values
+
+    h.each_with_index do |(key,value),i|
+      name = value
+      path = "#{include_path}/#{name}.yml"
+      logger.debug( "loading words #{key} (#{i+1}/#{h.size}) in '#{name}' (#{path})..." )
+      @words[ key ] = YAML.load( File.read_utf8( path ))
+    end
 
-    @words_en = fixtures_hash_to_words_ary( @fixtures_en )
-    @words_de = fixtures_hash_to_words_ary( @fixtures_de )
-    @words_es = fixtures_hash_to_words_ary( @fixtures_es )
-    
-    logger.debug "en - #{@words_en.size} words: #{@words_en}"
-    logger.debug "de - #{@words_de.size} words: #{@words_de}"
-    logger.debug "es - #{@words_es.size} words: #{@words_es}"
+    @classifier = TextUtils::Classifier.new
+    @words.each_with_index do |(key,value),i|
+      logger.debug "train classifier for #{key} (#{i+1}/#{@words.size})"
+      @classifier.train( key, value )
+    end
     
+    @classifier.dump  # for debugging dump all words
   end
 
-  attr_reader :words_en
-  attr_reader :words_de
-  attr_reader :words_es
-  attr_reader :lang
+  def classify( text )
+    @classifier.classify( text )
+  end
+
+  def classify_file( path )
+    @classifier.classify_file( path )
+  end
 
   def lang=(value)
     logger.debug "setting lang to #{value}"
     
     if @lang != value
+      
+      ### todo: make reset cached values into method/function for reuse (see load_words)
       # reset cached values on language change
       logger.debug "reseting cached lang values (lang changed from #{@lang} to #{value})"
 
-      @group = nil
-      @round = nil
-      @knockout_round = nil
-      @leg1 = nil
-      @leg2 = nil
-      
-      @regex_group = nil
-      @regex_round = nil
-      @regex_knockout_round = nil
-      @regex_leg1 = nil
-      @regex_leg2 = nil
+      @cache = {}
     end
     
     @lang = value
@@ -64,49 +71,50 @@ class Lang
 
 
   def group
-    @group ||= group_getter
+    @cache[ :group ] ||= group_getter
   end
   
   def round
-    @round ||= round_getter
+    @cache[ :round ] ||= round_getter
   end
- 
+
   def knockout_round
-    @knockout_round ||= knockout_round_getter
+    @cache[ :knockout_round ] ||= knockout_round_getter
   end
   
   def leg1
-    @leg1 ||= leg1_getter
+    @cache[ :leg1 ] ||= leg1_getter
   end
   
   def leg2
-    @leg2 ||= leg2_getter
+    @cache[ :leg2 ] ||= leg2_getter
   end
 
 
+
   def regex_group
-    @regex_group ||= regex_group_getter
+    @cache [ :regex_group ] ||= regex_group_getter
   end
-   
+
   def regex_round
-    @regex_round ||= regex_round_getter
+    @cache[ :regex_round ] ||= regex_round_getter
   end
-  
+
   def regex_knockout_round
-    @regex_knockout_round ||= regex_knockout_round_getter
+    @cache[ :regex_knockout_round ] ||= regex_knockout_round_getter
   end
   
   def regex_leg1
-    @regex_leg1 ||= regex_leg1_getter
+    @cache[ :regex_leg1 ] ||= regex_leg1_getter
   end
   
   def regex_leg2
-    @regex_leg2 ||= regex_leg2_getter
+    @cache[ :regex_leg2 ] ||= regex_leg2_getter
   end
 
 private
   def group_getter
-    h = @fixtures[ lang ]
+    h = @words[ lang ]
     values = ""   # NB: always construct a new string (do NOT use a reference to hash value)
     values << h['group']
     values
@@ -115,7 +123,7 @@ private
   def round_getter
     # e.g. Spieltag|Runde|Achtelfinale|Viertelfinale|Halbfinale|Finale
     
-    h = @fixtures[ lang ]
+    h = @words[ lang ]
     values = ""   # NB: always construct a new string (do NOT use a reference to hash value)
     values << h['round']
     values << "|" << h['matchday']  ## todo/check: fold round n matchday into one key? why? why not??
@@ -131,21 +139,21 @@ private
   end
 
   def leg1_getter
-    h = @fixtures[ lang ]
+    h = @words[ lang ]
     values = ""  # NB: always construct a new string (do NOT use a reference to hash value)
     values << h['leg1']
     values
   end
 
   def leg2_getter
-    h = @fixtures[ lang ]
+    h = @words[ lang ]
     values = ""  # NB: always construct a new string (do NOT use a reference to hash value)
     values << h['leg2']
     values
   end
 
   def knockout_round_getter
-    h = @fixtures[ lang ]
+    h = @words[ lang ]
     values = ""  # NB: always construct a new string (do NOT use a reference to hash value)
     values << h['round32']
     values << "|" << h['round16']
@@ -182,88 +190,7 @@ private
     /#{leg2}/
   end
 
-
-  def fixtures_hash_to_words_ary( hash )
-    ary = []
-    hash.each do |key_wild, values_wild|
-      key     = key_wild.to_s.strip
-      values  = values_wild.to_s.strip
-       
-      logger.debug "processing key >>#{key}<< with words >>#{values}<<"
-       
-      ary += values.split('|')
-    end
-    ary
-  end
-  
 end # class Lang
 
 
-class LangChecker
-
-  include LogUtils::Logging
-  
-  def initialize
-  end
-
-  def analyze( path )
-    # return lang code e.g. en, de, es
-
-    text = File.read_utf8( path )
-    
-    ### todo/fix:
-    # remove comment lines and end of line comments from text
-    
-    en = count_words_in_text( SportDb.lang.words_en, text )
-    de = count_words_in_text( SportDb.lang.words_de, text )
-    es = count_words_in_text( SportDb.lang.words_es, text )
-      
-    lang_counts = [
-      [ 'en', en ],
-      [ 'de', de ],
-      [ 'es', es ]
-    ]
-    
-    # sort by word count (reverse sort e.g. highest count goes first)
-    lang_counts = lang_counts.sort {|l,r| r[1] <=> l[1] }
-    
-    # dump stats
-    
-    logger.debug "lang checker:"
-    lang_counts.each_with_index do |item,index|
-      ## e.g. 1. en: 20 words
-      ##      2. de: 2 words
-      logger.debug " #{index+1}. #{item[0]}: #{item[1]}"
-    end
-    
-    logger.info "lang checker - using lang >>#{lang_counts[0][0]}<<"
-    
-    ## return lang code w/ highest count
-    lang_counts[0][0]
-  end
-
-private
-  def count_word_in_text( word, text )
-    count = 0
-    pos = text.index( word )
-    while pos.nil? == false
-      count += 1
-      logger.debug "bingo - found >>#{word}<< on pos #{pos}, count: #{count}"
-      ### todo: check if pos+word.length/size needs +1 or similar
-      pos = text.index( word, pos+word.length)
-    end
-    count
-  end
-  
-  def count_words_in_text( words, text )
-    count = 0
-    words.each do |word|
-      count += count_word_in_text( word, text )
-    end
-    count
-  end
-
-end # class LangChecker
-
-
 end # module SportDb

data/lib/sportdb/models/event.rb

@@ -30,11 +30,12 @@ class Event < ActiveRecord::Base
   end
   
   def title
-    league.title
+    "#{league.title} #{season.title}"
   end
 
   def full_title   # includes season (e.g. year)
-    "#{league.title} #{season.title}"
+    puts "*** depreciated API call Event#full_title; use Event#title instead; full_title will get removed"
+    "#{league.title} #{season.title}"    
   end
 
 

data/lib/sportdb/reader.rb

@@ -4,6 +4,8 @@ module SportDb
 
 module Matcher
 
+  include WorldDb::Matcher
+
   def match_leagues_for_country( name, &blk )
     match_xxx_for_country( name, 'leagues', blk )
   end
@@ -23,8 +25,6 @@ class Reader
 ## make models available in sportdb module by default with namespace
 #  e.g. lets you use Team instead of Models::Team 
   include SportDb::Models
-
-  include WorldDb::Matcher
   include SportDb::Matcher # lets us use match_teams_for_country etc.
 
 
@@ -272,6 +272,14 @@ class Reader
     reader = HashReaderV2.new( name, include_path )
 
     event_attribs = {}
+    
+    ## set default sources to basename by convention
+    #  e.g  2013_14/bl  => bl
+    #  etc.
+    # use fixtures/sources: to override default
+
+    event_attribs[ 'sources' ] = File.basename( name )
+    event_attribs[ 'config'  ] = File.basename( name )  # name a of .yml file
 
     reader.each_typed do |key, value|
 
@@ -325,13 +333,17 @@ class Reader
       elsif key == 'team3'
         ## for now always assume false  # todo: fix - use value and convert to boolean if not boolean
         event_attribs['team3'] = false
-      elsif key == 'fixtures'
-        ## skip fixtures for now (NOT yet stored in db; for now reload file to get fixtures)
+      elsif key == 'fixtures' || key == 'sources'
+        if value.kind_of?(Array)
+          event_attribs['sources'] = value.join(',') 
+        else # assume plain (single fixture) string
+          event_attribs['sources'] = value.to_s
+        end
       else
         ## todo: add a source location struct to_s or similar (file, line, col)
         logger.error "unknown event attrib #{key}; skipping attrib"
       end
-  
+
     end # each key,value
 
     league_id = event_attribs['league_id']
@@ -356,8 +368,11 @@ class Reader
   end  # load_event
 
 
+
   def load_fixtures_from_string( event_key, text )  # load from string (e.g. passed in via web form)
 
+    SportDb.lang.lang = SportDb.lang.classify( text )
+
     ## todo/fix: move code into LineReader e.g. use LineReader.fromString() - why? why not?
     reader = StringLineReader.new( text )
     
@@ -385,7 +400,7 @@ class Reader
 
     logger.info "parsing data '#{name}' (#{path})..."
     
-    SportDb.lang.lang = LangChecker.new.analyze( path )
+    SportDb.lang.lang = SportDb.lang.classify_file( path )
 
     reader = LineReader.new( path )
     

data/lib/sportdb/schema.rb

@@ -128,6 +128,12 @@ create_table :events do |t|
   t.date        :end_at   # make it required???  # NB: only use date (w/o time)
   t.boolean     :team3,    :null => false, :default => true   ## e.g. Champions League has no 3rd place (only 1st and 2nd/final)
 
+  #### track 1-n sources (from repos)  - # todo move to its own table later
+  ## NB: relative to event.yml - use mapper to "resolve" to full path w/ repo; use league+season keys
+  t.string      :sources    #  e.g. cup or bl,bl_ii   # NB: for now store all in on string separated by comma
+  t.string      :config  # e.g. cup or bl #  e.g assumes cup.yml, bl.yml etc. for now
+
+
   t.timestamps
 end
 

data/lib/sportdb/updater.rb

@@ -0,0 +1,122 @@
+
+module SportDb
+
+  class Updater
+
+    include LogUtils::Logging
+
+    ######
+    # NB: make models available in sportdb module by default with namespace
+    #  e.g. lets you use Team instead of Models::Team 
+    include SportDb::Models
+
+
+    def map_event_to_dlurl( event )
+
+      league_key = event.league.key
+      season_key = event.season.key
+
+      repo_path, folder_path = map_key_to_repo_n_folder_path( league_key )
+      return nil if repo_path.nil?   # no match/mapping found; cancel download
+
+      season_path = season_key.gsub( '/', '_')  # convert 2013/14 to 2013_14
+
+      ###
+      # e.g. https://raw.github.com/openfootball/at-austria/master/2013_14/bl.txt
+
+      dlurl = "https://raw.github.com/openfootball/#{repo_path}/master"
+      dlurl << "/#{folder_path}" if folder_path.present?
+      dlurl << "/#{season_path}"
+      dlurl
+    end
+
+
+    def map_key_to_repo_n_folder_path( key )
+
+      ### allow * for regex match w/ .+
+      map = [
+        [ 'at',      'at-austria'  ],
+        [ 'at.*',    'at-austria'  ],
+        [ 'de',      'de-deutschland' ],
+        [ 'be',      'europe', 'be-belgium' ], # NB: europe/be-belgium
+        [ 'cl',      'europe'      ],
+        [ 'world.*', 'world-cup'  ]]
+
+      map.each do |entry|
+         pattern = entry[0]
+         path    = [ entry[1], entry[2] ]  # repo n folder path
+         
+         if pattern.index( '*' ).nil?  # match just plain string (no wildcard *)
+           return path if key == pattern
+         else
+           # assume regex match
+           regex = pattern.gsub( '.', '\.' ).gsub( '*', '.+' )
+           return path if key =~ /#{regex}/
+         end
+      end
+      nil  # return nil; no match found
+    end
+
+    def update_event( event )
+      puts "update event >>#{event.title}<< (#{event.league.key}+#{event.season.key})"
+
+      dlbase = map_event_to_dlurl( event )
+      if dlbase.nil?
+        puts "  skip download; no download source mapping found"
+        return  # cancel download; no mapping found
+      end
+
+      puts "  using dlbase >>#{dlbase}<<"
+
+      if event.sources.nil?
+        puts "  skip download; no download event source configured/found"
+        return
+      end
+
+      sources = event.sources.gsub(' ','').split(',')   # NB: remove all blanks (leading,trailing,inside)
+      sources.each_with_index do |source,i|
+        dlurl = "#{dlbase}/#{source}.txt"
+        puts "   downloading source (#{i+1}/#{sources.length}) >>#{dlurl}<< ..."     # todo/check: use size for ary or length - does it matter?
+
+        # download fixtures into string
+        text = Fetcher.read( dlurl )
+
+        logger.debug "text.encoding.name (before): #{text.encoding.name}"
+        
+        ###
+        # NB: Net::HTTP will NOT set encoding UTF-8 etc.
+        #  will mostly be ASCII
+        #  - try to change encoding to UTF-8 ourselves
+
+        #####
+        # NB:  ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+
+        ## NB:
+        #  for now "hardcoded" to utf8 - what else can we do?
+        #  - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+        text = text.force_encoding( Encoding::UTF_8 )
+        logger.debug "text.encoding.name (after): #{text.encoding.name}"
+
+
+        puts "   importing/reading source..."
+        reader= Reader.new( '/tmp' )  # passing dummy include_path (not needed for reading from string)
+        
+        ## todo/fix: offer a version that lets us pass in event (not event.key)
+        #    no need for another event lookup
+        reader.load_fixtures_from_string( event.key, text )
+      end
+    end
+
+
+    def run
+      # for now update all events (fixtures only) - not *.yml
+
+      Event.all.each do |event|
+        update_event( event )
+      end
+
+    end
+    
+  end # class Updater
+  
+end # module SportDb

data/lib/sportdb/version.rb

@@ -1,6 +1,6 @@
 
 module SportDb
-  VERSION = '1.7.1'
+  VERSION = '1.7.2'
 end
 
 

metadata

@@ -1,29 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: sportdb
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.7.2
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-21 00:00:00.000000000 Z
+date: 2013-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: worlddb
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: fetcher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: gli
   requirement: !ruby/object:Gem::Requirement
@@ -81,10 +95,10 @@ files:
 - README.md
 - Rakefile
 - bin/sportdb
-- config/fixtures.de.yml
-- config/fixtures.en.yml
-- config/fixtures.es.yml
-- config/fixtures.pt.yml
+- config/fixtures/de.yml
+- config/fixtures/en.yml
+- config/fixtures/es.yml
+- config/fixtures/pt.yml
 - data/seasons.txt
 - data/setups/all.yml
 - lib/sportdb.rb
@@ -123,6 +137,7 @@ files:
 - lib/sportdb/schema.rb
 - lib/sportdb/stats.rb
 - lib/sportdb/title.rb
+- lib/sportdb/updater.rb
 - lib/sportdb/utils.rb
 - lib/sportdb/version.rb
 - tasks/test.rb