sportdb 0.9.1 → 0.9.2

data/lib/sportdb/console.rb

@@ -23,8 +23,8 @@ Season  = SportDB::Models::Season
 League  = SportDB::Models::League
 Badge   = SportDB::Models::Badge
 
-# Tag     = WorldDB::Models::Tag
-# Tagging = WorldDB::Models::Tagging
+Tag     = WorldDB::Models::Tag
+Tagging = WorldDB::Models::Tagging
 Country = WorldDB::Models::Country
 Region  = WorldDB::Models::Region
 City    = WorldDB::Models::City
@@ -49,6 +49,44 @@ puts "  #{'%5d' % Game.count} games"
 puts "  #{'%5d' % City.count} cities"
 puts "Ready."
 
+## add some predefined shortcuts
+
+##### some countries
+
+AT = Country.find_by_key( 'at' )
+DE = Country.find_by_key( 'de' )
+EN = Country.find_by_key( 'en' )
+
+US = Country.find_by_key( 'us' )
+CA = Country.find_by_key( 'ca' )
+MX = Country.find_by_key( 'mx' )
+
+#### some events
+
+EURO2008 = Event.find_by_key( 'euro.2008' )
+EURO2012 = Event.find_by_key( 'euro.2012' )
+EURO = EURO2012  # add alias
+
+BL = Event.find_by_key( 'de.2012/13' )
+PL = Event.find_by_key( 'en.2012/13' )
+
+### some club teams
+
+BARCA   = Team.find_by_key( 'barcelona' )
+MANU    = Team.find_by_key( 'manunited' )
+MUN = MANUNITED = MANU    # add alias
+BAYERN  = Team.find_by_key( 'bayern' )
+AUSTRIA = Team.find_by_key( 'austria' )
+
+### some national teams (three letter fifa codes)
+
+ESP = Team.find_by_key( 'esp' )
+GER = Team.find_by_key( 'ger' )
+AUT = Team.find_by_key( 'aut' )
+
+MEX = Team.find_by_key( 'mex' )
+ARG = Team.find_by_key( 'arg' )
+
 ## turn on activerecord logging to console
 
 ActiveRecord::Base.logger = Logger.new( STDOUT )

data/lib/sportdb/loader.rb

@@ -65,22 +65,6 @@ class Loader
 
 private
 
-  ##
-  # fix/todo: share helper w/ other readers
-  
-  # helper
-  #   change at/2012_13/bl           to at.2012/13.bl
-  #    or    quali_2012_13_europe_c  to quali.2012/13.europe.c
-  
-  def fixture_name_to_prop_key( name )
-    prop_key = name.gsub( '/', '.' )
-    prop_key = prop_key.gsub( /(\d{4})_(\d{2})/, '\1/\2' )  # 2012_13 => 2012/13
-    prop_key = prop_key.gsub( '_', '.' )
-    prop_key
-  end
-
-
-
 
   def load_fixtures_worker( code )
     

data/lib/sportdb/models/event.rb

@@ -40,56 +40,6 @@ class Event < ActiveRecord::Base
   #####################
   ## convenience helper for text parser/reader
 
-  ############
-  ### fix/todo: share helper for all text readers/parsers- where to put it?  
-  ###
-  
-  def title_esc_regex( title_unescaped )
-      
-      ##  escape regex special chars e.g. . to \. and ( to \( etc.
-      # e.g. Benfica Lis.
-      # e.g. Club Atlético Colón (Santa Fe)
-
-      ## NB: cannot use Regexp.escape! will escape space '' to '\ '
-      ## title = Regexp.escape( title_unescaped )
-      title = title_unescaped.gsub( '.', '\.' )
-      title = title.gsub( '(', '\(' )
-      title = title.gsub( ')', '\)' )
-
-      ##  match accented char with or without accents
-      ##  add (ü|ue) etc.
-      ## also make - optional change to (-| ) e.g. Blau-Weiss == Blau Weiss
-
-      ## todo: add some more
-      ## see http://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references  for more
-      ##
-      ##  reuse for all readers!
-      
-      alternatives = [
-        ['-', '(-| )'],
-        ['ß', '(ß|ss)'],
-        ['æ', '(æ|ae)'],
-        ['á', '(á|a)'],  ## e.g. Bogotá
-        ['ã', '(ã|a)'],  ## e.g  São Paulo
-        ['ä', '(ä|ae)'],  ## add a ?
-        ['Ö', '(Ö|Oe)'], ## e.g. Österreich
-        ['ö', '(ö|oe)'],  ## add o ?
-        ['ó', '(ó|o)'],  ## e.g. Colón
-        ['ü', '(ü|ue)'],  ## add u ?
-        ['é', '(é|e)'],  ## e.g. Vélez
-        ['ê', '(ê|e)'],  ## e.g. Grêmio
-        ['ñ', '(ñ|n)'],  ## e.g. Porteño
-        ['ú', '(ú|u)']  ## e.g. Fútbol
-      ]
-      
-      alternatives.each do |alt|
-        title = title.gsub( alt[0], alt[1] )
-      end
-
-      title
-  end
-
-
   def known_teams_table
     
     ## build known teams table w/ synonyms e.g.

data/lib/sportdb/reader.rb

@@ -40,11 +40,9 @@ class Reader
 
     puts "*** parsing data '#{name}' (#{path})..."
 
-    ## nb: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
-    ## - see sportdb/utils.rb
-    code = File.read_utf8( path )
+    reader = LineReader.new( logger, path )
     
-    load_fixtures_worker( event_key, code )
+    load_fixtures_worker( event_key, reader )
 
     Prop.create!( key: "db.#{fixture_name_to_prop_key(name)}.version", value: "file.txt.#{File.mtime(path).strftime('%Y.%m.%d')}" )
   end
@@ -54,31 +52,69 @@ class Reader
 
     puts "*** parsing data '#{name}' (#{path})..."
 
-    code = File.read_utf8( path )
-    
-    load_fixtures_worker( event_key, code )
+    reader = LineReader.new( logger, path )
+
+    load_fixtures_worker( event_key, reader )
     
     Prop.create!( key: "db.#{fixture_name_to_prop_key(name)}.version", value: "sport.txt.#{SportDB::VERSION}" )
   end
 
 
-private
+  def load_teams_builtin( name, more_values={} )
+    path = "#{SportDB.root}/db/#{name}.txt"
 
- ##
-  # fix/todo: share helper w/ other readers
-  
-  # helper
-  #   change at/2012_13/bl           to at.2012/13.bl
-  #    or    quali_2012_13_europe_c  to quali.2012/13.europe.c
-  
-  def fixture_name_to_prop_key( name )
-    prop_key = name.gsub( '/', '.' )
-    prop_key = prop_key.gsub( /(\d{4})_(\d{2})/, '\1/\2' )  # 2012_13 => 2012/13
-    prop_key = prop_key.gsub( '_', '.' )
-    prop_key
+    puts "*** parsing data '#{name}' (#{path})..."
+
+    reader = ValuesReader.new( logger, path, more_values )
+
+    load_teams_worker( reader )
+    
+    Prop.create!( key: "db.#{fixture_name_to_prop_key(name)}.version", value: "sport.txt.#{SportDB::VERSION}" )    
   end
 
-  def load_fixtures_worker( event_key, data )
+
+private
+
+  include SportDB::FixtureHelpers
+
+  def load_teams_worker( reader )
+ 
+    reader.each_line do |attribs, values|
+
+      ## check optional values
+      values.each_with_index do |value, index|
+        if value =~ /^city:/   ## city:
+          value_city_key = value[5..-1]  ## cut off city: prefix
+          value_city = City.find_by_key!( value_city_key )
+          attribs[ :city_id ] = value_city.id
+        elsif value =~ /^[A-Z]{3}$/  ## assume three-letter code e.g. FCB, RBS, etc.
+          attribs[ :code ] = value
+        elsif value =~ /^[a-z]{2}$/  ## assume two-letter country key e.g. at,de,mx,etc.
+          value_country = Country.find_by_key!( value )
+          attribs[ :country_id ] = value_country.id
+        else
+          ## todo: assume title2 ??
+          # issue warning: unknown type for value
+          puts "*** warning: unknown type for value >#{value}<"
+        end
+      end
+
+      rec = Team.find_by_key( attribs[ :key ] )
+      if rec.present?
+        puts "*** update Team #{rec.id}-#{rec.key}:"
+      else
+        puts "*** create Team:"
+        rec = Team.new
+      end
+      
+      puts attribs.to_json
+   
+      rec.update_attributes!( attribs )
+
+    end # each lines
+  end # method load_teams_worker
+
+  def load_fixtures_worker( event_key, reader )
    
     ## assume active activerecord connection
     ##
@@ -95,258 +131,11 @@ private
     
     @known_teams = @event.known_teams_table
     
-    parse_fixtures( data )
+    parse_fixtures( reader )
     
   end   # method load_fixtures
 
 
-  def is_round?( line )
-    line =~ /Spieltag|Runde|Achtelfinale|Viertelfinale|Halbfinale|Finale/
-  end
-  
-  def is_group?( line )
-    # NB: check after is_round? (round may contain group reference!)
-    line =~ /Gruppe|Group/
-  end
-  
-  ### todo: rename to is_knockout_round?
-  ##
-  def find_knockout_flag( line )
-    if line =~ /Achtelfinale|Viertelfinale|Halbfinale|Spiel um Platz 3|Finale|K\.O\.|Knockout/
-      puts "   setting knockout flag to true"
-      true
-    else
-      false
-    end
-  end
-  
-  def find_group_title_and_pos!( line )
-    ## group pos - for now support single digit e.g 1,2,3 or letter e.g. A,B,C
-    ## nb:  (?:)  = is for non-capturing group(ing)
-    regex = /(?:Group|Gruppe)\s+((?:\d{1}|[A-Z]{1}))\b/
-    
-    match = regex.match( line )
-    unless match.nil?
-      if match[1] == 'A'
-        pos = 1
-      elsif match[1] == 'B'
-        pos = 2
-      elsif match[1] == 'C'
-        pos = 3
-      elsif match[1] == 'D'
-        pos = 4
-      elsif match[1] == 'E'
-        pos = 5
-      elsif match[1] == 'F'
-        pos = 6
-      elsif match[1] == 'G'
-        pos = 7
-      elsif match[1] == 'H'
-        pos = 8
-      elsif match[1] == 'I'
-        pos = 9
-      elsif match[1] == 'J'
-        pos = 10
-      else
-        pos = match[1].to_i
-      end
-
-      title = match[0]
-
-      puts "   title: >#{title}<"
-      puts "   pos: >#{pos}<"
-      
-      line.sub!( regex, '[GROUP|TITLE+POS]' )
-
-      return [title,pos]
-    else
-      return [nil,nil]
-    end
-  end
-  
-  def find_round_pos!( line )
-    ## fix/todo:
-    ##  if no round found assume last_pos+1 ??? why? why not?
-
-    regex = /\b(\d+)\b/
-    
-    if line =~ regex
-      value = $1.to_i
-      puts "   pos: >#{value}<"
-      
-      line.sub!( regex, '[ROUND|POS]' )
-
-      return value
-    else
-      return nil
-    end    
-  end
-  
-  def find_date!( line )
-    # extract date from line
-    # and return it
-    # NB: side effect - removes date from line string
-    
-    # e.g. 2012-09-14 20:30   => YYYY-MM-DD HH:MM
-    regex_db = /\b(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2})\b/
-
-    # e.g. 14.09. 20:30  => DD.MM. HH:MM
-    regex_de = /\b(\d{2})\.(\d{2})\.\s+(\d{2}):(\d{2})\b/
-
-    if line =~ regex_db
-      value = "#{$1}-#{$2}-#{$3} #{$4}:#{$5}"
-      puts "   date: >#{value}<"
-
-      ## todo: lets you configure year
-      ##  and time zone (e.g. cet, eet, utc, etc.)
-      
-      line.sub!( regex_db, '[DATE.DB]' )
-
-      return DateTime.strptime( value, '%Y-%m-%d %H:%M' )
-    elsif line =~ regex_de
-      value = "2012-#{$2}-#{$1} #{$3}:#{$4}"
-      puts "   date: >#{value}<"
-
-      ## todo: lets you configure year
-      ##  and time zone (e.g. cet, eet, utc, etc.)
-      
-      line.sub!( regex_de, '[DATE.DE]' )
-
-      return DateTime.strptime( value, '%Y-%m-%d %H:%M' )
-    else
-      return nil
-    end
-  end
-
-
-  def find_game_pos!( line )
-    # extract optional game pos from line
-    # and return it
-    # NB: side effect - removes pos from line string
-
-    # e.g.  (1)   - must start line 
-    regex = /^[ \t]*\((\d{1,3})\)[ \t]+/
-    if line =~ regex
-      puts "   pos: >#{$1}<"
-      
-      line.sub!( regex, '[POS] ' )
-      return $1.to_i
-    else
-      return nil
-    end
-
-  end
-
-  def find_scores!( line )
-    # extract score from line
-    # and return it
-    # NB: side effect - removes date from line string
-    
-    # e.g. 1:2 or 0:2 or 3:3
-    regex = /\b(\d):(\d)\b/
-    
-    # e.g. 1:2nV  => overtime
-    regex_ot = /\b(\d):(\d)[ \t]?[nN][vV]\b/
-    
-    # e.g. 5:4iE  => penalty
-    regex_p = /\b(\d):(\d)[ \t]?[iI][eE]\b/
-    
-    scores = []
-    
-    if line =~ regex
-      puts "   score: >#{$1}-#{$2}<"
-      
-      line.sub!( regex, '[SCORE]' )
-
-      scores << $1.to_i
-      scores << $2.to_i
-      
-      if line =~ regex_ot
-        puts "   score.ot: >#{$1}-#{$2}<"
-      
-        line.sub!( regex_ot, '[SCORE.OT]' )
-
-        scores << $1.to_i
-        scores << $2.to_i
-      
-        if line =~ regex_p
-          puts "   score.p: >#{$1}-#{$2}<"
-      
-          line.sub!( regex_p, '[SCORE.P]' )
-
-          scores << $1.to_i
-          scores << $2.to_i
-        end
-      end
-    end
-    scores
-  end # methdod find_scores!
-  
-
-  def find_team_worker!( line, index )
-    regex = /@@oo([^@]+?)oo@@/     # e.g. everything in @@ .... @@ (use non-greedy +? plus all chars but not @, that is [^@])
-    
-    if line =~ regex
-      value = "#{$1}"
-      puts "   team#{index}: >#{value}<"
-      
-      line.sub!( regex, "[TEAM#{index}]" )
-
-      return $1
-    else
-      return nil
-    end
-  end
-  
-  def find_teams!( line )
-    counter = 1
-    teams = []
-    
-    team = find_team_worker!( line, counter )
-    while team.present?
-      teams << team
-      counter += 1
-      team = find_team_worker!( line, counter )
-    end
-    
-    teams
-  end
-
-  def find_team1!( line )
-    find_team_worker!( line, 1 )
-  end
-  
-  def find_team2!( line )
-    find_team_worker!( line, 2 )
-  end
-
-
-  def match_team_worker!( line, key, values )
-    values.each do |value|
-      ## nb: \b does NOT include space or newline for word boundry (only alphanums e.g. a-z0-9)
-      ## (thus add it, allows match for Benfica Lis.  for example - note . at the end)
-
-      ## check add $ e.g. (\b| |\t|$) does this work? - check w/ Benfica Lis.$
-      regex = /\b#{value}(\b| |\t|$)/   # wrap with world boundry (e.g. match only whole words e.g. not wac in wacker) 
-      if line =~ regex
-        puts "     match for team >#{key}< >#{value}<"
-        # make sure @@oo{key}oo@@ doesn't match itself with other key e.g. wacker, wac, etc.
-        line.sub!( regex, "@@oo#{key}oo@@ " )    # NB: add one space char at end
-        return true    # break out after first match (do NOT continue)
-      end
-    end
-    return false
-  end
-
-  def match_teams!( line )
-    @known_teams.each do |rec|
-      key    = rec[0]
-      values = rec[1]
-      match_team_worker!( line, key, values )
-    end # each known_teams    
-  end # method translate_teams!
-  
-  
   def parse_group( line )
     puts "parsing group line: >#{line}<"
     
@@ -357,9 +146,6 @@ private
 
     puts "  line: >#{line}<"
 
-    ## world2 = Group.create!( event: world, pos: 2, title: 'Gruppe 2' )
-    ## world2.add_teams_from_ary!( team_keys_world2 )
-
     group_attribs = {
       title: title
     }
@@ -393,7 +179,7 @@ private
     puts "parsing round line: >#{line}<"
     pos = find_round_pos!( line )
         
-    @knockout_flag = find_knockout_flag( line )
+    @knockout_flag = is_knockout_round?( line )
 
     group_title, group_pos = find_group_title_and_pos!( line )
 
@@ -501,25 +287,9 @@ private
   end
 
 
-  def parse_fixtures( data )
+  def parse_fixtures( reader )
       
-    data.each_line do |line|
-  
-      if line =~ /^\s*#/
-        # skip komments and do NOT copy to result (keep comments secret!)
-        logger.debug 'skipping comment line'
-        next
-      end
-        
-      if line =~ /^\s*$/ 
-        # kommentar oder leerzeile überspringen 
-        logger.debug 'skipping blank line'
-        next
-      end
-
-      # remove leading and trailing whitespace
-      line = line.strip
-
+    reader.each_line do |line|
       if is_round?( line )
         parse_round( line )
       elsif is_group?( line ) ## NB: group goes after round (round may contain group marker too)