greenmonster 0.1.0.pre

data/README.markdown

@@ -0,0 +1,57 @@
+Greenmonster
+============
+
+Greenmonster is a toolkit for baseball stat enthusiasts or sabermetricians to build a database of 
+play-by-play stats from MLB's [Gameday XML data](http://gd.mlb.com/components/game/).
+
+The pre-release gem is currently spiders games from MLB's servers. I will be extracting pieces out 
+of my toolkit to parse Gameday XML data to produce play-by-play database of MLB and MiLB stats.
+
+Usage 
+-----
+
+The spider utility has two public class methods: Spider.pull_day and Spider.pull_days. 
+
+Spider.pull_day takes an hash of options as an argument. The only mandatory argument option
+is to specify :games_folder, which is the local folder where you want to store the game files.
+Greenmonster will create subfolders by MLB "sport_code" (MLB games fall under 'mlb', various 
+minor league games and non-MLB/MiLB games fall under other sport code designations), and then
+children folders for years, months, days, and specific games.
+
+    # Pulls all MLB games for today
+    => Greenmonster::Spider.pull_day({:date => Date.today, :games_folder => './home/geoff/games'})
+
+    # Pulls all rookie league games for today
+    > Greenmonster::Spider.pull_day({:sport_code => 'rok', :date => Date.today, :games_folder => './home/geoff/games'})
+
+
+Spider.pull_days takes a range of dates to process as an argument, plus a hash of arguments to pass
+to Spider.pull.
+
+    # Pulls all MLB games for in April, 2012
+	=> Greenmonster::Spider.pull_days((Date.new(2012,4,1)..Date.new(2012,4,30)), {:games_folder => './home/geoff/games'})
+	
+
+Requirements
+------------
+- Ruby 1.9
+- Nokogiri
+- HTTParty
+
+Testing
+-------
+
+The test suite downloads a few days of data, so it is not fast to execute.
+
+
+License
+-------
+(The MIT License)
+
+Copyright © [Geoff Harcourt](http://github.com/geoffharcourt) 2012
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the ‘Software’), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED ‘AS IS’, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run Tests"
+task :default => :test

data/lib/greenmonster/spider.rb

@@ -0,0 +1,125 @@
+# The Gameday XML Spider utility
+class Greenmonster::Spider
+  include HTTParty
+    
+  ##
+  # Pull Gameday XML files for a given date. Default options for 
+  # the spider are to pull games with sport_code of 'mlb' (games 
+  # played by MLB games rather than MiLB teams or foreign teams) 
+  # and to pull games on the current date. You must specify a 
+  # :games_folder argument that is the root folder under  which 
+  # all games will be placed.
+  #
+  # Example:
+  #   # Pull games from July 4, 2011
+  #   >> Gameday::Spider.pull({:date => Date.new(2011,7,1), :games_location => '/Users/geoff/games'})
+  #
+  # Arguments:
+  #   args: (Hash)
+    
+  def self.pull_day(args = {})
+    args = {
+      :date => Date.today,
+      :league => 'mlb',
+      :print_games => true
+    }.merge(args)
+      
+    raise "Games folder location required." if args[:games_folder].nil?
+      
+    game_day_url_for_date = "http://gd2.mlb.com/components/game/#{args[:league]}/#{format_date_as_folder(args[:date])}"
+  
+    # Iterate through every hyperlink on the page.
+    # These links represent the individual game folders
+    # for each date. Reject any links that aren't to game
+    # folders or that are to what look like backup game
+    # folders.
+    (Nokogiri::XML(self.get(game_day_url_for_date))/"a").reject{|l| l.attribute('href').value[0,4] != "gid_" or l.attribute('href').value[-5,4] == "_bak"}.each do |e|
+      puts e.attribute('href').value.gsub('/','') if args[:print_games]
+        
+      paths = {
+        :localGameFolder => "#{args[:games_folder]}/#{args[:league]}/#{format_date_as_folder(args[:date])}/#{e.attribute('href').value}",
+        :mlbGameFolder => "#{game_day_url_for_date}/#{e.attribute('href').value}"
+      }
+        
+      FileUtils.mkdir_p paths[:localGameFolder] + 'inning'
+
+      begin
+        # Always copy linescore first. If we can't get this
+        # data, all other game data is useless.
+        copy_gameday_xml('linescore.xml',paths)
+      
+        if args[:date].year > 2007
+          copy_gameday_xml('inning_all.xml',paths)
+          copy_gameday_xml('inning_hit.xml',paths)
+        else
+          # Iterate through the inning files, but skip inning 
+          # files numbered 0 (some bad spring training data).
+          # Necessary for games prior to 2008 because there is
+          # no inning_all.xml file in older games.
+          (Nokogiri::XML(self.get("#{paths[:mlbGameFolder]}/inning/").body).search('a')).each do |ic|
+            copy_gameday_xml(ic.attribute('href').value,paths) if ic.attribute('href').value[-3,3]  == "xml" unless ic.attribute('href').value[-6,6] == "_0.xml" or ic.attribute('href').value.include?('Score')
+          end
+        end
+
+        # Copy base data files 
+        # (if inning data wasn't there, this gets skipped)
+        ['boxscore.xml','eventLog.xml','players.xml'].each do |file|
+          copy_gameday_xml(file,paths)
+        end
+      rescue StandardError => bang
+        puts "Unable to download some data for #{e.attribute('href').value}"  
+      end
+    end
+    
+    game_day_url_for_date
+  end
+
+  ##
+  # Pull Gameday XML files for a range of dates. The args hash
+  # passes arguments like games_folder location on to Spider.pull.
+  #
+  # Example:
+  #   # Pull all games in MLB in July 2011
+  #   >> Gameday::Spider.pull_days(Date.new(2011,7,1)..Date.new(2011,7,31), {:games_location => '/Users/geoff/games'})
+  #
+  # Arguments:
+  #   range: (Range)
+  #   args: (Hash)
+
+  def self.pull_days(range,args = {})
+    range.each {|day| self.pull_day(args.merge({:date => day}))}
+  end
+    
+    private
+    ##
+    # Copy XML files from the Gameday severs to your
+    # local machine for parsing and analysis. This method
+    # is used by Spider.pull to put files on your local
+    # machine in a similar layout to the one used by 
+    # Gameday. The paths argument gets built by Spider.pull
+    # during the pull process.
+    #
+    # Arguments: 
+    #   file_name: (String)
+    #   paths: (Hash)
+  
+    def self.copy_gameday_xml (file_name,paths)
+      open(paths[:localGameFolder] + "#{file_name =~ /inning/ ? 'inning/' : ''}" + file_name, 'w') do |file|
+        file.write(self.get(paths[:mlbGameFolder] + "#{file_name =~ /inning/ ? 'inning/' : ''}" + file_name).body)
+      end
+    end
+  
+    ##
+    # Output a folder format similar to the one used by Gameday.
+    # 
+    # Example:
+    #   Spider.format_date_as_folder(Date.new(2011,7,4))
+    #   => "year_2011/month_07/day_04"
+    # 
+    # Arguments:
+    #   date: (Date)
+  
+    def self.format_date_as_folder(date)
+      date.strftime("year_%Y/month_%m/day_%d")
+    end
+end

data/lib/greenmonster.rb

@@ -0,0 +1,10 @@
+require 'httparty'
+require 'nokogiri'
+require 'fileutils'
+
+module Greenmonster
+  
+  
+end
+
+require 'greenmonster/spider'

data/test/test_greenmonster.rb

@@ -0,0 +1,78 @@
+require 'minitest/autorun'
+require 'greenmonster'
+
+class SpiderTest < MiniTest::Unit::TestCase
+  def setup
+    @local_test_data_location = "./greenmonster_test_games"
+    FileUtils.mkdir_p @local_test_data_location
+  end
+  
+  def test_format_date_as_folder
+    assert_equal "year_2011/month_07/day_04", Greenmonster::Spider.format_date_as_folder(Date.new(2011,7,4))
+    assert_equal "year_2012/month_12/day_31", Greenmonster::Spider.format_date_as_folder(Date.new(2012,12,31))
+  end
+  
+  def test_copy_gameday_xml
+    paths = {
+      :localGameFolder => "#{@local_test_data_location}/mlb/year_2011/month_05/day_03/gid_2011_05_03_sfnmlb_nynmlb_1/",
+      :mlbGameFolder => "http://gd2.mlb.com/components/game/mlb/year_2011/month_05/day_03/gid_2011_05_03_sfnmlb_nynmlb_1/"
+    }
+    FileUtils.mkdir_p paths[:localGameFolder] + 'inning'
+    
+    %w(linescore.xml inning_all.xml).each do |f|
+      Greenmonster::Spider.copy_gameday_xml(f,paths)
+    end
+    
+    assert_equal 287337, Nokogiri::XML(open(paths[:localGameFolder] + 'linescore.xml')).search("game").first.attribute('game_pk').value.to_i
+    assert_equal 10, Nokogiri::XML(open(paths[:localGameFolder] + 'inning/inning_all.xml')).search("inning").count
+    assert_equal 400023, Nokogiri::XML(open(paths[:localGameFolder] + 'inning/inning_all.xml')).search("atbat").first.attribute('batter').value.to_i
+  end
+  
+  def test_pull_day
+    Greenmonster::Spider.pull_day({:print_games => false, :games_folder => @local_test_data_location, :date => Date.new(2011,6,7)})
+    
+    assert_equal 17, Dir.entries(@local_test_data_location + '/mlb/year_2011/month_06/day_07/').count
+    
+    game_location = @local_test_data_location + '/mlb/year_2011/month_06/day_07/gid_2011_06_07_bosmlb_nyamlb_1'
+    %w(linescore.xml boxscore.xml players.xml eventLog.xml inning).each do |f|
+      assert Dir.entries(game_location).include? f
+    end
+    
+    boxscore = Nokogiri::XML(open(game_location + '/boxscore.xml'))
+    eventlog = Nokogiri::XML(open(game_location + '/eventLog.xml'))
+    players = Nokogiri::XML(open(game_location + '/players.xml'))
+    innings = Nokogiri::XML(open(game_location + '/inning/inning_all.xml'))
+    
+    assert_equal 8, boxscore.search('pitcher').count
+    assert_equal 147, boxscore.search('boxscore').first.attribute('home_id').value.to_i
+    assert eventlog.search("event[number='19']").attribute('description').value.include?('Pedroia walks.')
+    assert_equal 'Magadan', players.search("team[type='away']").search("coach[position='batting_coach']").attribute('last').value
+    assert_equal 'Strikeout', innings.search('atbat').last.attribute('event').value
+  end
+  
+  def test_pull_minor_league_games
+    Greenmonster::Spider.pull_day({:league => 'aaa', :print_games => false, :games_folder => @local_test_data_location, :date => Date.new(2011,9,1)})
+    
+    assert_equal 13, Dir.entries(@local_test_data_location + '/aaa/year_2011/month_09/day_01/').count
+    %w(linescore.xml boxscore.xml players.xml eventLog.xml inning).each do |f|
+      assert Dir.entries(@local_test_data_location + '/aaa/year_2011/month_09/day_01/gid_2011_09_01_albaaa_mrbaaa_1/').include? f
+    end
+  end
+  
+  def test_pull_games_prior_to_2008
+    Greenmonster::Spider.pull_day({:print_games => false, :games_folder => @local_test_data_location, :date => Date.new(2007,4,15)})
+    
+    assert_equal 12, Dir.entries(@local_test_data_location + '/mlb/year_2007/month_04/day_15/gid_2007_04_15_detmlb_tormlb_1/inning/').count
+  end
+  
+  def test_pull_days
+    Greenmonster::Spider.pull_days((Date.new(2011,8,4)..Date.new(2011,8,5)), {:print_games => false, :games_folder => @local_test_data_location})
+    
+    assert_equal 4, Dir.entries(@local_test_data_location + '/mlb/year_2011/month_08/').count
+    assert_equal 17, Dir.entries(@local_test_data_location + '/mlb/year_2011/month_08/day_05').count
+  end
+  
+  def teardown
+    FileUtils.remove_dir @local_test_data_location
+  end
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: greenmonster
+version: !ruby/object:Gem::Version
+  version: 0.1.0.pre
+  prerelease: 6
+platform: ruby
+authors:
+- Geoff Harcourt
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70119326330320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *70119326330320
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: &70119326329200 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: *70119326329200
+description: A utility for spidering and parsing MLB Gameday XML files.
+email: geoff.harcourt@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Rakefile
+- README.markdown
+- lib/greenmonster/spider.rb
+- lib/greenmonster.rb
+- test/test_greenmonster.rb
+homepage: http://rubygems.org/gems/greenmonster
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: A utility for spidering and parsing MLB Gameday XML files.
+test_files: []