greenmonster 0.5.0 → 1.0.0

data/lib/greenmonster/day_spider.rb

@@ -0,0 +1,61 @@
+module Greenmonster
+  class DaySpider
+    def initialize(date:, sport_code: "mlb")
+      @date = date
+      @sport_code = sport_code
+    end
+
+    def pull
+      game_ids_for_date.each do |game_id|
+        Greenmonster::GameSpider.
+          new(sport_code: sport_code, game_id: game_id).
+          pull
+      end
+    end
+
+    protected
+
+    attr_reader :date, :sport_code
+
+    private
+
+    def date_path
+      "#{sport_code}/year_#{year}/month_#{month}/day_#{day}"
+    end
+
+    def fetch_game_list
+      HTTParty.
+        get("#{Greenmonster::REMOTE_DATA_ROOT}/#{date_path}").
+        response.
+        body
+    end
+
+    def html_links
+      Nokogiri::HTML(fetch_game_list).search("a")
+    end
+
+    def game_links_for_date
+      html_links.select do |link|
+        link.text.strip[0..3] == "gid_"
+      end
+    end
+
+    def game_ids_for_date
+      game_links_for_date.map do |link|
+        link.text.strip.gsub("/", "")
+      end
+    end
+
+    def year
+      date.strftime("%Y")
+    end
+
+    def month
+      date.strftime("%m")
+    end
+
+    def day
+      date.strftime("%d")
+    end
+  end
+end

data/lib/greenmonster/file_downloader.rb

@@ -0,0 +1,38 @@
+module Greenmonster
+  class FileDownloader
+    def initialize(file_name:, game_path:)
+      @file_name = file_name
+      @game_path = game_path
+    end
+
+    def pull
+      return false if fetch.code != "200"
+
+      write_file
+    end
+
+    protected
+
+    attr_reader :file_name, :game_path
+
+    private
+
+    def fetch
+      @fetch ||= HTTParty.get(remote_path).response
+    end
+
+    def local_path
+      "#{Greenmonster.local_data_location}/games/#{game_path}/#{file_name}"
+    end
+
+    def remote_path
+      "#{Greenmonster::REMOTE_DATA_ROOT}/#{game_path}/#{file_name}"
+    end
+
+    def write_file
+      File.open(local_path, "w") do |file|
+        file.write(fetch.body)
+      end
+    end
+  end
+end

data/lib/greenmonster/game_spider.rb

@@ -0,0 +1,100 @@
+module Greenmonster
+  class GameSpider
+    def initialize(game_id:, sport_code: "mlb")
+      @game_id = game_id
+      @sport_code = sport_code
+    end
+
+    def pull
+      make_folder
+      download_xml
+    end
+
+    protected
+
+    attr_reader :game_id, :sport_code
+
+    private
+
+    def make_folder
+      FileUtils.
+        mkdir_p("#{Greenmonster.local_data_location}/games/#{game_path}/inning")
+    end
+
+    def download_xml
+      download_innings
+      download_hit_locations
+      download_boxscore
+      download_linescore
+      download_players
+    end
+
+    def game_path
+      "#{sport_code}/year_#{year}/month_#{month}/day_#{day}/#{game_id}"
+    end
+
+    def download_boxscore
+      Greenmonster::FileDownloader.
+        new(game_path: game_path, file_name: "boxscore.xml").
+        pull
+    end
+
+    def download_hit_locations
+      Greenmonster::FileDownloader.
+        new(game_path: game_path, file_name: "inning/inning_hit.xml").
+        pull
+    end
+
+    def download_innings
+      if year.to_i >= 2008
+        download_all_innings_as_one_file
+      else
+        download_each_inning_file
+      end
+    end
+
+    def download_all_innings_as_one_file
+      Greenmonster::FileDownloader.
+        new(game_path: game_path, file_name: "inning/inning_all.xml").
+        pull
+    end
+
+    def download_each_inning_file
+      Greenmonster::InningsDownloader.new(game_path: game_path).pull
+    end
+
+    def download_linescore
+      Greenmonster::FileDownloader.
+        new(game_path: game_path, file_name: "linescore.xml").
+        pull
+    end
+
+    def download_players
+      Greenmonster::FileDownloader.
+        new(game_path: game_path, file_name: "players.xml").
+        pull
+    end
+
+    protected
+
+    attr_reader :date, :game_number, :sport_code
+
+    private
+
+    def date_segments
+      @date_segments ||= game_id.split("_")
+    end
+
+    def year
+      @year ||= date_segments[1]
+    end
+
+    def month
+      @month ||= date_segments[2]
+    end
+
+    def day
+      @day ||= date_segments[3]
+    end
+  end
+end

data/lib/greenmonster/innings_downloader.rb

@@ -0,0 +1,41 @@
+module Greenmonster
+  class InningsDownloader
+    def initialize(game_path:)
+      @game_path = game_path
+    end
+
+    def pull
+      inning_names.each do |inning_name|
+        Greenmonster::FileDownloader.new(
+          game_path: game_path,
+          file_name: "inning/#{inning_name}"
+        ).pull
+      end
+    end
+
+    protected
+
+    attr_reader :game_path
+
+    private
+
+    def inning_names
+      relevant_links.map do |link|
+        link.text.strip.gsub("/", "")
+      end
+    end
+
+    def innings_listing_html
+      HTTParty.
+        get("#{Greenmonster::REMOTE_DATA_ROOT}/#{game_path}/inning").
+        response.
+        body
+    end
+
+    def relevant_links
+      Nokogiri::HTML(innings_listing_html).search("a").select do |link|
+        link.text =~ /inning_\d+\.xml/
+      end
+    end
+  end
+end

data/lib/greenmonster/spider.rb

@@ -56,6 +56,7 @@ class Greenmonster::Spider
     range.each { |date| self.pull_day(date, sport_code) }
   end
 
+
   private
 
   def get_gameday_date_page(date, sport_code)

data/lib/greenmonster/version.rb

@@ -1,3 +1,3 @@
 module Greenmonster
-  VERSION = "0.5.0"
-end
+  VERSION = "1.0.0"
+end

data/lib/greenmonster.rb

@@ -1,91 +1,25 @@
-require "rubygems"
-require "bundler/setup"
-
-require 'httparty'
-require 'nokogiri'
-require 'fileutils'
-require 'date'
+require "date"
+require "fileutils"
+require "httparty"
+require "nokogiri"
+require "greenmonster/version"
+require "greenmonster/file_downloader"
+require "greenmonster/innings_downloader"
+require "greenmonster/game_spider"
+require "greenmonster/day_spider"
 
 module Greenmonster
-  SPORT_CODES = %w(aaa aax afa afx asx bbc fps hsb ind int jml nae naf nas nat naw oly rok win)
-
-  @@games_folder = nil
-
-  ##
-  # Set the default folder to which games
-  # are saved after being downloaded from
-  # the server.
-  #
-  # Example:
-  #  => Greenmonster.set_games_folder("/Users/geoff/game_data")
-  #
-  # Arguments:
-  #  location: (String)
-  #
-
-  def self.set_games_folder(location)
-    @@games_folder = Pathname.new(location)
-  end
-
-  ##
-  # Return the default games folder location
-  #
-  # Example:
-  #  >> Greenmonster.set_games_folder("/Users/geoff/game_data")
-  #  >> Greenmonster.games_folder
-  #  => #<Pathname:/Users/geoff/game_data>
-
-  def self.games_folder
-    @@games_folder
-  end
-
-  ##
-  # Walk the dates in a range of dates, and execute whatever
-  # methods on the date and argument set specified. Used when
-  # processing games and players.
-  #
-  #
+  REMOTE_DATA_ROOT = "http://gd2.mlb.com/components/game"
 
-  def self.traverse_dates(range, args)
-    range.each { |day| yield day,args }
-  end
-
-  ##
-  # Walk the game folders in a range of dates, and execute whatever
-  # methods on the game folder and argument set specified. Used when
-  # processing games and players.
-  #
-  #
+  def self.local_data_location
+    raise NoLocalDataLocationSet unless @@local_data_location
 
-  def self.traverse_folders_for_date(date, args)
-    game_folders_for_date_and_sport_code(date, args[:sport_code]).each do |gdir|
-      yield gdir, args
-    end
+    @@local_data_location
   end
 
-
-  private
-
-  def self.game_folders_for_date_and_sport_code(date, sport_code)
-    folders_for_date_and_sport_code(date, sport_code).select do |folder|
-      folder[0,3] == 'gid' && folder[-4,4] != '_bak'
-    end
-  end
-
-  def self.folders_for_date_and_sport_code(date, sport_code)
-    begin
-      Dir.entries(
-        Pathname.new(games_folder) + sport_code + format_date_as_folder(date)
-      ).sort
-    rescue Errno::ENOENT
-      []
-    end
-  end
-
-  def self.format_date_as_folder(date)
-    date.strftime("year_%Y/month_%m/day_%d")
+  def self.set_local_data_location(folder)
+    @@local_data_location = folder
   end
 
+  class NoLocalDataLocationSet < Exception; end
 end
-
-require 'greenmonster/spider'

data/spec/cassettes/aaa/year_2015/month_05/day_09.yml

@@ -0,0 +1,153 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://gd2.mlb.com/components/game/aaa/year_2015/month_05/day_09
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 301
+      message: Moved Permanently
+    headers:
+      Server:
+      - Apache/2.0.65 (Unix)
+      Location:
+      - http://gd2.mlb.com/components/game/aaa/year_2015/month_05/day_09/
+      Cache-Control:
+      - max-age=15
+      Expires:
+      - Sat, 09 May 2015 15:21:45 GMT
+      Content-Type:
+      - text/html; charset=iso-8859-1
+      Content-Length:
+      - '273'
+      Accept-Ranges:
+      - bytes
+      Date:
+      - Sat, 09 May 2015 15:21:30 GMT
+      X-Varnish:
+      - '1791922661'
+      Via:
+      - 1.1 varnish
+      Connection:
+      - keep-alive
+      X-Varnish-Cache:
+      - MISS
+      X-Served-By:
+      - w1cache12.web1.mlbam.net
+    body:
+      encoding: UTF-8
+      string: |
+        <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+        <html><head>
+        <title>301 Moved Permanently</title>
+        </head><body>
+        <h1>Moved Permanently</h1>
+        <p>The document has moved <a href="http://gd2.mlb.com/components/game/aaa/year_2015/month_05/day_09/">here</a>.</p>
+        </body></html>
+    http_version: 
+  recorded_at: Sat, 09 May 2015 15:21:30 GMT
+- request:
+    method: get
+    uri: http://gd2.mlb.com/components/game/aaa/year_2015/month_05/day_09/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - Apache/2.0.65 (Unix)
+      Expires:
+      - Sat, 09 May 2015 15:21:45 GMT
+      Access-Control-Allow-Origin:
+      - "*"
+      Edge-Control:
+      - max-age=15
+      Cache-Control:
+      - max-age=15
+      Content-Type:
+      - text/html;charset=ISO-8859-1
+      Content-Length:
+      - '3063'
+      Accept-Ranges:
+      - bytes
+      Date:
+      - Sat, 09 May 2015 15:21:30 GMT
+      X-Varnish:
+      - '1791922682'
+      Via:
+      - 1.1 varnish
+      Connection:
+      - keep-alive
+      X-Varnish-Cache:
+      - MISS
+      X-Served-By:
+      - w1cache12.web1.mlbam.net
+    body:
+      encoding: UTF-8
+      string: |
+        <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+        <html>
+         <head>
+          <title>Index of /components/game/aaa/year_2015/month_05/day_09</title>
+         </head>
+         <body>
+        <h1>Index of /components/game/aaa/year_2015/month_05/day_09</h1>
+        <ul><li><a href="/components/game/aaa/year_2015/month_05/"> Parent Directory</a></li>
+        <li><a href="gid_2015_05_09_albaaa_srcaaa_1/"> gid_2015_05_09_albaaa_srcaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_bufaaa_noraaa_1/"> gid_2015_05_09_bufaaa_noraaa_1/</a></li>
+        <li><a href="gid_2015_05_09_cdcaaa_yucaaa_1/"> gid_2015_05_09_cdcaaa_yucaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_chraaa_tolaaa_1/"> gid_2015_05_09_chraaa_tolaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_cspaaa_iowaaa_1/"> gid_2015_05_09_cspaaa_iowaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_dubaaa_syraaa_1/"> gid_2015_05_09_dubaaa_syraaa_1/</a></li>
+        <li><a href="gid_2015_05_09_gwiaaa_lhvaaa_1/"> gid_2015_05_09_gwiaaa_lhvaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_indaaa_swbaaa_1/"> gid_2015_05_09_indaaa_swbaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_lvgaaa_elpaaa_1/"> gid_2015_05_09_lvgaaa_elpaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_mxoaaa_tijaaa_1/"> gid_2015_05_09_mxoaaa_tijaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_mxoaaa_tijaaa_2/"> gid_2015_05_09_mxoaaa_tijaaa_2/</a></li>
+        <li><a href="gid_2015_05_09_nozaaa_mrbaaa_1/"> gid_2015_05_09_nozaaa_mrbaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_okcaaa_omaaaa_1/"> gid_2015_05_09_okcaaa_omaaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_orhaaa_omaaaa_1/"> gid_2015_05_09_orhaaa_omaaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_pawaaa_colaaa_1/"> gid_2015_05_09_pawaaa_colaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_pueaaa_oaxaaa_1/"> gid_2015_05_09_pueaaa_oaxaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_quiaaa_camaaa_1/"> gid_2015_05_09_quiaaa_camaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_renaaa_slcaaa_1/"> gid_2015_05_09_renaaa_slcaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_reyaaa_mtyaaa_1/"> gid_2015_05_09_reyaaa_mtyaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_rocaaa_louaaa_1/"> gid_2015_05_09_rocaaa_louaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_rreaaa_nasaaa_1/"> gid_2015_05_09_rreaaa_nasaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_sltaaa_aguaaa_1/"> gid_2015_05_09_sltaaa_aguaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_tacaaa_freaaa_1/"> gid_2015_05_09_tacaaa_freaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_vaqaaa_mvaaaa_1/"> gid_2015_05_09_vaqaaa_mvaaaa_1/</a></li>
+        <li><a href="gid_2015_05_09_vraaaa_tabaaa_1/"> gid_2015_05_09_vraaaa_tabaaa_1/</a></li>
+        <li><a href="master_scoreboard.json"> master_scoreboard.json</a></li>
+        <li><a href="master_scoreboard.xml"> master_scoreboard.xml</a></li>
+        <li><a href="miniscoreboard.html"> miniscoreboard.html</a></li>
+        <li><a href="miniscoreboard.xml"> miniscoreboard.xml</a></li>
+        <li><a href="scoreboard_android.xml"> scoreboard_android.xml</a></li>
+        <li><a href="scoreboard_ipad.xml"> scoreboard_ipad.xml</a></li>
+        <li><a href="scoreboard_iphone.xml"> scoreboard_iphone.xml</a></li>
+        <li><a href="uber_scoreboard.xml"> uber_scoreboard.xml</a></li>
+        </ul>
+        </body></html>
+    http_version: 
+  recorded_at: Sat, 09 May 2015 15:21:30 GMT
+recorded_with: VCR 2.9.3

data/spec/cassettes/gid_2007_10_01_sdnmlb_colmlb_1/innings.yml

@@ -0,0 +1,135 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://gd2.mlb.com/components/game/mlb/year_2007/month_10/day_01/gid_2007_10_01_sdnmlb_colmlb_1/inning
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 301
+      message: Moved Permanently
+    headers:
+      Server:
+      - Apache/2.0.65 (Unix)
+      Location:
+      - http://gd2.mlb.com/components/game/mlb/year_2007/month_10/day_01/gid_2007_10_01_sdnmlb_colmlb_1/inning/
+      Cache-Control:
+      - max-age=15
+      Expires:
+      - Sat, 09 May 2015 15:20:12 GMT
+      Content-Type:
+      - text/html; charset=iso-8859-1
+      Content-Length:
+      - '311'
+      Accept-Ranges:
+      - bytes
+      Date:
+      - Sat, 09 May 2015 15:19:57 GMT
+      X-Varnish:
+      - '2550137287'
+      Via:
+      - 1.1 varnish
+      Connection:
+      - keep-alive
+      X-Varnish-Cache:
+      - MISS
+      X-Served-By:
+      - w1cache10.web1.mlbam.net
+    body:
+      encoding: UTF-8
+      string: |
+        <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+        <html><head>
+        <title>301 Moved Permanently</title>
+        </head><body>
+        <h1>Moved Permanently</h1>
+        <p>The document has moved <a href="http://gd2.mlb.com/components/game/mlb/year_2007/month_10/day_01/gid_2007_10_01_sdnmlb_colmlb_1/inning/">here</a>.</p>
+        </body></html>
+    http_version: 
+  recorded_at: Sat, 09 May 2015 15:19:57 GMT
+- request:
+    method: get
+    uri: http://gd2.mlb.com/components/game/mlb/year_2007/month_10/day_01/gid_2007_10_01_sdnmlb_colmlb_1/inning/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - Apache/2.0.65 (Unix)
+      Expires:
+      - Sat, 09 May 2015 15:20:12 GMT
+      Access-Control-Allow-Origin:
+      - "*"
+      Edge-Control:
+      - max-age=15
+      Cache-Control:
+      - max-age=15
+      Content-Type:
+      - text/html;charset=ISO-8859-1
+      Content-Length:
+      - '1219'
+      Accept-Ranges:
+      - bytes
+      Date:
+      - Sat, 09 May 2015 15:19:57 GMT
+      X-Varnish:
+      - '2550137308'
+      Via:
+      - 1.1 varnish
+      Connection:
+      - keep-alive
+      X-Varnish-Cache:
+      - MISS
+      X-Served-By:
+      - w1cache10.web1.mlbam.net
+    body:
+      encoding: UTF-8
+      string: |
+        <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+        <html>
+         <head>
+          <title>Index of /components/game/mlb/year_2007/month_10/day_01/gid_2007_10_01_sdnmlb_colmlb_1/inning</title>
+         </head>
+         <body>
+        <h1>Index of /components/game/mlb/year_2007/month_10/day_01/gid_2007_10_01_sdnmlb_colmlb_1/inning</h1>
+        <ul><li><a href="/components/game/mlb/year_2007/month_10/day_01/gid_2007_10_01_sdnmlb_colmlb_1/"> Parent Directory</a></li>
+        <li><a href="inning_1.xml"> inning_1.xml</a></li>
+        <li><a href="inning_10.xml"> inning_10.xml</a></li>
+        <li><a href="inning_11.xml"> inning_11.xml</a></li>
+        <li><a href="inning_12.xml"> inning_12.xml</a></li>
+        <li><a href="inning_13.xml"> inning_13.xml</a></li>
+        <li><a href="inning_2.xml"> inning_2.xml</a></li>
+        <li><a href="inning_3.xml"> inning_3.xml</a></li>
+        <li><a href="inning_4.xml"> inning_4.xml</a></li>
+        <li><a href="inning_5.xml"> inning_5.xml</a></li>
+        <li><a href="inning_6.xml"> inning_6.xml</a></li>
+        <li><a href="inning_7.xml"> inning_7.xml</a></li>
+        <li><a href="inning_8.xml"> inning_8.xml</a></li>
+        <li><a href="inning_9.xml"> inning_9.xml</a></li>
+        <li><a href="inning_Scores.xml"> inning_Scores.xml</a></li>
+        <li><a href="inning_hit.xml"> inning_hit.xml</a></li>
+        </ul>
+        </body></html>
+    http_version: 
+  recorded_at: Sat, 09 May 2015 15:19:57 GMT
+recorded_with: VCR 2.9.3