espn_scraper 1.3.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4e71f14160cc29d944f5860b4eaa3f983d353890
-  data.tar.gz: 75353026bf2aa879d8151a4c1455314e69af8c78
+  metadata.gz: 92cd55c7052685e30d2dbc52d24a058cd7e7ef46
+  data.tar.gz: d4751ef0577a07dd87e2721af60dfd5041129656
 SHA512:
-  metadata.gz: 5f06578dd6ca1eeefa13910a35a03e84a4781499de5d478c924707db90a476780fb25a9276a6ede4d3b66e1135c31dd1b0c105ad3bd37cc213cd15ae050720f6
-  data.tar.gz: d2b1eda63a460fc734f7280ae171ce0e9260612d1513bc6507dda595d8d484d038a7dc3df213c5066fcfb51151799156cd81b51a07a75bd204cbec1d99c267e8
+  metadata.gz: ac9fcbc028381b407c01ecde9ea3a47d75658c6ca582bdb6024c03e7333e9bd0ec41e5d27a48664ce331a48ea6d541e150918eb13c74b8d9d7d2f201c3d4ef25
+  data.tar.gz: bac030d2c801f9749a85be6d58986062dc79f94d27d7110c1cd006f1bc23d43f64c7320881329780dca7c9f674815e81bc1b9f54762ab459f12b089ac39083f9

data/README.md

@@ -1,6 +1,6 @@
 # ESPN Scraper
 
-ESPN Scraper is a simple gem for scraping teams and scores from `ESPN`'s website. Please note that `ESPN` is not involved with this gem or me in any way. I chose `ESPN` because it is a leader in sports statistics and has a robust website. 
+ESPN Scraper is a simple gem for scraping teams and scores from `ESPN`'s website. Please note that `ESPN` is not involved with this gem or me in any way. I chose `ESPN` because it is a leader in sports statistics and has a robust website.
 
 ```ruby
 ESPN.responding?
@@ -55,11 +55,11 @@ You can get the teams in each league by acronym. It returns a hash of each divis
 ```ruby
 ESPN.get_teams_in('nba')
 # => {
-#   "atlantic"=> [ 
-#     { :name => "Boston Celtics", :data_name => "bos" },  
-#     { :name => "Brooklyn Nets", :data_name => "bkn" }, 
-#     { :name => "New York Knicks", :data_name => "ny" }, 
-#     { :name => "Philadelphia 76ers", :data_name => "phi" }, 
+#   "atlantic"=> [
+#     { :name => "Boston Celtics", :data_name => "bos" },
+#     { :name => "Brooklyn Nets", :data_name => "bkn" },
+#     { :name => "New York Knicks", :data_name => "ny" },
+#     { :name => "Philadelphia 76ers", :data_name => "phi" },
 #     { :name => "Toronto Raptors", :data_name => "tor" }
 #   ]
 #   "pacific" => ...
@@ -121,7 +121,7 @@ require 'espn_scraper'
 
 ## Contributing
 
-Please report back if something breaks on you! 
+Please report back if something breaks on you!
 
 Also please let me know if any of the data names get outdated. For instance a bunch of NFL data names were recently changed. You can make fixes temporarily with the following:
 
@@ -129,7 +129,19 @@ Also please let me know if any of the data names get outdated. For instance a bu
 ESPN::DATA_NAME_FIXES['nfl']['gnb'] = 'gb'
 ```
 
-Future plans:  
+Running tests:
+```
+rake test
+```
+
+Re-building the gem and installing locally:
+```
+gem build espn_scraper.gemspec
+gem uninstall espn_scraper
+gem install espn_scraper-x.x.x.gem
+```
+
+Future plans:
 - Get start and end dates of a season
 
 ### Thank You

data/Rakefile

@@ -1,4 +1,3 @@
-require "bundler/gem_tasks"
 require 'rake/testtask'
 
 Rake::TestTask.new do |t|

data/lib/espn_scraper/boilerplate.rb

@@ -3,11 +3,11 @@ require 'nokogiri'
 
 module ESPN
   class << self
-    
+
     def responding?
-      HTTParty.get('http://espn.go.com/').code == 200
+      HTTParty.get('http://www.espn.com').code == 200
     end
-    
+
     def down?
       !responding?
     end
@@ -15,9 +15,8 @@ module ESPN
     # Ex: ESPN.url('scores')
     #     ESPN.url('teams', 'nba')
     def url(*path)
-      subdomain = (path.first == 'scores') ? path.shift : nil
-      domain = [subdomain, 'espn', 'go', 'com'].compact.join('.')
-      ['http:/', domain, *path].join('/')
+      path.shift if path.first == 'scores'
+      ['http://www.espn.com', *path].join('/')
     end
 
     # Returns Nokogiri HTML document
@@ -29,22 +28,22 @@ module ESPN
         Nokogiri::HTML(response.body)
       else
         raise ArgumentError, error_message(url, path)
-      end      
+      end
     end
-    
+
     def dasherize(str)
       str.strip.downcase.gsub(/\s+/, '-')
     end
-    
-    
+
+
     private
-    
-    
-    
+
+
+
     def error_message(url, path)
       "The url #{url} from the path #{path} did not return a valid page."
     end
-        
+
   end
 end
 

data/lib/espn_scraper/scores.rb

@@ -4,14 +4,14 @@ require 'json'
 
 module ESPN
   SEASONS = {
-      preseason: 1,
-      regular_season: 2,
-      postseason: 3
+    preseason: 1,
+    regular_season: 2,
+    postseason: 3
   }
 
-  mlb_ignores = %w( 
-    florida-state u-of-south-florida georgetown fla.-southern northeastern boston-college 
-    miami-florida florida-intl canada hanshin yomiuri sacramento springfield corpus-christi 
+  mlb_ignores = %w(
+    florida-state u-of-south-florida georgetown fla.-southern northeastern boston-college
+    miami-florida florida-intl canada hanshin yomiuri sacramento springfield corpus-christi
     round-rock carolina manatee-cc mexico cincinnati-(f) atlanta-(f) frisco toledo norfolk
     fort-myers tampa-bay-(f) nl-all-stars al-all-stars
   )
@@ -23,7 +23,7 @@ module ESPN
     team-chara team-alfredsson
   )
 
-  ncf_ignores = %w( paul-quinn san-diego-christian ferris-st notre-dame-college chaminade 
+  ncf_ignores = %w( paul-quinn san-diego-christian ferris-st notre-dame-college chaminade
     w-new-mexico n-new-mexico tx-a&m-commerce nw-oklahoma-st )
 
   IGNORED_TEAMS = (mlb_ignores + nhl_ignores + nba_ignores + ncf_ignores).inject({}) do |h, team|
@@ -31,29 +31,29 @@ module ESPN
   end
 
   DATA_NAME_EXCEPTIONS = {
-      'nets' => 'bkn',
-      'supersonics' => 'okc',
-      'hornets' => 'no',
+    'nets' => 'bkn',
+    'supersonics' => 'okc',
+    'hornets' => 'no',
 
-      'marlins' => 'mia'
+    'marlins' => 'mia'
   }.merge(IGNORED_TEAMS)
 
   DATA_NAME_FIXES = {
-      'nfl' => {
-          'nwe' => 'ne',
-          'kan' => 'kc',
-          'was' => 'wsh',
-          'nor' => 'no',
-          'gnb' => 'gb',
-          'sfo' => 'sf',
-          'tam' => 'tb',
-          'sdg' => 'sd'
-      },
-      'mlb' => {},
-      'nba' => {},
-      'nhl' => {},
-      'ncf' => {},
-      'ncb' => {}
+    'nfl' => {
+      'nwe' => 'ne',
+      'kan' => 'kc',
+      'was' => 'wsh',
+      'nor' => 'no',
+      'gnb' => 'gb',
+      'sfo' => 'sf',
+      'tam' => 'tb',
+      'sdg' => 'sd'
+    },
+    'mlb' => {},
+    'nba' => {},
+    'nhl' => {},
+    'ncf' => {},
+    'ncb' => {}
   }
 
   # Example output:
@@ -68,13 +68,6 @@ module ESPN
 
   class << self
 
-    def get_superbowl_score
-      markup = Scores.markup_from_superbowl
-      scores = Scores.home_away_parse(markup, false)
-      add_league_and_fixes(scores, 'nfl')
-      scores
-    end
-
     def get_nfl_scores(year, week)
       markup = Scores.markup_from_year_and_week('nfl', year, week)
       scores = Scores.home_away_parse(markup)
@@ -84,7 +77,7 @@ module ESPN
 
     def get_mlb_scores(date)
       markup = Scores.markup_from_date('mlb', date)
-      scores = Scores.home_away_parse(markup)
+      scores = Scores.home_away_parse(markup, date)
       scores.each { |report| report[:league] = 'mlb' }
       scores
     end
@@ -104,7 +97,7 @@ module ESPN
     end
 
     def get_ncf_scores(year, week)
-      markup = Scores.markup_from_year_and_week('college-football', year, week)
+      markup = Scores.markup_from_year_and_week('college-football', year, week, 80)
       scores = Scores.ncf_parse(markup)
       scores.each { |report| report[:league] = 'college-football' }
       scores
@@ -112,32 +105,15 @@ module ESPN
 
     alias_method :get_college_football_scores, :get_ncf_scores
 
-    def get_ncb_scores(date, conference_id=nil, final_only=true)
-      if conference_id
-        markup = Scores.markup_from_date_and_conference('ncb', date, conference_id)
-      else
-        markup = Scores.markup_from_date('ncb', date)
-      end
-
-      scores = Scores.home_away_parse(markup, final_only)
-
-      scores.each do |report|
-        report[:league] ||= 'mens-college-basketball'
-        report[:game_date] ||= date
-      end
-
+    def get_ncb_scores(date, conference_id)
+      markup = Scores.markup_from_date_and_conference('ncb', date, conference_id)
+      scores = Scores.home_away_parse(markup, date)
+      scores.each { |report| report.merge! league: 'mens-college-basketball', game_date: date }
       scores
     end
 
     alias_method :get_college_basketball_scores, :get_ncb_scores
 
-    def get_ncb_abbreviations(date)
-      markup = Scores.markup_from_date('ncb', date)
-
-      teams = Scores.team_abbreviation_parse(markup)
-      teams
-    end
-
     def add_league_and_fixes(scores, league)
       scores.each do |report|
         report[:league] = league
@@ -156,12 +132,12 @@ module ESPN
 
       # Get Markup
 
-      def markup_from_superbowl
-        ESPN.get 'scores', 'nfl', "scoreboard/_/group/80/year/#{Time.now.year - 1}/seasontype/3/week/5"
-      end
-
-      def markup_from_year_and_week(league, year, week)
-        ESPN.get 'scores', league, "scoreboard/_/group/80/year/#{year}/seasontype/2/week/#{week}"
+      def markup_from_year_and_week(league, year, week, group=nil)
+        if group
+          ESPN.get 'scores', league, "scoreboard/_/group/#{group}/year/#{year}/seasontype/2/week/#{week}"
+        else
+          ESPN.get 'scores', league, "scoreboard/_/year/#{year}/seasontype/2/week/#{week}"
+        end
       end
 
       def markup_from_date(league, date)
@@ -176,7 +152,7 @@ module ESPN
 
       # parsing strategies
 
-      def home_away_parse(doc, final=true)
+      def home_away_parse(doc, date=nil)
         scores = []
         games = []
         espn_regex = /window\.espn\.scoreboardData \t= (\{.*?\});/
@@ -190,11 +166,16 @@ module ESPN
         games.each do |game|
           # Game must be regular or postseason
           next unless game['season']['type'] == SEASONS[:regular_season] || game['season']['type'] == SEASONS[:postseason]
+
+          # Game must not be suspended if it was supposed to start on the query date.
+          # This prevents fetching scores for suspended games which are not yet completed.
+          game_start = DateTime.parse(game['date']).to_time.utc + Time.zone_offset('EDT')
+          next if date && game['competitions'][0]['wasSuspended'] && game_start.to_date == date
+
           score = {}
           competition = game['competitions'].first
-
           # Score must be final
-          if !final || competition['status']['type']['detail'] =~ /^Final/
+          if competition['status']['type']['detail'] =~ /^Final/
             competition['competitors'].each do |competitor|
               if competitor['homeAway'] == 'home'
                 score[:home_team] = competitor['team']['abbreviation'].downcase
@@ -205,34 +186,12 @@ module ESPN
               end
             end
             score[:game_date] = DateTime.parse(game['date'])
-            score[:status] = competition['status']['type']['detail']
             scores << score
           end
         end
         scores
       end
 
-      def team_abbreviation_parse(doc)
-        games = []
-        teams = {}
-        espn_regex = /window\.espn\.scoreboardData \t= (\{.*?\});/
-        doc.xpath("//script").each do |script_section|
-          if script_section.content =~ espn_regex
-            espn_data = JSON.parse(espn_regex.match(script_section.content)[1])
-            games = espn_data['events']
-            break
-          end
-        end
-        games.each do |game|
-          competition = game['competitions'].first
-          competition['competitors'].each do |competitor|
-            teams[competitor['team']['displayName']] = competitor['team']['abbreviation'].downcase
-          end
-        end
-
-        teams
-      end
-
       def ncf_parse(doc)
         scores = []
         games = []
@@ -256,7 +215,7 @@ module ESPN
               if competitor['homeAway'] == 'home'
                 score[:home_team] = competitor['team']['id'].downcase
                 score[:home_score] = competitor['score'].to_i
-              else
+                else
                 score[:away_team] = competitor['team']['id'].downcase
                 score[:away_score] = competitor['score'].to_i
               end
@@ -308,4 +267,4 @@ module ESPN
 
     end
   end
-end
+end

data/lib/espn_scraper/version.rb

@@ -1,3 +1,3 @@
 module ESPN
-  VERSION = '1.3.1'
+  VERSION = '1.5.0'
 end

data/test/espn_scraper_test/espn_test.rb

@@ -1,34 +1,34 @@
 require 'test_helper'
 
 class BoilerplateTest < EspnTest
-  
+
   test 'espn is up' do
     assert ESPN.responding?
     assert !ESPN.down?
   end
-	
+
   test 'paths are working' do
-    assert_equal 'http://scores.espn.go.com', ESPN.url('scores')
-    assert_equal 'http://espn.go.com/nba/teams', ESPN.url('nba', 'teams')
+    assert_equal 'http://www.espn.com', ESPN.url('scores')
+    assert_equal 'http://www.espn.com/nba/teams', ESPN.url('nba', 'teams')
   end
-  
+
   test 'error message works' do
     assert_raises(ArgumentError) do
       ESPN.get('bad-api-keyword')
     end
   end
-	
+
   test 'get pages is working' do
     assert ESPN.get('scores')
   end
-	
+
   test 'dasherize strings' do
     assert_equal 'string-is-dashed', ESPN.send(:dasherize, 'String is dashed')
   end
-  
+
   test 'leagues' do
     leagues = 'nfl mlb nba nhl ncf ncb'.split
     assert_equal leagues, ESPN.leagues
   end
 
-end
+end

data/test/espn_scraper_test/mlb_test.rb

@@ -1,7 +1,7 @@
 require 'test_helper'
 
 class MlbTest < EspnTest
-  
+
   test 'mlb august 13th 2012 yankees beat rangers' do
     starts_at = DateTime.parse('2012-08-13T23:00:00+00:00')
     expected = {
@@ -29,12 +29,18 @@ class MlbTest < EspnTest
     scores = ESPN.get_mlb_scores(starts_at.to_date)
     assert scores.include?(expected), 'A known MLB final score cannot be found'
   end
-  
+
+  test 'mlb sept 5th 2014 suspended game pittsburgh pirates at chicago cubs' do
+    scores = ESPN.get_mlb_scores(Date.parse('2014-09-05'))
+    home_teams = scores.each { |s| s['home_team'] }
+    assert !home_teams.include?('chc'), 'A known suspended game was unexpectedly found'
+  end
+
   test 'random mlb days' do
     random_days.each do |day|
       scores = ESPN.get_mlb_scores(day)
       assert all_names_present?(scores), "Error on #{day} for mlb"
     end
   end
-  
-end
+
+end

data/test/espn_scraper_test/teams_test.rb

@@ -1,7 +1,7 @@
 require 'test_helper'
 
 class TeamsTest < EspnTest
-  
+
   test 'scrape nfl teams' do
     divisions = ESPN.get_teams_in('nfl')
     assert_equal 8, divisions.count
@@ -13,7 +13,7 @@ class TeamsTest < EspnTest
     assert_equal 32, teams.map{ |h| h[:data_name] }.uniq.count
     assert divisions['nfc-west'].include?({ name: 'Seattle Seahawks', data_name: 'sea' })
   end
-  
+
   test 'scrape mlb teams' do
     divisions = ESPN.get_teams_in('mlb')
     assert_equal 6, divisions.count
@@ -25,7 +25,7 @@ class TeamsTest < EspnTest
     assert_equal 30, teams.map{ |h| h[:data_name] }.uniq.count
     assert divisions['al-west'].include?({ name: 'Seattle Mariners', data_name: 'sea' })
   end
-  
+
   test 'scrape nba teams' do
     divisions = ESPN.get_teams_in('nba')
     assert_equal 6, divisions.count
@@ -37,18 +37,18 @@ class TeamsTest < EspnTest
     assert_equal 30, teams.map{ |h| h[:data_name] }.uniq.count
     assert divisions['atlantic'].include?({ name: 'Toronto Raptors', data_name: 'tor' })
   end
-  
+
   test 'scrape nhl teams' do
     divisions = ESPN.get_teams_in('nhl')
     assert_equal 4, divisions.count
     assert_equal 7, divisions['central'].count
     assert_equal 8, divisions['atlantic'].count
     teams = divisions.values.flatten
-    assert_equal 30, teams.map{ |h| h[:name] }.uniq.count
-    assert_equal 30, teams.map{ |h| h[:data_name] }.uniq.count
+    assert_equal 31, teams.map{ |h| h[:name] }.uniq.count
+    assert_equal 31, teams.map{ |h| h[:data_name] }.uniq.count
     assert divisions['atlantic'].include?({ name: 'Montreal Canadiens', data_name: 'mtl' })
   end
-  
+
   test 'scrape ncaa football teams' do
     divisions = ESPN.get_teams_in('college-football')
     assert_equal 25, divisions.count
@@ -57,23 +57,23 @@ class TeamsTest < EspnTest
     assert divisions['conference-usa'].include?({ name: 'Rice Owls', data_name: '242' })
     assert divisions['meac'].include?({ name: 'Bethune-Cookman Wildcats', data_name: '2065' })
     assert divisions['northeast'].include?({ name: 'St Francis Red Flash', data_name: '2598' })
-    assert divisions['swac'].include?({ name: 'Alabama A&M Bulldogs', data_name: '2010' })
+    assert divisions['swac'].include?({ name: 'Alabama State Hornets', data_name: '2011' })
   end
-  
+
   test 'scrape ncaa basketball teams' do
     divisions = ESPN.get_teams_in('mens-college-basketball')
     assert_equal 32, divisions.count
     assert_equal 15, divisions['acc'].count
     assert_equal 10, divisions['patriot-league'].count
-    
-    assert divisions['southland'].include?({ name: 'Texas A&M-CC Islanders', data_name: '357' })
+
+    assert divisions['southland'].include?({ name: 'Incarnate Word Cardinals', data_name: '2916' })
     assert divisions['atlantic-10'].include?({ name: "Saint Joe's Saint Joseph's Hawks", data_name: '2603' })
   end
 
   test 'scrape ncaa basketball conferences' do
     conferences = ESPN.get_conferences_in_ncb
-    assert_equal 32, conferences.count
+    assert_equal 33, conferences.count
     assert conferences.include?({ name: 'Mountain West', data_name: '44' })
   end
-	
-end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: espn_scraper
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.5.0
 platform: ruby
 authors:
 - aj0strow
@@ -84,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: ESPN Scraper