espn_scraper 1.3.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4e71f14160cc29d944f5860b4eaa3f983d353890
+  data.tar.gz: 75353026bf2aa879d8151a4c1455314e69af8c78
+SHA512:
+  metadata.gz: 5f06578dd6ca1eeefa13910a35a03e84a4781499de5d478c924707db90a476780fb25a9276a6ede4d3b66e1135c31dd1b0c105ad3bd37cc213cd15ae050720f6
+  data.tar.gz: d2b1eda63a460fc734f7280ae171ce0e9260612d1513bc6507dda595d8d484d038a7dc3df213c5066fcfb51151799156cd81b51a07a75bd204cbec1d99c267e8

data/.gitignore

@@ -0,0 +1,2 @@
+.ruby-version
+Gemfile.lock

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org/'
+
+gem 'rake', '~> 10.4.2'
+gem 'httparty'
+gem 'nokogiri'
+gem 'minitest', '~> 5.6.0'

data/README.md

@@ -0,0 +1,141 @@
+# ESPN Scraper
+
+ESPN Scraper is a simple gem for scraping teams and scores from `ESPN`'s website. Please note that `ESPN` is not involved with this gem or me in any way. I chose `ESPN` because it is a leader in sports statistics and has a robust website. 
+
+```ruby
+ESPN.responding?
+# => true
+```
+
+Lets begin...
+
+#### Supported leagues
+
+The gem only supports the following leagues:
+
+```ruby
+ESPN.leagues
+# => [ "nfl", "mlb", "nba", "nhl", "ncf", "ncb" ]
+```
+
+Which are the NFL, MLB, NBA, NHL, NCAA D1 Football, NCAA D1 Men's Basketball respectively.
+
+#### Scrape Divisions
+
+You can get all the divisions in each league.
+
+```ruby
+ESPN.get_divisions
+# => {
+#   "nfl" => [
+#     { :name => "NFC East", :data_name => "nfc-east" },
+#     { :name => "NFC West", :data_name => "nfc-west" },
+#     ...
+#   ],
+#   "mlb" => ...
+# }
+```
+
+#### Scrape Conferences (NCAA D1 Men's Basketball only)
+
+You can get all the conferences in NCAA D1 Men's Basketball.
+
+```ruby
+ESPN.get_conferences_in_ncb
+# => [{:name=>"America East", :data_name=>"1"},
+#      {:name=>"American", :data_name=>"62"},
+#      ...
+#      ]
+```
+
+#### Scrape teams
+
+You can get the teams in each league by acronym. It returns a hash of each division with an array of hashes for each team in the division.
+
+```ruby
+ESPN.get_teams_in('nba')
+# => {
+#   "atlantic"=> [ 
+#     { :name => "Boston Celtics", :data_name => "bos" },  
+#     { :name => "Brooklyn Nets", :data_name => "bkn" }, 
+#     { :name => "New York Knicks", :data_name => "ny" }, 
+#     { :name => "Philadelphia 76ers", :data_name => "phi" }, 
+#     { :name => "Toronto Raptors", :data_name => "tor" }
+#   ]
+#   "pacific" => ...
+# }
+```
+
+#### Scraping scores
+
+All score requests return an array of hashes. Here's an example NFL score hash:
+
+```ruby
+{
+  league: 'nfl',
+  game_date: #<DateTime: 2012-10-25>,
+  home_team: 'min',
+  home_score: 17,
+  away_team: 'tb',
+  away_score: 36
+}
+```
+
+You'll notice the teams are identified with the same `:data_name` from a `ESPN.get_teams_in` request. One issue with scraping scores is that football goes by year and week, and baseball, basketball, hockey go by date.
+
+###### weekly (football)
+
+Pattern is `ESPN.get_<league>_scores(year, week)`. This is for `nfl` and `ncf`:
+
+```ruby
+ESPN.get_nfl_scores(2012, 8)
+ESPN.get_ncf_scores(2011, 3)
+```
+
+###### daily (baseball, basketball, hockey)
+
+Pattern is `ESPN.get_<league>_scores(date)`. This is for `mlb`, `nba`, `nhl`, `ncb`:
+
+```ruby
+ESPN.get_mlb_scores( Date.parse('Aug 13, 2012') )
+ESPN.get_nba_scores( Date.parse('Dec 25, 2011') )
+ESPN.get_nhl_scores( Date.parse('Feb 14, 2009') )
+ESPN.get_ncb_scores( Date.parse('Mar 15, 2012') )
+```
+
+## Installing
+
+Add the gem to your `Gemfile`
+
+```ruby
+gem 'espn_scraper', git: 'git://github.com/aj0strow/espn-scraper.git'
+# or
+gem 'espn_scraper', github: 'aj0strow/espn-scraper'
+```
+
+..and then require it. I personally use it in rake tasks of a Rails app.
+
+```ruby
+require 'espn_scraper'
+```
+
+## Contributing
+
+Please report back if something breaks on you! 
+
+Also please let me know if any of the data names get outdated. For instance a bunch of NFL data names were recently changed. You can make fixes temporarily with the following:
+
+```ruby
+ESPN::DATA_NAME_FIXES['nfl']['gnb'] = 'gb'
+```
+
+Future plans:  
+- Get start and end dates of a season
+
+### Thank You
+
+* Dan Madere ([dgmdan](https://github.com/dgmdan))
+
+---
+
+**MIT License**

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList['test/espn_scraper_test/*_test.rb']
+end
+
+task default: :test

data/espn_scraper.gemspec

@@ -0,0 +1,19 @@
+$:.push File.expand_path("../lib", __FILE__)
+require 'espn_scraper/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'espn_scraper'
+  s.version     = ESPN::VERSION
+  s.date        = '2013-04-07'
+  s.summary     = 'ESPN Scraper'
+  s.description = 'a simple scraping api for espn stats and data'
+  s.authors     = %w[ aj0strow ]
+  s.email       = 'alexander.ostrow@gmail.com'
+  s.homepage    = 'http://github.com/aj0strow/espn-scraper'
+  
+  s.add_dependency 'httparty'
+  s.add_dependency 'nokogiri'
+  
+  s.files       = `git ls-files`.split("\n")
+  s.test_files  = `git ls-files -- test`.split("\n")
+end

data/lib/espn_scraper.rb

@@ -0,0 +1,3 @@
+%w[ boilerplate teams scores ].each do |file|
+  require "espn_scraper/#{ file }"
+end

data/lib/espn_scraper/boilerplate.rb

@@ -0,0 +1,50 @@
+require 'httparty'
+require 'nokogiri'
+
+module ESPN
+  class << self
+    
+    def responding?
+      HTTParty.get('http://espn.go.com/').code == 200
+    end
+    
+    def down?
+      !responding?
+    end
+
+    # Ex: ESPN.url('scores')
+    #     ESPN.url('teams', 'nba')
+    def url(*path)
+      subdomain = (path.first == 'scores') ? path.shift : nil
+      domain = [subdomain, 'espn', 'go', 'com'].compact.join('.')
+      ['http:/', domain, *path].join('/')
+    end
+
+    # Returns Nokogiri HTML document
+    # Ex: ESPN.get('teams', 'nba')
+    def get(*path)
+      http_url = self.url(*path)
+      response = HTTParty.get(http_url)
+      if response.code == 200
+        Nokogiri::HTML(response.body)
+      else
+        raise ArgumentError, error_message(url, path)
+      end      
+    end
+    
+    def dasherize(str)
+      str.strip.downcase.gsub(/\s+/, '-')
+    end
+    
+    
+    private
+    
+    
+    
+    def error_message(url, path)
+      "The url #{url} from the path #{path} did not return a valid page."
+    end
+        
+  end
+end
+

data/lib/espn_scraper/scores.rb

@@ -0,0 +1,311 @@
+require 'uri'
+require 'cgi'
+require 'json'
+
+module ESPN
+  SEASONS = {
+      preseason: 1,
+      regular_season: 2,
+      postseason: 3
+  }
+
+  mlb_ignores = %w( 
+    florida-state u-of-south-florida georgetown fla.-southern northeastern boston-college 
+    miami-florida florida-intl canada hanshin yomiuri sacramento springfield corpus-christi 
+    round-rock carolina manatee-cc mexico cincinnati-(f) atlanta-(f) frisco toledo norfolk
+    fort-myers tampa-bay-(f) nl-all-stars al-all-stars
+  )
+
+  nba_ignores = %w( west-all-stars east-all-stars )
+
+  nhl_ignores = %w(
+    hc-sparta frolunda hc-slovan ev-zug jokerit-helsinki hamburg-freezers adler-mannheim
+    team-chara team-alfredsson
+  )
+
+  ncf_ignores = %w( paul-quinn san-diego-christian ferris-st notre-dame-college chaminade 
+    w-new-mexico n-new-mexico tx-a&m-commerce nw-oklahoma-st )
+
+  IGNORED_TEAMS = (mlb_ignores + nhl_ignores + nba_ignores + ncf_ignores).inject({}) do |h, team|
+    h.merge team => false
+  end
+
+  DATA_NAME_EXCEPTIONS = {
+      'nets' => 'bkn',
+      'supersonics' => 'okc',
+      'hornets' => 'no',
+
+      'marlins' => 'mia'
+  }.merge(IGNORED_TEAMS)
+
+  DATA_NAME_FIXES = {
+      'nfl' => {
+          'nwe' => 'ne',
+          'kan' => 'kc',
+          'was' => 'wsh',
+          'nor' => 'no',
+          'gnb' => 'gb',
+          'sfo' => 'sf',
+          'tam' => 'tb',
+          'sdg' => 'sd'
+      },
+      'mlb' => {},
+      'nba' => {},
+      'nhl' => {},
+      'ncf' => {},
+      'ncb' => {}
+  }
+
+  # Example output:
+  # {
+  #   league: "nfl",
+  #   game_date: #<Date: 2013-01-05 ((2456298j,0s,0n),+0s,2299161j)>,
+  #   home_team: "sea",
+  #   home_score: 48,
+  #   away_team: "min",
+  #   away_score: 27
+  # }
+
+  class << self
+
+    def get_superbowl_score
+      markup = Scores.markup_from_superbowl
+      scores = Scores.home_away_parse(markup, false)
+      add_league_and_fixes(scores, 'nfl')
+      scores
+    end
+
+    def get_nfl_scores(year, week)
+      markup = Scores.markup_from_year_and_week('nfl', year, week)
+      scores = Scores.home_away_parse(markup)
+      add_league_and_fixes(scores, 'nfl')
+      scores
+    end
+
+    def get_mlb_scores(date)
+      markup = Scores.markup_from_date('mlb', date)
+      scores = Scores.home_away_parse(markup)
+      scores.each { |report| report[:league] = 'mlb' }
+      scores
+    end
+
+    def get_nba_scores(date)
+      markup = Scores.markup_from_date('nba', date)
+      scores = Scores.home_away_parse(markup)
+      scores.each { |report| report[:league] = 'nba' }
+      scores
+    end
+
+    def get_nhl_scores(date)
+      markup = Scores.markup_from_date('nhl', date)
+      scores = Scores.winner_loser_parse(markup, date)
+      scores.each { |report| report[:league] = 'nhl' }
+      scores
+    end
+
+    def get_ncf_scores(year, week)
+      markup = Scores.markup_from_year_and_week('college-football', year, week)
+      scores = Scores.ncf_parse(markup)
+      scores.each { |report| report[:league] = 'college-football' }
+      scores
+    end
+
+    alias_method :get_college_football_scores, :get_ncf_scores
+
+    def get_ncb_scores(date, conference_id=nil, final_only=true)
+      if conference_id
+        markup = Scores.markup_from_date_and_conference('ncb', date, conference_id)
+      else
+        markup = Scores.markup_from_date('ncb', date)
+      end
+
+      scores = Scores.home_away_parse(markup, final_only)
+
+      scores.each do |report|
+        report[:league] ||= 'mens-college-basketball'
+        report[:game_date] ||= date
+      end
+
+      scores
+    end
+
+    alias_method :get_college_basketball_scores, :get_ncb_scores
+
+    def get_ncb_abbreviations(date)
+      markup = Scores.markup_from_date('ncb', date)
+
+      teams = Scores.team_abbreviation_parse(markup)
+      teams
+    end
+
+    def add_league_and_fixes(scores, league)
+      scores.each do |report|
+        report[:league] = league
+        [:home_team, :away_team].each do |sym|
+          team = report[sym]
+          report[sym] = DATA_NAME_FIXES[league][team] || team
+        end
+      end
+    end
+  end
+
+
+
+  module Scores
+    class << self
+
+      # Get Markup
+
+      def markup_from_superbowl
+        ESPN.get 'scores', 'nfl', "scoreboard/_/group/80/year/#{Time.now.year - 1}/seasontype/3/week/5"
+      end
+
+      def markup_from_year_and_week(league, year, week)
+        ESPN.get 'scores', league, "scoreboard/_/group/80/year/#{year}/seasontype/2/week/#{week}"
+      end
+
+      def markup_from_date(league, date)
+        day = date.to_s.gsub(/[^\d]+/, '')
+        ESPN.get 'scores', league, "scoreboard?date=#{ day }"
+      end
+
+      def markup_from_date_and_conference(league, date, conference_id)
+        day = date.to_s.gsub(/[^\d]+/, '')
+        ESPN.get league, 'scoreboard', '_', 'group', conference_id.to_s, 'date', day
+      end
+
+      # parsing strategies
+
+      def home_away_parse(doc, final=true)
+        scores = []
+        games = []
+        espn_regex = /window\.espn\.scoreboardData \t= (\{.*?\});/
+        doc.xpath("//script").each do |script_section|
+          if script_section.content =~ espn_regex
+            espn_data = JSON.parse(espn_regex.match(script_section.content)[1])
+            games = espn_data['events']
+            break
+          end
+        end
+        games.each do |game|
+          # Game must be regular or postseason
+          next unless game['season']['type'] == SEASONS[:regular_season] || game['season']['type'] == SEASONS[:postseason]
+          score = {}
+          competition = game['competitions'].first
+
+          # Score must be final
+          if !final || competition['status']['type']['detail'] =~ /^Final/
+            competition['competitors'].each do |competitor|
+              if competitor['homeAway'] == 'home'
+                score[:home_team] = competitor['team']['abbreviation'].downcase
+                score[:home_score] = competitor['score'].to_i
+              else
+                score[:away_team] = competitor['team']['abbreviation'].downcase
+                score[:away_score] = competitor['score'].to_i
+              end
+            end
+            score[:game_date] = DateTime.parse(game['date'])
+            score[:status] = competition['status']['type']['detail']
+            scores << score
+          end
+        end
+        scores
+      end
+
+      def team_abbreviation_parse(doc)
+        games = []
+        teams = {}
+        espn_regex = /window\.espn\.scoreboardData \t= (\{.*?\});/
+        doc.xpath("//script").each do |script_section|
+          if script_section.content =~ espn_regex
+            espn_data = JSON.parse(espn_regex.match(script_section.content)[1])
+            games = espn_data['events']
+            break
+          end
+        end
+        games.each do |game|
+          competition = game['competitions'].first
+          competition['competitors'].each do |competitor|
+            teams[competitor['team']['displayName']] = competitor['team']['abbreviation'].downcase
+          end
+        end
+
+        teams
+      end
+
+      def ncf_parse(doc)
+        scores = []
+        games = []
+        espn_regex = /window\.espn\.scoreboardData \t= (\{.*?\});/
+        doc.xpath("//script").each do |script_section|
+          if script_section.content =~ espn_regex
+            espn_data = JSON.parse(espn_regex.match(script_section.content)[1])
+            games = espn_data['events']
+            break
+          end
+        end
+        games.each do |game|
+          score = { league: 'college-football' }
+          competition = game['competitions'].first
+          date = DateTime.parse(competition['startDate'])
+          date = date.new_offset('-06:00')
+          score[:game_date] = date.to_date
+          # Score must be final
+          if competition['status']['type']['detail'] =~ /^Final/
+            competition['competitors'].each do |competitor|
+              if competitor['homeAway'] == 'home'
+                score[:home_team] = competitor['team']['id'].downcase
+                score[:home_score] = competitor['score'].to_i
+              else
+                score[:away_team] = competitor['team']['id'].downcase
+                score[:away_score] = competitor['score'].to_i
+              end
+            end
+            scores << score
+          end
+        end
+        scores
+      end
+
+      def winner_loser_parse(doc, date)
+        doc.css('.mod-scorebox-final').map do |game|
+          game_info = { game_date: date }
+          teams = game.css('td.team-name:not([colspan])').map { |td| parse_data_name_from(td) }
+          game_info[:away_team], game_info[:home_team] = teams
+          scores = game.css('.team-score').map { |td| td.at_css('span').content.to_i }
+          game_info[:away_score], game_info[:home_score] = scores
+          game_info
+        end
+      end
+
+      # parsing helpers
+
+      def parse_data_name_from(container)
+        if container.at_css('a')
+          link = container.at_css('a')['href']
+          self.data_name_from(link)
+        else
+          if container.at_css('div')
+            name = container.at_css('div text()').content
+          elsif container.at_css('span')
+            name = container.at_css('span text()').content
+          else
+            name = container.at_css('text()').content
+          end
+          ESPN::DATA_NAME_EXCEPTIONS[ ESPN.dasherize(name) ]
+        end
+      end
+
+      def data_name_from(link)
+        encoded_link = URI::encode(link.strip)
+        query = URI::parse(encoded_link).query
+        if query
+          CGI::parse(query)['team'].first
+        else
+          link.split('/')[-2]
+        end
+      end
+
+    end
+  end
+end

data/lib/espn_scraper/teams.rb

@@ -0,0 +1,78 @@
+module ESPN
+  class << self
+    
+    def leagues
+      @leagues || %w(nfl mlb nba nhl ncf ncb)
+    end
+    
+    def leagues=(leagues)
+      @leagues = leagues
+    end
+    
+    def get_divisions
+      divisions = {}
+      leagues.each do |league|
+        divisions[league] = get_divisions_in(league)
+      end
+      divisions
+    end
+    
+    def get_divisions_in(league)
+      get_divs(league).map do |div|
+        name = parse_div_name(div)
+        { name: name, data_name: div_data_name(name) }
+      end
+    end
+
+    def get_conferences_in_ncb
+      get_ncb_conferences.map do |element|
+        name = element.content
+        data_name = $1 if element.children[0].attributes['href'].value =~ /confId=(\d+)/
+        { name: name, data_name: data_name }
+      end
+    end
+    
+    def get_teams_in(league)
+      divisions = {}
+	    get_divs(league.to_s.downcase).each do |division|
+        key = div_data_name parse_div_name(division)
+        divisions[key] = division.css('.mod-content li').map do |team|
+          team_elem = team.at_css('h5 a.bi')
+          team_name = team_elem.content
+          data_name, slug = team_elem['href'].split('/').last(2)
+          
+          slug.sub! dasherize(team_name), ''
+          team_name.sub! /\(.*\)/, ''
+          
+          name_adds = slug.split('-').reject(&:empty?).each(&:capitalize!)
+          name = name_adds.unshift(team_name).join(' ').strip.gsub(/\s+/, ' ')
+          { name: name, data_name: data_name }
+        end
+      end
+      divisions
+    end
+    
+    
+    
+    private
+    
+    
+    
+    def get_divs(league)
+      self.get(league, 'teams').css('.mod-teams-list-medium')
+    end
+
+    def get_ncb_conferences
+      self.get('ncb', 'conferences').css('.mod-content h5')
+    end
+    
+    def parse_div_name(div)
+      div.at_css('.mod-header h4 text()').content
+    end
+    
+    def div_data_name(div_name)
+      dasherize div_name.gsub(/division/i, '')
+    end
+
+  end
+end

data/lib/espn_scraper/version.rb

@@ -0,0 +1,3 @@
+module ESPN
+  VERSION = '1.3.1'
+end

data/test/espn_scraper_test/divisions_test.rb

@@ -0,0 +1,35 @@
+require 'test_helper'
+
+class DivisionsTest < EspnTest
+  
+  test 'nfl afc north' do
+    nfl = ESPN.get_divisions['nfl']
+    assert nfl.include?({ name: 'AFC North', data_name: 'afc-north' })
+  end
+  
+  test 'mlb al west' do
+    mlb = ESPN.get_divisions['mlb']
+    assert mlb.include?({ name: 'AL West', data_name: 'al-west' })
+  end
+  
+  test 'nba central' do
+    nba = ESPN.get_divisions['nba']
+    assert nba.include?({ name: 'Central', data_name: 'central' })
+  end
+  
+  test 'nhl pacific division' do
+    nhl = ESPN.get_divisions['nhl']
+    assert nhl.include?({ name: 'Pacific Division', data_name: 'pacific' })
+  end
+  
+  test 'ncf conference usa' do
+    ncf = ESPN.get_divisions['ncf']
+    assert ncf.include?({ name: 'Conference USA', data_name: 'conference-usa' })
+  end
+  
+  test 'ncb' do
+    ncb = ESPN.get_divisions['ncb']
+    assert ncb.include?({ name: 'ACC', data_name: 'acc' })
+  end
+  
+end

data/test/espn_scraper_test/espn_test.rb

@@ -0,0 +1,34 @@
+require 'test_helper'
+
+class BoilerplateTest < EspnTest
+  
+  test 'espn is up' do
+    assert ESPN.responding?
+    assert !ESPN.down?
+  end
+	
+  test 'paths are working' do
+    assert_equal 'http://scores.espn.go.com', ESPN.url('scores')
+    assert_equal 'http://espn.go.com/nba/teams', ESPN.url('nba', 'teams')
+  end
+  
+  test 'error message works' do
+    assert_raises(ArgumentError) do
+      ESPN.get('bad-api-keyword')
+    end
+  end
+	
+  test 'get pages is working' do
+    assert ESPN.get('scores')
+  end
+	
+  test 'dasherize strings' do
+    assert_equal 'string-is-dashed', ESPN.send(:dasherize, 'String is dashed')
+  end
+  
+  test 'leagues' do
+    leagues = 'nfl mlb nba nhl ncf ncb'.split
+    assert_equal leagues, ESPN.leagues
+  end
+
+end

data/test/espn_scraper_test/mlb_test.rb

@@ -0,0 +1,40 @@
+require 'test_helper'
+
+class MlbTest < EspnTest
+  
+  test 'mlb august 13th 2012 yankees beat rangers' do
+    starts_at = DateTime.parse('2012-08-13T23:00:00+00:00')
+    expected = {
+      league: 'mlb',
+      game_date: starts_at,
+      home_team: 'nyy',
+      home_score: 8,
+      away_team: 'tex',
+      away_score: 2
+    }
+    scores = ESPN.get_mlb_scores(starts_at.to_date)
+    assert scores.include?(expected), 'A known MLB final score cannot be found'
+  end
+
+  test 'mlb april 18th 2015 blue jays beat braves in extra innings' do
+    starts_at = DateTime.parse('2015-04-18T17:07:00+00:00')
+    expected = {
+        league: 'mlb',
+        game_date: starts_at,
+        home_team: 'tor',
+        home_score: 6,
+        away_team: 'atl',
+        away_score: 5
+    }
+    scores = ESPN.get_mlb_scores(starts_at.to_date)
+    assert scores.include?(expected), 'A known MLB final score cannot be found'
+  end
+  
+  test 'random mlb days' do
+    random_days.each do |day|
+      scores = ESPN.get_mlb_scores(day)
+      assert all_names_present?(scores), "Error on #{day} for mlb"
+    end
+  end
+  
+end

data/test/espn_scraper_test/nba_test.rb

@@ -0,0 +1,26 @@
+require 'test_helper'
+
+class NbaTest < EspnTest
+  
+  test 'nba december 25th celtics beat nets' do
+    day = Date.parse('Dec 25, 2012')
+    expected = {
+      league: 'nba',
+      game_date: DateTime.parse('2012-12-25T17:00:00+00:00'),
+      home_team: 'bkn',
+      home_score: 76,
+      away_team: 'bos',
+      away_score: 93
+    }
+    scores = ESPN.get_nba_scores(day)
+    assert_equal expected, scores.first
+  end
+  
+  test 'random nba days' do
+    random_days.each do |day|
+      scores = ESPN.get_nba_scores(day)
+      assert all_names_present?(scores), "Error on #{day} for nba"
+    end
+  end
+  
+end

data/test/espn_scraper_test/ncb_test.rb

@@ -0,0 +1,30 @@
+require 'test_helper'
+
+class NcbTest < EspnTest
+  
+  test 'mens college basketball march 15th murray state beats colorado state' do
+    day = Date.parse('2012-03-15')
+    league = 'mens-college-basketball'
+    expected = {
+      league: league,
+      game_date: day,
+      home_team: 'murr',
+      home_score: 58,
+      away_team: 'csu',
+      away_score: 41
+    }
+    mountain_west_conf = ESPN.get_conferences_in_ncb.select {|c| c[:name] == 'Mountain West' }.first
+    scores = ESPN.get_college_basketball_scores(day, mountain_west_conf[:data_name])
+    assert_equal expected, scores.first
+  end
+  
+
+  test 'random ncb dates' do
+    random_days.each do |day|
+      random_conf = ESPN.get_conferences_in_ncb.sample
+      scores = ESPN.get_college_basketball_scores(day, random_conf[:data_name])
+      assert all_names_present?(scores), "Error on #{day} for college basketball"
+    end
+  end
+  
+end

data/test/espn_scraper_test/ncf_test.rb

@@ -0,0 +1,26 @@
+require 'test_helper'
+
+class NcfTest < EspnTest
+  
+  test 'college football 2012 week 9 regular season' do
+    starts_at = DateTime.parse('2012-10-23T00:00:00+00:00')
+    expected = {
+      league: 'college-football',
+      game_date: starts_at,
+      home_team: '309',
+      home_score: 27,
+      away_team: '2032',
+      away_score: 50
+    }
+    scores = ESPN.get_college_football_scores(2012, 9)
+    assert_equal expected, scores.first
+  end
+  
+  test 'looking for a break' do
+    random_days.each do |week|
+      scores = ESPN.get_college_football_scores(2012, week)
+      assert all_names_present?(scores), "!!! error in week #{week}"
+    end
+  end
+
+end

data/test/espn_scraper_test/nfl_test.rb

@@ -0,0 +1,46 @@
+require 'test_helper'
+
+class NflTest < EspnTest
+  
+  test 'data names are fixed' do
+    scores = ESPN.get_nfl_scores(2012, 2)
+    assert scores.any?, 'scores parsing failed'
+    assert_equal 'gb', scores.first[:home_team]
+  end
+  
+  test 'nfl 2012 week 8 regular season' do
+    starts_at = DateTime.parse('2012-10-26T00:20Z')
+    expected = {
+      league: 'nfl',
+      game_date: starts_at,
+      home_team: 'min',
+      home_score: 17,
+      away_team: 'tb',
+      away_score: 36
+    }
+    scores = ESPN.get_nfl_scores(2012, 8)
+    assert_equal expected, scores.first
+  end
+  
+  test 'nfl 2012 week 7 regular season' do
+    starts_at = DateTime.parse('2012-10-23T00:30:00+00:00')
+    expected = {
+      league: 'nfl',
+      game_date: starts_at,
+      home_team: 'chi',
+      home_score: 13,
+      away_team: 'det',
+      away_score: 7
+    }
+    scores = ESPN.get_nfl_scores(2012, 7)
+    assert_equal expected, scores.last
+  end
+  
+  test 'looking for a break' do
+    random_weeks.each do |week|
+      scores = ESPN.get_nfl_scores(2012, week)
+      assert all_names_present?(scores), "!!! error in week #{week}"
+    end
+  end
+
+end

data/test/espn_scraper_test/nhl_test.rb

@@ -0,0 +1,26 @@
+require 'test_helper'
+
+class NhlTest < EspnTest
+  
+  test 'nhl rangers beat bruins on valentines day' do
+    day = Date.parse('Feb 14, 2012')
+    expected = {
+      league: 'nhl',
+      game_date: day,
+      home_team: 'bos',
+      home_score: 0,
+      away_team: 'nyr',
+      away_score: 3
+    }
+    scores = ESPN.get_nhl_scores(day)
+    assert_equal expected, scores.first
+  end
+  
+  test 'random nhl dates' do
+    random_days.each do |day|
+      scores = ESPN.get_nhl_scores(day)
+      assert all_names_present?(scores), "Error on #{day} for nhl"
+    end
+  end
+    
+end

data/test/espn_scraper_test/teams_test.rb

@@ -0,0 +1,79 @@
+require 'test_helper'
+
+class TeamsTest < EspnTest
+  
+  test 'scrape nfl teams' do
+    divisions = ESPN.get_teams_in('nfl')
+    assert_equal 8, divisions.count
+    divisions.each do |name, teams|
+      assert_equal 4, teams.count
+    end
+    teams = divisions.values.flatten
+    assert_equal 32, teams.map{ |h| h[:name] }.uniq.count
+    assert_equal 32, teams.map{ |h| h[:data_name] }.uniq.count
+    assert divisions['nfc-west'].include?({ name: 'Seattle Seahawks', data_name: 'sea' })
+  end
+  
+  test 'scrape mlb teams' do
+    divisions = ESPN.get_teams_in('mlb')
+    assert_equal 6, divisions.count
+    divisions.each do |name, teams|
+      assert_equal 5, teams.count
+    end
+    teams = divisions.values.flatten
+    assert_equal 30, teams.map{ |h| h[:name] }.uniq.count
+    assert_equal 30, teams.map{ |h| h[:data_name] }.uniq.count
+    assert divisions['al-west'].include?({ name: 'Seattle Mariners', data_name: 'sea' })
+  end
+  
+  test 'scrape nba teams' do
+    divisions = ESPN.get_teams_in('nba')
+    assert_equal 6, divisions.count
+    divisions.each do |name, teams|
+      assert_equal 5, teams.count
+    end
+    teams = divisions.values.flatten
+    assert_equal 30, teams.map{ |h| h[:name] }.uniq.count
+    assert_equal 30, teams.map{ |h| h[:data_name] }.uniq.count
+    assert divisions['atlantic'].include?({ name: 'Toronto Raptors', data_name: 'tor' })
+  end
+  
+  test 'scrape nhl teams' do
+    divisions = ESPN.get_teams_in('nhl')
+    assert_equal 4, divisions.count
+    assert_equal 7, divisions['central'].count
+    assert_equal 8, divisions['atlantic'].count
+    teams = divisions.values.flatten
+    assert_equal 30, teams.map{ |h| h[:name] }.uniq.count
+    assert_equal 30, teams.map{ |h| h[:data_name] }.uniq.count
+    assert divisions['atlantic'].include?({ name: 'Montreal Canadiens', data_name: 'mtl' })
+  end
+  
+  test 'scrape ncaa football teams' do
+    divisions = ESPN.get_teams_in('college-football')
+    assert_equal 25, divisions.count
+    assert_equal 12, divisions['pac-12'].count
+
+    assert divisions['conference-usa'].include?({ name: 'Rice Owls', data_name: '242' })
+    assert divisions['meac'].include?({ name: 'Bethune-Cookman Wildcats', data_name: '2065' })
+    assert divisions['northeast'].include?({ name: 'St Francis Red Flash', data_name: '2598' })
+    assert divisions['swac'].include?({ name: 'Alabama A&M Bulldogs', data_name: '2010' })
+  end
+  
+  test 'scrape ncaa basketball teams' do
+    divisions = ESPN.get_teams_in('mens-college-basketball')
+    assert_equal 32, divisions.count
+    assert_equal 15, divisions['acc'].count
+    assert_equal 10, divisions['patriot-league'].count
+    
+    assert divisions['southland'].include?({ name: 'Texas A&M-CC Islanders', data_name: '357' })
+    assert divisions['atlantic-10'].include?({ name: "Saint Joe's Saint Joseph's Hawks", data_name: '2603' })
+  end
+
+  test 'scrape ncaa basketball conferences' do
+    conferences = ESPN.get_conferences_in_ncb
+    assert_equal 32, conferences.count
+    assert conferences.include?({ name: 'Mountain West', data_name: '44' })
+  end
+	
+end

data/test/test_helper.rb

@@ -0,0 +1,36 @@
+ERROR_CHECKS = 1
+
+require 'minitest/autorun'
+require 'espn_scraper'
+
+class EspnTest < Minitest::Test
+
+  class << self
+    
+    def test(test_name, &block)
+      define_method("test_: #{ test_name }", &block)
+    end
+    
+  end
+
+  def all_names_present?(ary)
+    ary.map do |obj|
+      h, a = obj[:home_team], obj[:away_team]
+      test = h.nil? || (h && h.empty?) || a.nil? || (a && a.empty?)
+      puts h, a if test
+      test
+    end.count(true) == 0
+  end
+
+  def whole_year
+    Date.today..Date.today.prev_year
+  end
+
+  def random_days(amount = ERROR_CHECKS)
+    whole_year.to_a.sample(amount)
+  end
+
+  def random_weeks(amount = ERROR_CHECKS)
+    (1..17).to_a.sample(amount)
+  end
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: espn_scraper
+version: !ruby/object:Gem::Version
+  version: 1.3.1
+platform: ruby
+authors:
+- aj0strow
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: a simple scraping api for espn stats and data
+email: alexander.ostrow@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- espn_scraper.gemspec
+- lib/espn_scraper.rb
+- lib/espn_scraper/boilerplate.rb
+- lib/espn_scraper/scores.rb
+- lib/espn_scraper/teams.rb
+- lib/espn_scraper/version.rb
+- test/espn_scraper_test/divisions_test.rb
+- test/espn_scraper_test/espn_test.rb
+- test/espn_scraper_test/mlb_test.rb
+- test/espn_scraper_test/nba_test.rb
+- test/espn_scraper_test/ncb_test.rb
+- test/espn_scraper_test/ncf_test.rb
+- test/espn_scraper_test/nfl_test.rb
+- test/espn_scraper_test/nhl_test.rb
+- test/espn_scraper_test/teams_test.rb
+- test/test_helper.rb
+homepage: http://github.com/aj0strow/espn-scraper
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: ESPN Scraper
+test_files:
+- test/espn_scraper_test/divisions_test.rb
+- test/espn_scraper_test/espn_test.rb
+- test/espn_scraper_test/mlb_test.rb
+- test/espn_scraper_test/nba_test.rb
+- test/espn_scraper_test/ncb_test.rb
+- test/espn_scraper_test/ncf_test.rb
+- test/espn_scraper_test/nfl_test.rb
+- test/espn_scraper_test/nhl_test.rb
+- test/espn_scraper_test/teams_test.rb
+- test/test_helper.rb