vegas_insider_scraper 0.2.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d913d3440b91317653dc9fd161e9b9a5474cc7dfd80012682c74c9f6a2bb9e6
-  data.tar.gz: 43b029b0436db2dcf18bf472cf9784521b5546b8aaf383ab4a10b9fa2c36d618
+  metadata.gz: 69a23eb646b9085eac798a97bc6cf733f98df54c21b3011371e4f4fa9f5a2252
+  data.tar.gz: bd8fbd9fccc62b419800a757191e232120657cfa02b48d9397dd34ff9479da3b
 SHA512:
-  metadata.gz: 120c346e644548e23676daa0f4a264a75bcf9a12396e1510272239f96a4da832f41f2e8b460b5d2bb65cfa8eba944a7eaeee21c6e0251d4a6fdd31a38418c755
-  data.tar.gz: a2ed206492830f2b4e11ba348ef443ea5f608efa719a4764e0b06b2dfcb30d29a4cd170bc3690f8bedff769a1c33dab016cbc9f15a9f575907bcd95ecf3bb2a9
+  metadata.gz: 882881640914c68decd5b24d49343a73b4c2cb011d98f6bb2490866143dc80a28b5fca6bf069ca113ea0cf91298332e14bfb278d738ebca4d71bc8fd2b3d742f
+  data.tar.gz: 7f933d97c7ccabfc07e1186ea22ea27bc132c4ef7e6a0ccc55818ea80ee6e1192f73e3f45b71230877227dce5f4526dfcf015a98b763d577058697c8fbf6ca2d

data/lib/vegas_insider_scraper/mlb.rb

@@ -0,0 +1,15 @@
+
+module VegasInsiderScraper
+  class MLB < ScraperLeague
+    def initialize
+      @vegas_sport_identifier = 'mlb'
+      super
+      @moneyline_sport = true
+    end
+
+    def current_games
+      @current_games ||= get_lines(["http://www.vegasinsider.com/#{vegas_sport_identifier}/odds/las-vegas/run/", 
+        "http://www.vegasinsider.com/#{vegas_sport_identifier}/odds/las-vegas/"])
+    end
+  end
+end

data/lib/vegas_insider_scraper/nba.rb

@@ -0,0 +1,9 @@
+
+module VegasInsiderScraper
+  class NBA < ScraperLeague
+    def initialize
+      @vegas_sport_identifier = 'nba'
+      super
+    end
+  end
+end

data/lib/vegas_insider_scraper/ncaabb.rb

@@ -0,0 +1,9 @@
+
+module VegasInsiderScraper
+  class NCAABB < ScraperLeague
+    def initialize
+      @vegas_sport_identifier = 'college-basketball'
+      super
+    end
+  end
+end

data/lib/vegas_insider_scraper/ncaafb.rb

@@ -0,0 +1,13 @@
+
+module VegasInsiderScraper
+  class NCAAFB < ScraperLeague
+    def initialize
+      @vegas_sport_identifier = 'college-football'
+      super
+    end
+
+    def teams
+      @teams ||= scrape_teams
+    end
+  end
+end

data/lib/vegas_insider_scraper/nfl.rb

@@ -0,0 +1,9 @@
+
+module VegasInsiderScraper
+  class NFL < ScraperLeague
+    def initialize
+      @vegas_sport_identifier = 'nfl'
+      super
+    end
+  end
+end

data/lib/vegas_insider_scraper/nhl.rb

@@ -0,0 +1,15 @@
+
+module VegasInsiderScraper
+  class NHL < ScraperLeague
+    def initialize
+      @vegas_sport_identifier = 'nhl'
+      super
+      @moneyline_sport = true
+    end
+
+    def current_games
+      @current_games ||= get_lines(["http://www.vegasinsider.com/#{vegas_sport_identifier}/odds/las-vegas/puck/", 
+        "http://www.vegasinsider.com/#{vegas_sport_identifier}/odds/las-vegas/"])
+    end
+  end
+end

data/lib/{sports → vegas_insider_scraper}/scraper_league.rb

@@ -2,12 +2,14 @@ require 'nokogiri'
 require 'open-uri'
 
 class ScraperLeague
-
-  attr_reader :sport_id
-  attr_reader :sport_name
+  attr_reader :vegas_sport_identifier
   attr_reader :moneyline_sport
   attr_reader :teams
 
+  def sport_name
+    self.class.name.split('::').last.downcase
+  end
+
   def initialize
     @moneyline_sport = false
   end
@@ -22,27 +24,31 @@ class ScraperLeague
 
   # Gets the upcoming/current games for the sport
   def current_games
-    @current_games ||= get_lines(["http://www.vegasinsider.com/#{sport_name}/odds/las-vegas/","http://www.vegasinsider.com/#{sport_name}/odds/las-vegas/money/"])
+    @current_games ||= get_lines(["http://www.vegasinsider.com/#{vegas_sport_identifier}/odds/las-vegas/","http://www.vegasinsider.com/#{vegas_sport_identifier}/odds/las-vegas/money/"])
   end
 
   # Gets all of the schedule and results for each team
   def team_schedules
-    @team_schedules ||= teams.map { |team|
-      puts " ### GETTING GAMES FOR:   #{team[:info][:full_name]}"
-      url = "http://www.vegasinsider.com/#{sport_name}/teams/team-page.cfm/team/#{team[:info][:identifier]}"
-      scrape_team_page(url, team[:info][:identifier])
-    }
+    @team_schedules ||= teams.map do |team|
+      puts " >>>>>> GETTING GAMES FOR:   #{team[:info][:full_name]}"
+      team_schedule_for(team[:info][:identifier])
+    end
   end
 
-  def live_scores
-    @live_scores = get_live_scores("https://web.archive.org/web/20170704205945/http://www.vegasinsider.com/mlb/scoreboard/")
-    nil
+  def schedule_for(identifier)
+    url = "http://www.vegasinsider.com/#{vegas_sport_identifier}/teams/team-page.cfm/team/#{identifier}"
+    scrape_team_page(url, identifier)
   end
 
+  # def live_scores
+  #   @live_scores = get_live_scores("https://web.archive.org/web/20170704205945/http://www.vegasinsider.com/mlb/scoreboard/")
+  #   nil
+  # end
+
   private
 
   def scrape_teams
-    url = "http://www.vegasinsider.com/#{sport_name}/teams/"
+    url = "http://www.vegasinsider.com/#{vegas_sport_identifier}/teams/"
     doc = Nokogiri::HTML(open(url)).at_css('.main-content-cell')
 
     doc.css('a').map do |team_link|
@@ -62,7 +68,7 @@ class ScraperLeague
   # Gets the teams and scrapes the records for the teams
   def scrape_standings
     standings_teams = []
-    url = "http://www.vegasinsider.com/#{sport_name}/standings/"
+    url = "http://www.vegasinsider.com/#{vegas_sport_identifier}/standings/"
     doc = Nokogiri::HTML(open(url)).at_css('.main-content-cell')
     teams_doc = Nokogiri::HTML(open(url.gsub('standings','teams'))).at_css('.main-content-cell')
 
@@ -71,7 +77,7 @@ class ScraperLeague
       next if conference_title.nil?
 
       table = conference.css('.viBodyBorderNorm table')[standings_table_index]
-      table = conference.css('.viBodyBorderNorm table')[2] if (conference_title.content == 'Conference USA' && sport_name == 'college-football')
+      table = conference.css('.viBodyBorderNorm table')[2] if (conference_title.content == 'Conference USA' && vegas_sport_identifier == 'college-football')
 
       if table
         table.css('tr').each_with_index do |row, index|
@@ -86,13 +92,13 @@ class ScraperLeague
   # Utility method for scraping standings
   # * gets the standings table class
   def standings_table_class
-    sport_name == "college-football" ? '.SLTables1' : 'table' 
+    vegas_sport_identifier == "college-football" ? '.SLTables1' : 'table' 
   end
 
   # Utility method for scraping standings
   # * gets the index of the table
   def standings_table_index
-    sport_name == "college-football" ? 1 : 0
+    vegas_sport_identifier == "college-football" ? 1 : 0
   end
 
   # Utility method for scraping standings
@@ -110,19 +116,20 @@ class ScraperLeague
   # Utility method for scraping standings
   # * is a college sport?
   def college_sport?
-    ['college-football','college-basketball'].include?(sport_name)
+    ['college-football','college-basketball'].include?(vegas_sport_identifier)
   end
 
   # Utility method for scraping standings
   # * scrapes a row of the standings, chooses a helper method based on the league
   def scrape_standings_row(row, grouping, teams_doc)
     team_shell = { info: {}, record: {} }
-    team = case sport_id
-    when 0 then ncaafb_standings_row_parser(row, team_shell, teams_doc)
-    when 1 then ncaabb_standings_row_parser(row, team_shell, teams_doc)
-    when 2 then nfl_standings_row_parser(row, team_shell)
-    when 3,4 then pro_standings_row_parser(row, team_shell)
-    when 5 then hockey_standings_row_parser(row, team_shell)
+
+    team = case vegas_sport_identifier
+    when 'college-football' then ncaafb_standings_row_parser(row, team_shell, teams_doc)
+    when 'college-basketball' then ncaabb_standings_row_parser(row, team_shell, teams_doc)
+    when 'nfl' then nfl_standings_row_parser(row, team_shell)
+    when 'mlb', 'nba' then pro_standings_row_parser(row, team_shell)
+    when 'nhl' then hockey_standings_row_parser(row, team_shell)
     end
     team[:grouping] = grouping
     team

data/lib/vegas_insider_scraper/soccer.rb

@@ -0,0 +1,90 @@
+
+module VegasInsiderScraper
+  class Soccer < ScraperLeague
+    def initialize
+      @vegas_sport_identifier = 'soccer'
+      super
+      @moneyline_sport = true
+    end
+
+    def current_games
+      @current_games ||= get_lines([
+        "http://www.vegasinsider.com/#{vegas_sport_identifier}/odds/las-vegas/spread",
+        "http://www.vegasinsider.com/#{vegas_sport_identifier}/odds/las-vegas/"
+      ])
+    end
+
+    def teams
+      []
+    end
+
+    def standings
+      []
+    end
+
+    def team_schedules
+      []
+    end
+
+    def live_scores
+      []
+    end
+
+    private
+
+    def get_lines(urls)
+      games = []
+
+      urls.each { |url|
+        is_first_url = games.empty?
+        doc = Nokogiri::HTML(open(url))
+        doc.css('.viBodyBorderNorm .frodds-data-tbl tr').each do |game_row|
+          game_cell = game_row.at_css('td:first-child')
+          teams = game_cell_parser(game_cell)
+          game = Game.new(home_team: teams[1], away_team: teams[0]) 
+
+          if game.teams_found?
+            game.update(time: get_game_time(game_cell))
+            game.update(doubleheader: doubleheader_id(game_row.next&.next&.at_css('td:first-child')&.content))
+            is_first_url ? (games.push game) : (game = game.find_equal(games))
+            game.update(vegas_info: get_line(get_odds(game_row)))
+
+          elsif is_first_url
+            last_game = games.last
+            if last_game then last_game.update(notes: (last_game.notes ? "#{last_game.notes} / " : '') + game_cell.content) end
+          end
+        end
+      }
+      games
+    end
+
+    def game_cell_parser(cell)
+      cell.css('b').map(&:content)
+    end
+
+    def get_odds(odds_element)
+      (odds_element.at_css('td:nth-child(3)')&.content || '')
+        .gsub(' ', '')
+        .gsub(/[[:space:]]/, '')
+        .gsub('½','.5')
+        .gsub('¼', '.25')
+        .gsub('¾', '.75')
+        .strip
+    end
+
+    def get_odds_inner_html(odds_element)
+      ''
+      # ((odds_element.at_css('td:nth-child(3) a'))&.inner_html || '').encode('utf-8').gsub(" ","").gsub("½",".5").strip
+    end
+
+    def get_line(odds_string)
+      odds_string = odds_string.gsub('PK', '-0')
+      result = matchdata_to_hash(RegularExpressions::SOCCER_MONEYLINE_ODDS.match(odds_string) || 
+                                 RegularExpressions::SOCCER_ODDS.match(odds_string))
+
+
+      result.each { |k,v| result[k] = result[k].to_s.to_f if result[k] }
+      get_home_and_away(result)
+    end
+  end
+end

data/lib/vegas_insider_scraper.rb

@@ -2,27 +2,51 @@ require 'nokogiri'
 require 'open-uri'
 # require 'byebug'
 
-require 'sports/scraper_league'
-require 'sports/ncaafb'
-require 'sports/ncaabb'
-require 'sports/nba'
-require 'sports/nfl'
-require 'sports/mlb'
-require 'sports/nhl'
-require 'sports/soccer'
-
-class VegasInsiderScraper
-  attr_reader :sports
-
-  SPORTS = [NCAAFB, NCAABB, NFL, NBA, MLB, NHL]
-
-  def initialize
-    @sports = SPORTS.map { |sport_class| sport_class.new }
+require 'vegas_insider_scraper/scraper_league'
+require 'vegas_insider_scraper/ncaafb'
+require 'vegas_insider_scraper/ncaabb'
+require 'vegas_insider_scraper/nba'
+require 'vegas_insider_scraper/nfl'
+require 'vegas_insider_scraper/mlb'
+require 'vegas_insider_scraper/nhl'
+require 'vegas_insider_scraper/soccer'
+
+module VegasInsiderScraper
+  SPORTS = [
+    VegasInsiderScraper::NCAAFB,
+    VegasInsiderScraper::NCAABB,
+    VegasInsiderScraper::NFL,
+    VegasInsiderScraper::NBA,
+    VegasInsiderScraper::MLB,
+    VegasInsiderScraper::NHL,
+    VegasInsiderScraper::Soccer
+  ]
+
+  def self.ncaafb
+    VegasInsiderScraper::NCAAFB.new
+  end
+
+  def self.ncaabb
+    VegasInsiderScraper::NCAABB.new
+  end
+
+  def self.nfl
+    VegasInsiderScraper::NFL.new
+  end
+
+  def self.nba
+    VegasInsiderScraper::NBA.new
+  end
+
+  def self.mlb
+    VegasInsiderScraper::MLB.new
+  end
+
+  def self.nhl
+    VegasInsiderScraper::NHL.new
   end
 
-  SPORTS.each do |sport_class|
-    define_method(sport_class.to_s.downcase) do
-      @sports.select { |sport| sport.class == sport_class }.first
-    end
+  def self.soccer
+    VegasInsiderScraper::Soccer.new
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vegas_insider_scraper
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Matt Reitz
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-08 00:00:00.000000000 Z
+date: 2019-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -38,22 +38,22 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: A gem to scrape the web for sports statistics, teams, betting lines,
-  records, and results!
+description: A web scraping gem for teams, betting lines, records, and results for
+  the NCAA (football and basketball), NFL, NBA, MLB, NHL, and Soccer
 email: reitz1994@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/sports/mlb.rb
-- lib/sports/nba.rb
-- lib/sports/ncaabb.rb
-- lib/sports/ncaafb.rb
-- lib/sports/nfl.rb
-- lib/sports/nhl.rb
-- lib/sports/scraper_league.rb
-- lib/sports/soccer.rb
 - lib/vegas_insider_scraper.rb
+- lib/vegas_insider_scraper/mlb.rb
+- lib/vegas_insider_scraper/nba.rb
+- lib/vegas_insider_scraper/ncaabb.rb
+- lib/vegas_insider_scraper/ncaafb.rb
+- lib/vegas_insider_scraper/nfl.rb
+- lib/vegas_insider_scraper/nhl.rb
+- lib/vegas_insider_scraper/scraper_league.rb
+- lib/vegas_insider_scraper/soccer.rb
 homepage: https://github.com/mreitz94/betting-tools-scraper
 licenses:
 - MIT
@@ -73,10 +73,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.9
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
-summary: A gem to scrape the web for sports statistics, teams, betting lines, records,
-  and results!
+summary: A web scraping gem for teams, betting lines, records, and results for the
+  NCAA (football and basketball), NFL, NBA, MLB, NHL, and Soccer
 test_files: []

data/lib/sports/mlb.rb

@@ -1,16 +0,0 @@
-
-class MLB < ScraperLeague
-
-  def initialize
-    @sport_id = 4
-    @sport_name = :mlb
-    super
-    @moneyline_sport = true
-  end
-
-  def current_games
-    @current_games ||= get_lines(["http://www.vegasinsider.com/#{sport_name}/odds/las-vegas/run/", 
-      "http://www.vegasinsider.com/#{sport_name}/odds/las-vegas/"])
-  end
-
-end

data/lib/sports/nba.rb

@@ -1,10 +0,0 @@
-
-class NBA < ScraperLeague
-
-  def initialize
-    @sport_id = 3
-    @sport_name = :nba
-    super
-  end
-
-end

data/lib/sports/ncaabb.rb

@@ -1,22 +0,0 @@
-
-class NCAABB < ScraperLeague
-
-  def initialize
-    @sport_id = 1
-    @sport_name = 'college-basketball'
-    super
-  end
-
-  # def get_nicknames
-  #   start_time = Time.now
-  #   num_successes = 0
-  #   Team.ncaabb_teams.each_with_index do |team, i|
-  #     url = "http://www.vegasinsider.com/college-basketball/teams/team-page.cfm/team/#{team.vegas_insider_identifier}"
-  #     nickname = Scraper.scrape_team_page_for_nickname(team.vegas_insider_identifier, url)
-  #     team.nickname = nickname
-  #     team.save
-  #   end
-  #   Time.now - start_time
-  # end
-
-end

data/lib/sports/ncaafb.rb

@@ -1,84 +0,0 @@
-
-class NCAAFB < ScraperLeague
-
-  def initialize
-    @sport_id = 0
-    @sport_name = 'college-football'
-    super
-  end
-
-  def teams
-    @teams ||= scrape_teams
-  end
-
-  # def get_nicknames
-  #   start_time = Time.now
-  #   Team.where(sport_id: 0).each_with_index do |team, i|
-  #     next if team.nickname
-  #     url = "http://www.vegasinsider.com/college-football/teams/team-page.cfm/team/#{team.vegas_insider_identifier}"
-  #     nickname = Scraper.scrape_team_page_for_nickname(team.vegas_insider_identifier, url)
-  #     team.nickname = nickname
-  #     team.save
-  #   end
-  #   Time.now - start_time
-  # end
-
-  # def get_locations
-  #   start_time = Time.now
-  #   Team.where(sport_id: 0, custom_team_flag: 1).each_with_index do |team, i|
-  #     team.location = nil
-  #     team.save
-  #   end
-  #   Time.now - start_time
-  # end
-
-  # def scrape_custom_team_page_for_location(vegas_identifier, url)
-  #   doc = Nokogiri::HTML(open(url))
-  #   title = doc.at_css('h1.page_title').content.gsub(' Team Page', '')
-  #   return title
-  # end
-
-  # def remove_nickname_from_location
-  #   start_time = Time.now
-  #   Team.where(sport_id: 0).each_with_index do |team, i|
-  #     puts team.location
-  #     puts team.location.gsub(" #{team.nickname}", '')
-  #   end
-  #   Time.now - start_time
-  # end
-
-  # def scrape_fcs_teams
-  #   url = 'http://www.vegasinsider.com/college-football/teams/'
-  #   doc = Nokogiri::HTML(open(url))
-
-  #   current_conference = nil
-  #   fcs = []
-
-  #   doc.css('.main-content-cell table table table').each_with_index do |col,i|
-  #     col.css('tr').each do |row|
-  #       new_conference = row.at_css('td.viSubHeader1')
-
-  #       if new_conference
-  #         current_conference = new_conference.content
-  #       else
-  #         team = row.at_css('a')
-  #         if team 
-  #           team_formatted = { 
-  #             team_name: team.content,
-  #             team_url_id: team_url_parser(team.attribute('href')),
-  #             conference: current_conference,
-  #             league: sport_id
-  #           }
-  #           puts team_formatted
-  #           fcs.push team_formatted
-  #         end
-  #       end
-  #     end
-  #   end
-
-  #   Team.save_teams(fcs)
-  #   return true
-
-  # end
-
-end

data/lib/sports/nfl.rb

@@ -1,10 +0,0 @@
-
-class NFL < ScraperLeague
-
-  def initialize
-    @sport_id = 2
-    @sport_name = :nfl
-    super
-  end
-
-end

data/lib/sports/nhl.rb

@@ -1,15 +0,0 @@
-
-class NHL < ScraperLeague
-
-  def initialize
-    @sport_id = 5
-    @sport_name = :nhl
-    super
-    @moneyline_sport = true
-  end
-
-  def current_games
-    @current_games ||= get_lines(["http://www.vegasinsider.com/#{sport_name}/odds/las-vegas/puck/", 
-      "http://www.vegasinsider.com/#{sport_name}/odds/las-vegas/"])
-  end
-end

data/lib/sports/soccer.rb

@@ -1,89 +0,0 @@
-
-class Soccer < ScraperLeague
-  def initialize
-    @sport_id = 9
-    @sport_name = :soccer
-    super
-    @moneyline_sport = true
-  end
-
-  def current_games
-    @current_games ||= get_lines([
-      "http://www.vegasinsider.com/#{sport_name}/odds/las-vegas/spread",
-      "http://www.vegasinsider.com/#{sport_name}/odds/las-vegas/"
-    ])
-  end
-
-  def teams
-    []
-  end
-
-  def standings
-    []
-  end
-
-  def team_schedules
-    []
-  end
-
-  def live_scores
-    []
-  end
-
-  private
-
-  def get_lines(urls)
-    games = []
-
-    urls.each { |url|
-      is_first_url = games.empty?
-      doc = Nokogiri::HTML(open(url))
-      doc.css('.viBodyBorderNorm .frodds-data-tbl tr').each do |game_row|
-        game_cell = game_row.at_css('td:first-child')
-        teams = game_cell_parser(game_cell)
-        game = Game.new(home_team: teams[1], away_team: teams[0]) 
-
-        if game.teams_found?
-          game.update(time: get_game_time(game_cell))
-          game.update(doubleheader: doubleheader_id(game_row.next&.next&.at_css('td:first-child')&.content))
-          is_first_url ? (games.push game) : (game = game.find_equal(games))
-          game.update(vegas_info: get_line(get_odds(game_row)))
-
-        elsif is_first_url
-          last_game = games.last
-          if last_game then last_game.update(notes: (last_game.notes ? "#{last_game.notes} / " : '') + game_cell.content) end
-        end
-      end
-    }
-    games
-  end
-
-  def game_cell_parser(cell)
-    cell.css('b').map(&:content)
-  end
-
-  def get_odds(odds_element)
-    (odds_element.at_css('td:nth-child(3)')&.content || '')
-      .gsub(' ', '')
-      .gsub(/[[:space:]]/, '')
-      .gsub('½','.5')
-      .gsub('¼', '.25')
-      .gsub('¾', '.75')
-      .strip
-  end
-
-  def get_odds_inner_html(odds_element)
-    ''
-    # ((odds_element.at_css('td:nth-child(3) a'))&.inner_html || '').encode('utf-8').gsub(" ","").gsub("½",".5").strip
-  end
-
-  def get_line(odds_string)
-    odds_string = odds_string.gsub('PK', '-0')
-    result = matchdata_to_hash(RegularExpressions::SOCCER_MONEYLINE_ODDS.match(odds_string) || 
-                               RegularExpressions::SOCCER_ODDS.match(odds_string))
-
-
-    result.each { |k,v| result[k] = result[k].to_s.to_f if result[k] }
-    get_home_and_away(result)
-  end
-end