steam_scraper 1.1.7 → 1.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2b682929fb38924b8ad5b928ad27f98b4cc666f1
-  data.tar.gz: b1b40cebd77d5202362d006af504e47ed5f8b473
+  metadata.gz: 3eb0595031ddaac9c22e6501cc1aecba8d22a8bf
+  data.tar.gz: 399dbd713d76fb4f28b3a81bd20ee2fe978facf1
 SHA512:
-  metadata.gz: d88b7769ea0294bb94ac839f8597c4f461f3d9eb6bb6cf9af26c6cde55b2c8cbb931a3016c44c60417669962228b03f12ccbf9aec92f9d96baeca7421e79f2e2
-  data.tar.gz: edf0a0f2980c07e5802b4823392fc5c40e56e85dbe328299938948e22b739d1144ba9a52f548322a1cc67d161aa1617a3c2f0499e694316a70c5b5843275b5db
+  metadata.gz: d6827c7c7ba1c5127ef85ff87065cb8061ec4e7166ef856bc245f018a68ec453bbb4271fc398e9b7daf5f351d06aac0b512c5cfb66910e91206f06ee5057d447
+  data.tar.gz: a70456ea7d52266e01025b25b46e0ef0142f75f8ddacfcdb02523dbe6471f02e101af3f918e2def8a04189a00858d8e5ec888e83d182330e1d6799b859b85767

data/lib/steam_scraper.rb

@@ -13,6 +13,7 @@ module SteamScraper
 
     def scrape(first_page = 1, last_page = nil)
       scraped_game_list = @game_list_scraper.scrape(first_page, last_page)
+      puts 'Initial scrape complete.  Found ' + scraped_game_list.length.to_s + ' games.'
       final_game_list = @game_page_scraper.scrape(scraped_game_list)
 
       final_game_list

data/lib/steam_scraper/game_list_scraper.rb

@@ -5,10 +5,16 @@ class GameListScraper
   def initialize(*_args)
     @game_list = []
     @page_retriever = PageRetriever.new
+    init_last_page_num
+  end
+
+  def init_last_page_num
     current_page_contents = get_page_contents(site + 1.to_s)
-    pagination_contents = current_page_contents.xpath("//div[contains(@class, 'search_pagination_right')]") .text
-    last_page_number = pagination_contents.scan(/(\d+)/i) .flatten.last
-    @last_page_num = last_page_number.to_i
+    @last_page_num = current_page_contents.xpath("//div[contains(@class, 'search_pagination_right')]")
+                                          .text.scan(/(\d+)/i).flatten.last.to_i
+  rescue StandardError => e
+    puts 'Could not connect to Steam Store'
+    raise e
   end
 
   def site
@@ -22,13 +28,15 @@ class GameListScraper
   def search_results(page_number)
     current_page_contents = get_page_contents(site + page_number.to_s)
     current_page_contents.xpath("//div[@id='search_result_container']/div/a")
+  rescue
+    nil
   end
 
   def scrape(first_page = 1, last_page = nil)
     last_page ||= @last_page_num
-    # scrape each search page
     result = Parallel.map(first_page..last_page,
-                          progress: 'Scraping Steam Store pages ' + [first_page..last_page].join(' to ')) do |page|
+                          progress: 'Scraping Steam Store pages ' + [first_page..last_page].join(' to '),
+                          in_processes: 8) do |page|
       items_on_page = search_results(page)
       scrape_page(items_on_page)
     end
@@ -40,7 +48,6 @@ class GameListScraper
     current_page.each do |entry|
       entries.push(scrape_entry(entry))
     end
-
     entries
   end
 
@@ -54,22 +61,13 @@ class GameListScraper
 
   def scrape_price(entry)
     entry.xpath(".//div[contains(@class, 'search_price')
-                and not(contains(@class, 'search_price_discount_combined'))]")
-         .text
-         .strip
-         .split('$')
-         .last
+                and not(contains(@class, 'search_price_discount_combined'))]").text.strip.split('$').last
   end
 
   def scrape_release_date(entry)
-    date_node_text = entry.xpath(".//div[contains(@class, 'search_released')]").text
-    date = nil
-    begin
-      date = Date.parse(date_node_text)
-    rescue ArgumentError
-    end
-
-    date
+    Date.parse(entry.xpath(".//div[contains(@class, 'search_released')]").text)
+  rescue
+    nil
   end
 
   def scrape_platforms(entry)
@@ -78,7 +76,6 @@ class GameListScraper
     platforms.push('macOS') unless entry.xpath(".//span[contains(@class, 'mac')]").empty?
     platforms.push('Linux') unless entry.xpath(".//span[contains(@class, 'linux')]").empty?
     platforms.push('Steamplay') unless entry.xpath(".//span[contains(@class, 'steamplay')]").empty?
-
     platforms
   end
 
@@ -90,7 +87,6 @@ class GameListScraper
     node = entry.xpath(".//span[contains(@class, 'search_review_summary')]")
     result = nil
     result = node.attribute('data-store-tooltip').value unless node.empty?
-
     result
   end
 
@@ -98,7 +94,6 @@ class GameListScraper
     review_string = get_review_contents(entry)
     matches = /.*(\d\d)[%]/i.match(review_string)
     review_percentage = matches[1] unless matches.nil?
-
     review_percentage
   end
 
@@ -106,7 +101,6 @@ class GameListScraper
     review_string = get_review_contents(entry)
     matches = /.*\d\d[%] of the ([0-9,]*) user/i.match(review_string)
     num_reviews = matches[1] unless matches.nil?
-
     num_reviews
   end
 

data/lib/steam_scraper/game_page_scraper.rb

@@ -8,7 +8,9 @@ class GamePageScraper
   end
 
   def scrape(games_hash)
-    result = Parallel.map(games_hash, progress: 'Scraping additional per game data') do |game|
+    result = Parallel.map(games_hash,
+                          progress: 'Scraping additional per game data',
+                          in_processes: 8) do |game|
       url = game[:url]
       scrape_game(game, url) unless url.nil?
     end

data/lib/steam_scraper/page_retriever.rb

@@ -7,8 +7,7 @@ class PageRetriever
   def retrieve(url)
     page_contents = HTTParty.get(url)
     Nokogiri::HTML(page_contents)
-  rescue Error => e
-    puts e
+  rescue
     nil
   end
 end

data/lib/steam_scraper/version.rb

@@ -1,3 +1,3 @@
 module SteamScraper
-  VERSION = '1.1.7'.freeze
+  VERSION = '1.1.8'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: steam_scraper
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.1.8
 platform: ruby
 authors:
 - Robert Gardner
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-09-11 00:00:00.000000000 Z
+date: 2016-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler