steam_scraper 1.1.6 → 1.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d35d39b0ec4e9ee65403d6f213f02818db2bc493
-  data.tar.gz: ff9f3ec8ec11b965da0a3d8cf3d958f99536af83
+  metadata.gz: 2b682929fb38924b8ad5b928ad27f98b4cc666f1
+  data.tar.gz: b1b40cebd77d5202362d006af504e47ed5f8b473
 SHA512:
-  metadata.gz: 1c464a5c2d986221968cb6cbc1ab6ecfcab3c6e11a93a732eda686ef09722d1585e5649c6e399a53ee4486939a5c83e5c65ddcf5c87037c9bc2cf1840e91a8cd
-  data.tar.gz: c98b76070ddefc042c4871b328f27b6a7e416b8aaada35b451eedc16c1cfb0e4f1d475d14523b7bfa3ded45c9563c4b29e608ccd5210f9f93da2616032d6ffeb
+  metadata.gz: d88b7769ea0294bb94ac839f8597c4f461f3d9eb6bb6cf9af26c6cde55b2c8cbb931a3016c44c60417669962228b03f12ccbf9aec92f9d96baeca7421e79f2e2
+  data.tar.gz: edf0a0f2980c07e5802b4823392fc5c40e56e85dbe328299938948e22b739d1144ba9a52f548322a1cc67d161aa1617a3c2f0499e694316a70c5b5843275b5db

data/lib/steam_scraper/game_list_scraper.rb

@@ -1,5 +1,5 @@
 require_relative './page_retriever.rb'
-
+require 'parallel'
 # Core scraping class
 class GameListScraper
   def initialize(*_args)
@@ -27,19 +27,21 @@ class GameListScraper
   def scrape(first_page = 1, last_page = nil)
     last_page ||= @last_page_num
     # scrape each search page
-    puts 'Scraping Steam Store pages ' + [first_page..last_page].join(' to ')
-    (first_page..last_page).each do |page|
-      puts 'Scraping Page ' + page.to_s
+    result = Parallel.map(first_page..last_page,
+                          progress: 'Scraping Steam Store pages ' + [first_page..last_page].join(' to ')) do |page|
       items_on_page = search_results(page)
       scrape_page(items_on_page)
     end
-    @game_list
+    @game_list.push(result).flatten!
   end
 
   def scrape_page(current_page)
+    entries = []
     current_page.each do |entry|
-      scrape_entry(entry)
+      entries.push(scrape_entry(entry))
     end
+
+    entries
   end
 
   def scrape_url(entry)
@@ -65,7 +67,6 @@ class GameListScraper
     begin
       date = Date.parse(date_node_text)
     rescue ArgumentError
-      puts 'Invalid date found.  Probably in the future.  Date set to nil'
     end
 
     date
@@ -119,6 +120,6 @@ class GameListScraper
     new_game[:icon_url] = scrape_icon_url(entry)
     new_game[:review_score] = scrape_review_score(entry)
     new_game[:number_of_reviews] = scrape_number_of_reviews(entry)
-    @game_list.push new_game
+    new_game
   end
 end

data/lib/steam_scraper/game_page_scraper.rb

@@ -1,4 +1,5 @@
 require_relative './page_retriever.rb'
+require 'Parallel'
 
 # Class that scrapes a games actual page
 class GamePageScraper
@@ -7,16 +8,21 @@ class GamePageScraper
   end
 
   def scrape(games_hash)
-    games_hash.map! do |game|
+    result = Parallel.map(games_hash, progress: 'Scraping additional per game data') do |game|
       url = game[:url]
-      scrape_game!(game, url) unless url.nil?
+      scrape_game(game, url) unless url.nil?
     end
-    games_hash
+    games_hash.push(result).flatten!
   end
 
-  def scrape_game!(game, url)
-    puts 'Scraping additional data for ' + game[:name]
+  def scrape_game(game, url)
     page_contents = get_page_contents(url)
+    game = scrape_game_with_valid_contents(game, page_contents) unless page_contents.nil?
+
+    game
+  end
+
+  def scrape_game_with_valid_contents(game, page_contents)
     game[:metacritic] = scrape_metacritic(page_contents)
     game[:tags] = scrape_tags(page_contents)
     game[:genres] = scrape_genres(page_contents)

data/lib/steam_scraper/page_retriever.rb

@@ -7,5 +7,8 @@ class PageRetriever
   def retrieve(url)
     page_contents = HTTParty.get(url)
     Nokogiri::HTML(page_contents)
+  rescue Error => e
+    puts e
+    nil
   end
 end

data/lib/steam_scraper/version.rb

@@ -1,3 +1,3 @@
 module SteamScraper
-  VERSION = '1.1.6'.freeze
+  VERSION = '1.1.7'.freeze
 end

data/steam_scraper.gemspec

@@ -24,6 +24,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'nokogiri', '~> 1.6.8'
   spec.add_development_dependency 'httparty', '~> 0.14.0'
+  spec.add_development_dependency 'parallel', '~> 1.9.0'
+  spec.add_development_dependency 'ruby-progressbar', '~> 1.8.1'
+  spec.add_runtime_dependency 'ruby-progressbar', '~> 1.8.1'
+  spec.add_runtime_dependency 'parallel', '~> 1.9.0'
   spec.add_runtime_dependency 'nokogiri', '~> 1.6.8'
   spec.add_runtime_dependency 'httparty', '~> 0.14.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: steam_scraper
 version: !ruby/object:Gem::Version
-  version: 1.1.6
+  version: 1.1.7
 platform: ruby
 authors:
 - Robert Gardner
@@ -80,6 +80,62 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.14.0
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.9.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.9.0
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.9.0
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement