steam_scraper 1.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 129284f963ebd372b9afba769e1da4dc4e541f33
+  data.tar.gz: 1da952ef7ac1b425d837d79cac0a14cfcff76695
+SHA512:
+  metadata.gz: ce43a448f7707cfb5036b4815d7f4c66d19e5561aa65d4bfb8a302337ae20612083ee65a87d6828d931028168ca3dfcd8374f1f0b6ca6cf072ca0917b1648a37
+  data.tar.gz: 230a3c1e53a67efc3e6476e25b2807c88e6d7ec1f0f498089e9c0feccdfc83caef79e6c4153785ae73dca6fd56dd301eee330ca5412de889adcf2da33c51b14e

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rubocop.yml

@@ -0,0 +1,11 @@
+Style/IfUnlessModifier:
+  MaxLineLength: 120
+
+Style/WhileUntilModifier:
+  MaxLineLength: 120
+
+Metrics/LineLength:
+  Max: 120
+
+Metrics/AbcSize:
+  Max: 99999

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in steam_scraper.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Robert Gardner
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,49 @@
+# SteamScraper
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'steam_scraper'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install steam_scraper
+
+## Usage
+
+To scrape all pages from the steam store:
+
+```Ruby
+scraper = SteamScraper.new
+results = scraper.scrape
+```
+
+To scrape a range of pages from the steam store:
+
+```Ruby
+scraper = SteamScraper.new
+results = scraper.scrape(first_page, last_page)
+```
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ShriekBob/steam_scraper.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler/gem_tasks'
+task default: :spec

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'steam_scraper'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require 'pry'
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/steam_scraper/game_list_scraper.rb

@@ -0,0 +1,116 @@
+require_relative './page_retriever.rb'
+
+# Core scraping class
+class GameListScraper
+  def initialize(*_args)
+    @game_list = []
+    @page_retriever = PageRetriever.new
+    current_page_contents = get_page_contents(site + 1.to_s)
+    pagination_contents = current_page_contents.xpath("//div[contains(@class, 'search_pagination_right')]") .text
+    last_page_number = pagination_contents.scan(/(\d+)/i) .flatten.last
+    @last_page_num = last_page_number.to_i
+  end
+
+  def site
+    'http://store.steampowered.com/search?page='
+  end
+
+  def get_page_contents(url)
+    @page_retriever.retrieve(url)
+  end
+
+  def search_results(page_number)
+    current_page_contents = get_page_contents(site + page_number.to_s)
+    current_page_contents.xpath("//div[@id='search_result_container']/div/a")
+  end
+
+  def scrape(first_page = 1, last_page = nil)
+    last_page ||= @last_page_num
+    # scrape each search page
+    puts 'Scraping Steam Store pages ' + [first_page..last_page].join(' to ')
+    (first_page..last_page).each do |page|
+      puts 'Scraping Page ' + page.to_s
+      items_on_page = search_results(site + page.to_s)
+      scrape_page(items_on_page)
+      puts 'Page Complete'
+    end
+    puts 'Scraping Complete'
+    @game_list
+  end
+
+  def scrape_page(current_page)
+    current_page.each do |entry|
+      scrape_entry(entry)
+    end
+  end
+
+  def scrape_url(entry)
+    entry.attribute('href').value
+  end
+
+  def scrape_name(entry)
+    entry.xpath(".//span[@class='title']").text
+  end
+
+  def scrape_price(entry)
+    entry.xpath(".//div[contains(@class, 'search_price')
+                and not(contains(@class, 'search_price_discount_combined'))]")
+         .text
+         .strip
+         .split('$')
+         .last
+  end
+
+  def scrape_release_date(entry)
+    Date.parse(entry.xpath(".//div[contains(@class, 'search_released')]").text)
+  end
+
+  def scrape_platforms(entry)
+    platforms = []
+    platforms.push('Windows') unless entry.xpath(".//span[contains(@class, 'win')]").empty?
+    platforms.push('macOS') unless entry.xpath(".//span[contains(@class, 'mac')]").empty?
+    platforms.push('Linux') unless entry.xpath(".//span[contains(@class, 'linux')]").empty?
+    platforms.push('Steamplay') unless entry.xpath(".//span[contains(@class, 'steamplay')]").empty?
+
+    platforms
+  end
+
+  def scrape_icon_url(entry)
+    entry.xpath(".//div[contains(@class, 'search_capsule')]/img").attribute('src').value
+  end
+
+  def get_review_contents(entry)
+    entry.xpath(".//span[contains(@class, 'search_review_summary')]")
+         .attribute('data-store-tooltip')
+         .value
+  end
+
+  def scrape_review_score(entry)
+    review_string = get_review_contents(entry)
+    matches = /.*(\d\d)[%]/i.match(review_string)
+    review_percentage = matches[1] unless matches.nil?
+
+    review_percentage
+  end
+
+  def scrape_number_of_reviews(entry)
+    review_string = get_review_contents(entry)
+    matches = /.*\d\d[%] of the ([0-9,]*) user/i.match(review_string)
+    num_reviews = matches[1] unless matches.nil?
+
+    num_reviews
+  end
+
+  def scrape_entry(entry)
+    new_game = {}
+    new_game[:url] = scrape_url(entry)
+    new_game[:name] = scrape_name(entry)
+    new_game[:price] = scrape_price(entry)
+    new_game[:release_date] = scrape_release_date(entry)
+    new_game[:platforms] = scrape_platforms(entry)
+    new_game[:icon_url] = scrape_icon_url(entry)
+    new_game[:review_score] = scrape_review_score(entry)
+    new_game[:number_of_reviews] = scrape_number_of_reviews(entry)
+    @game_list.push new_game
+  end
+end

data/lib/steam_scraper/game_page_scraper.rb

@@ -0,0 +1,91 @@
+require_relative './page_retriever.rb'
+
+# Class that scrapes a games actual page
+class GamePageScraper
+  def initialize(*_args)
+    @page_retriever = PageRetriever.new
+  end
+
+  def scrape(games_hash)
+    games_hash.map! do |game|
+      url = game[:url]
+      scrape_game!(game, url) unless url.nil?
+    end
+    games_hash
+  end
+
+  def scrape_game!(game, url)
+    puts 'Scraping Page for ' + game[:name]
+    page_contents = get_page_contents(url)
+    game[:metacritic] = scrape_metacritic(page_contents)
+    game[:tags] = scrape_tags(page_contents)
+    game[:genres] = scrape_genres(page_contents)
+    game[:developer] = scrape_developer(page_contents)
+    game[:publisher] = scrape_publisher(page_contents)
+    game[:min_spec] = scrape_min_spec(page_contents)
+    game[:recommended_spec] = scrape_recommended_spec(page_contents)
+
+    game
+  end
+
+  def get_page_contents(url)
+    @page_retriever.retrieve(url)
+  end
+
+  def scrape_metacritic(page_contents)
+    score_element = page_contents.xpath("//div[@id='game_area_metascore']/span").first
+    score_element.text.to_i unless score_element.nil?
+  end
+
+  def scrape_tags(page_contents)
+    tags = []
+    page_contents.xpath("//div[contains(@class, 'popular_tags')]/a").each do |tag|
+      tags.push(tag.text.strip)
+    end
+
+    tags
+  end
+
+  def scrape_genres(page_contents)
+    genres = []
+    details = page_contents.xpath("//div[@class='details_block']")
+    details.xpath(".//a[contains(@href, 'genre')]").each do |genre|
+      genres.push(genre.text.strip)
+    end
+
+    genres
+  end
+
+  def scrape_developer(page_contents)
+    details = page_contents.xpath("//div[@class='details_block']")
+    details.xpath(".//a[contains(@href, 'developer')]").text.strip
+  end
+
+  def scrape_publisher(page_contents)
+    details = page_contents.xpath("//div[@class='details_block']")
+    details.xpath(".//a[contains(@href, 'publisher')]").text.strip
+  end
+
+  def scrape_min_spec(page_contents)
+    spec_block = page_contents.xpath("//div[@data-os='win']/div[@class='game_area_sys_req_leftCol']/ul/ul")
+    scrape_spec(spec_block)
+  end
+
+  def scrape_recommended_spec(page_contents)
+    spec_block = page_contents.xpath("//div[@data-os='win']/div[@class='game_area_sys_req_rightCol']/ul/ul")
+    scrape_spec(spec_block)
+  end
+
+  def scrape_spec(node)
+    spec_array = node.text.split "\r"
+    spec_hash = {}
+    spec_array.each do |entry|
+      value_pair = entry.split(':')
+      key = value_pair.first.to_sym
+      value = value_pair.last.strip
+      spec_hash[key] = value
+    end
+
+    spec_hash
+  end
+end

data/lib/steam_scraper/page_retriever.rb

@@ -0,0 +1,11 @@
+# Retrieves a URL and return the Nokogiri representation of that page
+require 'HTTParty'
+require 'nokogiri'
+
+# Class for retrieving a Nokogiri tree from a URL
+class PageRetriever
+  def retrieve(url)
+    page_contents = HTTParty.get(url)
+    Nokogiri::HTML(page_contents)
+  end
+end

data/lib/steam_scraper/version.rb

@@ -0,0 +1,3 @@
+module SteamScraper
+  VERSION = '1.1.0'.freeze
+end

data/lib/steam_scraper.rb

@@ -0,0 +1,21 @@
+require 'steam_scraper/version'
+require 'steam_scraper/game_list_scraper'
+require 'steam_scraper/game_page_scraper'
+
+# Module that provides all SteamScraper functionality
+module SteamScraper
+  # Class actually scrapes the Steam Store
+  class SteamScraper
+    def initialize(*_args)
+      @game_list_scraper = GameListScraper.new
+      @game_page_scraper = GamePageScraper.new
+    end
+
+    def scrape(first_page = 1, last_page = nil)
+      scraped_game_list = @game_list_scraper.scrape(first_page, last_page)
+      final_game_list = @game_page_scraper.scrape(scraped_game_list)
+
+      final_game_list
+    end
+  end
+end

data/steam_scraper.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'steam_scraper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'steam_scraper'
+  spec.version       = SteamScraper::VERSION
+  spec.authors       = ['Robert Gardner']
+  spec.email         = ['rantingbob@gmail.com']
+
+  spec.summary       = 'Webscraper for the Steam Store'
+  spec.description   = 'A webscraper that scrapes game data from store.steampowered.com'
+  spec.homepage      = 'https://github.com/ShriekBob/steam_scraper'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.12'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'nokogiri', '~> 1.6.8'
+  spec.add_development_dependency 'httparty', '~> 0.14.0'
+  spec.add_runtime_dependency 'nokogiri', '~> 1.6.8'
+  spec.add_runtime_dependency 'httparty', '~> 0.14.0'
+end

metadata

@@ -0,0 +1,156 @@
+--- !ruby/object:Gem::Specification
+name: steam_scraper
+version: !ruby/object:Gem::Version
+  version: 1.1.0
+platform: ruby
+authors:
+- Robert Gardner
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-09-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+description: A webscraper that scrapes game data from store.steampowered.com
+email:
+- rantingbob@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/steam_scraper.rb
+- lib/steam_scraper/game_list_scraper.rb
+- lib/steam_scraper/game_page_scraper.rb
+- lib/steam_scraper/page_retriever.rb
+- lib/steam_scraper/version.rb
+- steam_scraper.gemspec
+homepage: https://github.com/ShriekBob/steam_scraper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Webscraper for the Steam Store
+test_files: []