desuraify 0.0.5.pre

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7c1a3942027163f722a2cbd49e66bbe0602f431f
+  data.tar.gz: f650f2b1f69d5784eb0b241b515da28f48e5da0c
+SHA512:
+  metadata.gz: 0f20d841726544fbcbe8379236dd8d6b661e52e417ed343a9f9d5e45e8451117bd12426f8e890ccb605f7828be0fe1f48c2256ea4540f0c2b5ed3d7ef7187b41
+  data.tar.gz: 6a92c5b32271cf52ad48a4747f83f7377b71926ae6ee3a52d863bc35bcb4bba41add78339f31ff536aee28e3dbd241396d41435d9bb67d5d1e8798d91315128b

data/.gitignore

@@ -0,0 +1,23 @@
+*.gem
+*.rbc
+.bundle
+.config
+.DS_Store
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in desuraify.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 TODO: Write your name
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,92 @@
+# Desuraify
+
+Desura game store scraper
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'desuraify'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install desuraify
+
+## Usage
+  ```
+  require 'desuraify'
+  ```
+
+  \# get all information about Dominions 4: Thrones of Ascension
+  
+  \# (http://www.desura.com/games/dominions-4-thrones-of-ascensions)
+  ```
+  game = Desuraify::Game.new('dominions-4-thrones-of-ascensions')
+  game.update
+  ```
+
+  \# show current price
+  ```
+  puts game.price
+  => "$34.99"
+  ```
+
+  \# game title
+  ```
+  puts game.title
+  => "Dominions 4: Thrones of Ascension"
+  ```
+
+  \# game platforms
+  ```
+  game.platforms.each { |platform| puts platform }
+  => "Windows"
+  => "Mac"
+  => "Linux"
+  ```
+
+  \# game developers
+  ```
+  game.developers.each do |developer|
+    puts developer[:name]
+    puts developer[:company]
+    puts developer[:id]
+  end
+
+  => "Illwinter Game Design"
+  => true
+  => "illwinter-game-design"
+  ```
+
+  \# numerous other attributes are available (some partially or poorly implemented)
+  ```
+  puts Desuraify::Game::ATTRIBUTES.inspect
+  or
+  puts game.attributes.inspect
+  ```
+
+## TODO in no particular order
+
+  1. Maybe nothing, apparently the parent company of Desura filed for bankruptcy yesterday...
+  2. Gather Engine data (currently game data is all that is functional)
+  3. Gather Member data
+  4. Gather Company data
+  5. Documentation...
+  6. Tests (I know, I'm a horrible person)
+
+## Contributing
+
+  1. Fork it ( https://github.com/[my-github-username]/desuraify/fork )
+  2. Create your feature branch (`git checkout -b my-new-feature`)
+  3. Commit your changes (`git commit -am 'Add some feature'`)
+  4. Push to the branch (`git push origin my-new-feature`)
+  5. Create a new Pull Request
+
+## Credits
+
+  This project and much of the code was inspired by Market Bot:
+  https://github.com/chadrem/market_bot

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/desuraify.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'desuraify/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "desuraify"
+  spec.version       = Desuraify::VERSION
+  spec.authors       = ["jfrazx"]
+  spec.email         = ["staringblind@gmail.com"]
+  spec.summary       = %q{A simple Desura store scraper}
+  spec.description   = %q{}
+  spec.platform      = Gem::Platform::RUBY
+  spec.homepage      = "https://github.com/jfrazx/desuraify"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "nokogiri", ">= 1.6.6.2"
+  spec.add_runtime_dependency "typhoeus", ">= 0.7.1"
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake", "= 10.4.2"
+end

data/lib/desuraify/base.rb

@@ -0,0 +1,293 @@
+module Desuraify
+  # base class to inherit, hopefully reducing duplicate code
+
+  class Base
+
+    attr_reader :id
+    attr_reader :hydra
+    attr_reader :callback
+    attr_reader :error
+
+    def initialize(id, options={})
+      @id = id
+      @hydra = options[:hydra] || Desuraify.hydra
+      @request_opts = options[:request_opts] || {}
+      @callback = nil
+      @error = nil
+    end
+
+    def update
+      resp = Typhoeus::Request.get(url, @request_opts)
+      result = handle_response(resp)
+      update_callback(result)
+
+      self
+    end
+
+    def enqueue_update(&block)
+      @callback = block
+      @error = nil
+
+      request = Typhoeus::Request.new(url, @request_opts)
+
+      request.on_complete do |response|
+        result = nil
+
+        begin
+          result = handle_response(response)
+        rescue Exception => e
+          @error = e
+        end
+
+        update_callback(result)
+      end
+
+      hydra.queue(request)
+
+      self
+    end
+
+    def parse_headers(headers)
+      result = Hash.new
+
+      headers.each do |header|
+        next if header.text.strip.empty?
+
+        case header.text.strip
+        when /^Platforms?$/i
+
+          result[:platforms] = header.parent.children.search('a').select do |platform|
+            platform unless platform == header || platform.text.strip.empty?
+          end.map! {|platform| platform.text.strip }.uniq
+
+        when /^Engine$/i
+
+          result[:engines] = header.parent.children.select do |engine|
+            engine unless engine == header || engine.text.strip.empty?
+          end.map do |engine|
+            eng = Hash.new
+            eng[:name] = engine.text.strip
+            eng[:id] = engine.child.attribute('href').value.split('/').last rescue nil
+            eng
+          end.uniq
+
+        when "Engines"
+
+          result[:engine_count] = header.next.next.text.strip.to_i rescue nil
+
+        when /^Developers?/i
+
+          result[:developers] = header.parent.children.select do |developer|
+            developer unless developer == header || developer.text.strip.empty?
+          end.map do |develop|
+            developer = Hash.new
+            developer[:name] = develop.text.strip
+            href = develop.child.attribute('href').value.split('/')
+            developer[:company] = !!href.find{|company| company.match(/^company$/i)}
+            developer[:id] = href.last
+            developer
+          end.uniq
+
+        when /Publishers?$/i
+
+          result[:publishers] = header.parent.children.select do |publisher|
+            publisher unless publisher == header || publisher.text.strip.empty?
+          end.map do |pub|
+            publisher = Hash.new
+            publisher[:name] = pub.text.strip
+            href = pub.child.attribute('href').value.split('/')
+            publisher[:company] = !!href.find{|company| company.match(/^company$/i)}
+            publisher[:id] = href.last
+            publisher
+          end.uniq
+
+        when /Languages?/i
+
+          result[:languages] = header.parent.children.select do |language|
+            language unless language == header || language.text.strip.empty?
+          end.map! {|language| language.text.strip }.uniq
+
+        when /^Genres?/i
+
+          result[:genres] = header.parent.children.select do |genre|
+            genre unless genre == header || genre.text.strip.empty?
+          end.map {|genre| genre.text.strip }.uniq
+
+        when /^Themes?$/i
+
+          result[:themes] = header.parent.children.select do |theme|
+            theme unless theme == header || theme.text.strip.empty?
+          end.map {|theme| theme.text.strip }.uniq
+
+        when /^This game is an expansion for\s+?/i
+
+          match = header.text.strip.match(/^This game is an expansion for\s+?(.*)/i)
+          expansion = Hash.new
+          expansion[:title] = match[match.size-1]
+          expansion[:boxshot] = header.parent.search('img').attribute('src').value.strip
+          expansion[:id] = header.parent.search('a').attribute('href').value.split('/').last.strip
+          result[:expansion] = expansion
+
+        when /^Projects?$/
+
+          result[:project] = header.parent.children.select do |project|
+            project unless project == header || project.text.strip.empty?
+          end.map {|project| project.text.strip }.uniq
+
+        when "Players"
+
+          result[:players] = header.parent.children.select do |player|
+            player unless player == header || player.text.strip.empty?
+          end.map{|player| player.text.strip }.uniq
+
+        when "Boxshot"
+
+          result[:boxshot] = header.parent.search('a').attribute('href').value.strip rescue nil
+
+        when "Last Update"
+
+          result[:updated] = header.next.next.text.strip rescue nil
+
+        when "License"
+
+          result[:license] = header.next.next.text.strip rescue nil
+
+        when "Release Date"
+
+          result[:release_date] = header.next.next.text.strip rescue nil
+
+        when "Watchers"
+
+          result[:watchers] = header.next.next.text.strip rescue nil
+
+        when "Images"
+
+          result[:image_count] = header.next.next.text.strip.to_i rescue nil
+
+        when "Games"
+
+          result[:game_count] = header.next.next.text.strip.to_i   rescue nil
+
+        when "News"
+
+          result[:news_count] = header.next.next.text.strip.to_i rescue nil
+
+        when "Rank"
+
+          result[:rank] = header.next.next.text.strip rescue nil
+
+        when "Videos"
+
+          result[:video_count] = header.next.next.text.strip.to_i rescue nil
+
+        when "Visits"
+
+          result[:visits] = header.next.next.text.strip rescue nil
+
+        when "Official Page"
+
+          result[:official_page] = header.parent.search('a').attribute('href').value.strip rescue nil
+
+
+        when "Homepage"
+
+          result[:official_page] = header.parent.search('a').attribute('href').value.strip rescue nil
+
+        when "Members"
+
+          result[:member_count] = header.next.next.text.strip.to_i rescue nil
+
+        when "Established"
+
+            result[:established] = header.next.next.text.strip rescue nil
+
+        when "Company"
+
+          result[:company] = header.next.next.text.strip rescue nil
+
+        when "Office"
+
+            result[:office] = header.next.next.text.strip rescue nil
+
+        when "Address"
+
+          addresses = Array.new
+
+          header.parent.children.each do |child|
+            next if child == header || child.text.strip.empty?
+            break if child.text.strip == "Phone"
+            addresses << child.text.strip
+          end
+
+          result[:address] = addresses.map { |address| address.split("\n") }.flatten
+
+        when "Phone"
+
+          result[:phone] = header.next.next.text.strip rescue nil
+        end
+
+      end
+
+      result
+    end
+
+    def parse_similar(doc, img_count=0, vid_count=0)
+      result = Hash.new
+
+      result[:rating] = doc.at_css('.score').text.strip rescue nil
+
+      result[:videos] = doc.css('.videobox').search('a').map do |video|
+        values = video.attribute('href').value.strip.split('/')
+        values.pop
+        "http://www.desura.com#{values.join('/')}"
+      end rescue nil
+      result[:videos] = rss_update(video_rss) if result[:videos].size < vid_count
+
+      result[:images] = doc.css('.mediaitem').search('a').select{|item| item if item.attribute('href').value.match(/^https?/i) }.map{|pic| pic.attribute('href').value.strip }
+      result[:images] = rss_update(image_rss) if result[:images].size < img_count
+
+      result[:summary] = doc.at_css('.body.clear').search('p').map{ |paragraph| paragraph.text.strip }
+      result[:page_title] = doc.css('title').text.strip
+      result[:title] = doc.at_css('.title').css('h2').text.strip rescue nil
+
+      result
+    end
+
+    def to_s
+      "#{@title}" rescue "#{self.class}::#{self.object_id}"
+    end
+
+    def rss_update(url)
+      response = Typhoeus::Request.get(url, @request_opts)
+      if response.success?
+        xml = Nokogiri::XML(response.body)
+        data = xml.search('enclosure').map{|item| item.attribute('url').value.strip }
+
+        block_given? ? (yield data) : data
+      end
+    end
+
+    private
+
+    def handle_response(response)
+      if response.success?
+        parse(response.body)
+      else
+        raise Desuraify::ResponseError.new("Got unexpected response code: #{response.code}")
+      end
+    end
+
+    def update_callback(result)
+      unless @error
+        self.attributes.each do |a|
+          attr_name = "@#{a}"
+          attr_value = result[a]
+          instance_variable_set(attr_name, attr_value)
+        end
+      end
+
+      @callback.call(self) if @callback
+    end
+
+  end
+end

data/lib/desuraify/company.rb

@@ -0,0 +1,43 @@
+module Desuraify
+  class Company < Base
+
+    ATTRIBUTES = [
+      :members, :member_count, :html, :official_page, :company, :rank, :visits, :game_count, 
+      :office, :watchers, :address, :phone, :engine_count, :news_count, :images, 
+      :image_count, :videos, :video_count
+    ]
+
+    attr_reader *ATTRIBUTES
+
+    def initialize(id, options={})
+      super(id, options)
+    end
+
+    def parse(html)
+
+      doc = Nokogiri::HTML(html)
+
+      result = parse_headers(doc.css('h5'))
+      result.merge!(parse_similar(doc))
+      result[:html] = html
+
+      result
+    end
+
+    def url
+      "http://www.desura.com/company/#{@id}"
+    end
+
+    def image_rss
+      "http://rss.desura.com/company/#{@id}/images/feed/rss.xml"
+    end
+
+    def video_rss
+      "http://rss.desura.com/company/#{@id}/videos/feed/rss.xml"
+    end
+
+    def attributes
+      ATTRIBUTES
+    end
+  end
+end

data/lib/desuraify/engine.rb

@@ -0,0 +1,46 @@
+module Desuraify
+  class Engine < Base
+    ATTRIBUTES = [
+      :developers, :games, :game_count, :html, :images, :image_count, :license, :news, 
+      :news_count, :official_page, :page_title, :platforms, :publishers, :rank, 
+      :rating, :release_date, :reviews, :summary, :title, :updated, :videos, 
+      :video_count, :visits, :watchers
+    ]
+
+    attr_reader *ATTRIBUTES
+
+    def initialize(id, options={})
+      super(id, options)
+    end
+
+    def parse(html)
+
+      doc = Nokogiri::HTML(html)
+
+      result = parse_headers(doc.css('h5'))
+      result.merge!(parse_similar(doc, result[:image_count], result[:video_count]))
+      result[:html] = html
+
+      result
+
+    end
+
+    def url
+      "http://www.desura.com/engines/#{@id}"
+    end
+
+    def attributes
+      ATTRIBUTES
+    end
+
+    private
+
+    def image_rss
+      "http://rss.desura.com/engines/#{@id}/images/feed/rss.xml"
+    end
+
+    def video_rss
+      "http://rss.desura.com/engines/#{@id}/videos/feed/rss.xml"
+    end
+  end
+end

data/lib/desuraify/exception.rb

@@ -0,0 +1,4 @@
+module Desuraify
+  # Exception raised when an unsuccessful response is indicated by Typhoeus
+  class ResponseError < StandardError; end
+end

data/lib/desuraify/game.rb

@@ -0,0 +1,59 @@
+module Desuraify
+
+  class Game < Base
+    ATTRIBUTES = [
+      :boxshot, :developers, :engines, :expansion, :genres, :html, 
+      :image_count, :images, :languages, :news, :news_count, :official_page, 
+      :original_price, :page_title, :platforms, :players, :price, :project, 
+      :publishers, :rank, :rating, :reviews, :summary, :themes, :title, 
+      :updated, :video_count, :videos, :visits, :watchers
+    ]
+
+    attr_reader *ATTRIBUTES
+
+    def initialize(id, options={})
+      super(id, options)
+    end
+
+    def parse(html) 
+      doc = Nokogiri::HTML(html)
+
+      result = parse_headers(doc.css('h5'))
+      result.merge!(parse_similar(doc, result[:image_count], result[:video_count]))
+
+      # acquire prices
+      prices = doc.css('.price').children.select{ |price| price unless price.text.strip.empty? }.map{ |price| price.text.strip } rescue nil
+
+      result[:price] = prices.min unless prices.empty? rescue nil
+      result[:original_price] = prices.max unless prices.empty? rescue nil
+
+      result[:publishers] = result[:developers].map{ |dev| dev.dup } unless result[:publishers] if result[:developers]
+      result[:html] = html
+
+      result
+    end
+
+    def url
+      "http://www.desura.com/games/#{@id}"
+    end
+
+    def attributes
+      ATTRIBUTES
+    end
+
+    def to_s
+      "#{@title} for #{@platforms.join(', ')}" rescue "#{self.class}::#{self.object_id}"
+    end
+
+    private
+
+    def image_rss
+      "http://rss.desura.com/games/#{@id}/images/feed/rss.xml"
+    end
+
+    def video_rss
+      "http://rss.desura.com/games/#{@id}/videos/feed/rss.xml"
+    end
+
+  end # end of class
+end

data/lib/desuraify/member.rb

@@ -0,0 +1,18 @@
+module Desuraify
+  class Member < Base
+
+    ATTRIBUTES = [
+
+    ]
+
+    def initialize(id, options)
+      super(id, options)
+    end
+
+    def attributes
+      ATTRIBUTES
+    end
+  end
+end
+
+# http://rss.desura.com/games/arbalest-3035/images/feed/rss.xml

data/lib/desuraify/version.rb

@@ -0,0 +1,3 @@
+module Desuraify
+  VERSION = "0.0.5.pre"
+end

data/lib/desuraify.rb

@@ -0,0 +1,18 @@
+require "uri"
+require "nokogiri"
+require "typhoeus"
+
+# Dir.glob(File.join('path', '**', '*.rb'), &method(:require))
+require "desuraify/exception"
+require "desuraify/base"
+require "desuraify/game"
+require "desuraify/engine"
+require "desuraify/member"
+require "desuraify/company"
+require "desuraify/version"
+
+module Desuraify
+  def self.hydra
+    @hydra ||= Typhoeus::Hydra.new(:max_concurrency => 5)
+  end
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: desuraify
+version: !ruby/object:Gem::Version
+  version: 0.0.5.pre
+platform: ruby
+authors:
+- jfrazx
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-06-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 10.4.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 10.4.2
+description: ''
+email:
+- staringblind@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- desuraify.gemspec
+- lib/desuraify.rb
+- lib/desuraify/base.rb
+- lib/desuraify/company.rb
+- lib/desuraify/engine.rb
+- lib/desuraify/exception.rb
+- lib/desuraify/game.rb
+- lib/desuraify/member.rb
+- lib/desuraify/version.rb
+homepage: https://github.com/jfrazx/desuraify
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A simple Desura store scraper
+test_files: []