desuraify 0.0.5.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +23 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +92 -0
- data/Rakefile +2 -0
- data/desuraify.gemspec +27 -0
- data/lib/desuraify/base.rb +293 -0
- data/lib/desuraify/company.rb +43 -0
- data/lib/desuraify/engine.rb +46 -0
- data/lib/desuraify/exception.rb +4 -0
- data/lib/desuraify/game.rb +59 -0
- data/lib/desuraify/member.rb +18 -0
- data/lib/desuraify/version.rb +3 -0
- data/lib/desuraify.rb +18 -0
- metadata +114 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 7c1a3942027163f722a2cbd49e66bbe0602f431f
|
4
|
+
data.tar.gz: f650f2b1f69d5784eb0b241b515da28f48e5da0c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 0f20d841726544fbcbe8379236dd8d6b661e52e417ed343a9f9d5e45e8451117bd12426f8e890ccb605f7828be0fe1f48c2256ea4540f0c2b5ed3d7ef7187b41
|
7
|
+
data.tar.gz: 6a92c5b32271cf52ad48a4747f83f7377b71926ae6ee3a52d863bc35bcb4bba41add78339f31ff536aee28e3dbd241396d41435d9bb67d5d1e8798d91315128b
|
data/.gitignore
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
.bundle
|
4
|
+
.config
|
5
|
+
.DS_Store
|
6
|
+
.yardoc
|
7
|
+
Gemfile.lock
|
8
|
+
InstalledFiles
|
9
|
+
_yardoc
|
10
|
+
coverage
|
11
|
+
doc/
|
12
|
+
lib/bundler/man
|
13
|
+
pkg
|
14
|
+
rdoc
|
15
|
+
spec/reports
|
16
|
+
test/tmp
|
17
|
+
test/version_tmp
|
18
|
+
tmp
|
19
|
+
*.bundle
|
20
|
+
*.so
|
21
|
+
*.o
|
22
|
+
*.a
|
23
|
+
mkmf.log
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2015 TODO: Write your name
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
# Desuraify
|
2
|
+
|
3
|
+
Desura game store scraper
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'desuraify'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install desuraify
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
```
|
21
|
+
require 'desuraify'
|
22
|
+
```
|
23
|
+
|
24
|
+
\# get all information about Dominions 4: Thrones of Ascension
|
25
|
+
|
26
|
+
\# (http://www.desura.com/games/dominions-4-thrones-of-ascensions)
|
27
|
+
```
|
28
|
+
game = Desuraify::Game.new('dominions-4-thrones-of-ascensions')
|
29
|
+
game.update
|
30
|
+
```
|
31
|
+
|
32
|
+
\# show current price
|
33
|
+
```
|
34
|
+
puts game.price
|
35
|
+
=> "$34.99"
|
36
|
+
```
|
37
|
+
|
38
|
+
\# game title
|
39
|
+
```
|
40
|
+
puts game.title
|
41
|
+
=> "Dominions 4: Thrones of Ascension"
|
42
|
+
```
|
43
|
+
|
44
|
+
\# game platforms
|
45
|
+
```
|
46
|
+
game.platforms.each { |platform| puts platform }
|
47
|
+
=> "Windows"
|
48
|
+
=> "Mac"
|
49
|
+
=> "Linux"
|
50
|
+
```
|
51
|
+
|
52
|
+
\# game developers
|
53
|
+
```
|
54
|
+
game.developers.each do |developer|
|
55
|
+
puts developer[:name]
|
56
|
+
puts developer[:company]
|
57
|
+
puts developer[:id]
|
58
|
+
end
|
59
|
+
|
60
|
+
=> "Illwinter Game Design"
|
61
|
+
=> true
|
62
|
+
=> "illwinter-game-design"
|
63
|
+
```
|
64
|
+
|
65
|
+
\# numerous other attributes are available (some partially or poorly implemented)
|
66
|
+
```
|
67
|
+
puts Desuraify::Game::ATTRIBUTES.inspect
|
68
|
+
or
|
69
|
+
puts game.attributes.inspect
|
70
|
+
```
|
71
|
+
|
72
|
+
## TODO in no particular order
|
73
|
+
|
74
|
+
1. Maybe nothing, apparently the parent company of Desura filed for bankruptcy yesterday...
|
75
|
+
2. Gather Engine data (currently game data is all that is functional)
|
76
|
+
3. Gather Member data
|
77
|
+
4. Gather Company data
|
78
|
+
5. Documentation...
|
79
|
+
6. Tests (I know, I'm a horrible person)
|
80
|
+
|
81
|
+
## Contributing
|
82
|
+
|
83
|
+
1. Fork it ( https://github.com/[my-github-username]/desuraify/fork )
|
84
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
85
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
86
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
87
|
+
5. Create a new Pull Request
|
88
|
+
|
89
|
+
## Credits
|
90
|
+
|
91
|
+
This project and much of the code was inspired by Market Bot:
|
92
|
+
https://github.com/chadrem/market_bot
|
data/Rakefile
ADDED
data/desuraify.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'desuraify/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "desuraify"
|
8
|
+
spec.version = Desuraify::VERSION
|
9
|
+
spec.authors = ["jfrazx"]
|
10
|
+
spec.email = ["staringblind@gmail.com"]
|
11
|
+
spec.summary = %q{A simple Desura store scraper}
|
12
|
+
spec.description = %q{}
|
13
|
+
spec.platform = Gem::Platform::RUBY
|
14
|
+
spec.homepage = "https://github.com/jfrazx/desuraify"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0")
|
18
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
19
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
20
|
+
spec.require_paths = ["lib"]
|
21
|
+
|
22
|
+
spec.add_runtime_dependency "nokogiri", ">= 1.6.6.2"
|
23
|
+
spec.add_runtime_dependency "typhoeus", ">= 0.7.1"
|
24
|
+
|
25
|
+
spec.add_development_dependency "bundler", "~> 1.6"
|
26
|
+
spec.add_development_dependency "rake", "= 10.4.2"
|
27
|
+
end
|
@@ -0,0 +1,293 @@
|
|
1
|
+
module Desuraify
|
2
|
+
# base class to inherit, hopefully reducing duplicate code
|
3
|
+
|
4
|
+
class Base
|
5
|
+
|
6
|
+
attr_reader :id
|
7
|
+
attr_reader :hydra
|
8
|
+
attr_reader :callback
|
9
|
+
attr_reader :error
|
10
|
+
|
11
|
+
def initialize(id, options={})
|
12
|
+
@id = id
|
13
|
+
@hydra = options[:hydra] || Desuraify.hydra
|
14
|
+
@request_opts = options[:request_opts] || {}
|
15
|
+
@callback = nil
|
16
|
+
@error = nil
|
17
|
+
end
|
18
|
+
|
19
|
+
def update
|
20
|
+
resp = Typhoeus::Request.get(url, @request_opts)
|
21
|
+
result = handle_response(resp)
|
22
|
+
update_callback(result)
|
23
|
+
|
24
|
+
self
|
25
|
+
end
|
26
|
+
|
27
|
+
def enqueue_update(&block)
|
28
|
+
@callback = block
|
29
|
+
@error = nil
|
30
|
+
|
31
|
+
request = Typhoeus::Request.new(url, @request_opts)
|
32
|
+
|
33
|
+
request.on_complete do |response|
|
34
|
+
result = nil
|
35
|
+
|
36
|
+
begin
|
37
|
+
result = handle_response(response)
|
38
|
+
rescue Exception => e
|
39
|
+
@error = e
|
40
|
+
end
|
41
|
+
|
42
|
+
update_callback(result)
|
43
|
+
end
|
44
|
+
|
45
|
+
hydra.queue(request)
|
46
|
+
|
47
|
+
self
|
48
|
+
end
|
49
|
+
|
50
|
+
def parse_headers(headers)
|
51
|
+
result = Hash.new
|
52
|
+
|
53
|
+
headers.each do |header|
|
54
|
+
next if header.text.strip.empty?
|
55
|
+
|
56
|
+
case header.text.strip
|
57
|
+
when /^Platforms?$/i
|
58
|
+
|
59
|
+
result[:platforms] = header.parent.children.search('a').select do |platform|
|
60
|
+
platform unless platform == header || platform.text.strip.empty?
|
61
|
+
end.map! {|platform| platform.text.strip }.uniq
|
62
|
+
|
63
|
+
when /^Engine$/i
|
64
|
+
|
65
|
+
result[:engines] = header.parent.children.select do |engine|
|
66
|
+
engine unless engine == header || engine.text.strip.empty?
|
67
|
+
end.map do |engine|
|
68
|
+
eng = Hash.new
|
69
|
+
eng[:name] = engine.text.strip
|
70
|
+
eng[:id] = engine.child.attribute('href').value.split('/').last rescue nil
|
71
|
+
eng
|
72
|
+
end.uniq
|
73
|
+
|
74
|
+
when "Engines"
|
75
|
+
|
76
|
+
result[:engine_count] = header.next.next.text.strip.to_i rescue nil
|
77
|
+
|
78
|
+
when /^Developers?/i
|
79
|
+
|
80
|
+
result[:developers] = header.parent.children.select do |developer|
|
81
|
+
developer unless developer == header || developer.text.strip.empty?
|
82
|
+
end.map do |develop|
|
83
|
+
developer = Hash.new
|
84
|
+
developer[:name] = develop.text.strip
|
85
|
+
href = develop.child.attribute('href').value.split('/')
|
86
|
+
developer[:company] = !!href.find{|company| company.match(/^company$/i)}
|
87
|
+
developer[:id] = href.last
|
88
|
+
developer
|
89
|
+
end.uniq
|
90
|
+
|
91
|
+
when /Publishers?$/i
|
92
|
+
|
93
|
+
result[:publishers] = header.parent.children.select do |publisher|
|
94
|
+
publisher unless publisher == header || publisher.text.strip.empty?
|
95
|
+
end.map do |pub|
|
96
|
+
publisher = Hash.new
|
97
|
+
publisher[:name] = pub.text.strip
|
98
|
+
href = pub.child.attribute('href').value.split('/')
|
99
|
+
publisher[:company] = !!href.find{|company| company.match(/^company$/i)}
|
100
|
+
publisher[:id] = href.last
|
101
|
+
publisher
|
102
|
+
end.uniq
|
103
|
+
|
104
|
+
when /Languages?/i
|
105
|
+
|
106
|
+
result[:languages] = header.parent.children.select do |language|
|
107
|
+
language unless language == header || language.text.strip.empty?
|
108
|
+
end.map! {|language| language.text.strip }.uniq
|
109
|
+
|
110
|
+
when /^Genres?/i
|
111
|
+
|
112
|
+
result[:genres] = header.parent.children.select do |genre|
|
113
|
+
genre unless genre == header || genre.text.strip.empty?
|
114
|
+
end.map {|genre| genre.text.strip }.uniq
|
115
|
+
|
116
|
+
when /^Themes?$/i
|
117
|
+
|
118
|
+
result[:themes] = header.parent.children.select do |theme|
|
119
|
+
theme unless theme == header || theme.text.strip.empty?
|
120
|
+
end.map {|theme| theme.text.strip }.uniq
|
121
|
+
|
122
|
+
when /^This game is an expansion for\s+?/i
|
123
|
+
|
124
|
+
match = header.text.strip.match(/^This game is an expansion for\s+?(.*)/i)
|
125
|
+
expansion = Hash.new
|
126
|
+
expansion[:title] = match[match.size-1]
|
127
|
+
expansion[:boxshot] = header.parent.search('img').attribute('src').value.strip
|
128
|
+
expansion[:id] = header.parent.search('a').attribute('href').value.split('/').last.strip
|
129
|
+
result[:expansion] = expansion
|
130
|
+
|
131
|
+
when /^Projects?$/
|
132
|
+
|
133
|
+
result[:project] = header.parent.children.select do |project|
|
134
|
+
project unless project == header || project.text.strip.empty?
|
135
|
+
end.map {|project| project.text.strip }.uniq
|
136
|
+
|
137
|
+
when "Players"
|
138
|
+
|
139
|
+
result[:players] = header.parent.children.select do |player|
|
140
|
+
player unless player == header || player.text.strip.empty?
|
141
|
+
end.map{|player| player.text.strip }.uniq
|
142
|
+
|
143
|
+
when "Boxshot"
|
144
|
+
|
145
|
+
result[:boxshot] = header.parent.search('a').attribute('href').value.strip rescue nil
|
146
|
+
|
147
|
+
when "Last Update"
|
148
|
+
|
149
|
+
result[:updated] = header.next.next.text.strip rescue nil
|
150
|
+
|
151
|
+
when "License"
|
152
|
+
|
153
|
+
result[:license] = header.next.next.text.strip rescue nil
|
154
|
+
|
155
|
+
when "Release Date"
|
156
|
+
|
157
|
+
result[:release_date] = header.next.next.text.strip rescue nil
|
158
|
+
|
159
|
+
when "Watchers"
|
160
|
+
|
161
|
+
result[:watchers] = header.next.next.text.strip rescue nil
|
162
|
+
|
163
|
+
when "Images"
|
164
|
+
|
165
|
+
result[:image_count] = header.next.next.text.strip.to_i rescue nil
|
166
|
+
|
167
|
+
when "Games"
|
168
|
+
|
169
|
+
result[:game_count] = header.next.next.text.strip.to_i rescue nil
|
170
|
+
|
171
|
+
when "News"
|
172
|
+
|
173
|
+
result[:news_count] = header.next.next.text.strip.to_i rescue nil
|
174
|
+
|
175
|
+
when "Rank"
|
176
|
+
|
177
|
+
result[:rank] = header.next.next.text.strip rescue nil
|
178
|
+
|
179
|
+
when "Videos"
|
180
|
+
|
181
|
+
result[:video_count] = header.next.next.text.strip.to_i rescue nil
|
182
|
+
|
183
|
+
when "Visits"
|
184
|
+
|
185
|
+
result[:visits] = header.next.next.text.strip rescue nil
|
186
|
+
|
187
|
+
when "Official Page"
|
188
|
+
|
189
|
+
result[:official_page] = header.parent.search('a').attribute('href').value.strip rescue nil
|
190
|
+
|
191
|
+
|
192
|
+
when "Homepage"
|
193
|
+
|
194
|
+
result[:official_page] = header.parent.search('a').attribute('href').value.strip rescue nil
|
195
|
+
|
196
|
+
when "Members"
|
197
|
+
|
198
|
+
result[:member_count] = header.next.next.text.strip.to_i rescue nil
|
199
|
+
|
200
|
+
when "Established"
|
201
|
+
|
202
|
+
result[:established] = header.next.next.text.strip rescue nil
|
203
|
+
|
204
|
+
when "Company"
|
205
|
+
|
206
|
+
result[:company] = header.next.next.text.strip rescue nil
|
207
|
+
|
208
|
+
when "Office"
|
209
|
+
|
210
|
+
result[:office] = header.next.next.text.strip rescue nil
|
211
|
+
|
212
|
+
when "Address"
|
213
|
+
|
214
|
+
addresses = Array.new
|
215
|
+
|
216
|
+
header.parent.children.each do |child|
|
217
|
+
next if child == header || child.text.strip.empty?
|
218
|
+
break if child.text.strip == "Phone"
|
219
|
+
addresses << child.text.strip
|
220
|
+
end
|
221
|
+
|
222
|
+
result[:address] = addresses.map { |address| address.split("\n") }.flatten
|
223
|
+
|
224
|
+
when "Phone"
|
225
|
+
|
226
|
+
result[:phone] = header.next.next.text.strip rescue nil
|
227
|
+
end
|
228
|
+
|
229
|
+
end
|
230
|
+
|
231
|
+
result
|
232
|
+
end
|
233
|
+
|
234
|
+
def parse_similar(doc, img_count=0, vid_count=0)
|
235
|
+
result = Hash.new
|
236
|
+
|
237
|
+
result[:rating] = doc.at_css('.score').text.strip rescue nil
|
238
|
+
|
239
|
+
result[:videos] = doc.css('.videobox').search('a').map do |video|
|
240
|
+
values = video.attribute('href').value.strip.split('/')
|
241
|
+
values.pop
|
242
|
+
"http://www.desura.com#{values.join('/')}"
|
243
|
+
end rescue nil
|
244
|
+
result[:videos] = rss_update(video_rss) if result[:videos].size < vid_count
|
245
|
+
|
246
|
+
result[:images] = doc.css('.mediaitem').search('a').select{|item| item if item.attribute('href').value.match(/^https?/i) }.map{|pic| pic.attribute('href').value.strip }
|
247
|
+
result[:images] = rss_update(image_rss) if result[:images].size < img_count
|
248
|
+
|
249
|
+
result[:summary] = doc.at_css('.body.clear').search('p').map{ |paragraph| paragraph.text.strip }
|
250
|
+
result[:page_title] = doc.css('title').text.strip
|
251
|
+
result[:title] = doc.at_css('.title').css('h2').text.strip rescue nil
|
252
|
+
|
253
|
+
result
|
254
|
+
end
|
255
|
+
|
256
|
+
def to_s
|
257
|
+
"#{@title}" rescue "#{self.class}::#{self.object_id}"
|
258
|
+
end
|
259
|
+
|
260
|
+
def rss_update(url)
|
261
|
+
response = Typhoeus::Request.get(url, @request_opts)
|
262
|
+
if response.success?
|
263
|
+
xml = Nokogiri::XML(response.body)
|
264
|
+
data = xml.search('enclosure').map{|item| item.attribute('url').value.strip }
|
265
|
+
|
266
|
+
block_given? ? (yield data) : data
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
private
|
271
|
+
|
272
|
+
def handle_response(response)
|
273
|
+
if response.success?
|
274
|
+
parse(response.body)
|
275
|
+
else
|
276
|
+
raise Desuraify::ResponseError.new("Got unexpected response code: #{response.code}")
|
277
|
+
end
|
278
|
+
end
|
279
|
+
|
280
|
+
def update_callback(result)
|
281
|
+
unless @error
|
282
|
+
self.attributes.each do |a|
|
283
|
+
attr_name = "@#{a}"
|
284
|
+
attr_value = result[a]
|
285
|
+
instance_variable_set(attr_name, attr_value)
|
286
|
+
end
|
287
|
+
end
|
288
|
+
|
289
|
+
@callback.call(self) if @callback
|
290
|
+
end
|
291
|
+
|
292
|
+
end
|
293
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module Desuraify
|
2
|
+
class Company < Base
|
3
|
+
|
4
|
+
ATTRIBUTES = [
|
5
|
+
:members, :member_count, :html, :official_page, :company, :rank, :visits, :game_count,
|
6
|
+
:office, :watchers, :address, :phone, :engine_count, :news_count, :images,
|
7
|
+
:image_count, :videos, :video_count
|
8
|
+
]
|
9
|
+
|
10
|
+
attr_reader *ATTRIBUTES
|
11
|
+
|
12
|
+
def initialize(id, options={})
|
13
|
+
super(id, options)
|
14
|
+
end
|
15
|
+
|
16
|
+
def parse(html)
|
17
|
+
|
18
|
+
doc = Nokogiri::HTML(html)
|
19
|
+
|
20
|
+
result = parse_headers(doc.css('h5'))
|
21
|
+
result.merge!(parse_similar(doc))
|
22
|
+
result[:html] = html
|
23
|
+
|
24
|
+
result
|
25
|
+
end
|
26
|
+
|
27
|
+
def url
|
28
|
+
"http://www.desura.com/company/#{@id}"
|
29
|
+
end
|
30
|
+
|
31
|
+
def image_rss
|
32
|
+
"http://rss.desura.com/company/#{@id}/images/feed/rss.xml"
|
33
|
+
end
|
34
|
+
|
35
|
+
def video_rss
|
36
|
+
"http://rss.desura.com/company/#{@id}/videos/feed/rss.xml"
|
37
|
+
end
|
38
|
+
|
39
|
+
def attributes
|
40
|
+
ATTRIBUTES
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
module Desuraify
|
2
|
+
class Engine < Base
|
3
|
+
ATTRIBUTES = [
|
4
|
+
:developers, :games, :game_count, :html, :images, :image_count, :license, :news,
|
5
|
+
:news_count, :official_page, :page_title, :platforms, :publishers, :rank,
|
6
|
+
:rating, :release_date, :reviews, :summary, :title, :updated, :videos,
|
7
|
+
:video_count, :visits, :watchers
|
8
|
+
]
|
9
|
+
|
10
|
+
attr_reader *ATTRIBUTES
|
11
|
+
|
12
|
+
def initialize(id, options={})
|
13
|
+
super(id, options)
|
14
|
+
end
|
15
|
+
|
16
|
+
def parse(html)
|
17
|
+
|
18
|
+
doc = Nokogiri::HTML(html)
|
19
|
+
|
20
|
+
result = parse_headers(doc.css('h5'))
|
21
|
+
result.merge!(parse_similar(doc, result[:image_count], result[:video_count]))
|
22
|
+
result[:html] = html
|
23
|
+
|
24
|
+
result
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
def url
|
29
|
+
"http://www.desura.com/engines/#{@id}"
|
30
|
+
end
|
31
|
+
|
32
|
+
def attributes
|
33
|
+
ATTRIBUTES
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def image_rss
|
39
|
+
"http://rss.desura.com/engines/#{@id}/images/feed/rss.xml"
|
40
|
+
end
|
41
|
+
|
42
|
+
def video_rss
|
43
|
+
"http://rss.desura.com/engines/#{@id}/videos/feed/rss.xml"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module Desuraify
|
2
|
+
|
3
|
+
class Game < Base
|
4
|
+
ATTRIBUTES = [
|
5
|
+
:boxshot, :developers, :engines, :expansion, :genres, :html,
|
6
|
+
:image_count, :images, :languages, :news, :news_count, :official_page,
|
7
|
+
:original_price, :page_title, :platforms, :players, :price, :project,
|
8
|
+
:publishers, :rank, :rating, :reviews, :summary, :themes, :title,
|
9
|
+
:updated, :video_count, :videos, :visits, :watchers
|
10
|
+
]
|
11
|
+
|
12
|
+
attr_reader *ATTRIBUTES
|
13
|
+
|
14
|
+
def initialize(id, options={})
|
15
|
+
super(id, options)
|
16
|
+
end
|
17
|
+
|
18
|
+
def parse(html)
|
19
|
+
doc = Nokogiri::HTML(html)
|
20
|
+
|
21
|
+
result = parse_headers(doc.css('h5'))
|
22
|
+
result.merge!(parse_similar(doc, result[:image_count], result[:video_count]))
|
23
|
+
|
24
|
+
# acquire prices
|
25
|
+
prices = doc.css('.price').children.select{ |price| price unless price.text.strip.empty? }.map{ |price| price.text.strip } rescue nil
|
26
|
+
|
27
|
+
result[:price] = prices.min unless prices.empty? rescue nil
|
28
|
+
result[:original_price] = prices.max unless prices.empty? rescue nil
|
29
|
+
|
30
|
+
result[:publishers] = result[:developers].map{ |dev| dev.dup } unless result[:publishers] if result[:developers]
|
31
|
+
result[:html] = html
|
32
|
+
|
33
|
+
result
|
34
|
+
end
|
35
|
+
|
36
|
+
def url
|
37
|
+
"http://www.desura.com/games/#{@id}"
|
38
|
+
end
|
39
|
+
|
40
|
+
def attributes
|
41
|
+
ATTRIBUTES
|
42
|
+
end
|
43
|
+
|
44
|
+
def to_s
|
45
|
+
"#{@title} for #{@platforms.join(', ')}" rescue "#{self.class}::#{self.object_id}"
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def image_rss
|
51
|
+
"http://rss.desura.com/games/#{@id}/images/feed/rss.xml"
|
52
|
+
end
|
53
|
+
|
54
|
+
def video_rss
|
55
|
+
"http://rss.desura.com/games/#{@id}/videos/feed/rss.xml"
|
56
|
+
end
|
57
|
+
|
58
|
+
end # end of class
|
59
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Desuraify
|
2
|
+
class Member < Base
|
3
|
+
|
4
|
+
ATTRIBUTES = [
|
5
|
+
|
6
|
+
]
|
7
|
+
|
8
|
+
def initialize(id, options)
|
9
|
+
super(id, options)
|
10
|
+
end
|
11
|
+
|
12
|
+
def attributes
|
13
|
+
ATTRIBUTES
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
# http://rss.desura.com/games/arbalest-3035/images/feed/rss.xml
|
data/lib/desuraify.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
require "uri"
|
2
|
+
require "nokogiri"
|
3
|
+
require "typhoeus"
|
4
|
+
|
5
|
+
# Dir.glob(File.join('path', '**', '*.rb'), &method(:require))
|
6
|
+
require "desuraify/exception"
|
7
|
+
require "desuraify/base"
|
8
|
+
require "desuraify/game"
|
9
|
+
require "desuraify/engine"
|
10
|
+
require "desuraify/member"
|
11
|
+
require "desuraify/company"
|
12
|
+
require "desuraify/version"
|
13
|
+
|
14
|
+
module Desuraify
|
15
|
+
def self.hydra
|
16
|
+
@hydra ||= Typhoeus::Hydra.new(:max_concurrency => 5)
|
17
|
+
end
|
18
|
+
end
|
metadata
ADDED
@@ -0,0 +1,114 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: desuraify
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.5.pre
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- jfrazx
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-06-13 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: nokogiri
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.6.6.2
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 1.6.6.2
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: typhoeus
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.7.1
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.7.1
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.6'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.6'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 10.4.2
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 10.4.2
|
69
|
+
description: ''
|
70
|
+
email:
|
71
|
+
- staringblind@gmail.com
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- ".gitignore"
|
77
|
+
- Gemfile
|
78
|
+
- LICENSE.txt
|
79
|
+
- README.md
|
80
|
+
- Rakefile
|
81
|
+
- desuraify.gemspec
|
82
|
+
- lib/desuraify.rb
|
83
|
+
- lib/desuraify/base.rb
|
84
|
+
- lib/desuraify/company.rb
|
85
|
+
- lib/desuraify/engine.rb
|
86
|
+
- lib/desuraify/exception.rb
|
87
|
+
- lib/desuraify/game.rb
|
88
|
+
- lib/desuraify/member.rb
|
89
|
+
- lib/desuraify/version.rb
|
90
|
+
homepage: https://github.com/jfrazx/desuraify
|
91
|
+
licenses:
|
92
|
+
- MIT
|
93
|
+
metadata: {}
|
94
|
+
post_install_message:
|
95
|
+
rdoc_options: []
|
96
|
+
require_paths:
|
97
|
+
- lib
|
98
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
104
|
+
requirements:
|
105
|
+
- - ">"
|
106
|
+
- !ruby/object:Gem::Version
|
107
|
+
version: 1.3.1
|
108
|
+
requirements: []
|
109
|
+
rubyforge_project:
|
110
|
+
rubygems_version: 2.2.2
|
111
|
+
signing_key:
|
112
|
+
specification_version: 4
|
113
|
+
summary: A simple Desura store scraper
|
114
|
+
test_files: []
|