market_bot 0.17.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -1
- data/Guardfile +70 -0
- data/README.markdown +45 -95
- data/Rakefile +0 -89
- data/bin/app_categories +17 -0
- data/bin/benchmark_parser_app +24 -0
- data/bin/console +14 -0
- data/bin/setup +7 -0
- data/bin/update_test_data +30 -0
- data/lib/market_bot.rb +20 -15
- data/lib/market_bot/exceptions.rb +2 -1
- data/lib/market_bot/play/app.rb +188 -0
- data/lib/market_bot/play/app/constants.rb +33 -0
- data/lib/market_bot/play/chart.rb +118 -0
- data/lib/market_bot/play/chart/constants.rb +74 -0
- data/lib/market_bot/play/constants.rb +7 -0
- data/lib/market_bot/play/developer.rb +32 -0
- data/lib/market_bot/util.rb +17 -0
- data/lib/market_bot/version.rb +1 -1
- data/market_bot.gemspec +6 -3
- data/spec/market_bot/play/app_spec.rb +201 -0
- data/spec/market_bot/play/chart_spec.rb +126 -0
- data/spec/market_bot/play/data/app-com.bluefroggaming.popdat.txt +99 -0
- data/spec/market_bot/play/data/app-com.mg.android.txt +103 -0
- data/spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-0.txt +97 -0
- data/spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-1.txt +97 -0
- data/spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-2.txt +97 -0
- data/spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-3.txt +97 -0
- data/spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-4.txt +97 -0
- data/spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-5.txt +97 -0
- data/spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-6.txt +97 -0
- data/spec/market_bot/play/data/chart-topselling_paid-GAME_ARCADE-7.txt +97 -0
- data/spec/market_bot/play/data/developer-zynga.txt +97 -0
- data/spec/market_bot/play/developer_spec.rb +52 -0
- data/spec/market_bot_spec.rb +14 -0
- data/spec/spec_helper.rb +5 -11
- metadata +57 -52
- data/NOTES.txt +0 -61
- data/lib/market_bot/android/app.rb +0 -273
- data/lib/market_bot/android/developer.rb +0 -32
- data/lib/market_bot/android/leaderboard.rb +0 -232
- data/lib/market_bot/android/leaderboard/constants.rb +0 -59
- data/lib/market_bot/android/search_query.rb +0 -35
- data/lib/market_bot/movie/leaderboard.rb +0 -167
- data/lib/market_bot/movie/leaderboard/constants.rb +0 -30
- data/lib/market_bot/movie/search_query.rb +0 -32
- data/spec/market_bot/android/app_spec.rb +0 -253
- data/spec/market_bot/android/data/app_1.txt +0 -99
- data/spec/market_bot/android/data/app_2.txt +0 -100
- data/spec/market_bot/android/data/app_3.txt +0 -103
- data/spec/market_bot/android/data/app_4.txt +0 -117
- data/spec/market_bot/android/data/developer-zynga.txt +0 -97
- data/spec/market_bot/android/data/leaderboard-apps_editors_choice.txt +0 -97
- data/spec/market_bot/android/data/leaderboard-apps_topselling_paid-page1.txt +0 -97
- data/spec/market_bot/android/data/leaderboard-apps_topselling_paid-page2.txt +0 -97
- data/spec/market_bot/android/data/leaderboard-apps_topselling_paid-page3.txt +0 -97
- data/spec/market_bot/android/data/leaderboard-apps_topselling_paid-page4.txt +0 -97
- data/spec/market_bot/android/developer_spec.rb +0 -57
- data/spec/market_bot/android/leaderboard_spec.rb +0 -140
- data/spec/market_bot/android/search_query_spec.rb +0 -6
- data/spec/market_bot/movie/data/leaderboard-movies_comedy_topselling_paid.txt +0 -327
- data/spec/market_bot/movie/leaderboard_spec.rb +0 -67
@@ -1,32 +0,0 @@
|
|
1
|
-
module MarketBot
|
2
|
-
module Android
|
3
|
-
|
4
|
-
# Developer pages are extremely similar to leaderboard pages.
|
5
|
-
# Amazingly, this inheritence hack works!
|
6
|
-
#
|
7
|
-
# BUG: This code only retrieves the first page of results.
|
8
|
-
# This means you will only get the first 24 apps for a developer.
|
9
|
-
# Some developers have hundreds of apps so this needs fixed!!!
|
10
|
-
class Developer < MarketBot::Android::Leaderboard
|
11
|
-
def initialize(developer, options={})
|
12
|
-
super(developer, nil, options)
|
13
|
-
end
|
14
|
-
|
15
|
-
def market_urls(options={})
|
16
|
-
results = []
|
17
|
-
|
18
|
-
country = options[:country] || 'us'
|
19
|
-
|
20
|
-
url = "https://play.google.com/store/apps/developer?"
|
21
|
-
url << "id=#{URI.escape(identifier)}&"
|
22
|
-
url << "gl=#{country}&"
|
23
|
-
url << "hl=en"
|
24
|
-
|
25
|
-
results << url
|
26
|
-
|
27
|
-
return results
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
end
|
32
|
-
end
|
@@ -1,232 +0,0 @@
|
|
1
|
-
module MarketBot
|
2
|
-
module Android
|
3
|
-
|
4
|
-
class Leaderboard
|
5
|
-
attr_reader :identifier, :category
|
6
|
-
attr_reader :hydra
|
7
|
-
|
8
|
-
MAX_STARS = 5
|
9
|
-
PERCENT_DENOM = 100
|
10
|
-
|
11
|
-
def self.parse(html)
|
12
|
-
if html.include?('<title>Editor's Choice')
|
13
|
-
parse_editors_choice_page(html)
|
14
|
-
else
|
15
|
-
parse_normal_page(html)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
def self.parse_normal_page(html)
|
20
|
-
results = []
|
21
|
-
doc = Nokogiri::HTML(html)
|
22
|
-
|
23
|
-
doc.css('.card').each do |snippet_node|
|
24
|
-
result = {}
|
25
|
-
|
26
|
-
details_node = snippet_node.css('.details')
|
27
|
-
|
28
|
-
unless snippet_node.css('img').empty?
|
29
|
-
result[:icon_url] = snippet_node.css('img').first.attributes['src'].value
|
30
|
-
end
|
31
|
-
|
32
|
-
unless snippet_node.css('.current-rating').empty?
|
33
|
-
stars_style = snippet_node.css('.current-rating').first.attributes['style'].value
|
34
|
-
stars_width_percent = stars_style[/width:\s+([0-9.]+)%/, 1].to_f
|
35
|
-
result[:stars] = (MAX_STARS * stars_width_percent/PERCENT_DENOM).round(1).to_s
|
36
|
-
else
|
37
|
-
result[:stars] = nil
|
38
|
-
end
|
39
|
-
|
40
|
-
result[:title] = details_node.css('.title').first.attributes['title'].to_s
|
41
|
-
|
42
|
-
if (price_elem = details_node.css('.buy span').first)
|
43
|
-
result[:price_usd] = price_elem.text
|
44
|
-
end
|
45
|
-
|
46
|
-
result[:developer] = details_node.css('.subtitle').first.attributes['title'].to_s
|
47
|
-
result[:market_id] = details_node.css('.title').first.attributes['href'].to_s.gsub('/store/apps/details?id=', '').gsub(/&feature=.*$/, '')
|
48
|
-
result[:market_url] = "https://play.google.com/store/apps/details?id=#{result[:market_id]}&hl=en"
|
49
|
-
|
50
|
-
result[:price_usd] = '$0.00' if result[:price_usd] == 'Install'
|
51
|
-
|
52
|
-
results << result
|
53
|
-
end
|
54
|
-
|
55
|
-
results
|
56
|
-
end
|
57
|
-
|
58
|
-
def self.parse_editors_choice_page(html)
|
59
|
-
results = []
|
60
|
-
|
61
|
-
doc = Nokogiri::HTML(html)
|
62
|
-
|
63
|
-
doc.css('.fsg-snippet').each do |snippet_node|
|
64
|
-
result = {}
|
65
|
-
|
66
|
-
result[:title] = snippet_node.css('.title').text
|
67
|
-
result[:price_usd] = nil
|
68
|
-
result[:developer] = snippet_node.css('.attribution').text
|
69
|
-
result[:market_id] = snippet_node.attributes['data-docid'].text
|
70
|
-
result[:market_url] = "https://play.google.com/store/apps/details?id=#{result[:market_id]}&hl=en"
|
71
|
-
|
72
|
-
results << result
|
73
|
-
end
|
74
|
-
|
75
|
-
results
|
76
|
-
end
|
77
|
-
|
78
|
-
# This is the initializer method for the Leaderboard class.
|
79
|
-
#
|
80
|
-
# Leaderboard gets initialized by default with a specified identifier, an optional app category, along with optional
|
81
|
-
# request options.
|
82
|
-
#
|
83
|
-
# * *Args* :
|
84
|
-
# - +identifier+ -> The identifier is used to get the results for distinct leaderboards.
|
85
|
-
# Valid identifiers include:
|
86
|
-
# :topselling_paid
|
87
|
-
# :topselling_free
|
88
|
-
# :topselling_new_free
|
89
|
-
# :topselling_new_paid
|
90
|
-
# :editors_choice
|
91
|
-
# :topselling_paid_game
|
92
|
-
# :movers_shakers
|
93
|
-
# :featured
|
94
|
-
# :tablet_featured
|
95
|
-
# :topgrossing
|
96
|
-
# - +category+ -> The category switches between the actual categories, or genres, of apps within a given leaderboard.
|
97
|
-
# Valid categories include:
|
98
|
-
# :game
|
99
|
-
# :arcade
|
100
|
-
# :brain
|
101
|
-
# :cards
|
102
|
-
# :casual
|
103
|
-
# :game_wallpaper
|
104
|
-
# :racing
|
105
|
-
# :sports_games
|
106
|
-
# :game_widgets
|
107
|
-
# :application
|
108
|
-
# :books_and_reference
|
109
|
-
# :business
|
110
|
-
# :comics
|
111
|
-
# :communication
|
112
|
-
# :education
|
113
|
-
# :entertainment
|
114
|
-
# :finance
|
115
|
-
# :health_and_fitness
|
116
|
-
# :libraries_and_demo
|
117
|
-
# :lifestyle
|
118
|
-
# :app_wallpaper
|
119
|
-
# :media_and_video
|
120
|
-
# :medical
|
121
|
-
# :music_and_audio
|
122
|
-
# :news_and_magazines
|
123
|
-
# :personalization
|
124
|
-
# :photography
|
125
|
-
# :productivity
|
126
|
-
# :shopping
|
127
|
-
# :social
|
128
|
-
# :sports
|
129
|
-
# :tools
|
130
|
-
# :transportation
|
131
|
-
# :travel_and_local
|
132
|
-
# :weather
|
133
|
-
# :app_widgets
|
134
|
-
# - +options+ -> The optional options Hash contains keys :hydra and :request_opts. :hydra can be used to specify
|
135
|
-
# a custom Hydra instance, while :request_opts is a Hash containing further options for the Play
|
136
|
-
# Store HTTP request.
|
137
|
-
#
|
138
|
-
def initialize(identifier, category=nil, options={})
|
139
|
-
@identifier = identifier
|
140
|
-
@category = category
|
141
|
-
@hydra = options[:hydra] || MarketBot.hydra
|
142
|
-
@parsed_results = []
|
143
|
-
@pending_pages = []
|
144
|
-
@request_opts = options[:request_opts] || {}
|
145
|
-
@request_opts[:timeout] ||= MarketBot.timeout
|
146
|
-
@request_opts[:connecttimeout] ||= MarketBot.connecttimeout
|
147
|
-
end
|
148
|
-
|
149
|
-
def market_urls(options={})
|
150
|
-
results = []
|
151
|
-
|
152
|
-
min_page = options[:min_page] || 1
|
153
|
-
max_page = options[:max_page] || 25
|
154
|
-
country = options[:country] || 'us'
|
155
|
-
|
156
|
-
(min_page..max_page).each do |page|
|
157
|
-
start_val = (page - 1) * 24
|
158
|
-
|
159
|
-
url = 'https://play.google.com/store/apps'
|
160
|
-
url << "/category/#{category.to_s.upcase}" if category
|
161
|
-
url << "/collection/#{identifier.to_s}?"
|
162
|
-
url << "start=#{start_val}&"
|
163
|
-
url << "gl=#{country}&"
|
164
|
-
url << "num=24&hl=en"
|
165
|
-
|
166
|
-
results << url
|
167
|
-
end
|
168
|
-
|
169
|
-
results
|
170
|
-
end
|
171
|
-
|
172
|
-
def enqueue_update(options={},&block)
|
173
|
-
@callback = block
|
174
|
-
if @identifier.to_s.downcase == 'editors_choice' && category == nil
|
175
|
-
url = 'https://play.google.com/store/apps/collection/editors_choice?&hl=en'
|
176
|
-
process_page(url, 1)
|
177
|
-
else
|
178
|
-
min_rank = options[:min_rank] || 1
|
179
|
-
max_rank = options[:max_rank] || 500
|
180
|
-
country = options[:country] || 'us'
|
181
|
-
|
182
|
-
min_page = rank_to_page(min_rank)
|
183
|
-
max_page = rank_to_page(max_rank)
|
184
|
-
|
185
|
-
@parsed_results = []
|
186
|
-
|
187
|
-
urls = market_urls(:min_page => min_page, :max_page => max_page, :country => country)
|
188
|
-
urls.each_index{ |i| process_page(urls[i], i+1) }
|
189
|
-
end
|
190
|
-
|
191
|
-
self
|
192
|
-
end
|
193
|
-
|
194
|
-
def update(options={})
|
195
|
-
enqueue_update(options)
|
196
|
-
@hydra.run
|
197
|
-
|
198
|
-
self
|
199
|
-
end
|
200
|
-
|
201
|
-
def rank_to_page(rank)
|
202
|
-
((rank - 1) / 24) + 1
|
203
|
-
end
|
204
|
-
|
205
|
-
def results
|
206
|
-
raise 'Results do not exist yet.' unless @parsed_results
|
207
|
-
@parsed_results.reject{ |page| page.nil? || page.empty? }.flatten
|
208
|
-
end
|
209
|
-
|
210
|
-
private
|
211
|
-
def process_page(url, page_num)
|
212
|
-
@pending_pages << page_num
|
213
|
-
request = Typhoeus::Request.new(url, @request_opts)
|
214
|
-
request.on_complete do |response|
|
215
|
-
# HACK: Typhoeus <= 0.4.2 returns a response, 0.5.0pre returns the request.
|
216
|
-
response = response.response if response.is_a?(Typhoeus::Request)
|
217
|
-
|
218
|
-
result = Leaderboard.parse(response.body)
|
219
|
-
update_callback(result, page_num)
|
220
|
-
end
|
221
|
-
@hydra.queue(request)
|
222
|
-
end
|
223
|
-
|
224
|
-
def update_callback(result, page)
|
225
|
-
@parsed_results[page] = result
|
226
|
-
@pending_pages.delete(page)
|
227
|
-
@callback.call(self) if @callback and @pending_pages.empty?
|
228
|
-
end
|
229
|
-
end
|
230
|
-
|
231
|
-
end
|
232
|
-
end
|
@@ -1,59 +0,0 @@
|
|
1
|
-
module MarketBot
|
2
|
-
module Android
|
3
|
-
|
4
|
-
class Leaderboard
|
5
|
-
IDENTIFIERS = [
|
6
|
-
:editors_choice,
|
7
|
-
:featured,
|
8
|
-
:movers_shakers,
|
9
|
-
:tablet_featured,
|
10
|
-
:topgrossing,
|
11
|
-
:topselling_free,
|
12
|
-
:topselling_new_free,
|
13
|
-
:topselling_new_paid,
|
14
|
-
:topselling_paid,
|
15
|
-
:topselling_paid_game
|
16
|
-
]
|
17
|
-
|
18
|
-
CATEGORIES = [
|
19
|
-
:application,
|
20
|
-
:app_wallpaper,
|
21
|
-
:app_widgets,
|
22
|
-
:arcade,
|
23
|
-
:books_and_reference,
|
24
|
-
:brain,
|
25
|
-
:business,
|
26
|
-
:cards,
|
27
|
-
:casual,
|
28
|
-
:comics,
|
29
|
-
:communication,
|
30
|
-
:education,
|
31
|
-
:entertainment,
|
32
|
-
:finance,
|
33
|
-
:game,
|
34
|
-
:game_wallpaper,
|
35
|
-
:game_widgets,
|
36
|
-
:health_and_fitness,
|
37
|
-
:libraries_and_demo,
|
38
|
-
:lifestyle,
|
39
|
-
:media_and_video,
|
40
|
-
:medical,
|
41
|
-
:music_and_audio,
|
42
|
-
:news_and_magazines,
|
43
|
-
:personalization,
|
44
|
-
:photography,
|
45
|
-
:productivity,
|
46
|
-
:racing,
|
47
|
-
:shopping,
|
48
|
-
:social,
|
49
|
-
:sports,
|
50
|
-
:sports_games,
|
51
|
-
:tools,
|
52
|
-
:transportation,
|
53
|
-
:travel_and_local,
|
54
|
-
:weather
|
55
|
-
]
|
56
|
-
end
|
57
|
-
|
58
|
-
end
|
59
|
-
end
|
@@ -1,35 +0,0 @@
|
|
1
|
-
module MarketBot
|
2
|
-
module Android
|
3
|
-
|
4
|
-
# Search query pages are extremely similar to leaderboard pages.
|
5
|
-
# Amazingly, this inheritence hack works!
|
6
|
-
class SearchQuery < MarketBot::Android::Leaderboard
|
7
|
-
def initialize(query, options={})
|
8
|
-
super(query, nil, options)
|
9
|
-
end
|
10
|
-
|
11
|
-
def market_urls(options={})
|
12
|
-
results = []
|
13
|
-
|
14
|
-
min_page = options[:min_page] || 1
|
15
|
-
max_page = options[:max_page] || 25
|
16
|
-
country = options[:country] || 'us'
|
17
|
-
|
18
|
-
(min_page..max_page).each do |page|
|
19
|
-
start_val = (page - 1) * 24
|
20
|
-
|
21
|
-
url = "https://play.google.com/store/search?"
|
22
|
-
url << "q=#{URI.escape(identifier)}&"
|
23
|
-
url << "c=apps&start=#{start_val}&"
|
24
|
-
url << "gl=#{country}&"
|
25
|
-
url << "num=24&hl=en"
|
26
|
-
|
27
|
-
results << url
|
28
|
-
end
|
29
|
-
|
30
|
-
results
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
end
|
35
|
-
end
|
@@ -1,167 +0,0 @@
|
|
1
|
-
module MarketBot
|
2
|
-
module Movie
|
3
|
-
|
4
|
-
class Leaderboard
|
5
|
-
attr_reader :identifier, :category
|
6
|
-
attr_reader :hydra
|
7
|
-
|
8
|
-
MAX_STARS = 5
|
9
|
-
PERCENT_DENOM = 100
|
10
|
-
|
11
|
-
def self.parse(html)
|
12
|
-
results = []
|
13
|
-
doc = Nokogiri::HTML(html)
|
14
|
-
|
15
|
-
doc.css('.card-list').each do |snippet_node|
|
16
|
-
result = {}
|
17
|
-
|
18
|
-
details_node = snippet_node.css('.details')
|
19
|
-
|
20
|
-
unless snippet_node.css('.current-rating').empty?
|
21
|
-
stars_style = snippet_node.css('.current-rating').first.attributes['style'].value
|
22
|
-
stars_width_percent = stars_style[/width:\s+([0-9.]+)%/, 1].to_f
|
23
|
-
result[:stars] = (MAX_STARS * stars_width_percent/PERCENT_DENOM).round(1).to_s
|
24
|
-
else
|
25
|
-
result[:stars] = nil
|
26
|
-
end
|
27
|
-
|
28
|
-
result[:title] = details_node.css('.title').first.attributes['title'].to_s
|
29
|
-
|
30
|
-
if (price_elem = details_node.css('.buy span').first)
|
31
|
-
result[:price_usd] = price_elem.text
|
32
|
-
end
|
33
|
-
|
34
|
-
result[:genre] = details_node.css('.subtitle').first.attributes['title'].to_s
|
35
|
-
movie_detail_url = details_node.css('.title').first.attributes['href'].to_s.gsub('/store/movies/details', '')
|
36
|
-
result[:market_id] = movie_detail_url.split('?id=').last
|
37
|
-
result[:market_url] = "https://play.google.com/store/movies/details#{movie_detail_url}&hl=en"
|
38
|
-
|
39
|
-
result[:price_usd] = '$0.00' if result[:price_usd] == 'Install'
|
40
|
-
|
41
|
-
results << result
|
42
|
-
end
|
43
|
-
|
44
|
-
results
|
45
|
-
end
|
46
|
-
|
47
|
-
# This is the initializer method for the Leaderboard class.
|
48
|
-
#
|
49
|
-
# Leaderboard gets initialized by default with a specified identifier, an optional movies category, along with optional
|
50
|
-
# request options.
|
51
|
-
#
|
52
|
-
# * *Args* :
|
53
|
-
# - +identifier+ -> The identifier is used to get the results for distinct leaderboards.
|
54
|
-
# Valid identifiers include:
|
55
|
-
# :topselling_paid
|
56
|
-
# :topselling_paid_show
|
57
|
-
# - +category+ -> The category switches between the actual categories, or genres, of movies within a given leaderboard.
|
58
|
-
# Valid categories include:
|
59
|
-
#
|
60
|
-
# :action_and_adventure,
|
61
|
-
# :animation,
|
62
|
-
# :classics,
|
63
|
-
# :comedy,
|
64
|
-
# :crime,
|
65
|
-
# :documentary,
|
66
|
-
# :drama,
|
67
|
-
# :family,
|
68
|
-
# :horror,
|
69
|
-
# :independent,
|
70
|
-
# :indian_cinema,
|
71
|
-
# :music,
|
72
|
-
# :sci_fi_and_fantasy,
|
73
|
-
# :short_films,
|
74
|
-
# :sports,
|
75
|
-
# :world_cinema
|
76
|
-
## - +options+ -> The optional options Hash contains keys :hydra and :request_opts. :hydra can be used to specify
|
77
|
-
# a custom Hydra instance, while :request_opts is a Hash containing further options for the Play
|
78
|
-
# Store HTTP request.
|
79
|
-
#
|
80
|
-
def initialize(identifier, category=nil, options={})
|
81
|
-
@identifier = identifier
|
82
|
-
@category = category
|
83
|
-
@hydra = options[:hydra] || MarketBot.hydra
|
84
|
-
@request_opts = options[:request_opts] || {}
|
85
|
-
@parsed_results = []
|
86
|
-
@pending_pages = []
|
87
|
-
end
|
88
|
-
|
89
|
-
def market_urls(options={})
|
90
|
-
results = []
|
91
|
-
|
92
|
-
min_page = options[:min_page] || 1
|
93
|
-
max_page = options[:max_page] || 25
|
94
|
-
country = options[:country] || 'us'
|
95
|
-
|
96
|
-
(min_page..max_page).each do |page|
|
97
|
-
start_val = (page - 1) * 24
|
98
|
-
|
99
|
-
url = 'https://play.google.com/store/movies'
|
100
|
-
url << "/category/#{category.to_s.upcase}" if category
|
101
|
-
url << "/collection/#{identifier.to_s}?"
|
102
|
-
url << "start=#{start_val}&"
|
103
|
-
url << "gl=#{country}&"
|
104
|
-
url << "num=24&hl=en"
|
105
|
-
|
106
|
-
results << url
|
107
|
-
end
|
108
|
-
|
109
|
-
results
|
110
|
-
end
|
111
|
-
|
112
|
-
def enqueue_update(options={},&block)
|
113
|
-
@callback = block
|
114
|
-
min_rank = options[:min_rank] || 1
|
115
|
-
max_rank = options[:max_rank] || 500
|
116
|
-
country = options[:country] || 'us'
|
117
|
-
|
118
|
-
min_page = rank_to_page(min_rank)
|
119
|
-
max_page = rank_to_page(max_rank)
|
120
|
-
|
121
|
-
@parsed_results = []
|
122
|
-
|
123
|
-
urls = market_urls(:min_page => min_page, :max_page => max_page, :country => country)
|
124
|
-
urls.each_index{ |i| process_page(urls[i], i+1) }
|
125
|
-
|
126
|
-
self
|
127
|
-
end
|
128
|
-
|
129
|
-
def update(options={})
|
130
|
-
enqueue_update(options)
|
131
|
-
@hydra.run
|
132
|
-
|
133
|
-
self
|
134
|
-
end
|
135
|
-
|
136
|
-
def rank_to_page(rank)
|
137
|
-
((rank - 1) / 24) + 1
|
138
|
-
end
|
139
|
-
|
140
|
-
def results
|
141
|
-
raise 'Results do not exist yet.' unless @parsed_results
|
142
|
-
@parsed_results.reject{ |page| page.nil? || page.empty? }.flatten
|
143
|
-
end
|
144
|
-
|
145
|
-
private
|
146
|
-
def process_page(url, page_num)
|
147
|
-
@pending_pages << page_num
|
148
|
-
request = Typhoeus::Request.new(url, @request_opts)
|
149
|
-
request.on_complete do |response|
|
150
|
-
# HACK: Typhoeus <= 0.4.2 returns a response, 0.5.0pre returns the request.
|
151
|
-
response = response.response if response.is_a?(Typhoeus::Request)
|
152
|
-
|
153
|
-
result = Leaderboard.parse(response.body)
|
154
|
-
update_callback(result, page_num)
|
155
|
-
end
|
156
|
-
@hydra.queue(request)
|
157
|
-
end
|
158
|
-
|
159
|
-
def update_callback(result, page)
|
160
|
-
@parsed_results[page] = result
|
161
|
-
@pending_pages.delete(page)
|
162
|
-
@callback.call(self) if @callback and @pending_pages.empty?
|
163
|
-
end
|
164
|
-
end
|
165
|
-
|
166
|
-
end
|
167
|
-
end
|