pandata 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/pandata.rb +1 -1
- data/lib/pandata/scraper.rb +12 -5
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c9819160b7e403e619b6a5dfeae8e8c9199a74a4
|
|
4
|
+
data.tar.gz: b37522f882a3d40b7f0546bcea59f14aa52ec0d4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9f1e76bec614fc7b251a1b85ad8cd3720b6ac1ac02bfe037f6fa65f70f48651f530c705b0d697fea815d7aca77b638cbac99ef6a6c9a5d8d649600cc4e2fb43e
|
|
7
|
+
data.tar.gz: 15b8d23fab4a7f527da0f904a2dec68c4e760c018442f86f7b3f9c547aaa88798e8797421d77c1b583f2c3b4b8adcd73b2a551f17dad19aac0e6cc4224c1c746
|
data/lib/pandata.rb
CHANGED
data/lib/pandata/scraper.rb
CHANGED
|
@@ -156,7 +156,14 @@ module Pandata
|
|
|
156
156
|
|
|
157
157
|
while next_data_indices
|
|
158
158
|
html = Downloader.read_page(url)
|
|
159
|
+
|
|
160
|
+
# Sometimes Pandora returns the same next_data_indices as the previous page.
|
|
161
|
+
# If we don't check for this, an infinite loop occurs.
|
|
162
|
+
# This problem occurs with tconrad.
|
|
163
|
+
prev_next_data_indices = next_data_indices
|
|
159
164
|
next_data_indices = @parser.get_next_data_indices(html)
|
|
165
|
+
next_data_indices = false if prev_next_data_indices == next_data_indices
|
|
166
|
+
|
|
160
167
|
url = yield(html, next_data_indices)
|
|
161
168
|
end
|
|
162
169
|
end
|
|
@@ -165,11 +172,11 @@ module Pandata
|
|
|
165
172
|
# @param data_name [Symbol]
|
|
166
173
|
# @param next_data_indices [Symbol] query parameters to get the next set of data
|
|
167
174
|
def get_url(data_name, next_data_indices = {})
|
|
168
|
-
next_data_indices
|
|
169
|
-
nextStartIndex: 0,
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
175
|
+
if next_data_indices.empty?
|
|
176
|
+
next_data_indices = { nextStartIndex: 0, nextLikeStartIndex: 0, nextThumbStartIndex: 0 }
|
|
177
|
+
else
|
|
178
|
+
next_data_indices = next_data_indices.dup
|
|
179
|
+
end
|
|
173
180
|
|
|
174
181
|
next_data_indices[:webname] = @webname
|
|
175
182
|
next_data_indices[:pat] = Downloader.get_pat
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pandata
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Brian Ustas
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2015-06-28 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: nokogiri
|