list_spider 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/list_spider.rb +12 -7
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4b43422e1a52997518a53a38c021f68700ec6d3f
|
|
4
|
+
data.tar.gz: cefa5535e5870a221b04a47d6b1f6ec177392caa
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2c561ce1828e592a538e1747dccc1da39c1946f182ffc19212fd8284b90b0c1f43e7e584ff3bb1f416b984bc04328901d887bf4fd294f20448ad98a7d7b0c1dd
|
|
7
|
+
data.tar.gz: 55d6a8f01bb1a325d365d42f33123289881f37e985768ef75538a6f1a1f044b248cf42f5685f6974591f86f0ba140e78a693cac4af4981c6b564cd3a131568ef
|
data/lib/list_spider.rb
CHANGED
|
@@ -199,12 +199,12 @@ module ListSpider
|
|
|
199
199
|
if todo.empty?
|
|
200
200
|
stop_machine
|
|
201
201
|
else
|
|
202
|
-
if @@
|
|
202
|
+
if @@interval != 0
|
|
203
203
|
if success_list.size != 0 || failed_list.size != 0
|
|
204
|
-
if @@
|
|
204
|
+
if @@interval == RANDOM_TIME
|
|
205
205
|
sleep(rand(@random_time_range))
|
|
206
206
|
else
|
|
207
|
-
sleep(@@
|
|
207
|
+
sleep(@@interval)
|
|
208
208
|
end
|
|
209
209
|
end
|
|
210
210
|
end
|
|
@@ -239,13 +239,18 @@ module ListSpider
|
|
|
239
239
|
return need_down_list
|
|
240
240
|
end
|
|
241
241
|
|
|
242
|
-
def get_list(down_list,
|
|
242
|
+
def get_list(down_list, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
|
|
243
|
+
if interval.is_a?Range
|
|
244
|
+
@random_time_range = interval
|
|
245
|
+
interval = RANDOM_TIME
|
|
246
|
+
end
|
|
247
|
+
|
|
243
248
|
@@down_list = []
|
|
244
249
|
|
|
245
250
|
need_down_list = filter_list(down_list)
|
|
246
251
|
|
|
247
252
|
@@down_list = @@down_list + need_down_list
|
|
248
|
-
@@
|
|
253
|
+
@@interval = interval
|
|
249
254
|
@@max = max
|
|
250
255
|
@@max = @@down_list.size if @@max == NO_LIMIT_CONCURRENT
|
|
251
256
|
@@succeed_size = 0
|
|
@@ -255,8 +260,8 @@ module ListSpider
|
|
|
255
260
|
event_machine_start_list(get_next_task, method(:complete))
|
|
256
261
|
end
|
|
257
262
|
|
|
258
|
-
def get_one(task,
|
|
259
|
-
get_list([task],
|
|
263
|
+
def get_one(task, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
|
|
264
|
+
get_list([task], interval: interval, max: max)
|
|
260
265
|
end
|
|
261
266
|
|
|
262
267
|
def add_task(task)
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: list_spider
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.9
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Charles Zhang
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-05-
|
|
11
|
+
date: 2016-05-12 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: em-http-request
|