list_spider 0.1.8 → 0.1.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/list_spider.rb +12 -7
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4b43422e1a52997518a53a38c021f68700ec6d3f
|
4
|
+
data.tar.gz: cefa5535e5870a221b04a47d6b1f6ec177392caa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2c561ce1828e592a538e1747dccc1da39c1946f182ffc19212fd8284b90b0c1f43e7e584ff3bb1f416b984bc04328901d887bf4fd294f20448ad98a7d7b0c1dd
|
7
|
+
data.tar.gz: 55d6a8f01bb1a325d365d42f33123289881f37e985768ef75538a6f1a1f044b248cf42f5685f6974591f86f0ba140e78a693cac4af4981c6b564cd3a131568ef
|
data/lib/list_spider.rb
CHANGED
@@ -199,12 +199,12 @@ module ListSpider
|
|
199
199
|
if todo.empty?
|
200
200
|
stop_machine
|
201
201
|
else
|
202
|
-
if @@
|
202
|
+
if @@interval != 0
|
203
203
|
if success_list.size != 0 || failed_list.size != 0
|
204
|
-
if @@
|
204
|
+
if @@interval == RANDOM_TIME
|
205
205
|
sleep(rand(@random_time_range))
|
206
206
|
else
|
207
|
-
sleep(@@
|
207
|
+
sleep(@@interval)
|
208
208
|
end
|
209
209
|
end
|
210
210
|
end
|
@@ -239,13 +239,18 @@ module ListSpider
|
|
239
239
|
return need_down_list
|
240
240
|
end
|
241
241
|
|
242
|
-
def get_list(down_list,
|
242
|
+
def get_list(down_list, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
|
243
|
+
if interval.is_a?Range
|
244
|
+
@random_time_range = interval
|
245
|
+
interval = RANDOM_TIME
|
246
|
+
end
|
247
|
+
|
243
248
|
@@down_list = []
|
244
249
|
|
245
250
|
need_down_list = filter_list(down_list)
|
246
251
|
|
247
252
|
@@down_list = @@down_list + need_down_list
|
248
|
-
@@
|
253
|
+
@@interval = interval
|
249
254
|
@@max = max
|
250
255
|
@@max = @@down_list.size if @@max == NO_LIMIT_CONCURRENT
|
251
256
|
@@succeed_size = 0
|
@@ -255,8 +260,8 @@ module ListSpider
|
|
255
260
|
event_machine_start_list(get_next_task, method(:complete))
|
256
261
|
end
|
257
262
|
|
258
|
-
def get_one(task,
|
259
|
-
get_list([task],
|
263
|
+
def get_one(task, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
|
264
|
+
get_list([task], interval: interval, max: max)
|
260
265
|
end
|
261
266
|
|
262
267
|
def add_task(task)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: list_spider
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Charles Zhang
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-05-
|
11
|
+
date: 2016-05-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: em-http-request
|