list_spider 0.1.5 → 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/list_spider.rb +4 -2
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ffb9c26b0fc7db4fd6a2a3b831832964afbaa949
|
|
4
|
+
data.tar.gz: 2c829fb860ad3c0c8b3825c1575988cadfccaa15
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1673f2f5daed52295ee567c2d1c6b672f1bdf532fd31a334a704cd9ac5fbdcef9aea0f27e9828cd816769f74bfcbd8498294f1d190e73defb0fb6c15fb990e04
|
|
7
|
+
data.tar.gz: 31247ae66dfac70a69671c336dcc942812d26fd9572c2e527dbd511bf40ed680304fcb24d7f1fac01e51b52d666061ffacaaae49c960a51aecff61f07ddcf9be
|
data/lib/list_spider.rb
CHANGED
|
@@ -32,6 +32,8 @@ module ListSpider
|
|
|
32
32
|
|
|
33
33
|
RANDOM_TIME = -1
|
|
34
34
|
NO_LIMIT_CONCURRENT = -1
|
|
35
|
+
DEFAULT_CONCURRNET_MAX = 50
|
|
36
|
+
DEFAULT_INTERVAL = 0
|
|
35
37
|
|
|
36
38
|
@random_time_range = 3..10
|
|
37
39
|
@conver_to_utf8 = false
|
|
@@ -210,7 +212,7 @@ module ListSpider
|
|
|
210
212
|
return need_down_list
|
|
211
213
|
end
|
|
212
214
|
|
|
213
|
-
def get_list(down_list, inter_val:
|
|
215
|
+
def get_list(down_list, inter_val: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
|
|
214
216
|
@@down_list = []
|
|
215
217
|
|
|
216
218
|
need_down_list = filter_list(down_list)
|
|
@@ -226,7 +228,7 @@ module ListSpider
|
|
|
226
228
|
event_machine_start_list(get_next_task, method(:complete))
|
|
227
229
|
end
|
|
228
230
|
|
|
229
|
-
def get_one(task, inter_val:
|
|
231
|
+
def get_one(task, inter_val: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
|
|
230
232
|
get_list([task], inter_val: inter_val, max: max)
|
|
231
233
|
end
|
|
232
234
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: list_spider
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.6
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Charles Zhang
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-05-
|
|
11
|
+
date: 2016-05-08 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: em-http-request
|