list_spider 2.7.0 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +3 -3
  3. data/lib/list_spider.rb +6 -10
  4. data/lib/list_spider/version.rb +1 -1
  5. metadata +7 -7
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 95a8753dc4e29320bebe13a0ad93ee9bb16c6a80bd820d430835f41a00eca637
4
- data.tar.gz: 36aac688ddd264ba02028c415e60165684223a287cb0ce0ed94f44a7257dd668
3
+ metadata.gz: 82a3e44e95936f19117e2c76e6edc3e692ec1fc67da0b920d00d0220db611e0f
4
+ data.tar.gz: 11c76a651e6711bf4179585ac33388aef50e2e258e5bed39a1f8eb39b1a7e044
5
5
  SHA512:
6
- metadata.gz: a356472be61aa5f42c994668c0dc07a36a6a7e2a15b7bd90132e4ecd935a4e6d7c06c7f032b9ae3fa8e4e9edceca3467104c8608a5b50d82561706279d06779c
7
- data.tar.gz: cd0a5099a8043d941d60a65feb5b54433d0a5c90a88050f66b05682114f77dc03b8b5996e0499806524bbf55049b1b93de07ba36c2b865fd7b043c27f1a0b6fc
6
+ metadata.gz: 997374b0ae24023d05239f3253fc338ba1a814fb85a799bb75cda97a875a1d40c9fd73ca3bdda5fffb52a665a4c0296b0b27bafa6dcc818b5cb990cd7dfe83bf
7
+ data.tar.gz: e95b8c72c3c5429e4c46f266f13a22ca925ab8261501970f8d217ca118b9c4b9e3ad1542d07da62219f36430bee1c112a0e3446a75216406aaff840621426bef
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- list_spider (2.7.0)
4
+ list_spider (2.8.0)
5
5
  em-http-request (~> 1.1, >= 1.1.3)
6
6
  nokogiri (>= 1.10.8)
7
7
  rchardet (~> 1.6, >= 1.6.1)
@@ -23,9 +23,9 @@ GEM
23
23
  eventmachine (1.2.7)
24
24
  http_parser.rb (0.6.0)
25
25
  mini_portile2 (2.4.0)
26
- nokogiri (1.10.9)
26
+ nokogiri (1.10.10)
27
27
  mini_portile2 (~> 2.4.0)
28
- public_suffix (4.0.4)
28
+ public_suffix (4.0.6)
29
29
  rake (13.0.1)
30
30
  rchardet (1.8.0)
31
31
 
data/lib/list_spider.rb CHANGED
@@ -129,7 +129,7 @@ module ListSpider
129
129
  interval = RANDOM_TIME
130
130
  end
131
131
 
132
- @down_list = filter_list(down_list)
132
+ filter_list(down_list)
133
133
  @interval = interval
134
134
  @max = max
135
135
  @max = @down_list.size if @max == NO_LIMIT_CONCURRENT
@@ -146,11 +146,9 @@ module ListSpider
146
146
 
147
147
  def add_task(task)
148
148
  if task.is_a? Array
149
- need_down_list = filter_list(task)
150
- @down_list += need_down_list
149
+ filter_list(task)
151
150
  elsif task.is_a?TaskStruct
152
- need_down_list = filter_list([task])
153
- @down_list += need_down_list
151
+ filter_list([task])
154
152
  else
155
153
  puts "error task type:#{task.class}"
156
154
  end
@@ -291,17 +289,15 @@ module ListSpider
291
289
  end
292
290
 
293
291
  def filter_list(down_list)
294
- return down_list unless @save_file
295
-
296
- need_down_list = []
292
+ return unless @save_file
293
+
297
294
  down_list.each do |ts|
298
295
  if !ts.overwrite_exist && File.exist?(ts.local_path)
299
296
  call_parse_method(ts)
300
297
  elsif @local_path_set.add?(ts.local_path)
301
- need_down_list << ts
298
+ @down_list << ts
302
299
  end
303
300
  end
304
- need_down_list
305
301
  end
306
302
  end
307
303
 
data/lib/list_spider/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module ListSpider
2
- VERSION = '2.7.0'.freeze
2
+ VERSION = '2.8.0'.freeze
3
3
  end
metadata CHANGED
@@ -1,24 +1,24 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: list_spider
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.7.0
4
+ version: 2.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Charles Zhang
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-04-25 00:00:00.000000000 Z
11
+ date: 2021-01-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
15
+ prerelease: false
15
16
  requirement: !ruby/object:Gem::Requirement
16
17
  requirements:
17
18
  - - "~>"
18
19
  - !ruby/object:Gem::Version
19
20
  version: '1.16'
20
21
  type: :development
21
- prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
@@ -26,13 +26,13 @@ dependencies:
26
26
  version: '1.16'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: rake
29
+ prerelease: false
29
30
  requirement: !ruby/object:Gem::Requirement
30
31
  requirements:
31
32
  - - ">="
32
33
  - !ruby/object:Gem::Version
33
34
  version: 12.3.3
34
35
  type: :development
35
- prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
@@ -40,6 +40,7 @@ dependencies:
40
40
  version: 12.3.3
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: em-http-request
43
+ prerelease: false
43
44
  requirement: !ruby/object:Gem::Requirement
44
45
  requirements:
45
46
  - - "~>"
@@ -49,7 +50,6 @@ dependencies:
49
50
  - !ruby/object:Gem::Version
50
51
  version: 1.1.3
51
52
  type: :runtime
52
- prerelease: false
53
53
  version_requirements: !ruby/object:Gem::Requirement
54
54
  requirements:
55
55
  - - "~>"
@@ -60,13 +60,13 @@ dependencies:
60
60
  version: 1.1.3
61
61
  - !ruby/object:Gem::Dependency
62
62
  name: nokogiri
63
+ prerelease: false
63
64
  requirement: !ruby/object:Gem::Requirement
64
65
  requirements:
65
66
  - - ">="
66
67
  - !ruby/object:Gem::Version
67
68
  version: 1.10.8
68
69
  type: :runtime
69
- prerelease: false
70
70
  version_requirements: !ruby/object:Gem::Requirement
71
71
  requirements:
72
72
  - - ">="
@@ -74,6 +74,7 @@ dependencies:
74
74
  version: 1.10.8
75
75
  - !ruby/object:Gem::Dependency
76
76
  name: rchardet
77
+ prerelease: false
77
78
  requirement: !ruby/object:Gem::Requirement
78
79
  requirements:
79
80
  - - "~>"
@@ -83,7 +84,6 @@ dependencies:
83
84
  - !ruby/object:Gem::Version
84
85
  version: 1.6.1
85
86
  type: :runtime
86
- prerelease: false
87
87
  version_requirements: !ruby/object:Gem::Requirement
88
88
  requirements:
89
89
  - - "~>"