list_spider 2.7.0 → 2.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 95a8753dc4e29320bebe13a0ad93ee9bb16c6a80bd820d430835f41a00eca637
-  data.tar.gz: 36aac688ddd264ba02028c415e60165684223a287cb0ce0ed94f44a7257dd668
+  metadata.gz: 82a3e44e95936f19117e2c76e6edc3e692ec1fc67da0b920d00d0220db611e0f
+  data.tar.gz: 11c76a651e6711bf4179585ac33388aef50e2e258e5bed39a1f8eb39b1a7e044
 SHA512:
-  metadata.gz: a356472be61aa5f42c994668c0dc07a36a6a7e2a15b7bd90132e4ecd935a4e6d7c06c7f032b9ae3fa8e4e9edceca3467104c8608a5b50d82561706279d06779c
-  data.tar.gz: cd0a5099a8043d941d60a65feb5b54433d0a5c90a88050f66b05682114f77dc03b8b5996e0499806524bbf55049b1b93de07ba36c2b865fd7b043c27f1a0b6fc
+  metadata.gz: 997374b0ae24023d05239f3253fc338ba1a814fb85a799bb75cda97a875a1d40c9fd73ca3bdda5fffb52a665a4c0296b0b27bafa6dcc818b5cb990cd7dfe83bf
+  data.tar.gz: e95b8c72c3c5429e4c46f266f13a22ca925ab8261501970f8d217ca118b9c4b9e3ad1542d07da62219f36430bee1c112a0e3446a75216406aaff840621426bef

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    list_spider (2.7.0)
+    list_spider (2.8.0)
       em-http-request (~> 1.1, >= 1.1.3)
       nokogiri (>= 1.10.8)
       rchardet (~> 1.6, >= 1.6.1)
@@ -23,9 +23,9 @@ GEM
     eventmachine (1.2.7)
     http_parser.rb (0.6.0)
     mini_portile2 (2.4.0)
-    nokogiri (1.10.9)
+    nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
-    public_suffix (4.0.4)
+    public_suffix (4.0.6)
     rake (13.0.1)
     rchardet (1.8.0)
 

data/lib/list_spider.rb

@@ -129,7 +129,7 @@ module ListSpider
         interval = RANDOM_TIME
       end
 
-      @down_list = filter_list(down_list)
+      filter_list(down_list)
       @interval = interval
       @max = max
       @max = @down_list.size if @max == NO_LIMIT_CONCURRENT
@@ -146,11 +146,9 @@ module ListSpider
 
     def add_task(task)
       if task.is_a? Array
-        need_down_list = filter_list(task)
-        @down_list += need_down_list
+        filter_list(task)
       elsif task.is_a?TaskStruct
-        need_down_list = filter_list([task])
-        @down_list += need_down_list
+        filter_list([task])
       else
         puts "error task type:#{task.class}"
       end
@@ -291,17 +289,15 @@ module ListSpider
     end
 
     def filter_list(down_list)
-      return down_list unless @save_file
-      
-      need_down_list = []
+      return unless @save_file
+
       down_list.each do |ts|
         if !ts.overwrite_exist && File.exist?(ts.local_path)
           call_parse_method(ts)
         elsif @local_path_set.add?(ts.local_path)
-          need_down_list << ts
+          @down_list << ts
         end
       end
-      need_down_list
     end
   end
 

data/lib/list_spider/version.rb

@@ -1,3 +1,3 @@
 module ListSpider
-  VERSION = '2.7.0'.freeze
+  VERSION = '2.8.0'.freeze
 end

metadata

@@ -1,24 +1,24 @@
 --- !ruby/object:Gem::Specification
 name: list_spider
 version: !ruby/object:Gem::Version
-  version: 2.7.0
+  version: 2.8.0
 platform: ruby
 authors:
 - Charles Zhang
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-04-25 00:00:00.000000000 Z
+date: 2021-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.16'
   type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -26,13 +26,13 @@ dependencies:
         version: '1.16'
 - !ruby/object:Gem::Dependency
   name: rake
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 12.3.3
   type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -40,6 +40,7 @@ dependencies:
         version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: em-http-request
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -49,7 +50,6 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.1.3
   type: :runtime
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -60,13 +60,13 @@ dependencies:
         version: 1.1.3
 - !ruby/object:Gem::Dependency
   name: nokogiri
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.8
   type: :runtime
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -74,6 +74,7 @@ dependencies:
         version: 1.10.8
 - !ruby/object:Gem::Dependency
   name: rchardet
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -83,7 +84,6 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.6.1
   type: :runtime
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"