list_spider 2.0.1 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ef1d9cd5edd321718137a9b9055b6e223fb870f1209271df4142930c35dd6f5
-  data.tar.gz: 983055db2e6714337f4f76ec3d279a83f46d48e689be0334c9a944bce197599c
+  metadata.gz: 4a69a63919c971b077855f236631b5b91895e11836338c8970dec65214f92a4c
+  data.tar.gz: 6e0fefdd9a9eba8bfbead303bfbe2e27a0d98171bd60cdae89864f026289c987
 SHA512:
-  metadata.gz: dcccc6c7761931c93bd729e58a844e127d9ac7a63075f8163c92083f7749df2e978cc745071a2c319bcc48825d2dd53a2e043e97a02329884168cc50d21f7975
-  data.tar.gz: 33cd0806b64df7e88296a9ff8ce0a3b233f6702df9cdb4c87938c7cc0b64cde2d096970903a3242c228c308d7fc2d8a35f4eff85d2262e5034de83bf7d009e85
+  metadata.gz: 11a2591f94021e7bb3d06bb7a712f98cf8329142150cf09979d44cc65f1803c809e9077d85ad5d41c83457b20c2bf9694a3a65692898145eb7738e18460e40ba
+  data.tar.gz: 3e393c808733042ba3b13a950fcc63f7498641a55d16a5ccae11f3bf1100ca04558e32873bd9063602abd6c72194e8d90492b9b292e47cbd08bfc44476f5417f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    list_spider (2.0.0)
+    list_spider (2.0.1)
       em-http-request (~> 1.1, >= 1.1.3)
       nokogiri (~> 1.6, >= 1.6.7)
       rchardet (~> 1.6, >= 1.6.1)

data/lib/list_spider.rb

@@ -126,7 +126,7 @@ module ListSpider
 
           http_req.callback do
             s = http_req.response_header.status
-            puts s
+            puts "#{Time.now}, http status code: #{s}"
 
             if s == 200
               local_dir = File.dirname(task_struct.local_path)
@@ -149,27 +149,9 @@ module ListSpider
           end
 
           http_req.errback do
-            puts "errback:#{http_req.response_header},retry..."
-            puts task_struct.href
-            puts http_req.response_header.status
-
-            if task_struct.errback
-              task_struct.errback.call(task_struct, http_req)
-            # else
-            #   ret = false
-            #   if task_struct.http_method == :get
-            #     ret = SpiderHelper.direct_http_get(task_struct.href, task_struct.local_path, convert_to_utf8: @convert_to_utf8)
-            #   elsif task_struct.http_method == :post
-            #     ret = SpiderHelper.direct_http_post(task_struct.href, task_struct.local_path, task_struct.params, convert_to_utf8: @convert_to_utf8)
-            #   end
+            puts "#{Time.now}, #{task_struct.href}, error: #{http_req.error}"
 
-            #   if ret
-            #     call_parse_method(task_struct)
-            #     succeed_list << task_struct
-            #   else
-            #     failed_list << task_struct
-            #   end
-            end
+            task_struct.errback.call(task_struct, http_req) if task_struct.errback
           end
 
           begin

data/lib/list_spider/version.rb

@@ -1,3 +1,3 @@
 module ListSpider
-  VERSION = '2.0.1'.freeze
+  VERSION = '2.0.2'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: list_spider
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.2
 platform: ruby
 authors:
 - Charles Zhang
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-02-23 00:00:00.000000000 Z
+date: 2018-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler