list_spider 0.3.4 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5f95841ea2cee7bec53f2fec3ef4400148714653
-  data.tar.gz: 3be30dba8ca687741c0639e050f9eac5b35102b0
+  metadata.gz: 41ff55315cb0b0cbb03b8ad9a11001e0c410935d
+  data.tar.gz: c3a79e8a3bebddf021cb0c9e13bed61be086bd33
 SHA512:
-  metadata.gz: 98991f0a63168037d6007a56864e44b1c32a3764f7281f32051cf572ccd699e928bbd0adc8141217d2286092ddaa236b8e5bc3cc2f54e70393ed2d28d5cf8d2a
-  data.tar.gz: e04cd448af2fd6fc9f072ceefafd244ebcde6f0267b7be4d5a8c0fedd9c4dca3e2f7d813484b58f503d470639b27977f730fb97061d35a3d0a89cf82bed9e219
+  metadata.gz: 20a16743baa0b56ac3438493e8e4ecc6a84c6e4cd933cf23c6a2e7b16906ebd3e425702808813d6634f66af857155c14867a2a4edaeb2dabeaa3c006e847771e
+  data.tar.gz: 4259ce7eea88f064a946a424542d3db30a830e0ed379ffadde7a78b7fd89ffe99099ed1caa0770edb313b5e22e358c04bb1e76a30065662a07624f3abbeb45aa

data/lib/list_spider.rb

@@ -8,7 +8,6 @@ require File.expand_path('../file_filter', __FILE__)
 
 class TaskStruct
   def initialize(href, local_path, http_method: :get, params: {}, extra_data: nil, parse_method: nil, header: nil)
-    @origin_href = href
     @href = href
     @href = SpiderHelper.string_to_uri(@href) if @href.class == ''.class
     @local_path = local_path
@@ -23,7 +22,7 @@ class TaskStruct
     other.class == self.class && other.href == href && other.local_path == local_path && other.http_method == http_method && other.params == params && other.extra_data == extra_data && other.header == header
   end
 
-  attr_accessor :origin_href, :href, :local_path, :http_method, :params, :extra_data, :parse_method, :request_object, :header
+  attr_accessor :href, :local_path, :http_method, :params, :extra_data, :parse_method, :request_object, :header
 end
 
 module ListSpider
@@ -33,14 +32,14 @@ module ListSpider
   DEFAULT_INTERVAL = 0
 
   @random_time_range = 3..10
-  @conver_to_utf8 = false
+  @convert_to_utf8 = false
   @connection_opts = { connect_timeout: 60 }
   @overwrite_exist = false
   @max_redirects = 10
   @local_path_set = Set.new
 
   class << self
-    attr_accessor :conver_to_utf8, :overwrite_exist, :max_redirects
+    attr_accessor :convert_to_utf8, :overwrite_exist, :max_redirects
 
     def set_proxy(proxy_addr, proxy_port, username: nil, password: nil)
       @connection_opts = {
@@ -102,7 +101,7 @@ module ListSpider
               FileUtils.mkdir_p(local_dir) unless Dir.exist?(local_dir)
               begin
                 File.open(e.local_path, 'wb') do |f|
-                  f << if @conver_to_utf8 == true
+                  f << if @convert_to_utf8 == true
                          SpiderHelper.to_utf8(w.response)
                        else
                          w.response
@@ -115,15 +114,21 @@ module ListSpider
             end
           end
           w.errback do
-            puts "errback:#{w.response_header}"
-            puts e.origin_href
+            puts "errback:#{w.response_header},retry..."
             puts e.href
             puts w.response_header.status
-            failed_list << e
+
+            ret = false
             if e.http_method == :get
-              SpiderHelper.direct_http_get(e.href, e.local_path)
+              ret = SpiderHelper.direct_http_get(e.href, e.local_path, convert_to_utf8: @convert_to_utf8)
             elsif e.http_method == :post
-              SpiderHelper.direct_http_post(e.href, e.local_path, e.params)
+               ret = SpiderHelper.direct_http_post(e.href, e.local_path, e.params, convert_to_utf8: @convert_to_utf8)
+            end
+
+            if ret
+              succeed_list << e
+            else
+              failed_list << e
             end
           end
 

data/lib/spider_helper.rb

@@ -3,7 +3,7 @@ require 'net/http'
 
 module SpiderHelper
   class << self
-    def direct_http_get(href, local_path, params: nil, header: nil)
+    def direct_http_get(href, local_path, params: nil, header: nil, convert_to_utf8: false)
       href = string_to_uri(href) if href.class == ''.class
 
       begin
@@ -19,18 +19,23 @@ module SpiderHelper
         if res.is_a?(Net::HTTPSuccess)
           local_dir = File.dirname(local_path)
           FileUtils.mkdir_p(local_dir) unless Dir.exist?(local_dir)
-          File.write(local_path, res.body)
+          content = res.body
+          content = to_utf8(content) if convert_to_utf8
+          File.write(local_path, content)
           puts 'succeed'
+          return true
         else
           puts res
         end
       rescue => e
         puts e.backtrace
         puts e
+        false
       end
+      false
     end
 
-    def direct_http_post(href, local_path, params, header: nil)
+    def direct_http_post(href, local_path, params, header: nil, convert_to_utf8: false)
       href = string_to_uri(href) if href.class == ''.class
 
       begin
@@ -46,13 +51,19 @@ module SpiderHelper
         if res.is_a?(Net::HTTPSuccess)
           local_dir = File.dirname(local_path)
           FileUtils.mkdir_p(local_dir) unless Dir.exist?(local_dir)
-          File.write(local_path, res.body)
+          content = res.body
+          content = to_utf8(content) if convert_to_utf8s
+          File.write(local_path, content)
+          puts 'succeed'
+          return true
         else
           puts res
         end
       rescue => e
         puts e
+        false
       end
+      false
     end
 
     def extract_href_last(origin_href)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: list_spider
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.5
 platform: ruby
 authors:
 - Charles Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-19 00:00:00.000000000 Z
+date: 2016-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request
@@ -99,9 +99,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.4
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: List Spider
 test_files: []
-has_rdoc: