http_crawler 0.2.3.0 → 0.2.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 52f21f9a41ac75989b37da494ca3cf4321c7cf5e
-  data.tar.gz: e9fe5e402ac1f6333471adafaf997f81af2459c5
+  metadata.gz: e5a5b180d6051c3c0b343770e3d39f3711856b35
+  data.tar.gz: 4f74144ae22355a73e8775a41a85d5bba4a182ea
 SHA512:
-  metadata.gz: fa99a2f0bdcce2dab203238c0aa0fe20dc970d925d359e538f58027fe9b7670cfae48d70c8eb3154a9343d9aa75dbe6c7ada2a64f88e036be9393829cc3184e6
-  data.tar.gz: 1fc66e66594a5d9907170f4d71ce7e6b34e4c3691520ef33c4eb30bd778f744c1abf55fc249e0e125b0a1d7ca8f3dcacc159233da3e40f2653406c1ff786a90c
+  metadata.gz: 9a53292a05ad53fb701c1e9cb98075ff8a8daf185b0d4737a9ee821ae8e4a4ca3243120f2613feddb50ddc99a1ab10fcac4ddf363af8e5f627e5ffdedb3d8237
+  data.tar.gz: '09308f3773e6e9f8f6e30fd62c4adf8c2719d0297bf734e7726afcabaaa0851038ded52b27f1064d715d3d7d84359db252904a53554653e94592a3fa3bd4d8a1'

data/lib/http_crawler/http.rb

@@ -1,4 +1,3 @@
-
 module HttpCrawler
   class HTTP < Net::HTTP
 
@@ -59,7 +58,7 @@ module HttpCrawler
         Rails.logger.debug("@@proxy_list 为空进行更新")
         proxy_client = HttpCrawler::Proxy.for(proxy_api)
         proxy_r = proxy_client.get_proxy(key: proxy_key)
-        @@proxy_list << proxy_r.parsing
+        @@proxy_list << proxy_r.parsing unless proxy_r.parsing.blank?
         Rails.logger.debug("@@proxy_list => #{@@proxy_list}")
         sleep(1)
       end
@@ -125,13 +124,13 @@ module HttpCrawler
         Rails.logger.warn "Net::HTTPServerError  5XX to #{address}"
         server_error_sleep
         # 重新请求
-        get_fetch(uri_or_path, initheader, dest, &block)
+        get_fetch(uri_or_path, initheader, dest, limit - 1, &block)
       when Net::HTTPProxyAuthenticationRequired then
         Rails.logger.warn "Net::HTTPProxyAuthenticationRequired 407 to proxy:[#{@proxy_address}:#{@proxy_port}]  =>#{address}"
         if update_proxy?
           server_error_sleep
           # 重新请求
-          get_fetch(uri_or_path, initheader, dest, &block)
+          get_fetch(uri_or_path, initheader, dest, limit - 1, &block)
         else
           response.error!
         end

data/lib/http_crawler/net/response.rb

@@ -33,9 +33,10 @@ module Net
       # 通过 CharDet 判断编码格式
       encoding = CharDet.detect(@decoding_body)["encoding"] unless encoding
 
+
       # 进行转码
       begin
-        @decoding_body.force_encoding(encoding).encode!('utf-8') if encoding != @decoding_body.encoding
+        @decoding_body.force_encoding(encoding).encode!('utf-8') if encoding && encoding != @decoding_body.encoding
       rescue => e
         # 转码错误后再次使用 CharDet 判断编码格式后进行转码
         cd = CharDet.detect(@decoding_body)["encoding"]
@@ -43,6 +44,9 @@ module Net
           @decoding_body.force_encoding(cd).encode!('utf-8') if encoding != @decoding_body.encoding
         else
           # 还是转码错误则抛出异常
+          Rails.logger.debug "encoding => #{encoding}"
+          Rails.logger.debug "cd => #{cd}"
+          Rails.logger.debug "@decoding_body[0..200] => #{@decoding_body[0..200]}"
           raise e
         end
       end

data/lib/http_crawler/version.rb

@@ -1,3 +1,3 @@
 module HttpCrawler
-  VERSION = "0.2.3.0"
+  VERSION = "0.2.3.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: http_crawler
 version: !ruby/object:Gem::Version
-  version: 0.2.3.0
+  version: 0.2.3.1
 platform: ruby
 authors:
 - jagger
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-28 00:00:00.000000000 Z
+date: 2019-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec