http_crawler 0.3.0.1 → 0.3.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d3f0a5acf0654b1ccba309c40a7153866850ce8f5be508e06cf587f23cf8a1d
-  data.tar.gz: 38ca43bafdecec27eb57078de5b07dcc07a0d30b75807a44895eb7441c0860ff
+  metadata.gz: e47fc7ceac8e7335c7d873104a8ca7f504885af1c19a0802d23c1986d4ae5588
+  data.tar.gz: 392e793eae03814c1f3475e7515124d51b8adcdffdec9065873a90c800765225
 SHA512:
-  metadata.gz: 98d2057ce312a8beef8508ef411f2e1d3b7f65c0588c19f3dffc1a71f1c85a259e812e8f7fb48b66f0b25120a012eccafaba096e4d1ca1647e3e7fdb790fd6d8
-  data.tar.gz: 6f8b27afbf39767e1484ff62247e007fb666a8d49d6893968be85a93925b8b2f7207ef06e7843902d08d270af3041320f9b70e4f38248108841c4d1be3359b6c
+  metadata.gz: fb7ba4091d7320d1fcbb3926edb060fd55155156c34cf42b7ea1b67e1b8eba3c0cdf317a2f53d8094dee3672a17058dd57f688da6a89b4f86cfcdedad5bda42f
+  data.tar.gz: cd6001c16fbbff9023fe26c739fe270c62176849d3a4809d7bfa1aff4dd74856b6a8db95297c312d4fe56334dc7c8f04772d6eeb8a97f8d2de9a9df841c8a2ab

data/lib/http_crawler.rb

@@ -3,6 +3,7 @@ require 'json'
 require 'digest/md5'
 require 'nokogiri'
 
+require 'http_crawler/errors.rb'
 load 'http_crawler/common.rb'
 load 'http_crawler/client.rb'
 load 'http_crawler/web.rb'

data/lib/http_crawler/client.rb

@@ -232,21 +232,27 @@ module HttpCrawler
       n = max_error_num
       begin
         block.call
-      rescue HTTP::TimeoutError
-        # 超时错误切换代理
-        if self.update_proxy?
-          retry
-        else
-          raise error
-        end
       rescue => error
-        # 错误尝试次数
-        if n <= 0
-          raise error
+
+        case error
+        when HTTP::TimeoutError
+          # 超时错误切换代理
+          if self.update_proxy?
+            retry
+          else
+            raise error
+          end
+
         else
-          n -= 1
-          retry
+          # 错误尝试次数
+          if n <= 0
+            raise error
+          else
+            n -= 1
+            retry
+          end
         end
+
       end
     end
   end

data/lib/http_crawler/errors.rb

@@ -0,0 +1,9 @@
+
+module HttpCrawler
+  # 通用的错误类型
+  class Error < StandardError; end
+
+  # 验证码错误
+  class VerificationError < Error; end
+
+end

data/lib/http_crawler/http/response.rb

@@ -60,9 +60,10 @@ module HTTP
     def validation_page?
       # 正则匹配数组 validations 的所有匹配值
       validations.each do |regular|
-        if decoding_body[regular]
+        regular_num = decoding_body =~ regular
+        if regular_num
           Rails.logger.warn("触发验证信息")
-          Rails.logger.warn(decoding_body[(decoding_body =~ regular)..100])
+          Rails.logger.warn(decoding_body[regular_num..(regular_num + 100)])
           return true
         end
       end

data/lib/http_crawler/version.rb

@@ -1,3 +1,3 @@
 module HttpCrawler
-  VERSION = "0.3.0.1"
+  VERSION = "0.3.0.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: http_crawler
 version: !ruby/object:Gem::Version
-  version: 0.3.0.1
+  version: 0.3.0.2
 platform: ruby
 authors:
 - jagger
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-02-17 00:00:00.000000000 Z
+date: 2019-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -136,6 +136,7 @@ files:
 - lib/http_crawler/common.rb
 - lib/http_crawler/common/object.rb
 - lib/http_crawler/common/string.rb
+- lib/http_crawler/errors.rb
 - lib/http_crawler/http.rb
 - lib/http_crawler/http/response.rb
 - lib/http_crawler/proxy.rb