http_crawler 0.3.1.29 → 0.3.1.30

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 329e17b63dd792550707d01ce3a310a8bc81d2b04c5d2a6b5efc983a1b0a1eff
-  data.tar.gz: 1a0d14ccb1326181e9c929de5ce1f8e057c698c3b10ee88eab1a9685b6bf0c39
+  metadata.gz: 40f6c6a9c5d3225fe8b8c293e05306e8d1776c3648f82b4f7f8387310a55ec2d
+  data.tar.gz: ed8a2164e15d5d9f74672434cda4ee93975bfd8f80cb65b295451509bf25a6f9
 SHA512:
-  metadata.gz: 88a8389db13b3ed17a82772d9a66378c6184133636d53617e5550d6cdd27cd542c3a5ca1320faf94c3b1a2a20441b119e1a9b915ecd268802e0a993711149fcc
-  data.tar.gz: 8ac49deb0e1b0c3a813f2648d7d9a55e6135b883d6f9c73c7d962c37f5af364d79284c58e5069555bef433c84e4e91d71904ca3b695aa28daef2e503ee23625f
+  metadata.gz: 345b32732cb544585a3a12c7c6107ab8011c64ea1938043c82c74b66a0f2a12c96c5911c1533e6819c6d6809ddc73c95801c94faaf6f4dea0ddcf0fa58ab2594
+  data.tar.gz: e250dad59f16ae47d3866b0b5b9b9254d1441e2ce0c30fe8f689e83f1c07136b364f9a8c84b8e257c942f00c58400bdb5283aed99695036a22e0f9604a04194f

data/lib/http_crawler/client.rb

@@ -9,8 +9,8 @@ module HttpCrawler
       # web_name = "biquge_duquanben"
       # 返回 HttpCrawler::Web::BiqugeDuquanben::Client 实例
       #
-      def for(web_name)
-        "HttpCrawler::Web::#{web_name.camelize}::Client".constantize.new()
+      def for(web_name, args = {})
+        "HttpCrawler::Web::#{web_name.camelize}::Client".constantize.new(args)
       end
 
       #
@@ -18,8 +18,8 @@ module HttpCrawler
       # module_name = "HttpCrawler::Web::BiqugeDuquanben"
       # 返回 HttpCrawler::Web::BiqugeDuquanben::Client 实例
       #
-      def for_module(module_name, *args)
-        "#{module_name}::Client".constantize.new()
+      def for_module(module_name, args = {})
+        "#{module_name}::Client".constantize.new(args)
       end
 
       def for_uri(path)
@@ -54,6 +54,7 @@ module HttpCrawler
       # 初始化一些 client 自定义参数
       init_client
 
+      self.redirect = true
       # 初始化 代理参数
       @proxy_params = {key: "#{self.class.to_s.gsub(":", "_")}"}
     end
@@ -109,6 +110,8 @@ module HttpCrawler
       end
     end
 
+    attr_accessor :redirect
+
     attr_accessor :header
     # 头文件相关方法
     def header(parameter = {})
@@ -147,6 +150,11 @@ module HttpCrawler
 
     def update_cookies(parameter = {})
       parameter = parameter.symbolize_keys
+
+      @response.cookies.each do |cookie|
+        @cookies.add(cookie)
+      end unless @response.blank?
+
       nil
     end
 
@@ -244,10 +252,19 @@ module HttpCrawler
       nil
     end
 
-    # 初始化http请求前置条件
-    def http
+    # 创建时间: 2019/9/11 17:11
+    # 更新时间: 2019/9/11
+    # 作者: Jagger
+    # 方法名称: init_http
+    # 方法说明: 初始化http请求前置条件
+    # 调用方式: #init_http
+    #
+    # @return HTTP
+    #
+    def init_http
+      h = HTTP
       # 自动重定向。最大重定向次数 max_hops: 5
-      h = HTTP.follow(max_hops: 5)
+      h = h.follow(max_hops: 5) if self.redirect == true
 
       # 添加代理
       h = h.via(@proxy[:p_addr], @proxy[:p_port].to_i, @proxy[:p_user], @proxy[:p_pass]) unless (@proxy.blank?)
@@ -270,6 +287,11 @@ module HttpCrawler
       h
     end
 
+    # 初始化http请求前置条件
+    def http
+      init_http
+    end
+
 
     # 发送 get 请求
     def get(path, params = {}, limit = 3)
@@ -322,7 +344,7 @@ module HttpCrawler
       n = max_error_num
       begin
         r = block.call
-        if r.status.success?
+        if r.status.success? || (redirect == false && r.status.redirect?)
           return r
         else
           raise "请求失败(#{r.code}):#{r.uri.to_s}"

data/lib/http_crawler/version.rb

@@ -1,3 +1,3 @@
 module HttpCrawler
-  VERSION = "0.3.1.29"
+  VERSION = "0.3.1.30"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: http_crawler
 version: !ruby/object:Gem::Version
-  version: 0.3.1.29
+  version: 0.3.1.30
 platform: ruby
 authors:
 - jagger
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-07-19 00:00:00.000000000 Z
+date: 2019-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec