http_crawler 0.3.1.4 → 0.3.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e5fd1f93c080268b8ff29be71b4e0d7557fb6e489eceba9a342945997119b77e
-  data.tar.gz: 07ea9e9e7badc4f84275c306371e9ce8cd5ef6c855ba57e84141183a763dc2dd
+  metadata.gz: a7976a9fa4410e543a29dcbf761b79c2aa2b73f89604b7108daab863e3bf76d0
+  data.tar.gz: 349418d39a8346ac6844f3600541b982a6d7cbef445cd565b15ef2e8ef8cac2c
 SHA512:
-  metadata.gz: d174a147cad9851e140804d5f40383d536a3c1b1389d0d181e0e73d76b736ed4d6a5e8ebb0c1d6541abae37bf23fd1ef4340d085a9608f4d52977cc22a1cdeb3
-  data.tar.gz: e19bb87187a80c041f8884d2035d956c2dbe3aaa8e1ed1591036f8259f48847f5a3f36fc43b4a65ac3bbf713969091c3e8ddec0492a766129f2457a2969ff84f
+  metadata.gz: 932954f758d3136a124ea0b80da7bf1198cb8b86f0e1c52c1a838a119a307ad72bc572e78200b5d9e7ca25e4e3b8f2e645da4d1923e60b5f59294fa323ad5f94
+  data.tar.gz: e3b5598ddab7ed79bc18b445a120f0d91466fa701599339c480d5f882c4c305fc4aae966af5c64bbcfbfe989c6eee14494895fd89e512b710dede2705b086ea0

data/lib/http_crawler/client.rb

@@ -82,10 +82,12 @@ module HttpCrawler
     attr_accessor :header
     # 头文件相关方法
     def header(parameter = {})
+      parameter = parameter.symbolize_keys
       @header ||= init_header
     end
 
     def init_header(parameter = {})
+      parameter = parameter.symbolize_keys
       @header = {
           "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
           "Accept-Encoding": "gzip, br",
@@ -97,20 +99,24 @@ module HttpCrawler
     end
 
     def update_header(parameter = {})
+      parameter = parameter.symbolize_keys
       @header = init_header
     end
 
     attr_accessor :cookies
     # cookies相关方法
     def cookies(parameter = {})
+      parameter = parameter.symbolize_keys
       @cookies ||= init_cookies
     end
 
     def init_cookies(parameter = {})
+      parameter = parameter.symbolize_keys
       @cookies = {}
     end
 
     def update_cookies(parameter = {})
+      parameter = parameter.symbolize_keys
       nil
     end
 
@@ -136,10 +142,11 @@ module HttpCrawler
 
     # 调用代理 api使用的参数
     def proxy_params
-      @proxy_params ||= {"key": "default"}
+      @proxy_params ||= {key: "default"}
     end
 
     def update_proxy(proxy = {})
+      proxy = proxy.symbolize_keys
       if (proxy.blank?)
         @proxy = get_proxy
       else
@@ -167,7 +174,7 @@ module HttpCrawler
       begin
         Rails.logger.debug("开始获取代理IP")
         proxy_client = HttpCrawler::Proxy.for(proxy_api)
-        proxy_r = proxy_client.get_proxy(proxy_params)
+        proxy_r = proxy_client.get_proxy(proxy_params.symbolize_keys)
         proxy_ip = proxy_r.results unless proxy_r.results.blank?
         if proxy_ip.blank?
           Rails.logger.warn "无最新代理等待5秒后重新获取:proxy 为空"
@@ -176,10 +183,10 @@ module HttpCrawler
         end
         sleep(5)
       end while true
-
+      proxy_ip = proxy_ip.symbolize_keys
       Rails.logger.debug("当前IP => #{@proxy},获取最新代理 => #{proxy_ip}")
 
-      unless proxy_ip["p_addr"] && proxy_ip["p_port"]
+      unless proxy_ip[:p_addr] && proxy_ip[:p_port]
         Rails.logger.warn "无最新代理等待5秒后重新获取:p_addr 或 p_port 为空"
         sleep(5)
         proxy_ip = get_proxy
@@ -211,7 +218,7 @@ module HttpCrawler
       h = HTTP.follow(max_hops: 5)
 
       # 添加代理
-      h = h.via(@proxy["p_addr"], @proxy["p_port"].to_i, @proxy["p_user"], @proxy["p_pass"]) unless (@proxy.blank?)
+      h = h.via(@proxy[:p_addr], @proxy[:p_port].to_i, @proxy[:p_user], @proxy[:p_pass]) unless (@proxy.blank?)
 
       # 添加头文件
       h = h.headers(header) if header
@@ -236,6 +243,7 @@ module HttpCrawler
     #  继承类需要重定义 init_uri
     #
     def initialize(parameter = {})
+      parameter = parameter.symbolize_keys
       # 初始化 uri
       init_uri
 

data/lib/http_crawler/version.rb

@@ -1,3 +1,3 @@
 module HttpCrawler
-  VERSION = "0.3.1.4"
+  VERSION = "0.3.1.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: http_crawler
 version: !ruby/object:Gem::Version
-  version: 0.3.1.4
+  version: 0.3.1.5
 platform: ruby
 authors:
 - jagger
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-01 00:00:00.000000000 Z
+date: 2019-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec