http_crawler 0.3.0.5 → 0.3.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4deb4d5965b16f6f8edfbeea11855ec7c2af4d3a1d2337b8afba6fd7dfc34b28
-  data.tar.gz: 6ad1f67ff98bd61d7a7f105ea51ac2d692b9f81fbab3674404aa41f32978a1a3
+  metadata.gz: f9c1f28cef3cb0daf678534a97d54a4ee4ec583ebd91db8ae57be114a4579197
+  data.tar.gz: cdcdbb7d34d944409d56df437a38c8daae4a32dc27c02406df45f94b95c67b75
 SHA512:
-  metadata.gz: 144d4ed97e3d60c541d5e878b8a7252378effd929e778dc743af7aaf83dfae3e9c228759297d1c7fb4274c6680e920ac4a1fcb3711228d2ca75d99473fde771b
-  data.tar.gz: 44f5c97fc062d0a0424154f07bddb09078316df9a0c1aba27723083baeb4aebabed266f652b19d89f0448f08fe7c8db26871121ee6e6bdca9d4c29de7097f292
+  metadata.gz: 370d52a4216581172c92206465ef02f7ec99d3c507065cd87da4baec70700e6b84f89f8593c93e3f9282ca7213f78344791f3f649e533b7c13f65becc8710085
+  data.tar.gz: '06957bba7ccbe041ac18c25de19e36d69ac67dbc793ac4f6890ac8243cde38e53270e26bb149df98f3462ca7080e874bb388b7977f57cfa10ccd341c76b68d0d'

data/lib/http_crawler.rb

@@ -4,12 +4,9 @@ require 'nokogiri'
 
 # 此段代码用于解决 require_dependency 是 rails 的内置方法 必须要先引用 Rails的包才能用的bug
 class << self.class
-  def require_rename
-    # require 取别名 require_dependency
-    alias_method :require_dependency, :require
-  end
+  # require 取别名 require_dependency
+  alias_method :require_dependency, :require
 end
-self.class.require_rename
 
 # 千万不能使用 require 或者 load,这样的话 Rails 调试的时候就不能热加载了
 require_dependency 'http_crawler/errors.rb'

data/lib/http_crawler/client.rb

@@ -66,6 +66,7 @@ module HttpCrawler
       @connect_time = 5
       @write_time = 5
       @read_time = 5
+      @all_timeout = nil
     end
 
     # 初始化 ssl 协议
@@ -77,19 +78,28 @@ module HttpCrawler
       end
     end
 
+    attr_accessor :header
     # 头文件相关方法
     def header(parameter = {})
       @header ||= init_header
     end
 
     def init_header(parameter = {})
-      @header = {}
+      @header = {
+          "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+          "Accept-Encoding": "gzip, br",
+          "Accept-Language": "zh-CN,zh;q=0.9",
+          "Connection": "keep-alive",
+          "Upgrade-Insecure-Requests": "1",
+          "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36",
+      }
     end
 
     def update_header(parameter = {})
-      nil
+      @header = init_header
     end
 
+    attr_accessor :cookies
     # cookies相关方法
     def cookies(parameter = {})
       @cookies ||= init_cookies
@@ -103,6 +113,14 @@ module HttpCrawler
       nil
     end
 
+    # 字符串转换成cookies
+    # "abc=123; cd=412" => { "abc": "123", "cd": "412"}
+    def str_to_cookies(str)
+      str.scan(/([^=]*)=([^;]*);? ?/) do |m|
+        self.cookies[:"#{m[0]}"] = m[1]
+      end
+    end
+
     # 代理设置
     def auto_proxy=(value)
       Rails.logger.debug "自动更新代理"
@@ -206,7 +224,13 @@ module HttpCrawler
       h = h.cookies(cookies) if cookies
 
       # 添加超时时间
-      h = h.timeout(connect: @connect_time, write: @write_time, read: @read_time)
+      if(@all_timeout)
+        # 整体总计超时时间
+        h = h.timeout(@all_timeout)
+      else
+        # 指定每个处理超时时间
+        h = h.timeout(connect: @connect_time, write: @write_time, read: @read_time)
+      end
 
       h
     end

data/lib/http_crawler/http/response.rb

@@ -1,10 +1,8 @@
 module HTTP
   class Response
 
-
     # 解压并转码 body 数据
     def decoding_body
-
       return @decoding_body if @decoding_body
       return nil unless self.body
 
@@ -58,28 +56,67 @@ module HTTP
     end
 
     alias_method :dec, :decoding_body
-    #  def decoding_body
 
+
+    # 转换html格式
+    # @return [Nokogiri::HTML::Document]
     def html
-      @html ||= Nokogiri::HTML(decoding_body)
+      return @html if @html
+      self.html = self.dec
     end
 
+    # @return [Nokogiri::HTML::Document]
+    def html=(data)
+      if (Nokogiri::HTML::Document === data)
+        @html = data
+      else
+        @html = Nokogiri::HTML(data)
+      end
+      @html
+    end
+
+    # 转换json格式
+    # @return [Hash]
     def json
-      @json ||= JSON.parse(decoding_body)
-      @json = JSON.parse(@json) if String === @json
+      return @json if @json
+      self.json = self.dec
+    end
+
+    # @return [Hash]
+    def json=(data)
+      if (Hash === data)
+        @json = data
+      else
+        @json = JSON.parse(data)
+        @json = JSON.parse(@json) if String === @json
+      end
       @json
     end
 
     # 通过readability 解析数据
-    # [Readability::Document]
+    # @return [Readability::Document]
     def readability
-      @readability ||= Readability::Document.new(decoding_body, {do_not_guess_encoding: true})
+      return @readability if @readability
+      self.readability = self.dec
+    end
+
+    # @return [Readability::Document]
+    def readability=(data)
+      if (Readability::Document === data)
+        @readability = data
+      else
+        @readability = Readability::Document.new(data, {do_not_guess_encoding: true})
+      end
+      @readability
     end
 
+
     def content
       Nokogiri::HTML(readability.content).text
     end
+
     # 解析
+    # 默认使用 json 的值
     def parsing
       self.json
     end

data/lib/http_crawler/version.rb

@@ -1,3 +1,3 @@
 module HttpCrawler
-  VERSION = "0.3.0.5"
+  VERSION = "0.3.0.6"
 end

data/lib/http_crawler/web/baidu/client.rb

@@ -4,6 +4,11 @@ module HttpCrawler
     module Baidu
       class Client < HttpCrawler::Web::Client
 
+        def init_client
+          # 设置整体超时时间 3 秒
+          @all_timeout = 3
+        end
+
         def init_uri
           @uri = URI("https://www.baidu.com")
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: http_crawler
 version: !ruby/object:Gem::Version
-  version: 0.3.0.5
+  version: 0.3.0.6
 platform: ruby
 authors:
 - jagger
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-02-22 00:00:00.000000000 Z
+date: 2019-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec