http_crawler 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f034023a8c50c41be3d4e423d39fa2ad44e75930
+  data.tar.gz: 745c8a86a328387f8b8c7ef68e351c5d63d4d61a
+SHA512:
+  metadata.gz: aab1febfdc72a126e9edb1b496b661d236d3ae6689d5bf29350d4bffdbb4e9551e03487c2a7fc01d6ec7f15ff55d0872db8f90fb791474834982168aca88e531
+  data.tar.gz: 317de9a4ef0d5423b57de20cb9220cdecd598d431d67f03b025fd05c0a2ce2d013eec4e3da816ce50a35722c161aef01a90ee376add0e0ebc06c779b0e296370

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at 1336098842@qq.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,10 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+# gem 'rchardet', '~> 1.8'
+# gem 'nokogiri', '~> 1.8.4'
+#
+# gem "ruby-readability", :require => 'readability'
+
+# Specify your gem's dependencies in http_crawler.gemspec
+gemspec

data/README.md

@@ -0,0 +1,55 @@
+# HttpCrawler
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/http_crawler`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'http_crawler'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install http_crawler
+
+
+## 示例：百度爬虫维护
+
+
+### 通过对象调用
+
+```ruby
+client = HttpCrawler::Client::Baidu::Client.new
+client.index  # 抓取首页
+```
+
+### 通过别名调用
+```ruby
+client = HttpCrawler::Client.for("baidu") # 
+client.index   # 抓取首页
+```
+
+
+## 示例：测试API
+
+
+### 通过对象调用
+
+```ruby
+client = HttpCrawler::Proxy::TestProxyApi::Client.new
+client.index  # 抓取首页
+```
+
+### 通过别名调用
+```ruby
+client = HttpCrawler::Proxy.for("test_proxy_api") # 
+client.index   # 抓取首页
+```

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "http_crawler"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/http_crawler.gemspec

@@ -0,0 +1,45 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "http_crawler/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "http_crawler"
+  spec.version       = HttpCrawler::VERSION
+  spec.authors       = ["jagger"]
+  spec.email         = ["1336098842@qq.com"]
+
+  spec.summary       = %q{http 爬虫。}
+  spec.description   = %q{初级开发工程师，基于net/http 写的爬虫扩展包。}
+  spec.homepage      = "https://rubygems.org/gems/http_crawler"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata["allowed_push_host"] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
+  spec.files += Dir['lib/**/*.rb']
+
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "rspec", "~> 3.8"
+  spec.add_development_dependency "bundler", "~> 1.15"
+  spec.add_development_dependency "rake", "~> 10.0"
+
+  spec.add_dependency "rchardet", "~> 1.8"
+  spec.add_dependency "nokogiri", "~> 1.8"
+  spec.add_dependency "ruby-readability", "~> 0.7.0"
+  spec.add_dependency "brotli", "~> 0.2.1"
+
+end

data/lib/http_crawler/client.rb

@@ -0,0 +1,88 @@
+load File.dirname(__FILE__) + '/http.rb'
+load File.dirname(__FILE__) + '/object.rb'
+load File.dirname(__FILE__) + '/string.rb'
+
+module HttpCrawler
+  module Client
+
+    class << self
+
+      # 接收格式
+      # web_name = "biquge_duquanben"
+      # 返回 HttpCrawler::Web::BiqugeDuquanben::Client 实例
+      #
+      def for(web_name, *args)
+        "HttpCrawler::Web::#{web_name.camelize}::Client".constantize.new(*args)
+      end
+
+      #
+      # 接收格式
+      # module_name = "HttpCrawler::Web::BiqugeDuquanben"
+      # 返回 HttpCrawler::Web::BiqugeDuquanben::Client 实例
+      #
+      def for_module(module_name, *args)
+        "#{module_name}::Client".constantize.new(*args)
+      end
+    end
+
+    attr_reader :http, :uri
+
+    #
+    #  init_uri 如果未初始化@uri,则会报错
+    #  继承类需要重定义 init_uri
+    #
+    def initialize
+      raise "Client uri为空" unless init_uri
+      @http = Crawler::HTTP.new(uri.host, uri.port)
+
+      @http.use_ssl = (uri.scheme == "https")
+
+      @http.open_timeout = 5
+      @http.read_timeout = 5
+      @http.proxy_key = "#{self.class}"
+      init_http
+
+      Rails.logger.debug "proxy_key => #{@http.proxy_key}"
+    end
+
+    # 初始化http参数
+    def init_http
+
+    end
+
+    #  init_uri 如果未初始化@uri,则会报错
+    #  继承类需要实现 @uri = URI("http://host")
+    #
+    def init_uri
+      @uri = nil
+    end
+
+    def header
+      @header ||= init_header
+    end
+
+    def init_header
+      nil
+    end
+
+    def update_header(parameter = {})
+      nil
+    end
+
+    def update_proxy(proxy = {})
+      @http.update_proxy(proxy)
+    end
+
+    def auto_proxy=(value)
+      Rails.logger.debug "自动更新代理"
+      @http.auto_proxy = value
+      @http.update_proxy if (value == true && @http.proxy? == false)
+    end
+
+    # 是否验证码界面
+    def validation_page?(*arg)
+      false
+    end
+
+  end
+end

data/lib/http_crawler/http.rb

@@ -0,0 +1,211 @@
+load File.dirname(__FILE__) + '/net/http.rb'
+load File.dirname(__FILE__) + '/net/response.rb'
+
+module HttpCrawler
+  class HTTP < Net::HTTP
+
+    # 自动获取代理，true 表示自动获取代理 、false 表示不自动获取
+    attr_accessor :auto_proxy
+    # 代理API的别名 主要关联 Crawler::Proxy中维护的代理API
+    attr_accessor :proxy_api
+    # 调用自己的代理池所需要的主键 key
+    attr_accessor :proxy_key
+    # 请求错误后的重复最大请求次数
+    attr_accessor :max_error_num
+
+    def initialize(address, port = nil)
+      super(address, port)
+      @max_error_num = 2
+      @error_num = 0
+      @proxy_key = "default"
+    end
+
+    def http_error_sleep
+      sleep(0.5)
+    end
+
+    def server_error_sleep
+      sleep(3)
+    end
+
+    def proxy_api
+      @proxy_api ||= "my"
+    end
+    @@proxy_list = []
+    # 为 @http 重设代理
+    def proxy(p = {})
+
+      raise '代理设置 p_addr 不能为空' unless p["p_addr"]
+      raise '代理设置 p_port 不能为空' unless p["p_port"]
+
+      p["p_user"] ||= nil
+      p["p_pass"] ||= nil
+
+      Rails.logger.info("切换代理至 => #{p}")
+      # 设为 false 否则不会启用代理
+      @proxy_from_env = false
+
+      # 初始化代理数据
+      @proxy_address = p["p_addr"]
+      @proxy_port = p["p_port"]
+      @proxy_user = p["p_user"]
+      @proxy_pass = p["p_pass"]
+
+    end
+
+    # 通过调用 api 获取代理或者通过自定义设置代理
+    def get_proxy
+
+      while @@proxy_list.blank?
+        Rails.logger.debug("@@proxy_list 为空进行更新")
+        proxy_client = Crawler::Proxy.for(proxy_api)
+        proxy_r = proxy_client.get_proxy(key: proxy_key)
+        @@proxy_list << proxy_r.parsing
+        Rails.logger.debug("@@proxy_list => #{@@proxy_list}")
+        sleep(1)
+      end
+
+      p = @@proxy_list.delete_at(0)
+
+      Rails.logger.debug("当前IP => #{@proxy_address}:#{@proxy_port},获取最新代理 => #{p}")
+
+      unless p && p["p_addr"] && p["p_port"]
+        Rails.logger.warn "无最新代理等待5秒后重新获取"
+        sleep(5)
+        p = get_proxy
+      end
+
+      if (@proxy_address == p["p_addr"] && @proxy_port == p["p_port"])
+        Rails.logger.warn "无最新代理等待5秒后重新获取"
+        sleep(5)
+        p = get_proxy
+      end
+      p
+    end
+
+    def update_proxy(p = {})
+      if p.blank?
+        proxy(get_proxy)
+      else
+        proxy(p)
+      end
+    end
+
+    # 如果自动更新代理 则更新代理返回 true，否则返回false
+    def update_proxy?(p = {})
+      if auto_proxy
+        if p.blank?
+          proxy(get_proxy)
+        else
+          proxy(p)
+        end
+        return true
+      else
+        return false
+      end
+    end
+
+
+    # 重定向请求
+    def get_fetch(uri_or_path, initheader = nil, dest = nil, limit = 10, &block)
+      # You should choose a better exception.
+      raise ArgumentError, 'too many HTTP repeated' if limit == 0
+      # 更新uri_or_path
+      uri_or_path = URI.encode(uri_or_path) if String === uri_or_path && CharDet.detect(uri_or_path)["encoding"] != "ascii"
+
+      response = get(uri_or_path, initheader, dest, &block)
+      case response
+      when Net::HTTPSuccess then
+        response
+      when Net::HTTPRedirection then
+        location = response['location']
+        Rails.logger.warn "redirected to #{location}"
+        # 传入 location 进行跳转
+        get_fetch(location, initheader, dest, limit - 1, &block)
+      when Net::HTTPServerError then
+        Rails.logger.warn "Net::HTTPServerError  5XX to #{address}"
+        server_error_sleep
+        # 重新请求
+        get_fetch(uri_or_path, initheader, dest, &block)
+      else
+        server_error_sleep
+        response.error!
+      end
+    end
+
+    # 重定向请求
+    def post_fetch(uri_or_path, data, initheader = nil, dest = nil, &block)
+      # 更新uri_or_path 如果 uri_or_path 是 String类型 同时 又不是 ascii编码格式就进行转码
+      uri_or_path = URI.encode(uri_or_path) if String === uri_or_path && CharDet.detect(uri_or_path)["encoding"] != "ascii"
+      Rails.logger.debug "post_fetch => #{uri_or_path}"
+      response = post(uri_or_path, data, initheader, dest, &block)
+      case response
+      when Net::HTTPSuccess then
+        response
+      when Net::HTTPRedirection then
+        location = response['location']
+        Rails.logger.warn "redirected to #{location}"
+        # 传入 location 进行跳转
+        get_fetch(location, initheader, dest, 9, &block)
+      when Net::HTTPServerError then
+        Rails.logger.warn "Net::HTTPServerError 5XX to #{address}"
+        server_error_sleep
+        # 重新请求
+        post_fetch(uri_or_path, initheader, dest, &block)
+      else
+        server_error_sleep
+        response.error!
+      end
+    end
+
+    # def post_fetch
+
+    #
+    # 重写 发送请求的方法
+    #
+    def request(req, body = nil, &block)
+      begin
+        Rails.logger.debug("#{req.class} => #{use_ssl? ? "https://" : "http://" }#{address}:#{port}#{req.path}") if started?
+        super(req, body, &block)
+      rescue => error
+        if started?
+          # started? 是为了判断是否结束http请求，如果不添加则会处理2次异常
+          raise error
+        else
+          # 最大错误尝试次数
+          if @error_num < @max_error_num
+            @error_num += 1
+            http_error_sleep
+            retry # 这将把控制移到 begin 的开头
+          else
+            # 超过最大错误限制 判断错误类型
+            case error
+            when Net::HTTPFatalError
+              raise error
+            when EOFError
+              Rails.logger.warn "EOFError!"
+              if update_proxy?
+                proxy(get_proxy)
+                http_error_sleep
+                retry # 这将把控制移到 begin 的开头
+              else
+                raise error
+              end
+            when Timeout::Error
+              Rails.logger.warn "请求超时!"
+              if update_proxy?
+                @error_num = 0
+                http_error_sleep
+                retry # 这将把控制移到 begin 的开头
+              else
+                raise error
+              end
+            else
+              raise error
+            end
+          end
+        end
+      end # begin
+    end # def request(req, body = nil, &block)
+  end
+end

data/lib/http_crawler/net/http.rb

@@ -0,0 +1,7 @@
+module Net
+  class HTTP
+
+
+  end # class HTTP
+end # module Net
+

data/lib/http_crawler/net/response.rb

@@ -0,0 +1,96 @@
+module Net
+  class HTTPResponse
+
+    # 解压并转码 body 数据
+    def decoding_body
+
+      return @decoding_body if @decoding_body
+      return nil unless body
+
+      # 数据解压
+      case header['Content-Encoding']
+      when 'gzip' then
+        sio = StringIO.new(body)
+        gz = Zlib::GzipReader.new(sio)
+        @decoding_body = gz.read()
+      when 'br'
+        @decoding_body = Brotli.inflate(body)
+      when 'deflate'
+        # 可能错误代码 暂时没解决 deflate 编码格式
+        @decoding_body = Zlib::Inflate.inflate(body)
+      else
+        @decoding_body = body
+      end
+
+      # 判断解压后数据编码格式
+
+      # 从header取编码格式
+      encoding = header['Content-Type'][/charset=([^, ;"]*)/, 1]
+
+      # 从html中的 charset 取编码格式
+      encoding = @decoding_body[/charset=([^, ;"]*)/, 1] unless encoding
+
+      # 通过 CharDet 判断编码格式
+      encoding = CharDet.detect(@decoding_body)["encoding"] unless encoding
+
+      # 进行转码
+      begin
+        @decoding_body.force_encoding(encoding).encode!('utf-8') if encoding != @decoding_body.encoding
+      rescue => e
+        # 转码错误后再次使用 CharDet 判断编码格式后进行转码
+        cd = CharDet.detect(@decoding_body)["encoding"]
+        if (cd && cd != encoding)
+          @decoding_body.force_encoding(cd).encode!('utf-8') if encoding != @decoding_body.encoding
+        else
+          # 还是转码错误则抛出异常
+          raise e
+        end
+      end
+
+      @decoding_body
+    end
+
+    #  def decoding_body
+
+    def html
+      @html ||= Nokogiri::HTML(decoding_body)
+    end
+
+    def json
+      @json ||= JSON.parse(decoding_body)
+      @json = JSON.parse(@json) if String === @json
+      @json
+    end
+
+    # 通过readability 解析数据
+    def readability
+      @readability ||= Readability::Document.new(decoding_body, {do_not_guess_encoding: true})
+    end
+
+    # 解析
+    def parsing
+      nil
+    end
+
+    def get_date(str)
+      time = Time.now
+      case str
+      when /^(\d{1,2})小时前$/
+        time = time - $1.to_i.hours
+      when /^(\d{1,2})月(\d{1,2})日$/
+        time = Time.local(time.year, $1.to_i, $2.to_i)
+      when /^(\d{4})年(\d{1,2})月(\d{1,2})日$/
+        time = Time.local($1.to_i, $2.to_i, $3.to_i)
+      when /^(\d{1,2})月(\d{1,2})日[ ]{0,3}(\d{1,2}):(\d{1,2})$/ # 09月30日 12:04
+        time = Time.local(time.year, $1.to_i, $2.to_i, $3.to_i, $4.to_i)
+      end
+      return time
+    end
+
+
+    # 是否是网站验证  true表示正常数据、false表示弹出网站验证
+    def web_verify(*arg)
+      true
+    end
+  end # class Net::HTTPResponse
+end

data/lib/http_crawler/object.rb

@@ -0,0 +1,13 @@
+class Object
+  def blank_0(&block)
+    if (self.blank?)
+      nil
+    else
+      if block_given?
+        block.call self
+      else
+        self
+      end
+    end
+  end
+end

data/lib/http_crawler/proxy/README.md

@@ -0,0 +1,3 @@
+# 代理模块
+主要用于维护请求的代理
+

data/lib/http_crawler/proxy/client.rb

@@ -0,0 +1,7 @@
+module HttpCrawler
+  module Proxy
+    module Client
+
+    end
+  end
+end

data/lib/http_crawler/proxy/response.rb

@@ -0,0 +1,10 @@
+
+module HttpCrawler
+  module Proxy
+    module Response
+
+    end
+  end
+end
+
+

data/lib/http_crawler/proxy/test_proxy_api/README.md

@@ -0,0 +1,18 @@
+# 示例：测试API
+
+
+### 通过对象调用
+
+```ruby
+client = HttpCrawler::Proxy::TestProxyApi::Client.new
+client.get_proxy  # 获取代理
+```
+
+### 通过别名调用
+```ruby
+client = HttpCrawler::Proxy.for("test_proxy_api") # 
+client.get_proxy   # 获取代理
+```
+
+### response.rb
+用于维护不同响应结果的处理方法

data/lib/http_crawler/proxy/test_proxy_api/client.rb

@@ -0,0 +1,29 @@
+module HttpCrawler
+  module Proxy
+    module TestProxyApi
+      class Client
+
+        include(Crawler::Client)
+        include(Crawler::Proxy::Client)
+
+        class << self
+          def new(*args)
+            @client ||= super(*args)
+          end
+        end
+
+        def init_uri
+          @uri = URI("http://127.0.0.1:1111/")
+        end
+
+        # http://39.108.59.38:7772/Tools/proxyIP.ashx?OrderNumber=ccd4c8912691f28861a1ed048fec88dc&poolIndex=22717&cache=1&qty=2
+        def get_proxy(parameter = {})
+          r = http.get_fetch("/api/get_proxy")
+          r.extend(Crawler::Proxy::Laofu::Response::GetProxy)
+        end
+
+      end
+    end # module BiQuGe_DuQuanBen
+  end # module Web
+end # module Crawler
+

data/lib/http_crawler/proxy/test_proxy_api/response/get_proxy.rb

@@ -0,0 +1,24 @@
+# 查询
+module HttpCrawler
+  module Proxy
+    module Laofu
+      module Response
+        module TestProxyApi
+          def parsing
+            array = []
+            decoding_body.scan(/([^\n\r:]*):([^\n\r]*)/) do |v1, v2|
+              if v1 =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
+                array[array.length] = {"p_addr" => v1, "p_port" => v2, "p_user" => nil, "p_pass" => nil}
+              else
+                Rails.logger.warn decoding_body
+              end
+            end
+            array
+          end
+        end # module GetProxy
+      end # module Response
+    end # module Laofu
+  end # module Proxy
+end # module Crawler
+
+

data/lib/http_crawler/proxy/test_proxy_api/response.rb

@@ -0,0 +1,12 @@
+# Dir[File.dirname(__FILE__) + '/response/*.rb'].each {|file| require file}
+
+module HttpCrawler
+  module Proxy
+    module TestProxyApi
+      module Response
+
+      end
+    end
+  end
+end
+

data/lib/http_crawler/proxy.rb

@@ -0,0 +1,18 @@
+module HttpCrawler
+  module Proxy
+
+    class << self
+
+      # 接收格式
+      # web_name = "feilong"
+      # 返回 Crawler::Proxy::Feilong::Client 实例
+      #
+      def for(web_name, *arg)
+        "Crawler::Proxy::#{web_name.camelize}::Client".constantize.new(*arg)
+      end
+
+    end
+
+
+  end
+end

data/lib/http_crawler/string.rb

@@ -0,0 +1,9 @@
+
+class String
+  # 清除干扰数据
+  # 清除包含: 空格,回车
+  #
+  def del_inter
+    self.gsub(/(?:\n|\t| )/,"")
+  end
+end

data/lib/http_crawler/version.rb

@@ -0,0 +1,3 @@
+module HttpCrawler
+  VERSION = "0.2.0"
+end

data/lib/http_crawler/web/README.md

@@ -0,0 +1,4 @@
+# 爬虫维护
+
+
+主要用于维护爬虫的网站

data/lib/http_crawler/web/baidu/README.md

@@ -0,0 +1,19 @@
+# 示例：百度爬虫维护
+
+
+### 通过对象调用
+
+```ruby
+client = HttpCrawler::Client::Baidu::Client.new
+client.index  # 抓取首页
+```
+
+### 通过别名调用
+```ruby
+client = HttpCrawler::Client.for("baidu") # 
+client.index   # 抓取首页
+```
+
+
+### response.rb
+用于维护不同响应结果的处理方法

data/lib/http_crawler/web/baidu/client.rb

@@ -0,0 +1,25 @@
+module HttpCrawler
+  module Web
+    module Baidu
+      class Client
+        include(Crawler::Client)
+
+        def init_http
+          @http.open_timeout = 3
+          @http.read_timeout = 3
+        end
+
+        def init_uri
+          @uri = URI("https://www.baidu.com/")
+        end
+
+        def index(parameter = {})
+          r = http.get_fetch("/", header)
+          r.extend(HttpCrawler::Web::Baidu::Response::Index)
+        end
+
+      end
+    end # module Baidu
+  end # module Web
+end # module Crawler
+

data/lib/http_crawler/web/baidu/response/index.rb

@@ -0,0 +1,16 @@
+# 查询
+module HttpCrawler
+  module Web
+    module Baidu
+      module Response
+        module Index
+          def parsing(parameter = {})
+            html
+          end
+        end # module Search
+      end # module Response
+    end # module Qichacha
+  end # module Web
+end # module Crawler
+
+

data/lib/http_crawler/web/baidu/response.rb

@@ -0,0 +1,10 @@
+# Dir[File.dirname(__FILE__) + '/response/*.rb'].each {|file| require file}
+
+module HttpCrawler
+  module Web
+    module Baidu
+      module Response
+      end
+    end
+  end
+end

data/lib/http_crawler/web.rb

@@ -0,0 +1,7 @@
+# Dir[File.dirname(__FILE__) + '/web/*.rb'].each {|file| require file}
+
+module HttpCrawler
+  module Web
+
+  end
+end

data/lib/http_crawler.rb

@@ -0,0 +1,9 @@
+require 'net/http'
+require 'json'
+require 'digest/md5'
+require 'nokogiri'
+
+
+module HttpCrawler
+  # Your code goes here...
+end

metadata

@@ -0,0 +1,175 @@
+--- !ruby/object:Gem::Specification
+name: http_crawler
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- jagger
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-12-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rchardet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: ruby-readability
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+- !ruby/object:Gem::Dependency
+  name: brotli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+description: 初级开发工程师，基于net/http 写的爬虫扩展包。
+email:
+- 1336098842@qq.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".idea/vcs.xml"
+- ".rspec"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- http_crawler.gemspec
+- lib/http_crawler.rb
+- lib/http_crawler/client.rb
+- lib/http_crawler/http.rb
+- lib/http_crawler/net/http.rb
+- lib/http_crawler/net/response.rb
+- lib/http_crawler/object.rb
+- lib/http_crawler/proxy.rb
+- lib/http_crawler/proxy/README.md
+- lib/http_crawler/proxy/client.rb
+- lib/http_crawler/proxy/response.rb
+- lib/http_crawler/proxy/test_proxy_api/README.md
+- lib/http_crawler/proxy/test_proxy_api/client.rb
+- lib/http_crawler/proxy/test_proxy_api/response.rb
+- lib/http_crawler/proxy/test_proxy_api/response/get_proxy.rb
+- lib/http_crawler/string.rb
+- lib/http_crawler/version.rb
+- lib/http_crawler/web.rb
+- lib/http_crawler/web/README.md
+- lib/http_crawler/web/baidu/README.md
+- lib/http_crawler/web/baidu/client.rb
+- lib/http_crawler/web/baidu/response.rb
+- lib/http_crawler/web/baidu/response/index.rb
+homepage: https://rubygems.org/gems/http_crawler
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: http 爬虫。
+test_files: []