http_proxy_pool 0.0.2

data/README.md

@@ -0,0 +1,81 @@
+# http-proxy-pool 
+
+在爬取网页数据、批量投票，点赞等日常中，经常需要更换ip信息，需要大量代理。http-proxy-pool可用于收集网络上免费代理，供其它脚本程序使用。http-proxy-pool可以通过自定义爬取脚本来收集网络代理信息。
+
+## 安装
+
+`gem install http-proxy-pool`
+
+
+## 使用
+
+##### 1.命令行
+
+* 初始化资源  
+`proxypool crawl`
+
+* 查看当前已收集状态  
+`proxypool status`
+
+* 随机获取一个可用代理，默认强制检查代理是否可用  
+`proxypool get`
+
+更多参数，参看`proxypool help`
+
+##### 2.在脚本中引用
+
+    require 'http-proxy-pool'  
+
+    pool = HttpProxyPool::ProxyPool.new  
+    pool.query(:ip => "=~ /^111/", :proxy_type => "== 'HTTP'") do |proxy|
+      # do what you want ...
+    end
+
+query查询出proxy资源不会强制，校验是否可用。可用checker通过来校验:
+
+    pool.checker(proxy)
+
+## 定义爬取脚本
+http-proxy-pool默认脚本会安装到**[USER\_PATH]/http\_proxy\_pool/script**中，可以自己修改已有脚本，或者在此目录添加新脚本，目前自带以下网站（站点信息源自搜索引擎）爬取脚本:  
+
+* [ip.izmoney.com](http://ip.izmoney.com)
+* [kuaidaili.com](http://www.kuaidaili.com)
+* [proxy360.cn](http://www.proxy360.cn)
+* [goubanjia.com](http://proxy.goubanjia.com)
+
+##### 一个样例：
+	
+	# 开始抓取地址
+    sitetask("start_page_url") do
+      nextpage do
+        # nextpage 最终返回下一页URL
+        # 此部分需判断是否需要是否是最后页
+        # 如果未定义nextpage部分，程序默认只会爬去第一页
+      end
+
+      parser do
+        # 此部分，最终返回一个Proxy实例的数组
+        # 此block中，可以通过解析当前Mechanize页面，通过dom数据生成多个Proxy
+      end
+    end
+
+##### 创建Proxy:
+
+    HttpProxyPool::Proxy.new {
+      :ip => '127.0.0.1', 				# IP地址
+      :port => 8080,					# 端口
+      :username => 'jiyaping',			# 认证用户名
+      :password => 'xxxxxx',			# 认证密码
+      :proxy_level => 'high',			# 代理等级（匿名、透明代理）
+      :proxy_type => 'http',			# 代理类型（HTTP、HTTPS、SOCKS）
+      :speed => '0.5',					# 代理速度
+      :added_time => DateTime.now,		# 添加时间
+      :last_access_time => DateTime.now,# 上次使用时间
+      :nation => 'cn',					# 国家
+      :province => 'guangdong',			# 省份/州
+      :src_from => 'xxxxxx.com'			# 获取来源
+    }
+
+## 最后
+
+就酱紫 ...

data/Rakefile

@@ -0,0 +1,9 @@
+gem "minitest"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc 'Run tests'
+task :default => :test

data/bin/proxypool

@@ -0,0 +1,60 @@
+#! ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'thor'
+require "http_proxy_pool"
+
+class HttpProxyPoolApp < Thor
+  @@proxy_pool = HttpProxyPool::ProxyPool.new(
+                    :data_path=> File.join(HttpProxyPool.home, 'ips.yaml'),
+                    :script   => Dir["#{HttpProxyPool.home}/script/*.site"],
+                    :logger   => HttpProxyPool.logger
+                  )
+
+  desc 'status', 'show proxy pool status.'
+  def status
+    @@proxy_pool.status
+  end
+
+  desc 'crawl [WAY]', 'gather ip source store to local file through WAY.'
+  method_option :lastest, :aliases  => '-l',
+                          :type     => :boolean,
+                          :default  => true,
+                          :desc     => 'only crawl recently ip.'
+  method_option :check, :aliases  => '-c',
+                        :type     => :boolean,
+                        :default  => false,
+                        :desc     => 'store it after check if available.' 
+  def crawl(way = 'script')
+    puts "wait...."
+
+    if way == 'script'
+      lastest = options[:lastest]
+      check   = options[:check]
+
+      @@proxy_pool.crawling(lastest, check)
+    end
+
+    puts "done."
+  end
+
+  desc 'get', 'get ip from local storage.'
+  method_option :force_check, :aliases  => '-fc',
+                              :type     => :boolean,
+                              :default  => true,
+                              :desc     => 'check the ip if ready to use.'
+  method_option :thread_num, :aliases  => '-t',
+                              :type     => :numeric,
+                              :default  => 10,
+                              :desc     => 'num of search thread.'
+  def get
+    force_check = options[:force_check]
+    thread_num  = options[:thread_num]
+
+    puts @@proxy_pool.get_random_proxy(force_check, thread_num)
+  end
+end
+
+HttpProxyPoolApp.start

data/lib/http_proxy_pool.rb

@@ -0,0 +1,25 @@
+#encoding : utf-8
+
+require 'mechanize'
+
+require 'http_proxy_pool/error'
+require 'http_proxy_pool/utils'
+require 'http_proxy_pool/basetask'
+require 'http_proxy_pool/proxy'
+require 'http_proxy_pool/proxy_pool'
+require 'http_proxy_pool/version'
+
+module HttpProxyPool
+  # will support some configure
+  @config = {}
+
+  @home = File.join(Dir.home, 'http_proxy_pool')
+  Dir.mkdir(@home) unless Dir.exists? @home
+
+  @script_path = File.join(@home, 'script')
+  Dir.mkdir(@script_path) unless Dir.exists? @script_path
+  
+  @logger = Logger.new(File.join(@home, 'proxy.log'), 2_000_000)
+
+  init_default_script
+end

data/lib/http_proxy_pool/basetask.rb

@@ -0,0 +1,81 @@
+#encoding : utf-8
+
+module HttpProxyPool
+  class Basetask
+    attr_accessor :agent,
+                  :url,
+                  :logger,
+                  :page_parser,
+                  :next_page
+
+    def initialize(opts = {})
+      @agent  = opts[:agent]
+      @logger = opts[:logger]
+      @url    = opts[:url]
+    end
+
+    def sitetask(url, opts = {})
+      raise ScriptError.new("script do not specify a url!") unless url
+
+      @url        = url
+      @agent      = opts[:agent] || Mechanize.new
+      @logger   ||= opts[:logger]
+
+      #for debug
+      #@agent.set_proxy '127.0.0.1', 8888
+
+      yield
+    end
+
+    def ips(lastest = true)
+      uri = @url
+
+      loop do
+        @logger.info("start crawling page [#{uri}] ...")
+        @agent.get(uri)
+        # get all page need sleep a random time
+        rand_sleep unless lastest
+
+        begin
+          instance_eval(&page_parser).each do |field|
+            yield field
+          end  
+        rescue Exception => e
+          @logger.error("parsing page error[#{uri}]. #{e.to_s}")
+          break
+        end
+
+        begin
+          break unless @next_page
+          uri = instance_eval(&next_page)
+          break unless uri
+        rescue => e
+          @logger.error("error occoured when get next page[#{uri}]. #{e.to_s}")
+          break
+        end
+
+        break if lastest
+      end
+    end
+
+    def parser(&block)
+      @page_parser = block if block_given?
+    end
+
+    def nextpage(&block)
+      @next_page = block if block_given?
+    end
+
+    def curr_page
+      @agent.page.uri
+    end
+
+    def sitename
+      URI.parse(URI.encode(@url)).host
+    end
+
+    def rand_sleep(max_tick = 2)
+      sleep rand(max_tick)
+    end
+  end
+end

data/lib/http_proxy_pool/error.rb

@@ -0,0 +1,8 @@
+#encoding : utf-8
+
+module HttpProxyPool
+  class BaseError < StandardError; end
+  class ScriptError < BaseError; end
+  class TaskError < BaseError; end
+  class QueryError < BaseError; end
+end

data/lib/http_proxy_pool/example/izmoney_china_hight.site

@@ -0,0 +1,22 @@
+sitetask("http://ip.izmoney.com/search/china/high/index.html") do
+  parser do
+    ips = []
+
+    agent.page.search("tbody").search("tr").each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:ip]         = tds[0].text
+      fields[:port]       = tds[1].text
+      fields[:nation]     = tds[2].text
+      fields[:proxy_level]= tds[4].text
+      fields[:proxy_type] = tds[5].text
+      fields[:added_time] = DateTime.now
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/izmoney_china_normal.site

@@ -0,0 +1,22 @@
+sitetask("http://ip.izmoney.com/search/china/normal/index.html") do
+  parser do
+    ips = []
+
+    agent.page.search("tbody").search("tr").each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:ip]         = tds[0].text
+      fields[:port]       = tds[1].text
+      fields[:nation]     = tds[2].text
+      fields[:proxy_level]= tds[4].text
+      fields[:proxy_type] = tds[5].text
+      fields[:added_time] = DateTime.now
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/izmoney_foreign_high.site

@@ -0,0 +1,22 @@
+sitetask("http://ip.izmoney.com/search/foreign/high/index.html") do
+  parser do
+    ips = []
+
+    agent.page.search("tbody").search("tr").each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:ip]         = tds[0].text
+      fields[:port]       = tds[1].text
+      fields[:nation]     = tds[2].text
+      fields[:proxy_level]= tds[4].text
+      fields[:proxy_type] = tds[5].text
+      fields[:added_time] = DateTime.now
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/izmoney_foreign_normal.site

@@ -0,0 +1,22 @@
+sitetask("http://ip.izmoney.com/search/foreign/normal/index.html") do
+  parser do
+    ips = []
+
+    agent.page.search("tbody").search("tr").each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:ip]         = tds[0].text
+      fields[:port]       = tds[1].text
+      fields[:nation]     = tds[2].text
+      fields[:proxy_level]= tds[4].text
+      fields[:proxy_type] = tds[5].text
+      fields[:added_time] = DateTime.now
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/kuaidaili_inha.site

@@ -0,0 +1,36 @@
+sitetask("http://www.kuaidaili.com/free/inha/") do
+  nextpage do
+    curr_idx = 0
+
+    if agent.page.at('.active')
+      curr_idx = agent.page.at('.active').text.to_i
+    end
+
+    last_page = agent.page.at("#listnav").search("a[href*='/free']").last.text.to_i
+    return if curr_idx == last_page
+
+    File.join(url, (curr_idx + 1).to_s)
+  end
+
+  parser do
+    ips = []
+
+    agent.page.search("tbody").search("tr").each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:ip]         = tds[0].text
+      fields[:port]       = tds[1].text
+      fields[:proxy_level]= tds[2].text
+      fields[:proxy_type] = tds[3].text
+      fields[:province]   = tds[4].at('a').text if tds[4].at('a')
+      fields[:speed]      = tds[5].text
+      fields[:added_time] = tds[6].text
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/kuaidaili_intr.site

@@ -0,0 +1,36 @@
+sitetask("http://www.kuaidaili.com/free/intr/") do
+  nextpage do
+    curr_idx = 0
+
+    if agent.page.at('.active')
+      curr_idx = agent.page.at('.active').text.to_i
+    end
+
+    last_page = agent.page.at("#listnav").search("a[href*='/free']").last.text.to_i
+    return if curr_idx == last_page
+
+    File.join(url, (curr_idx + 1).to_s)
+  end
+
+  parser do
+    ips = []
+
+    agent.page.search("tbody").search("tr").each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:ip]         = tds[0].text
+      fields[:port]       = tds[1].text
+      fields[:proxy_level]= tds[2].text
+      fields[:proxy_type] = tds[3].text
+      fields[:province]   = tds[4].at('a').text if tds[4].at('a')
+      fields[:speed]      = tds[5].text
+      fields[:added_time] = tds[6].text
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/kuaidaili_outha.site

@@ -0,0 +1,36 @@
+sitetask("http://www.kuaidaili.com/free/outha/") do
+  nextpage do
+    curr_idx = 0
+
+    if agent.page.at('.active')
+      curr_idx = agent.page.at('.active').text.to_i
+    end
+
+    last_page = agent.page.at("#listnav").search("a[href*='/free']").last.text.to_i
+    return if curr_idx == last_page
+
+    File.join(url, (curr_idx + 1).to_s)
+  end
+
+  parser do
+    ips = []
+
+    agent.page.search("tbody").search("tr").each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:ip]         = tds[0].text
+      fields[:port]       = tds[1].text
+      fields[:proxy_level]= tds[2].text
+      fields[:proxy_type] = tds[3].text
+      fields[:province]   = tds[4].at('a').text if tds[4].at('a')
+      fields[:speed]      = tds[5].text
+      fields[:added_time] = tds[6].text
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/kuaidaili_outtr.site

@@ -0,0 +1,36 @@
+sitetask("http://www.kuaidaili.com/free/outtr/") do
+  nextpage do
+    curr_idx = 0
+
+    if agent.page.at('.active')
+      curr_idx = agent.page.at('.active').text.to_i
+    end
+
+    last_page = agent.page.at("#listnav").search("a[href*='/free']").last.text.to_i
+    return if curr_idx == last_page
+
+    File.join(url, (curr_idx + 1).to_s)
+  end
+
+  parser do
+    ips = []
+
+    agent.page.search("tbody").search("tr").each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:ip]         = tds[0].text
+      fields[:port]       = tds[1].text
+      fields[:proxy_level]= tds[2].text
+      fields[:proxy_type] = tds[3].text
+      fields[:province]   = tds[4].at('a').text if tds[4].at('a')
+      fields[:speed]      = tds[5].text
+      fields[:added_time] = tds[6].text
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/proxy360.site

@@ -0,0 +1,21 @@
+sitetask("http://www.proxy360.cn/default.aspx") do
+  parser do
+    ips = []
+
+    agent.page.search(".proxylistitem").each do |node|
+      tds = node.search('.tbBottomLine')
+      fields = {}
+
+      fields[:ip]         = tds[0].text.strip
+      fields[:port]       = tds[1].text.strip
+      fields[:proxy_level]= tds[2].text.strip
+      fields[:nation]     = tds[3].text.strip
+      fields[:added_time] = tds[4].text.strip
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/proxy_goubanjia_gngn.site

@@ -0,0 +1,23 @@
+sitetask("http://proxy.goubanjia.com/free/gngn/index.shtml") do
+  parser do
+    ips = []
+
+    agent.page.search("tbody").search("tr").each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:ip]         = tds[0].search(":not(p[style='display: none;'])").text
+      fields[:port]       = tds[1].text
+      fields[:proxy_level]= tds[2].text
+      fields[:proxy_type] = tds[3].text
+      fields[:nation]     = tds[4].text
+      fields[:province]   = tds[5].text
+      fields[:added_time] = DateTime.now
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/proxy_goubanjia_gnpt.site

@@ -0,0 +1,23 @@
+sitetask("http://proxy.goubanjia.com/free/gnpt/index.shtml") do
+  parser do
+    ips = []
+
+    agent.page.search("tbody").search("tr").each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:ip]         = tds[0].search(":not(p[style='display: none;'])").text
+      fields[:port]       = tds[1].text
+      fields[:proxy_level]= tds[2].text
+      fields[:proxy_type] = tds[3].text
+      fields[:nation]     = tds[4].text
+      fields[:province]   = tds[5].text
+      fields[:added_time] = DateTime.now
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/proxy_goubanjia_gwgn.site

@@ -0,0 +1,23 @@
+sitetask("http://proxy.goubanjia.com/free/gwgn/index.shtml") do
+  parser do
+    ips = []
+
+    agent.page.search("tbody").search("tr").each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:ip]         = tds[0].search(":not(p[style='display: none;'])").text
+      fields[:port]       = tds[1].text
+      fields[:proxy_level]= tds[2].text
+      fields[:proxy_type] = tds[3].text
+      fields[:nation]     = tds[4].text
+      fields[:province]   = tds[5].text
+      fields[:added_time] = DateTime.now
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/proxy_goubanjia_gwpt.site

@@ -0,0 +1,23 @@
+sitetask("http://proxy.goubanjia.com/free/gwpt/index.shtml") do
+  parser do
+    ips = []
+
+    agent.page.search("tbody").search("tr").each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:ip]         = tds[0].search(":not(p[style='display: none;'])").text
+      fields[:port]       = tds[1].text
+      fields[:proxy_level]= tds[2].text
+      fields[:proxy_type] = tds[3].text
+      fields[:nation]     = tds[4].text
+      fields[:province]   = tds[5].text
+      fields[:added_time] = DateTime.now
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/xicidaili_nn.site

@@ -0,0 +1,37 @@
+sitetask("http://www.xicidaili.com/nn/") do
+  nextpage do
+    curr_idx = 0
+
+    if agent.page.at('.current')
+      curr_idx = agent.page.at('.current').text.to_i
+    end
+
+    last_page = agent.page.at(".pagination").search("a[href*='/nn/']").last.text.to_i
+    return if curr_idx == last_page
+
+    File.join(url, (curr_idx + 1).to_s)
+  end
+
+  parser do
+    ips = []
+
+    agent.page.search("#ip_list").search("tr")[1..-1].each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:nation]     = tds[1].at('img')['alt'] if tds[1].at('img')
+      fields[:ip]         = tds[2].text
+      fields[:port]       = tds[3].text
+      fields[:province]   = tds[4].at('a').text if tds[4].at('a')
+      fields[:proxy_level]= tds[5].text
+      fields[:proxy_type] = tds[6].text
+      fields[:speed]      = tds[7].at('div')["title"] if tds[7].at('div')
+      fields[:added_time] = tds[9].text
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/xicidaili_nt.site

@@ -0,0 +1,37 @@
+sitetask("http://www.xicidaili.com/nt/") do
+  nextpage do
+    curr_idx = 0
+
+    if agent.page.at('.current')
+      curr_idx = agent.page.at('.current').text.to_i
+    end
+
+    last_page = agent.page.at(".pagination").search("a[href*='/nt/']").last.text.to_i
+    return if curr_idx == last_page
+
+    File.join(url, (curr_idx + 1).to_s)
+  end
+
+  parser do
+    ips = []
+
+    agent.page.search("#ip_list").search("tr")[1..-1].each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:nation]     = tds[1].at('img')['alt'] if tds[1].at('img')
+      fields[:ip]         = tds[2].text
+      fields[:port]       = tds[3].text
+      fields[:province]   = tds[4].at('a').text if tds[4].at('a')
+      fields[:proxy_level]= tds[5].text
+      fields[:proxy_type] = tds[6].text
+      fields[:speed]      = tds[7].at('div')["title"] if tds[7].at('div')
+      fields[:added_time] = tds[9].text
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/xicidaili_qq.site

@@ -0,0 +1,37 @@
+sitetask("http://www.xicidaili.com/qq/") do
+  nextpage do
+    curr_idx = 0
+
+    if agent.page.at('.current')
+      curr_idx = agent.page.at('.current').text.to_i
+    end
+
+    last_page = agent.page.at(".pagination").search("a[href*='/qq/']").last.text.to_i
+    return if curr_idx == last_page
+
+    File.join(url, (curr_idx + 1).to_s)
+  end
+
+  parser do
+    ips = []
+
+    agent.page.search("#ip_list").search("tr")[1..-1].each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:nation]     = tds[1].at('img')['alt'] if tds[1].at('img')
+      fields[:ip]         = tds[2].text
+      fields[:port]       = tds[3].text
+      fields[:province]   = tds[4].at('a').text if tds[4].at('a')
+      fields[:proxy_level]= tds[5].text
+      fields[:proxy_type] = tds[6].text
+      fields[:speed]      = tds[7].at('div')["title"] if tds[7].at('div')
+      fields[:added_time] = tds[9].text
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/xicidaili_wn.site

@@ -0,0 +1,37 @@
+sitetask("http://www.xicidaili.com/wn/") do
+  nextpage do
+    curr_idx = 0
+
+    if agent.page.at('.current')
+      curr_idx = agent.page.at('.current').text.to_i
+    end
+
+    last_page = agent.page.at(".pagination").search("a[href*='/wn/']").last.text.to_i
+    return if curr_idx == last_page
+
+    File.join(url, (curr_idx + 1).to_s)
+  end
+
+  parser do
+    ips = []
+
+    agent.page.search("#ip_list").search("tr")[1..-1].each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:nation]     = tds[1].at('img')['alt'] if tds[1].at('img')
+      fields[:ip]         = tds[2].text
+      fields[:port]       = tds[3].text
+      fields[:province]   = tds[4].at('a').text if tds[4].at('a')
+      fields[:proxy_level]= tds[5].text
+      fields[:proxy_type] = tds[6].text
+      fields[:speed]      = tds[7].at('div')["title"] if tds[7].at('div')
+      fields[:added_time] = tds[9].text
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/example/xicidaili_wt.site

@@ -0,0 +1,37 @@
+sitetask("http://www.xicidaili.com/wt/") do
+  nextpage do
+    curr_idx = 0
+
+    if agent.page.at('.current')
+      curr_idx = agent.page.at('.current').text.to_i
+    end
+
+    last_page = agent.page.at(".pagination").search("a[href*='/wt/']").last.text.to_i
+    return if curr_idx == last_page
+
+    File.join(url, (curr_idx + 1).to_s)
+  end
+
+  parser do
+    ips = []
+
+    agent.page.search("#ip_list").search("tr")[1..-1].each do |node|
+      tds = node.search('td')
+      fields = {}
+
+      fields[:nation]     = tds[1].at('img')['alt'] if tds[1].at('img')
+      fields[:ip]         = tds[2].text
+      fields[:port]       = tds[3].text
+      fields[:province]   = tds[4].at('a').text if tds[4].at('a')
+      fields[:proxy_level]= tds[5].text
+      fields[:proxy_type] = tds[6].text
+      fields[:speed]      = tds[7].at('div')["title"] if tds[7].at('div')
+      fields[:added_time] = tds[9].text
+      fields[:src_from]   = sitename
+
+      ips << fields
+    end
+
+    ips
+  end
+end

data/lib/http_proxy_pool/proxy.rb

@@ -0,0 +1,43 @@
+# encoding : utf-8
+
+module HttpProxyPool
+  class Proxy
+    attr_accessor :ip,
+                  :port,
+                  :username,
+                  :password,
+                  :proxy_level,
+                  :proxy_type,
+                  :speed,
+                  :added_time,
+                  :last_access_time,
+                  :nation,
+                  :province,
+                  :src_from,
+                  :try_times
+
+    def initialize(args = {})
+      @ip         = args[:ip]
+      @port       = args[:port]
+      @username   = args[:username] || ''
+      @password   = args[:password] || ''
+      @proxy_type = args[:proxy_type]
+      @proxy_level= args[:proxy_level]
+      @speed      = args[:speed]
+      @added_time = args[:added_time]
+      @last_access= args[:last_access]
+      @nation     = args[:nation]
+      @province   = args[:province]
+      @src_from   = args[:src_from]
+      @try_times  = args[:try_times] || 0
+    end
+
+    def to_arr
+      [@ip, @port, @proxy_type, @proxy_level, @nation, @province]
+    end
+
+    def to_s
+      "#{@ip}\t#{@port}"
+    end
+  end
+end

data/lib/http_proxy_pool/proxy_pool.rb

@@ -0,0 +1,202 @@
+#encoding : utf-8
+
+module HttpProxyPool
+  class ProxyPool
+    attr_accessor :proxys, :logger
+
+    def initialize(args = {})
+      @data_path  = args[:data_path] || File.join(HttpProxyPool.home, 'ips.yaml')
+      @script     = args[:script]    || Dir["#{HttpProxyPool.home}/script/*.site"]
+      @logger     = args[:logger]    || HttpProxyPool.logger
+      @proxys     = []
+
+      @agent      = Mechanize.new
+      @agent.user_agent_alias = get_agent_alias
+
+      load_proxy if File.exists? @data_path
+    end
+
+    def status
+      puts "proxy count : #{@proxys.size}"
+    end
+
+    # query interface
+    def query(args = {})
+      begin
+        selected_proxy = @proxys.select do |proxy|
+                           instance_eval(build_query_parameter('proxy', args))
+                         end
+      rescue => e
+        raise QueryError.new("query parameter error!")
+      end
+
+      return selected_proxy unless block_given?
+
+      selected_proxy.each do |proxy|
+        yield proxy
+      end
+    end
+
+    def build_query_parameter(prefix = 'proxy', args)
+      condition_str = ''
+
+      args = query_key_filter(args)
+
+      args.each do |key, express|
+        condition_str << "#{prefix}.#{key} #{express} && "
+      end
+
+      condition_str.sub!(/\s?&&\s?$/, '')
+
+      condition_str
+    end
+
+    def query_key_filter(args)
+      proxy = Proxy.new
+      args.select{ |k| proxy.respond_to? k }
+    end
+
+    def get_random_proxy(check = true, thread_num = 10)
+      mutex       = Mutex.new
+      result      = nil
+      thread_list = []
+
+      begin
+        thread_num.times do |thread|
+          thread_list  << Thread.new do
+                            while(!result)
+                              proxy = @proxys[rand(@proxys.size)]
+                              @logger.info("using #{proxy}.")
+                              proxy = checker(proxy) if check
+
+                              if proxy.is_a? Proxy
+                                mutex.synchronize do
+                                  result = proxy
+                                end
+                              end
+                            end
+                          end
+        end
+
+        thread_list.each { |t| t.join }
+      rescue => e
+        @logger.error("find proxy error. #{e.to_s}")
+      ensure
+        save_proxy
+      end
+
+      result
+    end
+
+    def crawling(lastest = true, check = false)
+      @script.each do |file|
+        begin
+          task = Basetask.new(:agent => @agent,:logger => @logger)
+          task.instance_eval(read_taskfile(file))
+
+          task.ips(lastest) do |fields|
+            proxy = Proxy.new(fields)
+            (next unless checker(proxy)) if check
+            @proxys << proxy unless include?(proxy)
+          end
+        rescue => e
+          @logger.error(e)
+        ensure
+          save_proxy
+        end
+      end
+    end
+
+    def include?(proxy)
+      @proxys.select{ |p| p.ip == proxy.ip}.size > 0
+    end
+
+    def save_proxy
+      file = File.open(@data_path, 'w')
+      YAML.dump(@proxys, file)
+      file.close
+    end
+
+    def load_proxy
+      @proxys = YAML.load_file(@data_path)
+    end
+
+    def read_taskfile(file)
+      cnt = ''
+      File.open(file) do |f|
+        while(line = f.gets)
+          cnt << line
+        end
+      end
+
+      cnt
+    end
+
+    def get_agent_alias
+      agent_arr = [
+                  'Linux Firefox',
+                  'Linux Mozilla',
+                  'Mac Firefox',
+                  'Mac Mozilla',
+                  'Mac Safari',
+                  'Windows Chrome',
+                  'Windows IE 7',
+                  'Windows IE 8',
+                  'Windows IE 9',
+                  'Windows Mozilla',
+                  'iPhone',
+                  'iPad',
+                  'Android']
+
+      agent_arr[rand(agent_arr.size)]                    
+    end
+
+    def checker(proxy)
+      if proxy.is_a? Array
+        checker_batch(proxy)
+      else
+        checker_single(proxy)
+      end
+    end
+
+    def checker_batch(proxys, task_count = 5)
+      result = []
+      mutex = Mutex.new
+      thread_count = (proxys.size / task_count.to_f).ceil
+
+      thread_count.times do |thread_idx|
+        (Thread.new do
+          start_idx = thread_idx * task_count
+          end_idx   = (thread_idx + 1) * task_count 
+          end_idx   = proxys.size if end_idx > proxys.size
+
+          proxys[start_idx..end_idx].each do |proxy|
+            p = checker_single(proxy)
+
+            mutex.synchronize  do
+              result<< p if p
+            end
+          end
+        end).join
+      end
+
+      result
+    end
+
+    def checker_single(proxy, timeout = 0.05)
+      http = Net::HTTP.new('baidu.com', 80, proxy.ip, proxy.port)
+      http.open_timeout = timeout
+      http.read_timeout = timeout * 10
+
+      begin
+        return proxy if http.get('/').code =~ /^[1|2|3|4]/
+      rescue => e
+        @logger.info("can not connect proxy.[#{proxy}].#{e.to_s}")
+        @proxys.delete(proxy)
+        @logger.info("delete disabled proxy [#{proxy}].")
+      end
+
+      false
+    end
+  end
+end

data/lib/http_proxy_pool/utils.rb

@@ -0,0 +1,30 @@
+#encoding : utf-8
+
+module HttpProxyPool
+  module_function
+
+  def init_default_script
+    
+    target_dir = Dir.new(@script_path)
+
+    src_dir = File.join(File.dirname(__FILE__), 'example')
+    Dir.entries(src_dir).each do |src|
+      next unless src.end_with? '.site'
+
+      FileUtils.cp File.join(src_dir, src),
+                  target_dir.path unless target_dir.include? src
+    end
+  end
+
+  def home
+    @home
+  end
+
+  def script_path
+    @script_path
+  end
+
+  def logger
+    @logger
+  end
+end

data/lib/http_proxy_pool/version.rb

@@ -0,0 +1,5 @@
+#encoding : utf-8
+
+module HttpProxyPool
+  VERSION = '0.0.1'
+end

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification
+name: http_proxy_pool
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- jiyaping
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: &10417392 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: *10417392
+description: Gather free http proxy data
+email: jiyaping0802@gmail.com
+executables:
+- proxypool
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/http_proxy_pool/basetask.rb
+- lib/http_proxy_pool/error.rb
+- lib/http_proxy_pool/example/izmoney_china_hight.site
+- lib/http_proxy_pool/example/izmoney_china_normal.site
+- lib/http_proxy_pool/example/izmoney_foreign_high.site
+- lib/http_proxy_pool/example/izmoney_foreign_normal.site
+- lib/http_proxy_pool/example/kuaidaili_inha.site
+- lib/http_proxy_pool/example/kuaidaili_intr.site
+- lib/http_proxy_pool/example/kuaidaili_outha.site
+- lib/http_proxy_pool/example/kuaidaili_outtr.site
+- lib/http_proxy_pool/example/proxy360.site
+- lib/http_proxy_pool/example/proxy_goubanjia_gngn.site
+- lib/http_proxy_pool/example/proxy_goubanjia_gnpt.site
+- lib/http_proxy_pool/example/proxy_goubanjia_gwgn.site
+- lib/http_proxy_pool/example/proxy_goubanjia_gwpt.site
+- lib/http_proxy_pool/example/xicidaili_nn.site
+- lib/http_proxy_pool/example/xicidaili_nt.site
+- lib/http_proxy_pool/example/xicidaili_qq.site
+- lib/http_proxy_pool/example/xicidaili_wn.site
+- lib/http_proxy_pool/example/xicidaili_wt.site
+- lib/http_proxy_pool/proxy.rb
+- lib/http_proxy_pool/proxy_pool.rb
+- lib/http_proxy_pool/utils.rb
+- lib/http_proxy_pool/version.rb
+- lib/http_proxy_pool.rb
+- Rakefile
+- README.md
+- !binary |-
+  YmluL3Byb3h5cG9vbA==
+homepage: https://github.com/jiyaping/http-proxy-pool
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.16
+signing_key: 
+specification_version: 3
+summary: http proxy crawling from web
+test_files: []