get_proxy_list 0.0.2

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NjZiNTkyNWI3NjJmMTE4MjEwYjU0YzY2ZDllMWVlOTA0N2E5ZTQ2MA==
+  data.tar.gz: !binary |-
+    ODliZWIwYzhhYWNmOGRiOWQwZmY2MGI4ODhhNTY3ZWJlNmZkOTA3NQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    YzczYWY5NTM2ODQyYTUwZGFlOTRkZjg5YjNiYTc2ODJlNTFjMWI4YTVhZjY1
+    MjAwZjJhOTg3MGZkOWE2YmYyMWYzZTQzMmQxMTc5ODlmZDFiM2FlM2QyODdl
+    NTlhMGFlODJkZTU1ZjQ5YjczMmZiMzg4OWQwNzBhYTQwN2VlNmI=
+  data.tar.gz: !binary |-
+    OGQ0MTczZjIwODAzZjY4ZDBmYTlkMGFhY2VhNmFjM2YzOGMzNjYxZDdlMGU5
+    ZDljM2EyN2YxMDFlMjE4ZWY0MDUyNGQxMjQ5ZDQwNWVjYWM2MWQ1MTVjZmZi
+    ZTlkNzY3ZWZhMzM0YzMzMTEyNDU2MzU0ZDlhMWUwZTkzNTU2YzA=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in get_proxy_list.gemspec
+gem 'nokogiri'
+gemspec
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 hahazql
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# GetProxyList
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'get_proxy_list'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install get_proxy_list
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/get_proxy_list.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'get_proxy_list/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "get_proxy_list"
+  spec.version       = GetProxyList::VERSION
+  spec.authors       = ["hahazql"]
+  spec.email         = ["hahazhouqunli@gmail.com"]
+  spec.description   = %q{use for get a proxy list}
+  spec.summary       = %q{use for get a proxy list}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/lib/get_proxy_list/base.rb

@@ -0,0 +1,55 @@
+require 'nokogiri'
+
+module GetProxyList
+  class Base
+    # To change this template use File | Settings | File Templates.
+
+    attr_writer :save_path
+
+    def initialize(save_path=nil)
+      @save_path = save_path
+      unless @save_path
+        @save_path = File.expand_path("../proxylist", __FILE__)
+      end
+    end
+
+
+
+
+    #获取符合时限的代理
+    #limit 时限
+    #proxylist 待筛选的代理列表
+    def get_proxy_in_time_limit(limit,proxylist)
+      limit_list = []
+      proxylist.each do |proxy|
+        url = "http://" + proxy["ip"] + ":" + proxy["port"]
+        time_start = Time.now.to_i
+        begin
+          timeout(limit+1) do
+            doc = Nokogiri::HTML(open("http://www.baidu.com",:proxy=> url))
+          end
+          time_end = Time.now.to_i
+          time_use = time_end - time_start
+          p  "url   use_time:#{time_use}"
+        rescue Exception =>e
+          case e
+            when Errno::ETIMEDOUT
+              p "Use #{url} timeout"
+            when Timeout::Error
+              p "Use #{url} timeout"
+            when Errno::ECONNREFUSED
+              p "Use #{url} Error connection"
+            else
+              p "Use #{url} Error:#{e.to_s}"
+          end
+          time_use = -1
+        end
+        if(time_use > 0 &&time_use < limit)
+          limit_list << url
+        end
+      end
+      limit_list
+    end
+
+  end
+end

data/lib/get_proxy_list/from_proxy_cn.rb

@@ -0,0 +1,61 @@
+require 'nokogiri'
+
+
+module GetProxyList
+  class FromProxyCn
+    # To change this template use File | Settings | File Templates.
+    include GetProxyList
+    attr_reader :site
+
+    def initialize
+      @site = "http://www.cnproxy.com"
+    end
+
+
+    #获取代理列表
+    def _getproxylist(url)
+      begin
+        doc = Nokogiri::HTML(open('#{url}'))
+        proxylist = []
+        p "Doc: #{docs}"
+        doc.css('table').each do |_table|
+           _table.css('tr').each do |tr|
+             unless !tr.css('td')[0].eql?("IP:Port")||tr.css('td').empty?
+               proxy=tr.css('td')[0]
+               p "Proxy: #{proxy}"
+               proxy = proxy.split("document.write")
+               _proxy = proxy.split(":")
+               proxylist << {ip:_proxy[0],port:_proxy[1]}
+             end
+           end
+        end
+        p "ProxyList: #{proxylist}"
+        return proxylist
+      rescue => err
+        raise "Get Proxy list Error! Class:'FromProxyCn' Gem:'get_proxy_list' Error:'#{err.to_s}'"
+      end
+    end
+
+
+    #根据需要的页数获取URL列表
+    #page 将获取的页数
+    def get_url_list(page)
+      urls = []
+      1.upto(page.to_i).each do |_page|
+        urls << @site+"/proxy#{_page.to_s}.html"
+      end
+      return urls
+    end
+
+
+    def get_proxylist(page)
+      urls = get_url_list(page)
+      proxylist = []
+      urls.each do |url|
+        proxylist =proxylist+proxylist|_getproxylist(url)
+      end
+      return proxylist
+    end
+
+  end
+end

data/lib/get_proxy_list/from_proxycn.rb

@@ -0,0 +1,7 @@
+
+
+module GetProxyList
+  class FromProxycn
+    # To change this template use File | Settings | File Templates.
+  end
+end

data/lib/get_proxy_list/version.rb

@@ -0,0 +1,3 @@
+module GetProxyList
+  VERSION = "0.0.2"
+end

data/lib/get_proxy_list.rb

@@ -0,0 +1,14 @@
+require 'get_proxy_list'
+require "get_proxy_list/version"
+require "get_proxy_list/from_proxy_cn"
+require "get_proxy_list/base"
+
+module GetProxyList
+  def self.get_list(limit,page)
+    fromProxyCn = FromProxyCn.new()
+    base = Base.new()
+    proxylist = fromProxyCn.get_proxylist(page)
+    proxylist = base.get_proxy_in_time_limit(limit,proxylist)
+    return proxylist
+  end
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: get_proxy_list
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- hahazql
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: use for get a proxy list
+email:
+- hahazhouqunli@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- get_proxy_list.gemspec
+- lib/get_proxy_list.rb
+- lib/get_proxy_list/base.rb
+- lib/get_proxy_list/from_proxy_cn.rb
+- lib/get_proxy_list/from_proxycn.rb
+- lib/get_proxy_list/version.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0
+signing_key: 
+specification_version: 4
+summary: use for get a proxy list
+test_files: []
+has_rdoc: