get_proxy_list 0.0.4 → 0.0.7

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZDUyYjQyMmFlNWI2OWMwZjBiYzkwMWY1ODVhMDQ0ZDliMDExZTEwOA==
+    NWU5MDYwYjNhZGYzMTgyN2Y4M2EzMWMwOTliMDliNmJhMGQyMTM2Mw==
   data.tar.gz: !binary |-
-    OTU0OTlmZjE5NDMxZGZkMDRmYTQyYjRmZWVkZTk4YjJlZmVkYzRiYw==
+    MGMyZTI2YjBjYmExYzZiZTNiOGYzMDkyYWY0ZWNiMGM5MGQzMDEyNw==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    OTkzNzIxODc4ZTZkNzdmMzU2YWZiYzhiODY3NjI4NTgwMjk3ODlmZTlhMzdm
-    ZDE4MDczYjI3MDIxMDYyMzJmM2JmODg1YWU5MTZlMDZlMzhkMDcxMzc5ZjBm
-    MjY2MjM5ZDk5OGJmMDRhMjgzZTU3NWNmZjg3MGE1YWVhMDlhMGU=
+    NjU1OWRiY2QwODczNDNkMmMxMmYzZGIzN2M2MWI5YTEyM2I1YTI5YjY1Yjg3
+    NjI3Njg4ZDYxNjVjMzA3NjVlZmIyNjkxNWQ5MjQ3MzczOTliMWI0MDIzNmNj
+    ZjRiNTlkYzhiYmUwMDBjN2VjYTVkYzU0ZDIyMDQxOGQxYTlhMjU=
   data.tar.gz: !binary |-
-    YTM1YzJjYmUzMWZkZjc0ZDQyNzkwZjQyNzk4OTk1ZGE0YzYzNTNjZmY2MTMy
-    YjM4YzIyZjIxMWRlNGYwNjZjMDllMGI3ZDNhMjVkMzM0NDE2MjNmZTBmYzU2
-    YWUwMjYxMDE3OGVlNDI0MjhlNWQ1MGNjY2UxYTM4N2I5Y2NkZDQ=
+    MzhjYjVkNmJkOGYzMDMzZGU0YmU1MGM4ZDQ4M2UwZjY4MTE5YzJmYWJiNmU0
+    ZmZmMGRjYTI1ZjQ1M2U1NzFlYjZkNzNiOTNiNzFjZWFkOWZkY2JkMTVmZTQ2
+    ZTgwY2M1NGFhYjM4YzUyOTdjMjM4NzBiMmRlZTBiMjE5YTY1ZDk=

data/Gemfile

@@ -1,6 +1,4 @@
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in get_proxy_list.gemspec
-gem 'nokogiri'
 gemspec
-

data/README.md

@@ -1,4 +1,4 @@
-# GetProxyList
+# Getproxylist
 
 TODO: Write a gem description
 
@@ -6,7 +6,7 @@ TODO: Write a gem description
 
 Add this line to your application's Gemfile:
 
-    gem 'get_proxy_list'
+    gem 'getproxylist'
 
 And then execute:
 
@@ -14,7 +14,7 @@ And then execute:
 
 Or install it yourself as:
 
-    $ gem install get_proxy_list
+    $ gem install getproxylist
 
 ## Usage
 

data/{get_proxy_list.gemspec → getproxylist.gemspec}

@@ -5,11 +5,11 @@ require 'get_proxy_list/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "get_proxy_list"
-  spec.version       = GetProxyList::VERSION
+  spec.version       = Getproxylist::VERSION
   spec.authors       = ["hahazql"]
   spec.email         = ["hahazhouqunli@gmail.com"]
-  spec.description   = %q{use for get a proxy list}
-  spec.summary       = %q{use for get a proxy list}
+  spec.description   = %q{""}
+  spec.summary       = %q{""}
   spec.homepage      = ""
   spec.license       = "MIT"
 

data/lib/debug.rb

@@ -0,0 +1,12 @@
+require File.expand_path("../get_proxy_list/version",__FILE__)
+require File.expand_path("../get_proxy_list/from_proxy_cn",__FILE__)
+require File.expand_path("../get_proxy_list/base",__FILE__)
+require File.expand_path("../get_proxy_list", __FILE__)
+
+class Debug
+  # To change this template use File | Settings | File Templates.
+  include GetProxyList
+  list= GetProxyList.get_list(5,4)
+  p "Size: #{list.size}"
+  p "list: #{list}"
+end

data/lib/debug.rb~

@@ -0,0 +1,12 @@
+require 'getproxylist'
+require "getproxylist/version"
+require "getproxylist/from_proxy_cn"
+require "getproxylist/base"
+
+class Debug
+  # To change this template use File | Settings | File Templates.
+  include GetProxyList
+  list= GetProxyList.get_list(5,4)
+  p "Size: #{list.size}"
+  p "list: #{list}"
+end

data/lib/get_proxy_list.rb

@@ -1,7 +1,6 @@
-require 'get_proxy_list'
-require "get_proxy_list/version"
-require "get_proxy_list/from_proxy_cn"
-require "get_proxy_list/base"
+require File.expand_path("../get_proxy_list/version",__FILE__)
+require File.expand_path("../get_proxy_list/from_proxy_cn",__FILE__)
+require File.expand_path("../get_proxy_list/base",__FILE__)
 
 module GetProxyList
   def self.get_list(limit,page)
@@ -11,4 +10,4 @@ module GetProxyList
     proxylist = base.get_proxy_in_time_limit(limit,proxylist)
     return proxylist
   end
-end
+end

data/lib/get_proxy_list/base.rb

@@ -14,8 +14,6 @@ module GetProxyList
     end
 
 
-
-
     #获取符合时限的代理
     #limit 时限
     #proxylist 待筛选的代理列表
@@ -27,6 +25,7 @@ module GetProxyList
         begin
           timeout(limit+1) do
             doc = Nokogiri::HTML(open("http://www.baidu.com",:proxy=> url))
+            x = doc.css("em")
           end
           time_end = Time.now.to_i
           time_use = time_end - time_start
@@ -52,4 +51,4 @@ module GetProxyList
     end
 
   end
-end
+end

data/lib/get_proxy_list/from_proxy_cn.rb

@@ -1,5 +1,5 @@
 require 'nokogiri'
-
+require 'open-uri'
 
 module GetProxyList
   class FromProxyCn
@@ -15,23 +15,41 @@ module GetProxyList
     #获取代理列表
     def _getproxylist(url)
       begin
-        doc = Nokogiri::HTML(open(url))
+        hash_port = {"c"=>"1","a"=>"2","z"=>"3","m"=>"4","b"=>"5","w"=>"6","i"=>"7","x"=>"8","l"=>"9","f"=>"0"}
+        html = open(url ).read
+        html.force_encoding("gbk")
+        html.encode!("utf-8", :undef => :replace, :replace => "", :invalid => :replace)
+        doc=Nokogiri::HTML.parse html
+        #doc = Nokogiri::HTML(open(url))
         proxylist = []
         doc.css('table').each do |_table|
-           _table.css('tr').each do |tr|
-             unless !tr.css('td')[0].eql?("IP:Port")||tr.css('td').empty?
-               proxy=tr.css('td')[0]
-               p "Proxy: #{proxy}"
-               proxy = proxy.split("document.write")
-               _proxy = proxy.split(":")
-               proxylist << {ip:_proxy[0],port:_proxy[1]}
-             end
-           end
+          _table.css('tr').each do |tr|
+            if !tr.css('td')[0].content.eql?("IP:Port")&&!tr.css('td')[0].content.empty?
+              proxy=tr.css('td')[0].content
+              p "Proxy: #{proxy[1]}"
+              proxy = proxy.split("document.write")
+              _proxy = proxy[1].split(":")
+              port = _proxy[1].gsub(")","")
+              p "port : #{port}"
+              port=port.split("+")
+              _port = ""
+              1.upto(port.length-1).each do |index|
+               p "Hash:#{hash_port[port[index]]}"
+              	unless hash_port[port[index]].nil?
+                	_port+=hash_port[port[index]]
+                else
+                  _port=8080 
+                  break
+                end
+              end
+              proxylist << {"ip"=>proxy[0],"port"=>_port}
+            end
+          end
         end
         p "ProxyList: #{proxylist}"
         return proxylist
-      rescue => err
-        raise "Get Proxy list Error! Class:'FromProxyCn' Gem:'get_proxy_list' Error:'#{err.to_s}'"
+        #rescue => err
+        #  raise "Get Proxy list Error! Class:'FromProxyCn' Gem:'get_proxy_list' Error:'#{err.to_s}'"
       end
     end
 
@@ -57,4 +75,4 @@ module GetProxyList
     end
 
   end
-end
+end

data/lib/get_proxy_list/from_proxy_cn.rb~

@@ -0,0 +1,78 @@
+require 'nokogiri'
+require 'open-uri'
+
+module GetProxyList
+  class FromProxyCn
+    # To change this template use File | Settings | File Templates.
+    include GetProxyList
+    attr_reader :site
+
+    def initialize
+      @site = "http://www.cnproxy.com"
+    end
+
+
+    #获取代理列表
+    def _getproxylist(url)
+      begin
+        hash_port = {"c"=>"1","a"=>"2","z"=>"3","m"=>"4","b"=>"5","w"=>"6","i"=>"7","x"=>"8","l"=>"9","f"=>"0"}
+        html = open(url ).read
+        html.force_encoding("gbk")
+        html.encode!("utf-8", :undef => :replace, :replace => "", :invalid => :replace)
+        doc=Nokogiri::HTML.parse html
+        #doc = Nokogiri::HTML(open(url))
+        proxylist = []
+        doc.css('table').each do |_table|
+          _table.css('tr').each do |tr|
+            if !tr.css('td')[0].content.eql?("IP:Port")&&!tr.css('td')[0].content.empty?
+              proxy=tr.css('td')[0].content
+              p "Proxy: #{proxy[1]}"
+              proxy = proxy.split("document.write")
+              _proxy = proxy[1].split(":")
+              port = _proxy[1].gsub(")","")
+              p "port : #{port}"
+              port=port.split("+")
+              _port = ""
+              1.upto(port.length-1).each do |index|
+               p "Hash:#{hash_port[port[index]]}"
+              	unless hash_port[port[index]].nil?
+                	_port+=hash_port[port[index]]
+                else
+                  _port=8080 
+                  break
+                end
+              end
+              proxylist << {"ip"=>proxy[0],"port"=>_port}
+            end
+          end
+        end
+        p "ProxyList: #{proxylist}"
+        return proxylist
+        #rescue => err
+        #  raise "Get Proxy list Error! Class:'FromProxyCn' Gem:'get_proxy_list' Error:'#{err.to_s}'"
+      end
+    end
+
+
+    #根据需要的页数获取URL列表
+    #page 将获取的页数
+    def get_url_list(page)
+      urls = []
+      1.upto(page.to_i).each do |_page|
+        urls << @site+"/proxy#{_page.to_s}.html"
+      end
+      return urls
+    end
+
+
+    def get_proxylist(page)
+      urls = get_url_list(page)
+      proxylist = []
+      urls.each do |url|
+        proxylist =proxylist+proxylist|_getproxylist(url)
+      end
+      return proxylist
+    end
+
+  end
+end

data/lib/get_proxy_list/from_proxycn.rb

@@ -4,4 +4,4 @@ module GetProxyList
   class FromProxycn
     # To change this template use File | Settings | File Templates.
   end
-end
+end

data/lib/get_proxy_list/version.rb

@@ -1,3 +1,3 @@
-module GetProxyList
-  VERSION = "0.0.4"
+module Getproxylist
+  VERSION = "0.0.7"
 end

data/lib/getproxylist.rb~

@@ -0,0 +1,14 @@
+require 'getproxylist'
+require "getproxylist/version"
+require "getproxylist/from_proxy_cn"
+require "getproxylist/base"
+
+module GetProxyList
+  def self.get_list(limit,page)
+    fromProxyCn = FromProxyCn.new()
+    base = Base.new()
+    proxylist = fromProxyCn.get_proxylist(page)
+    proxylist = base.get_proxy_in_time_limit(limit,proxylist)
+    return proxylist
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: get_proxy_list
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.7
 platform: ruby
 authors:
 - hahazql
@@ -38,7 +38,7 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: use for get a proxy list
+description: ! '""'
 email:
 - hahazhouqunli@gmail.com
 executables: []
@@ -50,12 +50,16 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- get_proxy_list.gemspec
+- getproxylist.gemspec
+- lib/debug.rb
+- lib/debug.rb~
 - lib/get_proxy_list.rb
 - lib/get_proxy_list/base.rb
 - lib/get_proxy_list/from_proxy_cn.rb
+- lib/get_proxy_list/from_proxy_cn.rb~
 - lib/get_proxy_list/from_proxycn.rb
 - lib/get_proxy_list/version.rb
+- lib/getproxylist.rb~
 homepage: ''
 licenses:
 - MIT
@@ -79,6 +83,6 @@ rubyforge_project:
 rubygems_version: 2.0.0
 signing_key: 
 specification_version: 4
-summary: use for get a proxy list
+summary: ! '""'
 test_files: []
 has_rdoc: