proxy_fetcher 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 04cf23c5f1bb6abfd29e5d6180a5a9e167a2d147
-  data.tar.gz: 5590d391eee582e511027a8282ef38917aa653a9
+  metadata.gz: 7a36eb7a048bc985836e08d8e380412ceadd6e4b
+  data.tar.gz: cdceec65f99f25a4bfe8b8ae3531a9d203fb8710
 SHA512:
-  metadata.gz: ccd0ef56339916919c8bf1b70dd3332d9972d6db1cc5b676b28b16691161955cffafbfa25bb310ba5b1d78a7b02dc7e8e440bb0a019736442b8bcbcfa3f8a004
-  data.tar.gz: e72892832780fddb424adc29dcb3b500ede2998d452c333aeffcb2a4766483a079288ec3e52e1eeca4ff61f7801ddae3ba80cddac2d527eaeabed09cbd8f6736
+  metadata.gz: 39954c3255c4ea3642023b4ceb0a9c03d4623d2cecd2b75237f4b219af71a292408583602ca5d7e3ce06ec54a8ac39c4e0c83b7d98ecceaea76ca8d86ddc5f81
+  data.tar.gz: 362d73388a1ed8b9d674e571c55763c265d808415efbc5db8c96e94c8ab110fffde2b137436ffe521f54b9940805df2c846a392c2a3316153f235a47f47ae96f

data/README.md

@@ -5,14 +5,16 @@
 [![License](http://img.shields.io/badge/license-MIT-brightgreen.svg)](#license)
 
 This gem can help your Ruby application to make HTTP(S) requests from proxy server, fetching and validating
-current proxy lists from the [HideMyAss](http://hidemyass.com/) service.
+current proxy lists from the different proxy services like [HideMyAss](http://hidemyass.com/) or Hide My Name.
+
+**IMPORTANT** currently HideMyAss service closed free proxy list service, but it will be open soon and gem will be updated.
 
 ## Installation
 
 If using bundler, first add 'proxy_fetcher' to your Gemfile:
 
 ```ruby
-gem 'proxy_fetcher', '~> 0.1'
+gem 'proxy_fetcher', '~> 0.2'
 ```
 
 or if you want to use the latest version (from `master` branch), then:
@@ -30,7 +32,7 @@ bundle install
 Otherwise simply install the gem:
 
 ```sh
-gem install proxy_fetcher -v '0.1'
+gem install proxy_fetcher -v '0.2'
 ```
 
 ## Example of usage
@@ -79,9 +81,9 @@ Every proxy is a `ProxyFetcher::Proxy` object that has next readers:
 * `port`
 * `country` (USA or Brazil for example)
 * `response_time` (5217 for example)
-* `connection_time` (rank from 0 to 100, where 0 — slow, 100 — high)
-* `speed` (rank from 0 to 100, where 0 — slow, 100 — high)
-* `type` (URI schema, HTTP for example)
+* `connection_time` (rank from 0 to 100, where 0 — slow, 100 — high. **Note** depends on the proxy provider)
+* `speed` (rank from 0 to 100, where 0 — slow, 100 — high. **Note** depends on the proxy provider)
+* `type` (URI schema, HTTP or HTTPS)
 * `anonimity` (Low or High +KA for example)
 
 Also you can call next instance method for every Proxy object:
@@ -109,16 +111,45 @@ You can sort or find any proxy by speed using next 3 instance methods:
 * `medium?`
 * `slow?`'
 
-To change open/read timeout for `cleanup!` and `connectable?` methods yu need to change ProxyFetcher::Manager config:
+To change open/read timeout for `cleanup!` and `connectable?` methods yu need to change ProxyFetcher.config:
 
 ```ruby
-ProxyFetcher::Manager.config.read_timeout = 1 # default is 3
-ProxyFetcher::Manager.config.open_timeout = 1 # default is 3
+ProxyFetcher.config.read_timeout = 1 # default is 3
+ProxyFetcher.config.open_timeout = 1 # default is 3
 
 manager = ProxyFetcher::Manager.new
 manager.cleanup!
 ```
 
+## Providers
+
+Currently ProxyFetcher can deal with next proxy providers:
+
+* Hide My Name (default one)
+* Free Proxy List
+* HideMyAss
+
+If you wanna use one of them just setup required in the config:
+
+ 
+```ruby
+ProxyFetcher.config.provider = :free_proxy_list
+
+manager = ProxyFetcher::Manager.new
+manager.proxies
+ #=> ...
+```
+
+Also you can write your own provider. All you need is to create a class, that would be inherited from the
+`ProxyFetcher::Providers::Base` class, and register your provider like this:
+
+```ruby
+ProxyFetcher::Configuration.register_provider(:your_provider, YourProviderClass)
+```
+
+Provider class must implement `self.load_proxy_list` and `#parse!(html_entry)` methods that will load and parse
+provider HTML page with proxy list. Take a look at the samples in the `proxy_fetcher/providers` directory.
+
 ## TODO
 
 * Proxy filters

data/lib/proxy_fetcher.rb

@@ -5,6 +5,15 @@ require 'nokogiri'
 require 'proxy_fetcher/configuration'
 require 'proxy_fetcher/proxy'
 require 'proxy_fetcher/manager'
+require 'proxy_fetcher/providers/base'
+require 'proxy_fetcher/providers/hide_my_ass'
+require 'proxy_fetcher/providers/hide_my_name'
+require 'proxy_fetcher/providers/free_proxy_list'
 
 module ProxyFetcher
+  class << self
+    def config
+      @config ||= ProxyFetcher::Configuration.new
+    end
+  end
 end

data/lib/proxy_fetcher/configuration.rb

@@ -1,10 +1,33 @@
 module ProxyFetcher
   class Configuration
-    attr_accessor :open_timeout, :read_timeout
+    UnknownProvider = Class.new(StandardError)
+    ProviderRegistered = Class.new(StandardError)
+
+    attr_accessor :open_timeout, :read_timeout, :provider
+
+    class << self
+      def providers
+        @providers ||= {}
+      end
+
+      def register_provider(name, klass)
+        raise ProviderRegistered, "#{name} provider already registered!" if providers.key?(name.to_sym)
+
+        providers[name.to_sym] = klass
+      end
+    end
 
     def initialize
       @open_timeout = 3
       @read_timeout = 3
+
+      self.provider = :hide_my_name # currently default one
+    end
+
+    def provider=(name)
+      @provider = self.class.providers[name.to_sym]
+
+      raise UnknownProvider, "unregistered proxy provider (#{name})!" if @provider.nil?
     end
   end
 end

data/lib/proxy_fetcher/manager.rb

@@ -1,12 +1,6 @@
 module ProxyFetcher
   class Manager
-    PROXY_PROVIDER_URL = 'http://proxylist.hidemyass.com/'.freeze
-
-    class << self
-      def config
-        @config ||= ProxyFetcher::Configuration.new
-      end
-    end
+    EmptyProxyList = Class.new(StandardError)
 
     attr_reader :proxies
 
@@ -22,9 +16,7 @@ module ProxyFetcher
 
     # Update current proxy list from the provider
     def refresh_list!
-      doc = Nokogiri::HTML(load_html(PROXY_PROVIDER_URL))
-      rows = doc.xpath('//table[@id="listable"]/tbody/tr')
-
+      rows = ProxyFetcher.config.provider.load_proxy_list
       @proxies = rows.map { |row| Proxy.new(row) }
     end
 
@@ -45,11 +37,11 @@ module ProxyFetcher
     # Pop first valid proxy (and back it to the end of the proxy list)
     # Invalid proxies will be removed from the list
     def get!
-      index = @proxies.find_index(&:connectable?)
+      index = proxies.find_index(&:connectable?)
       return if index.nil?
 
-      proxy = @proxies.delete_at(index)
-      tail = @proxies[index..-1]
+      proxy = proxies.delete_at(index)
+      tail = proxies[index..-1]
 
       @proxies = tail << proxy
 
@@ -65,7 +57,12 @@ module ProxyFetcher
 
     alias validate! cleanup!
 
-    # Just schema + host + port
+    # Return random proxy
+    def random
+      proxies.sample
+    end
+
+    # Returns array of proxy URLs (just schema + host + port)
     def raw_proxies
       proxies.map(&:url)
     end
@@ -74,15 +71,5 @@ module ProxyFetcher
     def inspect
       to_s
     end
-
-    private
-
-    # Get HTML from the requested URL
-    def load_html(url)
-      uri = URI.parse(url)
-      http = Net::HTTP.new(uri.host, uri.port)
-      response = http.get(uri.request_uri)
-      response.body
-    end
   end
 end

data/lib/proxy_fetcher/providers/base.rb

@@ -0,0 +1,30 @@
+module ProxyFetcher
+  module Providers
+    class Base
+      attr_reader :proxy
+
+      def initialize(proxy_instance)
+        @proxy = proxy_instance
+      end
+
+      def set!(name, value)
+        @proxy.instance_variable_set(:"@#{name}", value)
+      end
+
+      class << self
+        def parse_entry(entry, proxy_instance)
+          new(proxy_instance).parse!(entry)
+        end
+
+        # Get HTML from the requested URL
+        def load_html(url)
+          uri = URI.parse(url)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true if uri.scheme == 'https'
+          response = http.get(uri.request_uri)
+          response.body
+        end
+      end
+    end
+  end
+end

data/lib/proxy_fetcher/providers/free_proxy_list.rb

@@ -0,0 +1,47 @@
+module ProxyFetcher
+  module Providers
+    class FreeProxyList < Base
+      PROVIDER_URL = 'https://free-proxy-list.net/'.freeze
+
+      class << self
+        def load_proxy_list
+          doc = Nokogiri::HTML(load_html(PROVIDER_URL))
+          doc.xpath('//table[@id="proxylisttable"]/tbody/tr')
+        end
+      end
+
+      def parse!(html_entry)
+        html_entry.xpath('td').each_with_index do |td, index|
+          case index
+          when 0
+            set!(:addr, td.content.strip)
+          when 1 then
+            set!(:port, Integer(td.content.strip))
+          when 3 then
+            set!(:country, td.content.strip)
+          when 4
+            set!(:anonymity, td.content.strip)
+          when 6
+            set!(:type, parse_type(td))
+          else
+            # nothing
+          end
+        end
+      end
+
+      private
+
+      def parse_type(td)
+        type = td.content.strip
+
+        if type && type.downcase.include?('yes')
+          'HTTPS'
+        else
+          'HTTP'
+        end
+      end
+    end
+  end
+end
+
+ProxyFetcher::Configuration.register_provider(:free_proxy_list, ProxyFetcher::Providers::FreeProxyList)

data/lib/proxy_fetcher/providers/hide_my_ass.rb

@@ -0,0 +1,70 @@
+module ProxyFetcher
+  module Providers
+    class HideMyAss < Base
+      PROVIDER_URL = 'http://proxylist.hidemyass.com/'.freeze
+
+      class << self
+        def load_proxy_list
+          doc = Nokogiri::HTML(load_html(PROVIDER_URL))
+          doc.xpath('//table[@id="listable"]/tbody/tr')
+        end
+      end
+
+      def parse!(html_entry)
+        html_entry.xpath('td').each_with_index do |td, index|
+          case index
+          when 1
+            set!(:addr, parse_addr(td))
+          when 2 then
+            set!(:port, Integer(td.content.strip))
+          when 3 then
+            set!(:country, td.content.strip)
+          when 4
+            set!(:response_time, parse_response_time(td))
+            set!(:speed, parse_indicator_value(td))
+          when 5
+            set!(:connection_time, parse_indicator_value(td))
+          when 6 then
+            set!(:type, td.content.strip)
+          when 7
+            set!(:anonymity, td.content.strip)
+          else
+            # nothing
+          end
+        end
+      end
+
+      private
+
+      def parse_addr(html_doc)
+        good = []
+        bytes = []
+        css = html_doc.at_xpath('span/style/text()').to_s
+        css.split.each { |l| good << Regexp.last_match(1) if l =~ /\.(.+?)\{.*inline/ }
+
+        html_doc.xpath('span/span | span | span/text()').each do |span|
+          if span.is_a?(Nokogiri::XML::Text)
+            bytes << Regexp.last_match(1) if span.content.strip =~ /\.{0,1}(.+)\.{0,1}/
+          elsif (span['style'] && span['style'] =~ /inline/) ||
+                (span['class'] && good.include?(span['class'])) ||
+                (span['class'] =~ /^[0-9]/)
+
+            bytes << span.content
+          end
+        end
+
+        bytes.join('.').gsub(/\.+/, '.')
+      end
+
+      def parse_response_time(html_doc)
+        Integer(html_doc.at_xpath('div')['rel'])
+      end
+
+      def parse_indicator_value(html_doc)
+        Integer(html_doc.at('.indicator').attr('style').match(/width: (\d+)%/i)[1])
+      end
+    end
+  end
+end
+
+ProxyFetcher::Configuration.register_provider(:hide_my_ass, ProxyFetcher::Providers::HideMyAss)

data/lib/proxy_fetcher/providers/hide_my_name.rb

@@ -0,0 +1,49 @@
+module ProxyFetcher
+  module Providers
+    class HideMyName < Base
+      PROVIDER_URL = 'https://hidemy.name/en/proxy-list/?type=hs#list'.freeze
+
+      class << self
+        def load_proxy_list
+          doc = Nokogiri::HTML(load_html(PROVIDER_URL))
+          doc.xpath('//table[@class="proxy__t"]/tbody/tr')
+        end
+      end
+
+      def parse!(html_entry)
+        html_entry.xpath('td').each_with_index do |td, index|
+          case index
+          when 0
+            set!(:addr, td.content.strip)
+          when 1 then
+            set!(:port, Integer(td.content.strip))
+          when 2 then
+            set!(:country, td.at_xpath('*//span[1]/following-sibling::text()[1]').content.strip)
+          when 3
+            set!(:response_time, Integer(td.at('p').content.strip[/\d+/]))
+          when 4
+            set!(:type, parse_type(td))
+          when 5
+            set!(:anonymity, td.content.strip)
+          else
+            # nothing
+          end
+        end
+      end
+
+      private
+
+      def parse_type(td)
+        schemas = td.content.strip
+
+        if schemas && schemas.downcase.include?('https')
+          'HTTPS'
+        else
+          'HTTP'
+        end
+      end
+    end
+  end
+end
+
+ProxyFetcher::Configuration.register_provider(:hide_my_name, ProxyFetcher::Providers::HideMyName)

data/lib/proxy_fetcher/proxy.rb

@@ -1,10 +1,10 @@
 module ProxyFetcher
   class Proxy
     attr_reader :addr, :port, :country, :response_time,
-                :connection_time, :speed, :type, :anonimity
+                :connection_time, :speed, :type, :anonymity
 
     def initialize(html_row)
-      parse_row!(html_row)
+      ProxyFetcher.config.provider.parse_entry(html_row, self)
 
       self
     end
@@ -12,13 +12,13 @@ module ProxyFetcher
     def connectable?
       connection = Net::HTTP.new(addr, port)
       connection.use_ssl = true if https?
-      connection.open_timeout = ProxyFetcher::Manager.config.open_timeout
-      connection.read_timeout = ProxyFetcher::Manager.config.read_timeout
+      connection.open_timeout = ProxyFetcher.config.open_timeout
+      connection.read_timeout = ProxyFetcher.config.read_timeout
 
       connection.start { |http| return true if http.request_head('/') }
 
       false
-    rescue Timeout::Error, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ECONNABORTED
+    rescue Timeout::Error, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EOFError
       false
     end
 
@@ -51,60 +51,5 @@ module ProxyFetcher
     def url
       uri.to_s
     end
-
-    private
-
-    # HideMyAss proxy list rows parsing by columns
-    def parse_row!(html)
-      html.xpath('td').each_with_index do |td, index|
-        case index
-        when 1
-          @addr = parse_addr(td)
-        when 2 then
-          @port = Integer(td.content.strip)
-        when 3 then
-          @country = td.content.strip
-        when 4
-          @response_time = parse_response_time(td)
-          @speed = parse_indicator_value(td)
-        when 5
-          @connection_time = parse_indicator_value(td)
-        when 6 then
-          @type = td.content.strip
-        when 7
-          @anonymity = td.content.strip
-        else
-          # nothing
-        end
-      end
-    end
-
-    def parse_addr(html)
-      good = []
-      bytes = []
-      css = html.at_xpath('span/style/text()').to_s
-      css.split.each { |l| good << Regexp.last_match(1) if l =~ /\.(.+?)\{.*inline/ }
-
-      html.xpath('span/span | span | span/text()').each do |span|
-        if span.is_a?(Nokogiri::XML::Text)
-          bytes << Regexp.last_match(1) if span.content.strip =~ /\.{0,1}(.+)\.{0,1}/
-        elsif (span['style'] && span['style'] =~ /inline/) ||
-              (span['class'] && good.include?(span['class'])) ||
-              (span['class'] =~ /^[0-9]/)
-
-          bytes << span.content
-        end
-      end
-
-      bytes.join('.').gsub(/\.+/, '.')
-    end
-
-    def parse_response_time(html)
-      Integer(html.at_xpath('div')['rel'])
-    end
-
-    def parse_indicator_value(html)
-      Integer(html.at('.indicator').attr('style').match(/width: (\d+)%/i)[1])
-    end
   end
 end