proxy_fetcher 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: db8683f8d34f01ebb03c71753999649cdcbeba61
-  data.tar.gz: 480d289e2fe9bb038b04f080d005492fdfee7f7a
+  metadata.gz: 6c2d4076c9f73303b305c364d3dcf672b6e8eba7
+  data.tar.gz: b692847fab646ec4bd230909ac1cba9eb77c1c7f
 SHA512:
-  metadata.gz: 29129bcd201af8e5f8cf003331b5abeda3a590cf6a4d2cc91c95fa23622f513d66d5867bfcb6a5278f6dd706041c95b0d07d71e6173c8dd10c82fab7955fd550
-  data.tar.gz: e84e36c60f0a22018677c1896dffa51b45b76f9518e7edc8b62d3aeb4908525cfae12d3a8b60c9d547e7ef8217f75ed27fd8ac9d5ae149f5ef2b4b3d82346a51
+  metadata.gz: cd70341cd18eea64dad58d63b8888f36e403251bb08e9d0289c584de1bc2201b45ff97f48fbac26697b25ee52328611b1e37de8d393b3b50ed7b3885192d0885
+  data.tar.gz: aed0b7532628796cb2b9f397be8bd8d87108c4d5a4e4efd2d30a468e2b8c9c203fbe7ac1fd4b56c7e571a67748b681297c56e2c880fddddb20c463bc117789e6

data/.gitignore

@@ -17,16 +17,6 @@ Gemfile.lock
 certs
 gemfiles/*.gemfile.lock
 
-# TODO Comment out this rule if you are OK with secrets being uploaded to the repo
-config/initializers/secret_token.rb
-
-# Only include if you have production secrets in this file, which is no longer a Rails default
-# config/secrets.yml
-
-# dotenv
-# TODO Comment out this rule if environment variables can be committed
-.env
-
 ## Environment normalization:
 /.bundle
 /vendor/bundle
@@ -47,3 +37,5 @@ bower.json
 
 # Ignore Byebug command history file.
 .byebug_history
+.yardoc/
+doc/

data/.travis.yml

@@ -2,9 +2,10 @@ language: ruby
 before_install: gem install bundler
 bundler_args: --without yard guard benchmarks
 script: "rake spec"
-env:
-  global:
-    - "JRUBY_OPTS='$JRUBY_OPTS --debug'"
+env: JRUBY_OPTS="$JRUBY_OPTS --debug"
+gemfile:
+  - gemfiles/oga.gemfile
+  - gemfiles/nokogiri.gemfile
 gemfile:
   - gemfiles/oga.gemfile
   - gemfiles/nokogiri.gemfile
@@ -13,8 +14,10 @@ rvm:
   - 2.1
   - 2.2.4
   - 2.3.3
-  - 2.4.2
+  - 2.4.3
+  - 2.5.0
   - ruby-head
+  - jruby-9.1.15.0
 matrix:
   allow_failures:
     - rvm: ruby-head

data/README.md

@@ -1,16 +1,17 @@
-# Ruby lib for managing proxies
+# Ruby / JRuby lib for managing proxies
 [![Gem Version](https://badge.fury.io/rb/proxy_fetcher.svg)](http://badge.fury.io/rb/proxy_fetcher)
 [![Build Status](https://travis-ci.org/nbulaj/proxy_fetcher.svg?branch=master)](https://travis-ci.org/nbulaj/proxy_fetcher)
 [![Coverage Status](https://coveralls.io/repos/github/nbulaj/proxy_fetcher/badge.svg)](https://coveralls.io/github/nbulaj/proxy_fetcher)
 [![Code Climate](https://codeclimate.com/github/nbulaj/proxy_fetcher/badges/gpa.svg)](https://codeclimate.com/github/nbulaj/proxy_fetcher)
+[![Inline docs](http://inch-ci.org/github/nbulaj/proxy_fetcher.png?branch=master)](http://inch-ci.org/github/nbulaj/proxy_fetcher)
 [![License](http://img.shields.io/badge/license-MIT-brightgreen.svg)](#license)
 
-This gem can help your Ruby application to make HTTP(S) requests using proxy by fetching and validating actual
-proxy lists from multiple providers.
+This gem can help your Ruby / JRuby application to make HTTP(S) requests using
+proxy by fetching and validating actual proxy lists from multiple providers.
 
 It gives you a special `Manager` class that can load proxy lists, validate them and return random or specific proxies.
-It also has a `Client` class that encapsulates all the logic for the sending HTTP requests using proxies.
-Take a look at the documentation below to find all the gem features.
+It also has a `Client` class that encapsulates all the logic for sending HTTP requests using proxies, automatically
+fetched and validated by the gem. Take a look at the documentation below to find all the gem features.
 
 Also this gem can be used with any other programming language (Go / Python / etc) as standalone solution for downloading and
 validating proxy lists from the different providers. [Checkout examples](#standalone) of usage below.
@@ -32,13 +33,14 @@ validating proxy lists from the different providers. [Checkout examples](#standa
 
 ## Dependencies
 
-ProxyFetcher gem itself requires only Ruby `>= 2.0.0`.
+ProxyFetcher gem itself requires only Ruby `>= 2.0.0` (or [JRuby](http://jruby.org/) `> 9.0`, but maybe earlier too,
+[see Travis build matrix](.travis.yml)).
 
 However, it requires an adapter to parse HTML. If you do not specify any specific adapter, then it will use
 default one - [Nokogiri](https://github.com/sparklemotion/nokogiri). It's OK for any Ruby on Rails project
-(because they uses it by default).
+(because they use it by default).
 
-But if you want to use some specific adapter (for example your Ruby application uses [Oga](https://gitlab.com/yorickpeterse/oga),
+But if you want to use some specific adapter (for example your application uses [Oga](https://gitlab.com/yorickpeterse/oga),
 then you need to manually add your dependencies to your project and configure ProxyFetcher to use another adapter. Moreover,
 you can implement your own adapter if it your use-case. Take a look at the [Configuration](#configuration) section for more details.
 
@@ -65,7 +67,7 @@ bundle install
 Otherwise simply install the gem:
 
 ```sh
-gem install proxy_fetcher -v '0.5'
+gem install proxy_fetcher -v '0.6'
 ```
 
 ## Example of usage

data/lib/proxy_fetcher/client/client.rb

@@ -1,26 +1,121 @@
 module ProxyFetcher
+  # ProxyFetcher HTTP client that encapsulates all the logic for sending
+  # HTTP(S) requests using proxies, automatically fetched and validated by the gem.
   module Client
     class << self
+      # Sends HTTP GET request.
+      #
+      # @param url [String]
+      #   Requested URL
+      #
+      # @param headers [Hash]
+      #   HTTP headers that will be used in the request
+      #
+      # @param options [Hash]
+      #   Additional options used by <code>ProxyFetcher::Client</code>
+      #
+      # @return [String]
+      #   HTML body from the URL.
+      #
       def get(url, headers: {}, options: {})
         request_without_payload(:get, url, headers, options)
       end
 
+      # Sends HTTP HEAD request.
+      #
+      # @param url [String]
+      #   Requested URL
+      #
+      # @param headers [Hash]
+      #   HTTP headers that will be used in the request
+      #
+      # @param options [Hash]
+      #   Additional options used by <code>ProxyFetcher::Client</code>
+      #
+      # @return [String]
+      #   HTML body from the URL.
+      #
       def head(url, headers: {}, options: {})
         request_without_payload(:head, url, headers, options)
       end
 
+      # Sends HTTP POST request.
+      #
+      # @param url [String]
+      #   Requested URL
+      #
+      # @param payload [String, Hash]
+      #   HTTP payload
+      #
+      # @param headers [Hash]
+      #   HTTP headers that will be used in the request
+      #
+      # @param options [Hash]
+      #   Additional options used by <code>ProxyFetcher::Client</code>
+      #
+      # @return [String]
+      #   HTML body from the URL.
+      #
       def post(url, payload, headers: {}, options: {})
         request_with_payload(:post, url, payload, headers, options)
       end
 
+      # Sends HTTP DELETE request.
+      #
+      # @param url [String]
+      #   Requested URL
+      #
+      # @param headers [Hash]
+      #   HTTP headers that will be used in the request
+      #
+      # @param options [Hash]
+      #   Additional options used by <code>ProxyFetcher::Client</code>
+      #
+      # @return [String]
+      #   HTML body from the URL.
+      #
       def delete(url, headers: {}, options: {})
         request_without_payload(:delete, url, headers, options)
       end
 
+      # Sends HTTP PUT request.
+      #
+      # @param url [String]
+      #   Requested URL
+      #
+      # @param payload [String, Hash]
+      #   HTTP payload
+      #
+      # @param headers [Hash]
+      #   HTTP headers that will be used in the request
+      #
+      # @param options [Hash]
+      #   Additional options used by <code>ProxyFetcher::Client</code>
+      #
+      # @return [String]
+      #   HTML body from the URL.
+      #
       def put(url, payload, headers: {}, options: {})
         request_with_payload(:put, url, payload, headers, options)
       end
 
+      # Sends HTTP PATCH request.
+      #
+      # @param url [String]
+      #   Requested URL
+      #
+      # @param payload [String, Hash]
+      #   HTTP payload
+      #
+      # @param headers [Hash]
+      #   HTTP headers that will be used in the request
+      #
+      # @param options [Hash]
+      #   Additional options used by <code>ProxyFetcher::Client</code>
+      #
+      # @return [String]
+      #   HTML body from the URL.
+      #
       def patch(url, payload, headers: {}, options: {})
         request_with_payload(:patch, url, payload, headers, options)
       end

data/lib/proxy_fetcher/client/request.rb

@@ -1,10 +1,15 @@
 module ProxyFetcher
   module Client
+    # ProxyFetcher::Client HTTP request abstraction.
     class Request
+      # URL encoding HTTP headers.
       URL_ENCODED = {
         'Content-Type' => 'application/x-www-form-urlencoded'
       }.freeze
 
+      # Default SSL options that will be used for connecting to resources
+      # the uses secure connection. By default ProxyFetcher wouldn't verify
+      # SSL certs.
       DEFAULT_SSL_OPTIONS = {
         verify_mode: OpenSSL::SSL::VERIFY_NONE
       }.freeze
@@ -46,7 +51,7 @@ module ProxyFetcher
         return if payload.nil?
 
         if payload.is_a?(Hash)
-          headers.merge(URL_ENCODED)
+          headers.merge!(URL_ENCODED)
           URI.encode_www_form(payload)
         else
           payload

data/lib/proxy_fetcher/configuration.rb

@@ -1,11 +1,23 @@
 module ProxyFetcher
+  # ProxyFetcher configuration. Stores all the options for dealing
+  # with HTTP requests, adapters, custom classes.
+  #
   class Configuration
     attr_accessor :timeout, :pool_size, :user_agent
     attr_reader :adapter, :http_client, :proxy_validator, :providers
 
-    # rubocop:disable Metrics/LineLength
-    DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112 Safari/537.36'.freeze
-
+    # User-Agent string that will be used by the ProxyFetcher HTTP client (to
+    # send requests via proxy) and to fetch proxy lists from the sources.
+    #
+    # Default is Google Chrome 60, but can be changed in <code>ProxyFetcher.config</code>.
+    #
+    DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 ' \
+                         '(KHTML, like Gecko) Chrome/60.0.3112 Safari/537.36'.freeze
+
+    # HTML parser adapter name.
+    #
+    # Default is Nokogiri, but can be changed in <code>ProxyFetcher.config</code>.
+    #
     DEFAULT_ADAPTER = :nokogiri
 
     class << self
@@ -13,6 +25,15 @@ module ProxyFetcher
         @registry ||= ProvidersRegistry.new
       end
 
+      # Register new proxy provider. Requires provider name and class
+      # that will process proxy list.
+      #
+      # @param name [String, Symbol]
+      #   name of the provider
+      #
+      # @param klass [Class]
+      #   Class that will fetch and process proxy list
+      #
       def register_provider(name, klass)
         providers_registry.register(name, klass)
       end

data/lib/proxy_fetcher/document.rb

@@ -1,4 +1,7 @@
 module ProxyFetcher
+  # HTML document abstraction class. Used to work with different HTML parser adapters
+  # such as Nokogiri, Oga or a custom one. Stores <i>backend</i< that will handle all
+  # the DOM manipulation logic.
   class Document
     class << self
       def parse(data)

data/lib/proxy_fetcher/exceptions.rb

@@ -1,7 +1,10 @@
 module ProxyFetcher
+  # Base exception class for all the ProxyFetcher exceptions.
   Error = Class.new(StandardError)
 
+  # ProxyFetcher exceptions namespace
   module Exceptions
+    # Exception for wrong custom classes (such as ProxyValidator or HTTP Client).
     class WrongCustomClass < Error
       def initialize(klass, methods)
         required_methods = Array(methods).join(', ')
@@ -9,36 +12,48 @@ module ProxyFetcher
       end
     end
 
+    # Exception for wrong provider name, that raises when configured provider
+    # that is not registered via <code>register_provider</code> interface.
     class UnknownProvider < Error
       def initialize(provider_name)
         super("unregistered proxy provider `#{provider_name}`")
       end
     end
 
+    # Exception for cases when user tries to register already existing provider.
     class RegisteredProvider < Error
       def initialize(name)
         super("`#{name}` provider already registered!")
       end
     end
 
+    # Exception for cases when HTTP client reached maximum count of redirects
+    # trying to process HTTP request.
     class MaximumRedirectsReached < Error
       def initialize(*)
         super('maximum redirects reached')
       end
     end
 
+    # Exception for cases when HTTP client reached maximum count of retries
+    # trying to process HTTP request. Can occur when request failed by timeout
+    # multiple times.
     class MaximumRetriesReached < Error
       def initialize(*)
         super('reached the maximum number of retries')
       end
     end
 
+    # Exception for cases when user tries to set wrong HTML parser adapter
+    # in the configuration.
     class UnknownAdapter < Error
       def initialize(name)
         super("unknown adapter '#{name}'")
       end
     end
 
+    # Exception for cases when user tries to set <code>nil</code> HTML parser adapter
+    # in the configuration (or just forget to change it).
     class BlankAdapter < Error
       def initialize(*)
         super(<<-MSG.strip.squeeze
@@ -49,6 +64,8 @@ module ProxyFetcher
       end
     end
 
+    # Exception for cases when HTML parser adapter can't be installed.
+    # It will print the reason (backtrace) of the exception that caused an error.
     class AdapterSetupError < Error
       def initialize(adapter_name, reason)
         adapter = demodulize(adapter_name.gsub('Adapter', ''))

data/lib/proxy_fetcher/manager.rb

@@ -1,4 +1,5 @@
 module ProxyFetcher
+  # ProxyFetcher Manager class for interacting with proxy lists from various providers.
   class Manager
     attr_reader :proxies
 

data/lib/proxy_fetcher/providers/base.rb

@@ -1,5 +1,6 @@
 module ProxyFetcher
   module Providers
+    # Base class for all the ProxyFetcher providers.
     class Base
       # Loads proxy provider page content, extract proxy list from it
       # and convert every entry to proxy object.
@@ -8,6 +9,7 @@ module ProxyFetcher
       end
 
       class << self
+        # Just synthetic sugar to make it easier to call #fetch_proxies! method.
         def fetch_proxies!(*args)
           new.fetch_proxies!(*args)
         end
@@ -26,7 +28,12 @@ module ProxyFetcher
         ProxyFetcher::Document.parse(html)
       end
 
-      # Get HTML elements with proxy info
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
+      # to return all the proxy entries (HTML nodes).
+      #
+      # Abstract method.
+      #
       def load_proxy_list(*)
         raise NotImplementedError, "#{__method__} must be implemented in a descendant class!"
       end

data/lib/proxy_fetcher/providers/free_proxy_list.rb

@@ -1,6 +1,8 @@
 module ProxyFetcher
   module Providers
+    # FreeProxyList provider class.
     class FreeProxyList < Base
+      # Provider URL to fetch proxy list
       PROVIDER_URL = 'https://free-proxy-list.net/'.freeze
 
       # [NOTE] Doesn't support filtering
@@ -9,6 +11,15 @@ module ProxyFetcher
         doc.xpath('//table[@id="proxylisttable"]/tbody/tr')
       end
 
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
       def to_proxy(html_node)
         ProxyFetcher::Proxy.new.tap do |proxy|
           proxy.addr = html_node.content_at('td[1]')
@@ -21,6 +32,14 @@ module ProxyFetcher
 
       private
 
+      # Parses HTML node to extract proxy type.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [String]
+      #   Proxy type
+      #
       def parse_type(html_node)
         https = html_node.content_at('td[6]')
         https && https.casecmp('yes').zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP

data/lib/proxy_fetcher/providers/free_proxy_list_ssl.rb

@@ -1,14 +1,32 @@
 module ProxyFetcher
   module Providers
+    # FreeProxyListSSL provider class.
     class FreeProxyListSSL < Base
+      # Provider URL to fetch proxy list
       PROVIDER_URL = 'https://www.sslproxies.org/'.freeze
 
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
+      # to return all the proxy entries (HTML nodes).
+      #
+      # @return [Array<ProxyFetcher::Document::Node>]
+      #   Collection of extracted HTML nodes with full proxy info
+      #
       # [NOTE] Doesn't support filtering
       def load_proxy_list(*)
         doc = load_document(PROVIDER_URL, {})
         doc.xpath('//table[@id="proxylisttable"]/tbody/tr')
       end
 
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
       def to_proxy(html_node)
         ProxyFetcher::Proxy.new.tap do |proxy|
           proxy.addr = html_node.content_at('td[1]')

data/lib/proxy_fetcher/providers/gather_proxy.rb

@@ -2,14 +2,32 @@ require 'json'
 
 module ProxyFetcher
   module Providers
+    # GatherProxy provider class.
     class GatherProxy < Base
+      # Provider URL to fetch proxy list
       PROVIDER_URL = 'http://www.gatherproxy.com/'.freeze
 
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
+      # to return all the proxy entries (HTML nodes).
+      #
+      # @return [Array<ProxyFetcher::Document::Node>]
+      #   Collection of extracted HTML nodes with full proxy info
+      #
       def load_proxy_list(*)
         doc = load_document(PROVIDER_URL)
         doc.xpath('//div[@class="proxy-list"]/table/script')
       end
 
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
       def to_proxy(html_node)
         json = parse_json(html_node)
 

data/lib/proxy_fetcher/providers/http_tunnel.rb

@@ -1,13 +1,31 @@
 module ProxyFetcher
   module Providers
+    # HTTPTunnel provider class.
     class HTTPTunnel < Base
+      # Provider URL to fetch proxy list
       PROVIDER_URL = 'http://www.httptunnel.ge/ProxyListForFree.aspx'.freeze
 
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
+      # to return all the proxy entries (HTML nodes).
+      #
+      # @return [Array<ProxyFetcher::Document::Node>]
+      #   Collection of extracted HTML nodes with full proxy info
+      #
       def load_proxy_list(*)
         doc = load_document(PROVIDER_URL)
         doc.xpath('//table[contains(@id, "GridView")]/tr[(count(td)>2)]')
       end
 
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
       def to_proxy(html_node)
         ProxyFetcher::Proxy.new.tap do |proxy|
           uri = parse_proxy_uri(html_node)
@@ -22,15 +40,39 @@ module ProxyFetcher
 
       private
 
+      # Parses HTML node to extract URI object with proxy host and port.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [URI]
+      #   URI object
+      #
       def parse_proxy_uri(html_node)
         full_addr = html_node.content_at('td[1]')
         URI.parse("http://#{full_addr}")
       end
 
+      # Parses HTML node to extract proxy country.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [String]
+      #   Country code
+      #
       def parse_country(html_node)
         html_node.find('.//img').attr('title')
       end
 
+      # Parses HTML node to extract proxy anonymity level.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [String]
+      #   Anonymity level
+      #
       def parse_anonymity(html_node)
         transparency = html_node.content_at('td[5]').to_sym
 

data/lib/proxy_fetcher/providers/proxy_docker.rb

@@ -1,14 +1,32 @@
 module ProxyFetcher
   module Providers
+    # ProxyDocker provider class.
     class ProxyDocker < Base
-      PROVIDER_URL = 'https://www.proxydocker.com/'.freeze
+      # Provider URL to fetch proxy list
+      PROVIDER_URL = 'https://www.proxydocker.com/en/proxylist/'.freeze
 
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
+      # to return all the proxy entries (HTML nodes).
+      #
+      # @return [Array<ProxyFetcher::Document::Node>]
+      #   Collection of extracted HTML nodes with full proxy info
+      #
       # [NOTE] Doesn't support direct filters
       def load_proxy_list(*)
         doc = load_document(PROVIDER_URL, {})
         doc.xpath('//table[contains(@class, "table")]/tr[(not(@id="proxy-table-header")) and (count(td)>2)]')
       end
 
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
       def to_proxy(html_node)
         ProxyFetcher::Proxy.new.tap do |proxy|
           uri = URI("//#{html_node.content_at('td[1]')}")

data/lib/proxy_fetcher/providers/proxy_list.rb

@@ -2,14 +2,32 @@ require 'base64'
 
 module ProxyFetcher
   module Providers
+    # ProxyList provider class.
     class ProxyList < Base
+      # Provider URL to fetch proxy list
       PROVIDER_URL = 'https://proxy-list.org/english/index.php'.freeze
 
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
+      # to return all the proxy entries (HTML nodes).
+      #
+      # @return [Array<ProxyFetcher::Document::Node>]
+      #   Collection of extracted HTML nodes with full proxy info
+      #
       def load_proxy_list(filters = {})
         doc = load_document(PROVIDER_URL, filters)
         doc.css('.table-wrap .table ul')
       end
 
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
       def to_proxy(html_node)
         ProxyFetcher::Proxy.new.tap do |proxy|
           uri = parse_proxy_uri(html_node)
@@ -24,6 +42,14 @@ module ProxyFetcher
 
       private
 
+      # Parses HTML node to extract URI object with proxy host and port.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [URI]
+      #   URI object
+      #
       def parse_proxy_uri(html_node)
         full_addr = ::Base64.decode64(html_node.at_css('li script').html.match(/'(.+)'/)[1])
         URI.parse("http://#{full_addr}")

data/lib/proxy_fetcher/providers/xroxy.rb

@@ -1,13 +1,31 @@
 module ProxyFetcher
   module Providers
+    # XRoxy provider class.
     class XRoxy < Base
+      # Provider URL to fetch proxy list
       PROVIDER_URL = 'http://www.xroxy.com/proxylist.php'.freeze
 
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
+      # to return all the proxy entries (HTML nodes).
+      #
+      # @return [Array<ProxyFetcher::Document::Node>]
+      #   Collection of extracted HTML nodes with full proxy info
+      #
       def load_proxy_list(filters = { type: 'All_http' })
         doc = load_document(PROVIDER_URL, filters)
         doc.xpath('//div[@id="content"]/table[1]/tr[contains(@class, "row")]')
       end
 
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
       def to_proxy(html_node)
         ProxyFetcher::Proxy.new.tap do |proxy|
           proxy.addr = html_node.content_at('td[2]')
@@ -21,6 +39,14 @@ module ProxyFetcher
 
       private
 
+      # Parses HTML node to extract proxy type.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [String]
+      #   Proxy type
+      #
       def parse_type(html_node)
         https = html_node.content_at('td[5]')
         https.casecmp('true').zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP

data/lib/proxy_fetcher/proxy.rb

@@ -1,7 +1,9 @@
 module ProxyFetcher
+  # Proxy object
   class Proxy
     attr_accessor :addr, :port, :type, :country, :response_time, :anonymity
 
+    # Proxy type
     TYPES = [
       HTTP = 'HTTP'.freeze,
       HTTPS = 'HTTPS'.freeze,
@@ -9,12 +11,22 @@ module ProxyFetcher
       SOCKS5 = 'SOCKS5'.freeze
     ].freeze
 
+    # Proxy type predicates (#socks4?, #https?)
+    #
+    # @return [Boolean]
+    #   true if proxy of requested type, otherwise false.
+    #
     TYPES.each do |proxy_type|
       define_method "#{proxy_type.downcase}?" do
         !type.nil? && type.upcase.include?(proxy_type)
       end
     end
 
+    # Returns true if proxy is secure (works through https, socks4 or socks5).
+    #
+    # @return [Boolean]
+    #   true if proxy is secure, otherwise false.
+    #
     def ssl?
       https? || socks4? || socks5?
     end
@@ -25,16 +37,32 @@ module ProxyFetcher
       end
     end
 
+    # Checks if proxy object is connectable? (can be used as a proxy for
+    # network requests).
+    #
+    # @return [Boolean]
+    #   true if proxy connectable, otherwise false.
+    #
     def connectable?
       ProxyFetcher.config.proxy_validator.connectable?(addr, port)
     end
 
     alias valid? connectable?
 
+    # Returns <code>URI::Generic</code> object with host and port values of the proxy.
+    #
+    # @return [URI::Generic]
+    #   URI object.
+    #
     def uri
       URI::Generic.build(host: addr, port: port)
     end
 
+    # Returns <code>String</object> object with <i>addr:port<i> values of the proxy.
+    #
+    # @return [String]
+    #   true if proxy connectable, otherwise false.
+    #
     def url
       "#{addr}:#{port}"
     end

data/lib/proxy_fetcher/utils/http_client.rb

@@ -1,4 +1,7 @@
 module ProxyFetcher
+  # Default ProxyFetcher HTTP client used to fetch proxy lists from
+  # the different providers. Uses ProxyFetcher configuration options
+  # for sending HTTP requests to providers URLs.
   class HTTPClient
     attr_reader :uri, :http
 

data/lib/proxy_fetcher/utils/proxy_validator.rb

@@ -1,5 +1,9 @@
 module ProxyFetcher
+  # Default ProxyFetcher proxy validator that checks either proxy
+  # connectable or not. It tries to send HEAD request to default
+  # URL to check if proxy can be used (aka connectable?).
   class ProxyValidator
+    # Default URL that will be used to check if proxy can be used.
     URL_TO_CHECK = 'https://google.com'.freeze
 
     def initialize(proxy_addr, proxy_port)

data/lib/proxy_fetcher/version.rb

@@ -1,15 +1,19 @@
 module ProxyFetcher
+  ##
+  # ProxyFetcher gem version.
   def self.gem_version
     Gem::Version.new VERSION::STRING
   end
 
+  ##
+  # ProxyFetcher gem semantic versioning.
   module VERSION
     # Major version number
     MAJOR = 0
     # Minor version number
     MINOR = 6
     # Smallest version number
-    TINY  = 1
+    TINY  = 2
 
     # Full version number
     STRING = [MAJOR, MINOR, TINY].compact.join('.')

data/lib/proxy_fetcher.rb

@@ -22,7 +22,10 @@ require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters/abstract_adap
 require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters/nokogiri_adapter'
 require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters/oga_adapter'
 
+##
+# Ruby / JRuby lib for managing proxies
 module ProxyFetcher
+  # ProxyFetcher providers namespace
   module Providers
     require File.dirname(__FILE__) + '/proxy_fetcher/providers/base'
     require File.dirname(__FILE__) + '/proxy_fetcher/providers/free_proxy_list'
@@ -34,11 +37,33 @@ module ProxyFetcher
     require File.dirname(__FILE__) + '/proxy_fetcher/providers/xroxy'
   end
 
+  # Main ProxyFetcher module.
   class << self
+    ##
+    # Returns ProxyFetcher configuration.
+    #
+    # @return [ProxyFetcher::Configuration]
+    #   Configuration object.
+    #
+    # @example
+    #   ProxyFetcher.config
+    #
+    #   #=> #<ProxyFetcher::Configuration:0x0000000241eec8 @user_agent="Mozilla/5.0, ...", @pool_size=10,
+    #           @timeout=3, @http_client=ProxyFetcher::HTTPClient, @proxy_validator=ProxyFetcher::ProxyValidator,
+    #           @providers=[:free_proxy_list, ...], @adapter=ProxyFetcher::Document::NokogiriAdapter>
+    #
     def config
       @config ||= ProxyFetcher::Configuration.new
     end
 
+    ##
+    # Configures ProxyFetcher and yields config object for additional manipulations.
+
+    # @yieldreturn [optional, types, ...] description
+    #
+    # @return [ProxyFetcher::Configuration]
+    #   Configuration object.
+    #
     def configure
       yield config
     end

data/proxy_fetcher.gemspec

@@ -5,7 +5,7 @@ require 'proxy_fetcher/version'
 Gem::Specification.new do |gem|
   gem.name = 'proxy_fetcher'
   gem.version = ProxyFetcher.gem_version
-  gem.date = '2017-12-11'
+  gem.date = '2017-12-27'
   gem.summary = 'Ruby gem for dealing with proxy lists from different providers'
   gem.description = 'This gem can help your Ruby application to make HTTP(S) requests ' \
                     'using proxies by fetching and validating proxy lists from the different providers.'

data/spec/proxy_fetcher/proxy_spec.rb

@@ -24,13 +24,20 @@ describe ProxyFetcher::Proxy do
     proxy.type = ProxyFetcher::Proxy::HTTP
     expect(proxy.http?).to be_truthy
     expect(proxy.https?).to be_falsey
+    expect(proxy.ssl?).to be_falsey
 
     proxy.type = ProxyFetcher::Proxy::HTTPS
     expect(proxy.https?).to be_truthy
     expect(proxy.http?).to be_truthy
+    expect(proxy.ssl?).to be_truthy
+
+    proxy.type = ProxyFetcher::Proxy::SOCKS4
+    expect(proxy.socks4?).to be_truthy
+    expect(proxy.ssl?).to be_truthy
 
     proxy.type = ProxyFetcher::Proxy::SOCKS5
     expect(proxy.socks5?).to be_truthy
+    expect(proxy.ssl?).to be_truthy
   end
 
   it 'not connectable if IP addr is wrong' do
@@ -51,6 +58,9 @@ describe ProxyFetcher::Proxy do
 
   it 'returns URI::Generic' do
     expect(proxy.uri).to be_a(URI::Generic)
+
+    expect(proxy.uri.host).not_to be_empty
+    expect(proxy.uri.port).not_to be_nil
   end
 
   it 'returns URL' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-11 00:00:00.000000000 Z
+date: 2017-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -107,7 +107,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Ruby gem for dealing with proxy lists from different providers