proxy_fetcher 0.6.2 → 0.6.3

data/lib/proxy_fetcher/document.rb

@@ -1,24 +1,47 @@
+# frozen_string_literal: true
+
 module ProxyFetcher
   # HTML document abstraction class. Used to work with different HTML parser adapters
   # such as Nokogiri, Oga or a custom one. Stores <i>backend</i< that will handle all
   # the DOM manipulation logic.
   class Document
-    class << self
-      def parse(data)
-        new(ProxyFetcher.config.adapter.parse(data))
-      end
-    end
-
+    # @!attribute [r] backend
+    #   @return [Object] Backend object that handles DOM processing
     attr_reader :backend
 
+    # Parses raw HTML data to abstract ProxyFetcher document.
+    #
+    # @param data [String] HTML
+    #
+    # @return [ProxyFetcher::Document]
+    #   ProxyFetcher document model
+    #
+    def self.parse(data)
+      new(ProxyFetcher.config.adapter.parse(data))
+    end
+
+    # Initialize abstract ProxyFetcher HTML Document
+    #
+    # @return [Document]
+    #
     def initialize(backend)
       @backend = backend
     end
 
+    # Searches elements by XPath selector.
+    #
+    # @return [Array<ProxyFetcher::Document::Node>]
+    #   collection of nodes
+    #
     def xpath(*args)
       backend.xpath(*args).map { |node| backend.proxy_node.new(node) }
     end
 
+    # Searches elements by CSS selector.
+    #
+    # @return [Array<ProxyFetcher::Document::Node>]
+    #   collection of nodes
+    #
     def css(*args)
       backend.css(*args).map { |node| backend.proxy_node.new(node) }
     end

data/lib/proxy_fetcher/document/adapters.rb

@@ -1,10 +1,30 @@
+# frozen_string_literal: true
+
 module ProxyFetcher
   class Document
+    # ProxyFetcher HTML parser adapters.
+    #
+    # ProxyFetcher default supported adapters are:
+    #
+    # * Nokogiri
+    # * Oga
+    #
+    # Any custom adapter can be used and must be inherited from
+    # <code>ProxyFetcher::Document::AbstractAdapter</code>.
     class Adapters
+      # Adapters class name suffix
       ADAPTER = 'Adapter'.freeze
       private_constant :ADAPTER
 
       class << self
+        # Returns HTML parser adapter by it's name or class.
+        # If name is provided, then it looks for predefined classes
+        # in <code>ProxyFetcher::Document</code> namespace. Otherwise
+        # it just returns the passed class.
+        #
+        # @param name_or_class [String, Class]
+        #   Adapter name or class
+        #
         def lookup(name_or_class)
           raise Exceptions::BlankAdapter if name_or_class.nil? || name_or_class.to_s.empty?
 

data/lib/proxy_fetcher/document/adapters/abstract_adapter.rb

@@ -1,26 +1,55 @@
+# frozen_string_literal: true
+
 module ProxyFetcher
   class Document
+    # Abstract HTML parser adapter class.
+    # Handles document manipulations.
     class AbstractAdapter
+      # @!attribute [r] events
+      #   @return [Hash] A hash with events registered within a bus
       attr_reader :document
 
+      # Initialize adapter
+      #
+      # @return [AbstractAdapter]
+      #
       def initialize(document)
         @document = document
       end
 
       # You can override this method in your own adapter class
+      #
+      # @param selector [String]
+      #   XPath selector
+      #
       def xpath(selector)
         document.xpath(selector)
       end
 
       # You can override this method in your own adapter class
+      #
+      # @param selector [String]
+      #   CSS selector
+      #
       def css(selector)
         document.css(selector)
       end
 
+      # Returns <code>Node</code> class that will handle HTML
+      # nodes for particular adapter.
+      #
+      # @return [ProxyFetcher::Document::Node]
+      #   node
+      #
       def proxy_node
         self.class.const_get('Node')
       end
 
+      # Installs adapter requirements.
+      #
+      # @raise [Exceptions::AdapterSetupError]
+      #   adapter can't be install due to some error
+      #
       def self.setup!(*args)
         install_requirements!(*args)
       rescue LoadError => error

data/lib/proxy_fetcher/document/adapters/nokogiri_adapter.rb

@@ -1,23 +1,49 @@
+# frozen_string_literal: true
+
 module ProxyFetcher
   class Document
+    # HTML parser adapter that uses Nokogiri as a backend.
     class NokogiriAdapter < AbstractAdapter
+      # Requires Nokogiri gem to the application.
       def self.install_requirements!
         require 'nokogiri'
       end
 
+      # Parses raw HTML content with specific gem.
+      #
+      # @param data [String]
+      #   HTML content
+      #
+      # @return [ProxyFetcher::Document::NokogiriAdapter]
+      #   Object with parsed document
+      #
       def self.parse(data)
         new(::Nokogiri::HTML(data))
       end
 
+      # Nokogiri DOM node
       class Node < ProxyFetcher::Document::Node
+        # Returns HTML node attribute value.
+        #
+        # @return [String] attribute value
+        #
         def attr(*args)
           clear(node.attr(*args))
         end
 
+        # Returns HTML node inner text value clean from
+        # whitespaces, tabs, etc.
+        #
+        # @return [String] node inner text
+        #
         def content
           clear(node.content)
         end
 
+        # Returns node inner HTML.
+        #
+        # @return [String] inner HTML
+        #
         def html
           node.inner_html
         end

data/lib/proxy_fetcher/document/adapters/oga_adapter.rb

@@ -1,23 +1,49 @@
+# frozen_string_literal: true
+
 module ProxyFetcher
   class Document
+    # HTML parser adapter that uses Oga as a backend.
     class OgaAdapter < AbstractAdapter
+      # Requires Oga gem to the application.
       def self.install_requirements!
         require 'oga'
       end
 
+      # Parses raw HTML content with specific gem.
+      #
+      # @param data [String]
+      #   HTML content
+      #
+      # @return [ProxyFetcher::Document::OgaAdapter]
+      #   Object with parsed document
+      #
       def self.parse(data)
         new(::Oga.parse_html(data))
       end
 
+      # Oga DOM node
       class Node < ProxyFetcher::Document::Node
+        # Returns HTML node attribute value.
+        #
+        # @return [String] attribute value
+        #
         def attr(*args)
           clear(node.attribute(*args).value)
         end
 
+        # Returns HTML node inner text value clean from
+        # whitespaces, tabs, etc.
+        #
+        # @return [String] node inner text
+        #
         def content
           clear(node.text)
         end
 
+        # Returns node inner HTML.
+        #
+        # @return [String] inner HTML
+        #
         def html
           node.to_xml
         end

data/lib/proxy_fetcher/document/node.rb

@@ -1,38 +1,85 @@
+# frozen_string_literal: true
+
 module ProxyFetcher
   class Document
+    # Abstract class for storing HTML elements that was parsed by
+    # one of the <code>ProxyFetcher::Document</code> adapters class.
     class Node
+      # @!attribute [r] node
+      #   @return [Object] original DOM node, parsed by adapter backend
       attr_reader :node
 
+      # Initialize new HTML node
+      #
+      # @return [Node]
+      #
       def initialize(node)
         @node = node
       end
 
+      # Searches for node in children using some selector (CSS or XPath).
+      #
+      # @param selector [String] selector (CSS or XPath)
+      #
+      # @return [Node] child node
+      #
       def find(selector, method = :at_xpath)
         self.class.new(node.public_send(method, selector))
       end
 
+      # Searches exact HTML element by XPath. Returns only one element.
+      #
+      # @return [ProxyFetcher::Document::Node]
+      #   node
+      #
       def at_xpath(*args)
         self.class.new(node.at_xpath(*args))
       end
 
+      # Searches exact HTML element by CSS. Returns only one element.
+      #
+      # @return [ProxyFetcher::Document::Node]
+      #   node
+      #
       def at_css(*args)
         self.class.new(node.at_css(*args))
       end
 
+      # Returns clean content (text) for the specific element.
+      #
+      # @return [String]
+      #   HTML node content
+      #
       def content_at(*args)
         clear(find(*args).content)
       end
 
+      # Returns HTML node content.
+      #
+      # Abstract method, must be implemented for specific adapter class.
+      #
       def content
         raise "`#{__method__}` must be implemented for specific adapter class!"
       end
 
+      # Returns HTML node inner HTML.
+      #
+      # Abstract method, must be implemented for specific adapter class.
+      #
       def html
         raise "`#{__method__}` must be implemented for specific adapter class!"
       end
 
       protected
 
+      # Removes whitespaces, tabulation and other "garbage" for the text.
+      #
+      # @param text [String]
+      #   text to clear
+      #
+      # @return [String]
+      #   clean text
+      #
       def clear(text)
         return if text.nil? || text.empty?
 

data/lib/proxy_fetcher/exceptions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ProxyFetcher
   # Base exception class for all the ProxyFetcher exceptions.
   Error = Class.new(StandardError)
@@ -6,6 +8,10 @@ module ProxyFetcher
   module Exceptions
     # Exception for wrong custom classes (such as ProxyValidator or HTTP Client).
     class WrongCustomClass < Error
+      # Initialize new exception
+      #
+      # @return [WrongCustomClass]
+      #
       def initialize(klass, methods)
         required_methods = Array(methods).join(', ')
         super("#{klass} must respond to [#{required_methods}] class methods!")
@@ -15,6 +21,12 @@ module ProxyFetcher
     # Exception for wrong provider name, that raises when configured provider
     # that is not registered via <code>register_provider</code> interface.
     class UnknownProvider < Error
+      # Initialize new exception
+      #
+      # @param provider_name [String] provider name
+      #
+      # @return [UnknownProvider]
+      #
       def initialize(provider_name)
         super("unregistered proxy provider `#{provider_name}`")
       end
@@ -22,6 +34,12 @@ module ProxyFetcher
 
     # Exception for cases when user tries to register already existing provider.
     class RegisteredProvider < Error
+      # Initialize new exception
+      #
+      # @param name [String, Symbol] provider name
+      #
+      # @return [RegisteredProvider]
+      #
       def initialize(name)
         super("`#{name}` provider already registered!")
       end
@@ -30,6 +48,10 @@ module ProxyFetcher
     # Exception for cases when HTTP client reached maximum count of redirects
     # trying to process HTTP request.
     class MaximumRedirectsReached < Error
+      # Initialize new exception
+      #
+      # @return [MaximumRedirectsReached]
+      #
       def initialize(*)
         super('maximum redirects reached')
       end
@@ -39,6 +61,10 @@ module ProxyFetcher
     # trying to process HTTP request. Can occur when request failed by timeout
     # multiple times.
     class MaximumRetriesReached < Error
+      # Initialize new exception
+      #
+      # @return [MaximumRetriesReached]
+      #
       def initialize(*)
         super('reached the maximum number of retries')
       end
@@ -47,6 +73,12 @@ module ProxyFetcher
     # Exception for cases when user tries to set wrong HTML parser adapter
     # in the configuration.
     class UnknownAdapter < Error
+      # Initialize new exception
+      #
+      # @param name [String] configured adapter name
+      #
+      # @return [UnknownAdapter]
+      #
       def initialize(name)
         super("unknown adapter '#{name}'")
       end
@@ -55,6 +87,10 @@ module ProxyFetcher
     # Exception for cases when user tries to set <code>nil</code> HTML parser adapter
     # in the configuration (or just forget to change it).
     class BlankAdapter < Error
+      # Initialize new exception
+      #
+      # @return [BlankAdapter]
+      #
       def initialize(*)
         super(<<-MSG.strip.squeeze
           you need to specify adapter for HTML parsing: ProxyFetcher.config.adapter = :nokogiri.
@@ -67,14 +103,28 @@ module ProxyFetcher
     # Exception for cases when HTML parser adapter can't be installed.
     # It will print the reason (backtrace) of the exception that caused an error.
     class AdapterSetupError < Error
-      def initialize(adapter_name, reason)
+      # Initialize new exception
+      #
+      # @param adapter_name [String] configured adapter name
+      # @param error [String] full setup error (backtrace)
+      #
+      # @return [AdapterSetupError]
+      #
+      def initialize(adapter_name, error)
         adapter = demodulize(adapter_name.gsub('Adapter', ''))
 
-        super("can't setup '#{adapter}' adapter during the following error:\n\t#{reason}'")
+        super("can't setup '#{adapter}' adapter during the following error:\n\t#{error}'")
       end
 
       private
 
+      # Returns just class name removing it's namespace.
+      #
+      # @param path [String]
+      #   full class name
+      #
+      # @return [String] demodulized class name
+      #
       def demodulize(path)
         path = path.to_s
         index = path.rindex('::')

data/lib/proxy_fetcher/manager.rb

@@ -1,10 +1,19 @@
+# frozen_string_literal: true
+
 module ProxyFetcher
   # ProxyFetcher Manager class for interacting with proxy lists from various providers.
   class Manager
+    # @!attribute [r] proxies
+    #   @return [Array<ProxyFetcher::Proxy>] An array of proxies
     attr_reader :proxies
 
+    # Initialize ProxyFetcher Manager instance for managing proxies
+    #
     # refresh: true - load proxy list from the remote server on initialization
     # refresh: false - just initialize the class, proxy list will be empty ([])
+    #
+    # @return [Manager]
+    #
     def initialize(refresh: true, validate: false, filters: {})
       if refresh
         refresh_list!(filters)
@@ -15,7 +24,10 @@ module ProxyFetcher
       cleanup! if validate
     end
 
-    # Update current proxy list from the provider
+    # Update current proxy list using configured providers.
+    #
+    # @param filters [Hash] providers filters
+    #
     def refresh_list!(filters = nil)
       @proxies = []
 
@@ -29,7 +41,11 @@ module ProxyFetcher
 
     alias fetch! refresh_list!
 
-    # Pop just first proxy (and back it to the end of the proxy list)
+    # Pop just first proxy (and back it to the end of the proxy list).
+    #
+    # @return [Proxy]
+    #   proxy object from the list
+    #
     def get
       return if @proxies.empty?
 
@@ -43,6 +59,10 @@ module ProxyFetcher
 
     # Pop first valid proxy (and back it to the end of the proxy list)
     # Invalid proxies will be removed from the list
+    #
+    # @return [Proxy]
+    #   proxy object from the list
+    #
     def get!
       index = proxies.find_index(&:connectable?)
       return if index.nil?
@@ -76,7 +96,11 @@ module ProxyFetcher
 
     alias validate! cleanup!
 
-    # Return random proxy
+    # Returns random proxy
+    #
+    # @return [Proxy]
+    #   random proxy from the loaded list
+    #
     def random_proxy
       proxies.sample
     end
@@ -84,11 +108,15 @@ module ProxyFetcher
     alias random random_proxy
 
     # Returns array of proxy URLs (just schema + host + port)
+    #
+    # @return [Array<String>]
+    #   collection of proxies
+    #
     def raw_proxies
       proxies.map(&:url)
     end
 
-    # No need to put all the attr_readers
+    # @private No need to put all the attr_readers to the output
     def inspect
       to_s
     end