proxy_fetcher 0.15.0 → 0.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf6591a8ede2c142fca7f433f3edceb407ed884c1dee6a98e3ca33efbae392e2
-  data.tar.gz: 7d7259da718adec21b8d749d04faf594fa926389e570be6c55bbe9ebf3695783
+  metadata.gz: 071dd5bfeaf01174bf803981de6147f999e75b99a2d03c32e034b72dd5dc7c82
+  data.tar.gz: 1675ce07491e0cbce401c2a113ee7eccdcba0c4cf42ba7e380ad63be988ec91a
 SHA512:
-  metadata.gz: 90eed5f390d122ae58f9bd6c7553b7a1ff6bbca8e65c05406593f53b7a00a4ed160120ae62d0ce8289965810aaa3795287769ee9ef12f9404d805694a8ac0cff
-  data.tar.gz: 31a5b4fcd6199dfc75ca3e89c7bddcc82533ca171f86954e50d4ff44716d73f1ad014f921a05e9eb2fd38a83c5e603eec1fbbf9530b4c01153f6780877388501
+  metadata.gz: 42037b0b55109b535e25284815a096154c79bca2f58b107311f5c53da5b9ec0242ab39a6dd9bae48ef3e863c7f43251167180266578ac177887bd3a63215ef1e
+  data.tar.gz: 188dd298f8ea03e5985d89e97f99a12ef5d5384eae03a78f9b77e5172c28ae5990dec64795817d52ea6895395679a6c2f2deafafa3f8617154afb32fc06bc972

data/CHANGELOG.md

@@ -6,6 +6,16 @@ Reverse Chronological Order:
 
 ...
 
+## `0.15.1` (2021-02-17)
+
+* Support for Ruby 3.0
+
+## `0.15.0` (2021-01-26)
+
+* Removed failing providers
+* Added new
+* Specs refactoring
+
 ## `0.14.0` (2020-05-11)
 
 * Add MTPro provider

data/Gemfile

@@ -6,9 +6,11 @@ gemspec
 
 gem "nokogiri", "~> 1.8"
 gem "oga", "~> 3.2"
-gem "rubocop", "~> 0.80"
+gem "rubocop", "~> 1.0"
 
 group :test do
   gem "coveralls", require: false
+  # Until I find a way to introduce other MITM proxy
+  gem "webrick", "1.4.2"
   gem "evil-proxy", "~> 0.2"
 end

data/gemfiles/nokogiri.gemfile

@@ -8,6 +8,6 @@ gem "nokogiri", "~> 1.8"
 
 group :test do
   gem "coveralls", require: false
+  gem "webrick"
   gem "evil-proxy", "~> 0.2"
-  gem "rspec", "~> 3.9"
 end

data/gemfiles/oga.gemfile

@@ -8,6 +8,6 @@ gem "oga", "~> 3.0"
 
 group :test do
   gem "coveralls", require: false
+  gem "webrick"
   gem "evil-proxy", "~> 0.2"
-  gem "rspec", "~> 3.9"
 end

data/lib/proxy_fetcher.rb

@@ -4,47 +4,47 @@ require "uri"
 require "http"
 require "logger"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/version"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/version"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/exceptions"
-require File.dirname(__FILE__) + "/proxy_fetcher/configuration"
-require File.dirname(__FILE__) + "/proxy_fetcher/configuration/providers_registry"
-require File.dirname(__FILE__) + "/proxy_fetcher/proxy"
-require File.dirname(__FILE__) + "/proxy_fetcher/manager"
-require File.dirname(__FILE__) + "/proxy_fetcher/null_logger"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/exceptions"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/configuration"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/configuration/providers_registry"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/proxy"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/manager"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/null_logger"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/utils/http_client"
-require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_validator"
-require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_list_validator"
-require File.dirname(__FILE__) + "/proxy_fetcher/client/client"
-require File.dirname(__FILE__) + "/proxy_fetcher/client/request"
-require File.dirname(__FILE__) + "/proxy_fetcher/client/proxies_registry"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/http_client"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/proxy_validator"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/proxy_list_validator"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/client/client"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/client/request"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/client/proxies_registry"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/document"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/node"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/abstract_adapter"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/nokogiri_adapter"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/oga_adapter"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/node"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/abstract_adapter"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/nokogiri_adapter"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/oga_adapter"
 
 ##
 # Ruby / JRuby lib for managing proxies
 module ProxyFetcher
   # ProxyFetcher providers namespace
   module Providers
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/base"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_socks"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_us"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/mtpro"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxypedia"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_http"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_socks4"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_socks5"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/base"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_socks"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_ssl"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_us"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/http_tunnel"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/mtpro"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxy_list"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxypedia"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_http"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_socks4"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_socks5"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/xroxy"
   end
 
   @__config_access_lock__ = Mutex.new

data/lib/proxy_fetcher/client/request.rb

@@ -41,15 +41,15 @@ module ProxyFetcher
       # @return [String]
       #   response body (requested resource content)
       #
-      def self.execute(args)
-        new(args).execute
+      def self.execute(**args)
+        new(**args).execute
       end
 
       # Initialize new HTTP request
       #
       # @return [Request]
       #
-      def initialize(args)
+      def initialize(**args)
         raise ArgumentError, "args must be a Hash!" unless args.is_a?(Hash)
 
         @url = args.fetch(:url)

data/lib/proxy_fetcher/configuration.rb

@@ -114,7 +114,7 @@ module ProxyFetcher
 
     # Sets default configuration options
     def reset!
-      @logger = Logger.new(STDOUT)
+      @logger = Logger.new($stdout)
       @user_agent = DEFAULT_USER_AGENT
       @pool_size = 10
       @client_timeout = 3

data/lib/proxy_fetcher/document/node.rb

@@ -83,7 +83,7 @@ module ProxyFetcher
       def clear(text)
         return "" if text.nil? || text.empty?
 
-        text.strip.gsub(/[\t]/i, "")
+        text.strip.gsub(/\t/i, "")
       end
     end
   end

data/lib/proxy_fetcher/utils/http_client.rb

@@ -41,8 +41,8 @@ module ProxyFetcher
     # @return [String]
     #   resource content
     #
-    def self.fetch(*args)
-      new(*args).fetch
+    def self.fetch(*args, **kwargs, &block)
+      new(*args, **kwargs, &block).fetch
     end
 
     # Initialize HTTP client instance
@@ -51,15 +51,17 @@ module ProxyFetcher
     #
     def initialize(url, method: :get, params: {}, headers: {})
       @url = url.to_s
-      @method = method
+      @method = method.to_sym
       @params = params
       @headers = headers
 
-      @http = HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout)
-      @timeout = ProxyFetcher.config.provider_proxies_load_timeout
+      unless HTTP::Request::METHODS.include?(@method)
+        raise ArgumentError, "'#{@method}' is a wrong HTTP method name"
+      end
 
-      @ssl_ctx = OpenSSL::SSL::SSLContext.new
-      @ssl_ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      @timeout = ProxyFetcher.config.provider_proxies_load_timeout
+      @http = build_http_engine
+      @ssl_ctx = build_ssl_context
     end
 
     # Fetches resource content by sending HTTP request to it.
@@ -67,8 +69,10 @@ module ProxyFetcher
     # @return [String]
     #   response body
     #
-    def fetch
-      response = process_http_request
+    def fetch(**options)
+      response = perform_http_request
+      return response if options.fetch(:raw, false)
+
       response.body.to_s
     rescue StandardError => e
       ProxyFetcher.config.logger.warn("Failed to process request to #{url} (#{e.message})")
@@ -77,13 +81,20 @@ module ProxyFetcher
 
     protected
 
-    def process_http_request(http_method: method, http_params: params)
-      unless HTTP::Request::METHODS.include?(http_method)
-        raise ArgumentError, "'#{http_method}' is a wrong HTTP method name!"
+    def build_ssl_context
+      OpenSSL::SSL::SSLContext.new.tap do |context|
+        context.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
+    end
+
+    def build_http_engine
+      HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout)
+    end
 
+    def perform_http_request(http_method: method, http_params: params)
       http.public_send(
-        http_method.to_sym, url,
+        http_method,
+        url,
         form: http_params,
         ssl_context: ssl_ctx
       )

data/lib/proxy_fetcher/version.rb

@@ -15,7 +15,7 @@ module ProxyFetcher
     # Minor version number
     MINOR = 15
     # Smallest version number
-    TINY  = 0
+    TINY  = 1
 
     # Full version number
     STRING = [MAJOR, MINOR, TINY].compact.join(".")

data/proxy_fetcher.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |gem|
   gem.version = ProxyFetcher.gem_version
   gem.summary = "Ruby gem for dealing with proxy lists from different providers"
   gem.description = <<-TEXT.strip.gsub(/[\s\n]+/, " ")
-    This gem can help your Ruby application to make HTTP(S) requests 
-    using proxies by fetching and validating proxy lists from 
+    This gem can help your Ruby application to make HTTP(S) requests
+    using proxies by fetching and validating proxy lists from
     the different providers.
   TEXT
   gem.authors = ["Nikita Bulai"]

data/spec/proxy_fetcher/client/client_spec.rb

@@ -3,6 +3,11 @@
 require "spec_helper"
 require "json"
 
+begin
+  require "webrick"
+rescue LoadError
+  # nop
+end
 require "evil-proxy"
 require "evil-proxy/async"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.15.1
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-26 00:00:00.000000000 Z
+date: 2021-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
@@ -143,8 +143,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.9
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Ruby gem for dealing with proxy lists from different providers