proxy_fetcher 0.10.2 → 0.15.0

data/spec/proxy_fetcher/document/adapters_spec.rb

@@ -1,26 +1,26 @@
 # frozen_string_literal: true
 
-require 'spec_helper'
+require "spec_helper"
 
 describe ProxyFetcher::Document::Adapters do
-  describe '#lookup' do
-    it 'returns predefined adapters if symbol or string passed' do
-      expect(described_class.lookup('nokogiri')).to eq(ProxyFetcher::Document::NokogiriAdapter)
+  describe "#lookup" do
+    it "returns predefined adapters if symbol or string passed" do
+      expect(described_class.lookup("nokogiri")).to eq(ProxyFetcher::Document::NokogiriAdapter)
 
       expect(described_class.lookup(:oga)).to eq(ProxyFetcher::Document::OgaAdapter)
     end
 
-    it 'returns self if class passed' do
+    it "returns self if class passed" do
       expect(described_class.lookup(Struct)).to eq(Struct)
     end
 
-    it 'raises an exception if passed value is blank' do
+    it "raises an exception if passed value is blank" do
       expect { described_class.lookup(nil) }.to raise_error(ProxyFetcher::Exceptions::BlankAdapter)
-      expect { described_class.lookup('') }.to raise_error(ProxyFetcher::Exceptions::BlankAdapter)
+      expect { described_class.lookup("") }.to raise_error(ProxyFetcher::Exceptions::BlankAdapter)
     end
 
     it "raises an exception if adapter doesn't exist" do
-      expect { described_class.lookup('wrong') }.to raise_error(ProxyFetcher::Exceptions::UnknownAdapter)
+      expect { described_class.lookup("wrong") }.to raise_error(ProxyFetcher::Exceptions::UnknownAdapter)
     end
   end
 end

data/spec/proxy_fetcher/document/node_spec.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require 'spec_helper'
+require "spec_helper"
 
 describe ProxyFetcher::Document::Node do
-  context 'overridable methods' do
-    it 'raises an error' do
-      node = ProxyFetcher::Document::Node.new('')
+  context "overridable methods" do
+    it "raises an error" do
+      node = ProxyFetcher::Document::Node.new("")
 
       %w[content html].each do |method|
         expect { node.public_send(method) }.to raise_error do |error|

data/spec/proxy_fetcher/manager_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe ProxyFetcher::Manager do
+  it "can initialize with a proxies from file(s)" do
+    manager = described_class.new(refresh: false, file: "spec/fixtures/proxies.txt")
+
+    expect(manager.proxies.size).to be(14)
+
+    manager = described_class.new(
+      refresh: false,
+      file: ["spec/fixtures/proxies.txt", "spec/fixtures/proxies.txt"]
+    )
+
+    expect(manager.proxies.size).to be(14)
+  end
+end

data/spec/proxy_fetcher/providers/base_spec.rb

@@ -1,37 +1,37 @@
 # frozen_string_literal: true
 
-require 'spec_helper'
+require "spec_helper"
 
 describe ProxyFetcher::Providers::Base do
   before { ProxyFetcher.config.reset! }
   after { ProxyFetcher.config.reset! }
 
-  it 'does not allows to use not implemented methods' do
+  it "does not allows to use not implemented methods" do
     NotImplementedCustomProvider = Class.new(ProxyFetcher::Providers::Base)
 
     ProxyFetcher::Configuration.register_provider(:provider_without_methods, NotImplementedCustomProvider)
     ProxyFetcher.config.provider = :provider_without_methods
 
     expect { ProxyFetcher::Manager.new }.to raise_error(NotImplementedError) do |error|
-      expect(error.message).to include('load_proxy_list')
+      expect(error.message).to include("provider_url")
     end
 
     # implement one of the methods
     NotImplementedCustomProvider.class_eval do
-      def load_proxy_list(*)
-        [1, 2, 3]
+      def provider_url
+        "http://provider.com"
       end
     end
 
     expect { ProxyFetcher::Manager.new }.to raise_error(NotImplementedError) do |error|
-      expect(error.message).to include('to_proxy')
+      expect(error.message).to include("xpath")
     end
   end
 
-  it 'logs failed to load proxy providers' do
+  it "logs failed to load proxy providers" do
     CustomProvider = Class.new(ProxyFetcher::Providers::Base) do
       def load_proxy_list(*)
-        doc = load_document('https://google.com', {})
+        doc = load_document("https://google.com", {})
         doc.xpath('//table[contains(@class, "table")]/tr[(not(@id="proxy-table-header")) and (count(td)>2)]')
       end
     end
@@ -44,7 +44,7 @@ describe ProxyFetcher::Providers::Base do
 
     allow_any_instance_of(HTTP::Client).to receive(:get).and_raise(StandardError)
 
-    expect(logger).to receive(:warn).with(/Failed to process request to http[s:\/]/)
+    expect(logger).to receive(:warn).with(%r{Failed to process request to http[s:/]})
 
     ProxyFetcher::Manager.new
   end

data/spec/proxy_fetcher/providers/multiple_providers_spec.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
-require 'spec_helper'
+require "spec_helper"
 
-describe 'Multiple proxy providers' do
+describe "Multiple proxy providers" do
   before { ProxyFetcher.config.reset! }
   after { ProxyFetcher.config.reset! }
 
-  it 'combine proxies from multiple providers' do
-    proxy_stub = ProxyFetcher::Proxy.new(addr: '192.168.1.1', port: 8080)
+  it "combine proxies from multiple providers" do
+    proxy_stub = ProxyFetcher::Proxy.new(addr: "192.168.1.1", port: 8080)
 
     # Each proxy provider will return 2 proxies
     ProxyFetcher::Configuration.providers_registry.providers.each do |_name, klass|

data/spec/proxy_fetcher/providers/proxy_classes_spec.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe "Proxy classes" do
+  [
+    [:free_proxy_list,       "FreeProxyList"],
+    [:free_proxy_list_socks, "FreeProxyListSocks"],
+    [:free_proxy_list_ssl,   "FreeProxyListSSL"],
+    [:free_proxy_list_us,    "FreeProxyListUS"],
+    [:http_tunnel,           "HTTPTunnel"],
+    [:mtpro,                 "MTPro"],
+    [:proxy_list,            "ProxyList"],
+    [:proxypedia,            "Proxypedia"],
+    [:proxyscrape_http,      "ProxyscrapeHTTP"],
+    [:proxyscrape_socks4,    "ProxyscrapeSOCKS4"],
+    [:proxyscrape_socks5,    "ProxyscrapeSOCKS5"],
+    [:xroxy,                 "XRoxy"]
+  ].each do |(provider_name, provider_klass)|
+    describe Object.const_get("ProxyFetcher::Providers::#{provider_klass}") do
+      before :all do
+        ProxyFetcher.config.provider = provider_name
+      end
+
+      it_behaves_like "a manager"
+    end
+  end
+end

data/spec/proxy_fetcher/proxy_spec.rb

@@ -1,20 +1,20 @@
 # frozen_string_literal: true
 
-require 'spec_helper'
+require "spec_helper"
 
 describe ProxyFetcher::Proxy do
-  let(:proxy) { described_class.new(addr: '192.169.1.1', port: 8080, type: 'HTTP') }
+  let(:proxy) { described_class.new(addr: "192.169.1.1", port: 8080, type: "HTTP") }
 
-  it 'can initialize a new proxy object' do
-    proxy = described_class.new(addr: '192.169.1.1', port: 8080, type: 'HTTP')
+  it "can initialize a new proxy object" do
+    proxy = described_class.new(addr: "192.169.1.1", port: 8080, type: "HTTP")
 
     expect(proxy).not_to be_nil
-    expect(proxy.addr).to eq('192.169.1.1')
+    expect(proxy.addr).to eq("192.169.1.1")
     expect(proxy.port).to eq(8080)
-    expect(proxy.type).to eq('HTTP')
+    expect(proxy.type).to eq("HTTP")
   end
 
-  it 'checks schema' do
+  it "checks schema" do
     proxy.type = ProxyFetcher::Proxy::HTTP
     expect(proxy.http?).to be_truthy
     expect(proxy.https?).to be_falsey
@@ -34,28 +34,28 @@ describe ProxyFetcher::Proxy do
     expect(proxy.ssl?).to be_truthy
   end
 
-  it 'not connectable if IP addr is wrong' do
-    proxy.addr = '192.168.1.0'
+  it "not connectable if IP addr is wrong" do
+    proxy.addr = "192.168.1.0"
     expect(proxy.connectable?).to be_falsey
   end
 
-  it 'not connectable if there are some error during connection request' do
+  it "not connectable if there are some error during connection request" do
     allow_any_instance_of(HTTP::Client).to receive(:head).and_raise(HTTP::TimeoutError)
     expect(proxy.connectable?).to be_falsey
   end
 
-  it 'returns URI::Generic' do
+  it "returns URI::Generic" do
     expect(proxy.uri).to be_a(URI::Generic)
 
     expect(proxy.uri.host).not_to be_empty
     expect(proxy.uri.port).not_to be_nil
   end
 
-  it 'returns URL' do
+  it "returns URL" do
     expect(proxy.url).to be_a(String)
   end
 
-  it 'returns URL with scheme' do
-    expect(proxy.url(scheme: true)).to include('://')
+  it "returns URL with scheme" do
+    expect(proxy.url(scheme: true)).to include("://")
   end
 end

data/spec/proxy_fetcher/version_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 RSpec.describe ProxyFetcher::VERSION do
   it { expect(ProxyFetcher::VERSION::STRING).to match(/^\d+\.\d+\.\d+(\.\w+)?$/) }
 end

data/spec/spec_helper.rb

@@ -1,24 +1,24 @@
 # frozen_string_literal: true
 
-require 'simplecov'
-SimpleCov.add_filter 'spec'
-SimpleCov.add_filter 'version'
+require "simplecov"
+SimpleCov.add_filter "spec"
+SimpleCov.add_filter "version"
 
-if ENV['CI'] || ENV['TRAVIS'] || ENV['COVERALLS'] || ENV['JENKINS_URL']
-  require 'coveralls'
+if ENV["CI"] || ENV["TRAVIS"] || ENV["COVERALLS"] || ENV["JENKINS_URL"]
+  require "coveralls"
   Coveralls.wear!
 else
   SimpleCov.start
 end
 
-require 'bundler/setup'
+require "bundler/setup"
 Bundler.setup
 
-require 'proxy_fetcher'
+require "proxy_fetcher"
 
-Dir['./spec/support/**/*.rb'].sort.each { |f| require f }
+Dir["./spec/support/**/*.rb"].sort.each { |f| require f }
 
-adapter = ENV['BUNDLE_GEMFILE'][/.+\/(.+)\.gemfile/i, 1] || :nokogiri
+adapter = ENV["BUNDLE_GEMFILE"][%r{.+/(.+)\.gemfile}i, 1] || :nokogiri
 puts "Configured adapter: '#{adapter}'"
 
 ProxyFetcher.configure do |config|
@@ -26,5 +26,5 @@ ProxyFetcher.configure do |config|
 end
 
 RSpec.configure do |config|
-  config.order = 'random'
+  config.order = "random"
 end

data/spec/support/manager_examples.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
 
-RSpec.shared_examples 'a manager' do
-  it 'loads proxy list on initialization by default' do
-    manager = ProxyFetcher::Manager.new
-    expect(manager.proxies).not_to be_empty
+RSpec.shared_examples "a manager" do
+  before :all do
+    @cached_manager = ProxyFetcher::Manager.new
+  end
+
+  it "loads proxy list on initialization by default" do
+    expect(@cached_manager.proxies).not_to be_empty
   end
 
   it "doesn't load proxy list on initialization if `refresh` argument was set to false" do
@@ -11,11 +14,10 @@ RSpec.shared_examples 'a manager' do
     expect(manager.proxies).to be_empty
   end
 
-  it 'returns valid Proxy objects' do
-    manager = ProxyFetcher::Manager.new
-    expect(manager.proxies).to all(be_a(ProxyFetcher::Proxy))
+  it "returns valid Proxy objects" do
+    expect(@cached_manager.proxies).to all(be_a(ProxyFetcher::Proxy))
 
-    manager.proxies.each do |proxy|
+    @cached_manager.proxies.each do |proxy|
       expect(proxy.addr).to match(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/i)
       expect(proxy.port).to be_a_kind_of(Numeric)
       expect(proxy.type).not_to be_empty
@@ -25,12 +27,11 @@ RSpec.shared_examples 'a manager' do
     end
   end
 
-  it 'returns raw proxies (HOST:PORT)' do
-    manager = ProxyFetcher::Manager.new
-    expect(manager.raw_proxies).to all(be_a(String))
+  it "returns raw proxies (HOST:PORT)" do
+    expect(@cached_manager.raw_proxies).to all(be_a(String))
   end
 
-  it 'cleanup proxy list from dead servers' do
+  it "cleanup proxy list from dead servers" do
     allow_any_instance_of(ProxyFetcher::Proxy).to receive(:connectable?).and_return(false)
 
     manager = ProxyFetcher::Manager.new
@@ -45,7 +46,7 @@ RSpec.shared_examples 'a manager' do
     expect(manager.inspect).to eq(manager.to_s)
   end
 
-  it 'returns first proxy' do
+  it "returns first proxy" do
     manager = ProxyFetcher::Manager.new
 
     first_proxy = manager.proxies.first
@@ -54,16 +55,16 @@ RSpec.shared_examples 'a manager' do
     expect(manager.proxies.first).not_to eq(first_proxy)
   end
 
-  it 'returns first valid proxy' do
+  it "returns first valid proxy" do
     manager = ProxyFetcher::Manager.new(refresh: false)
 
-    proxies = Array.new(5) { instance_double('ProxyFetcher::Proxy', connectable?: false) }
+    proxies = Array.new(5) { instance_double("ProxyFetcher::Proxy", connectable?: false) }
     manager.instance_variable_set(:@proxies, proxies)
 
-    connectable_proxy = instance_double('ProxyFetcher::Proxy')
+    connectable_proxy = instance_double("ProxyFetcher::Proxy")
     allow(connectable_proxy).to receive(:connectable?).and_return(true)
 
-    manager.proxies[0..2].each { |proxy| proxy.instance_variable_set(:@addr, '192.168.1.1') }
+    manager.proxies[0..2].each { |proxy| proxy.instance_variable_set(:@addr, "192.168.1.1") }
     manager.proxies[2] = connectable_proxy
 
     expect(manager.get!).to eq(connectable_proxy)
@@ -73,15 +74,14 @@ RSpec.shared_examples 'a manager' do
     expect(manager.proxies.size).to be(1)
   end
 
-  it 'returns nothing if proxy list is empty' do
+  it "returns nothing if proxy list is empty" do
     manager = ProxyFetcher::Manager.new(refresh: false)
 
     expect(manager.get).to be_nil
     expect(manager.get!).to be_nil
   end
 
-  it 'returns random proxy' do
-    manager = ProxyFetcher::Manager.new
-    expect(manager.random).to be_an_instance_of(ProxyFetcher::Proxy)
+  it "returns random proxy" do
+    expect(@cached_manager.random).to be_an_instance_of(ProxyFetcher::Proxy)
   end
 end

metadata

@@ -1,29 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.10.2
+  version: 0.15.0
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-15 00:00:00.000000000 Z
+date: 2021-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '5'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -44,14 +50,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.9'
 description: This gem can help your Ruby application to make HTTP(S) requests using
   proxies by fetching and validating proxy lists from the different providers.
 email: bulajnikita@gmail.com
@@ -88,10 +94,16 @@ files:
 - lib/proxy_fetcher/null_logger.rb
 - lib/proxy_fetcher/providers/base.rb
 - lib/proxy_fetcher/providers/free_proxy_list.rb
+- lib/proxy_fetcher/providers/free_proxy_list_socks.rb
 - lib/proxy_fetcher/providers/free_proxy_list_ssl.rb
-- lib/proxy_fetcher/providers/gather_proxy.rb
+- lib/proxy_fetcher/providers/free_proxy_list_us.rb
 - lib/proxy_fetcher/providers/http_tunnel.rb
+- lib/proxy_fetcher/providers/mtpro.rb
 - lib/proxy_fetcher/providers/proxy_list.rb
+- lib/proxy_fetcher/providers/proxypedia.rb
+- lib/proxy_fetcher/providers/proxyscrape_http.rb
+- lib/proxy_fetcher/providers/proxyscrape_socks4.rb
+- lib/proxy_fetcher/providers/proxyscrape_socks5.rb
 - lib/proxy_fetcher/providers/xroxy.rb
 - lib/proxy_fetcher/proxy.rb
 - lib/proxy_fetcher/utils/http_client.rb
@@ -99,18 +111,15 @@ files:
 - lib/proxy_fetcher/utils/proxy_validator.rb
 - lib/proxy_fetcher/version.rb
 - proxy_fetcher.gemspec
+- spec/fixtures/proxies.txt
 - spec/proxy_fetcher/client/client_spec.rb
 - spec/proxy_fetcher/configuration_spec.rb
 - spec/proxy_fetcher/document/adapters_spec.rb
 - spec/proxy_fetcher/document/node_spec.rb
+- spec/proxy_fetcher/manager_spec.rb
 - spec/proxy_fetcher/providers/base_spec.rb
-- spec/proxy_fetcher/providers/free_proxy_list_spec.rb
-- spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb
-- spec/proxy_fetcher/providers/gather_proxy_spec.rb
-- spec/proxy_fetcher/providers/http_tunnel_spec.rb
 - spec/proxy_fetcher/providers/multiple_providers_spec.rb
-- spec/proxy_fetcher/providers/proxy_list_spec.rb
-- spec/proxy_fetcher/providers/xroxy_spec.rb
+- spec/proxy_fetcher/providers/proxy_classes_spec.rb
 - spec/proxy_fetcher/proxy_spec.rb
 - spec/proxy_fetcher/version_spec.rb
 - spec/spec_helper.rb
@@ -127,14 +136,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.2
+rubyforge_project: 
+rubygems_version: 2.7.9
 signing_key: 
 specification_version: 4
 summary: Ruby gem for dealing with proxy lists from different providers