proxy_fetcher 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d57af14d443aa759d7a82f20f68f073d1cf7f864204b9ee70233c8dfad83903c
-  data.tar.gz: 69fcafad4c583fae0d192cf40d0687377158af6145ad3444e8e74d10e7c98bb7
+  metadata.gz: '09772e9bc018d8accb01401b2e8b1897804e0a948c67a7eeeef8b1a2e9fcd245'
+  data.tar.gz: 16b49036c2ecdd06e23f53c4fe55982ec7ceb1531e9e223f31590eb83417c4e0
 SHA512:
-  metadata.gz: 9023c9bc9b6634e9d8b0e58af0a86c918deff988afbac0fd7872cdc6c85a5fca6b41d92b179687241f2af54d1b0bfbf3e9bd8c97610c7fe3db837b4b91504c66
-  data.tar.gz: 5da728c2e6d57d458df3aa93ef689c4ab1e892e183a2dc780026643cfcc82e017ecf626dc635d5fdd65b95204be784b83d620c188dba9129c30345d3939a28f6
+  metadata.gz: 526c8fdcfb1171c09f2e9bf6a4e13dbbc6e837963ce1c1ef7e2e48ed1ec3052e3dbc6a4d04bb33a0b3f083914149c22af4b4238a6c558825a91e3f26e112a378
+  data.tar.gz: 8dddea87295d1825910e4eecb4a5a16836b06e0070cc07f823a4902d6952550ae3b016bfb2e29d0ce8cc16ba3add12ad28a6ab6fec8eccd8a1656e368d88be5d

data/CHANGELOG.md

@@ -6,6 +6,13 @@ Reverse Chronological Order:
 
 * Add your description here
 
+## `0.13.0` (2020-03-09)
+
+* Fix GatherProxy provider
+* Fix XRoxy provider
+* Allow ability to load proxies from files
+* Fix Proxy object comparators
+
 ## `0.12.0` (2020-01-28)
 
 * Fix XRoxy provider

data/Gemfile

@@ -5,7 +5,7 @@ source "https://rubygems.org"
 gemspec
 
 gem "nokogiri", "~> 1.8"
-gem "oga", "~> 2.0"
+gem "oga", "~> 3.2"
 gem "rubocop", "~> 0.74"
 
 group :test do

data/lib/proxy_fetcher/configuration.rb

@@ -51,7 +51,7 @@ module ProxyFetcher
     attr_reader :proxy_validator
 
     # @!attribute [r] providers
-    #   @return [Array<String>, Array<Symbol>] proxy providers list to be used
+    #   @return [Array<String, Symbol>] proxy providers list to be used
     attr_reader :providers
 
     # User-Agent string that will be used by the ProxyFetcher HTTP client (to
@@ -95,7 +95,7 @@ module ProxyFetcher
 
       # Returns registered providers names.
       #
-      # @return [Array<String>, Array<Symbol>]
+      # @return [Array<String, Symbol>]
       #   registered providers names
       #
       def registered_providers
@@ -144,7 +144,7 @@ module ProxyFetcher
 
     # Setups collection of providers that will be used to fetch proxies.
     #
-    # @param value [String, Symbol, Array<String>, Array<Symbol>]
+    # @param value [String, Symbol, Array<String, Symbol>]
     #   provider names
     #
     def providers=(value)

data/lib/proxy_fetcher/manager.rb

@@ -3,6 +3,16 @@
 module ProxyFetcher
   # ProxyFetcher Manager class for interacting with proxy lists from various providers.
   class Manager
+    REFRESHER_LOCK = Mutex.new
+
+    class << self
+      def from_files(files, **options)
+        new(**options.merge(files: Array(files)))
+      end
+
+      alias from_file from_files
+    end
+
     # @!attribute [r] proxies
     #   @return [Array<ProxyFetcher::Proxy>] An array of proxies
     attr_reader :proxies
@@ -14,14 +24,17 @@ module ProxyFetcher
     #
     # @return [Manager]
     #
-    def initialize(refresh: true, validate: false, filters: {})
-      if refresh
-        refresh_list!(filters)
+    def initialize(**options)
+      if options.fetch(:refresh, true)
+        refresh_list!(options.fetch(:filters, {}))
       else
         @proxies = []
       end
 
-      cleanup! if validate
+      files = Array(options.fetch(:file, options.fetch(:files, [])))
+      load_proxies_from_files!(files) if files&.any?
+
+      cleanup! if options.fetch(:validate, false)
     end
 
     # Update current proxy list using configured providers.
@@ -30,9 +43,7 @@ module ProxyFetcher
     #
     def refresh_list!(filters = nil)
       @proxies = []
-
       threads = []
-      lock = Mutex.new
 
       ProxyFetcher.config.providers.each do |provider_name|
         threads << Thread.new do
@@ -40,7 +51,7 @@ module ProxyFetcher
           provider_filters = filters && filters.fetch(provider_name.to_sym, filters)
           provider_proxies = provider.fetch_proxies!(provider_filters)
 
-          lock.synchronize do
+          REFRESHER_LOCK.synchronize do
             @proxies.concat(provider_proxies)
           end
         end
@@ -89,6 +100,26 @@ module ProxyFetcher
 
     alias pop! get!
 
+    # Loads proxies from files.
+    #
+    # @param proxy_files [String, Array<String,Pathname>]
+    #   file path of list of files to load
+    #
+    def load_proxies_from_files!(proxy_files)
+      proxy_files = Array(proxy_files)
+      return if proxy_files.empty?
+
+      proxy_files.each do |proxy_file|
+        File.foreach(proxy_file, chomp: true) do |proxy_string|
+          addr, port = proxy_string.split(":", 2)
+          port = Integer(port) if port
+          @proxies << Proxy.new(addr: addr, port: port)
+        end
+      end
+
+      @proxies.uniq!
+    end
+
     # Clean current proxy list from dead proxies (that doesn't respond by timeout)
     #
     # @return [Array<ProxyFetcher::Proxy>]

data/lib/proxy_fetcher/providers/base.rb

@@ -108,7 +108,8 @@ module ProxyFetcher
         to_proxy(*args)
       rescue StandardError => e
         ProxyFetcher.logger.warn(
-          "Failed to build Proxy object for #{self.class.name} due to error: #{e.message}"
+          "Failed to build Proxy for #{self.class.name.split("::").last} " \
+          "due to error: #{e.message}"
         )
 
         nil

data/lib/proxy_fetcher/providers/gather_proxy.rb

@@ -8,7 +8,7 @@ module ProxyFetcher
     class GatherProxy < Base
       # Provider URL to fetch proxy list
       def provider_url
-        "http://www.gatherproxy.com/"
+        "https://proxygather.com"
       end
 
       def xpath

data/lib/proxy_fetcher/providers/xroxy.rb

@@ -24,12 +24,12 @@ module ProxyFetcher
       #
       def to_proxy(html_node)
         ProxyFetcher::Proxy.new.tap do |proxy|
-          proxy.addr = html_node.content_at("td[2]")
-          proxy.port = Integer(html_node.content_at("td[3]").gsub(/^0+/, ""))
-          proxy.anonymity = html_node.content_at("td[4]")
-          proxy.country = html_node.content_at("td[6]")
-          proxy.response_time = Integer(html_node.content_at("td[7]"))
-          proxy.type = html_node.content_at("td[4]")
+          proxy.addr = html_node.content_at("td[1]")
+          proxy.port = Integer(html_node.content_at("td[2]").gsub(/^0+/, ""))
+          proxy.anonymity = html_node.content_at("td[3]")
+          proxy.country = html_node.content_at("td[5]")
+          proxy.response_time = Integer(html_node.content_at("td[6]"))
+          proxy.type = html_node.content_at("td[3]")
         end
       end
     end

data/lib/proxy_fetcher/proxy.rb

@@ -104,5 +104,17 @@ module ProxyFetcher
         URI::Generic.build(host: addr, port: port).to_s
       end
     end
+
+    def ==(other)
+      other.is_a?(Proxy) && addr == other.addr && port == other.port
+    end
+
+    def eql?(other)
+      hash.eql?(other.hash)
+    end
+
+    def hash
+      [addr.hash, port.hash].hash
+    end
   end
 end

data/lib/proxy_fetcher/version.rb

@@ -13,7 +13,7 @@ module ProxyFetcher
     # Major version number
     MAJOR = 0
     # Minor version number
-    MINOR = 12
+    MINOR = 13
     # Smallest version number
     TINY  = 0
 

data/lib/proxy_fetcher.rb

@@ -46,7 +46,6 @@ module ProxyFetcher
 
   # Main ProxyFetcher module.
   class << self
-
     ##
     # Returns ProxyFetcher configuration.
     #
@@ -57,9 +56,9 @@ module ProxyFetcher
     #   ProxyFetcher.config
     #
     #   #=> #<ProxyFetcher::Configuration:0x0000000241eec8 @user_agent="Mozilla/5.0, ...", @pool_size=10,
-    #           @client_timeout=3, @proxy_validation_timeout=3, @provider_proxies_load_timeout=30,
-    #           @http_client=ProxyFetcher::HTTPClient, @proxy_validator=ProxyFetcher::ProxyValidator,
-    #           @providers=[:free_proxy_list, ...], @adapter=ProxyFetcher::Document::NokogiriAdapter>
+    #          @client_timeout=3, @proxy_validation_timeout=3, @provider_proxies_load_timeout=30,
+    #          @http_client=ProxyFetcher::HTTPClient, @proxy_validator=ProxyFetcher::ProxyValidator,
+    #          @providers=[:free_proxy_list, ...], @adapter=ProxyFetcher::Document::NokogiriAdapter>
     #
     def config
       @__config_definition_lock__.synchronize do

data/spec/fixtures/proxies.txt

@@ -0,0 +1,14 @@
+139.162.59.9:3128
+176.62.187.158:56351
+144.217.22.142:8080
+176.55.108.21:3128
+157.225.214.251:3128
+202.51.49.52:48298
+104.244.75.26:8080
+163.172.28.20:80
+187.56.191.12:3128
+129.194.12.26:80
+178.128.39.39:8080
+181.30.28.15:8080
+51.181.96.185:8080
+148.134.10.13

data/spec/proxy_fetcher/manager_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe ProxyFetcher::Manager do
+  it "can initialize with a proxies from file(s)" do
+    manager = described_class.new(refresh: false, file: "spec/fixtures/proxies.txt")
+
+    expect(manager.proxies.size).to be(14)
+
+    manager = described_class.new(
+      refresh: false,
+      file: ["spec/fixtures/proxies.txt", "spec/fixtures/proxies.txt"]
+    )
+
+    expect(manager.proxies.size).to be(14)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-28 00:00:00.000000000 Z
+date: 2020-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
@@ -105,10 +105,12 @@ files:
 - lib/proxy_fetcher/utils/proxy_validator.rb
 - lib/proxy_fetcher/version.rb
 - proxy_fetcher.gemspec
+- spec/fixtures/proxies.txt
 - spec/proxy_fetcher/client/client_spec.rb
 - spec/proxy_fetcher/configuration_spec.rb
 - spec/proxy_fetcher/document/adapters_spec.rb
 - spec/proxy_fetcher/document/node_spec.rb
+- spec/proxy_fetcher/manager_spec.rb
 - spec/proxy_fetcher/providers/base_spec.rb
 - spec/proxy_fetcher/providers/free_proxy_list_spec.rb
 - spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb