proxy_fetcher 0.7.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 59ba9fd7169617766f99b048a1c889e0213ac57c
-  data.tar.gz: 03e3d51d71fb5c06c0c7c7d5808a8a43c79c50a4
+  metadata.gz: ad122ebb9d6241313981ef764c41a1b940dfdc12
+  data.tar.gz: 2cf2d359bfd74122fbfdf9d97d94c1e70b008cd5
 SHA512:
-  metadata.gz: 495ccb751c9f927f1ef31e1b2536c600aec0f68189ea11d1f8061d59bfe41caf2dd0e818d9b780a5b8279c66b20c05873ffba4976cadb101c4aff2a7a449bc94
-  data.tar.gz: a2aae32a4a3649b10399e23520fbf0a19df3ca564a3519290352bd51366cda6f8a709aa7a1783de31e5e474ab23bd86ea2fb41fff320a81eb57a16b3ef111b30
+  metadata.gz: 4ebe17a6ebfb52a14267856053ad750676d4b28e85e138aa172d778b85368ec85ccd04af55cdceb53e1aaad3db5cdab09d42ee87767d56b1fc5578c1c6ba114b
+  data.tar.gz: 6133dfc0b9810831ae7e79c2795ad3214a4b1c78df71e7a2a910429dd21d6b67a7a2f22c29b284fc30d68fff4ea9907fc39edefbb630027eb9a1e2cd4eb2ae24

data/CHANGELOG.md

@@ -2,6 +2,24 @@
 
 Reverse Chronological Order:
 
+## `master`
+
+* Add your changelog here
+
+## `0.8.0` (2018-11-12)
+
+* Improve speed of proxy list loading.
+* Improve speed of proxies cleanup. 
+* Fix ProxyDocker provider
+
+## `0.7.2` (2018-08-13)
+
+* Fix XRoxy provider
+
+## `0.7.1` (2018-07-13)
+
+* Fix XRoxy provider
+
 ## `0.7.0` (2018-06-04)
 
 * Migrate to `HTTP.rb` instead of `Net::HTTP`

data/lib/proxy_fetcher.rb

@@ -15,6 +15,7 @@ require File.dirname(__FILE__) + '/proxy_fetcher/null_logger'
 
 require File.dirname(__FILE__) + '/proxy_fetcher/utils/http_client'
 require File.dirname(__FILE__) + '/proxy_fetcher/utils/proxy_validator'
+require File.dirname(__FILE__) + '/proxy_fetcher/utils/proxy_list_validator'
 require File.dirname(__FILE__) + '/proxy_fetcher/client/client'
 require File.dirname(__FILE__) + '/proxy_fetcher/client/request'
 require File.dirname(__FILE__) + '/proxy_fetcher/client/proxies_registry'

data/lib/proxy_fetcher/manager.rb

@@ -31,12 +31,24 @@ module ProxyFetcher
     def refresh_list!(filters = nil)
       @proxies = []
 
+      threads = []
+      lock = Mutex.new
+
       ProxyFetcher.config.providers.each do |provider_name|
-        provider = ProxyFetcher::Configuration.providers_registry.class_for(provider_name)
-        provider_filters = filters && filters.fetch(provider_name.to_sym, filters)
+        threads << Thread.new do
+          provider = ProxyFetcher::Configuration.providers_registry.class_for(provider_name)
+          provider_filters = filters && filters.fetch(provider_name.to_sym, filters)
+          provider_proxies = provider.fetch_proxies!(provider_filters)
 
-        @proxies.concat(provider.fetch_proxies!(provider_filters))
+          lock.synchronize do
+            @proxies.concat(provider_proxies)
+          end
+        end
       end
+
+      threads.each(&:join)
+
+      @proxies
     end
 
     alias fetch! refresh_list!
@@ -78,20 +90,12 @@ module ProxyFetcher
     alias pop! get!
 
     # Clean current proxy list from dead proxies (that doesn't respond by timeout)
+    #
+    # @return [Array<ProxyFetcher::Proxy>]
+    #   list of valid proxies
     def cleanup!
-      lock = Mutex.new
-
-      proxies.dup.each_slice(ProxyFetcher.config.pool_size) do |proxy_group|
-        threads = proxy_group.map do |group_proxy|
-          Thread.new(group_proxy, proxies) do |proxy, proxies|
-            lock.synchronize { proxies.delete(proxy) } unless proxy.connectable?
-          end
-        end
-
-        threads.each(&:join)
-      end
-
-      @proxies
+      valid_proxies = ProxyListValidator.new(@proxies).validate
+      @proxies &= valid_proxies
     end
 
     alias validate! cleanup!

data/lib/proxy_fetcher/providers/proxy_docker.rb

@@ -17,7 +17,7 @@ module ProxyFetcher
       # [NOTE] Doesn't support direct filters
       def load_proxy_list(*)
         doc = load_document(PROVIDER_URL, {})
-        doc.xpath('//table[contains(@class, "table")]/tr[(not(@id="proxy-table-header")) and (count(td)>2)]')
+        doc.xpath('//table[contains(@class, "table")]/tbody/tr[(count(td)>2)]')
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>

data/lib/proxy_fetcher/providers/xroxy.rb

@@ -5,7 +5,7 @@ module ProxyFetcher
     # XRoxy provider class.
     class XRoxy < Base
       # Provider URL to fetch proxy list
-      PROVIDER_URL = 'https://www.xroxy.com/proxylist.php'.freeze
+      PROVIDER_URL = 'https://www.xroxy.com/free-proxy-lists/'.freeze
 
       # Fetches HTML content by sending HTTP request to the provider URL and
       # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
@@ -16,7 +16,7 @@ module ProxyFetcher
       #
       def load_proxy_list(filters = { type: 'All_http' })
         doc = load_document(PROVIDER_URL, filters)
-        doc.xpath('//div[@id="content"]/table[1]/tr[contains(@class, "row")]')
+        doc.xpath('//div/table/tbody/tr')
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
@@ -30,29 +30,14 @@ module ProxyFetcher
       #
       def to_proxy(html_node)
         ProxyFetcher::Proxy.new.tap do |proxy|
-          proxy.addr = html_node.content_at('td[2]')
-          proxy.port = Integer(html_node.content_at('td[3]').gsub(/^0+/, ''))
-          proxy.anonymity = html_node.content_at('td[4]')
-          proxy.country = html_node.content_at('td[6]')
-          proxy.response_time = Integer(html_node.content_at('td[7]'))
-          proxy.type = parse_type(html_node)
+          proxy.addr = html_node.content_at('td[1]')
+          proxy.port = Integer(html_node.content_at('td[2]').gsub(/^0+/, ''))
+          proxy.anonymity = html_node.content_at('td[3]')
+          proxy.country = html_node.content_at('td[5]')
+          proxy.response_time = Integer(html_node.content_at('td[6]'))
+          proxy.type = html_node.content_at('td[3]')
         end
       end
-
-      private
-
-      # Parses HTML node to extract proxy type.
-      #
-      # @param html_node [Object]
-      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
-      #
-      # @return [String]
-      #   Proxy type
-      #
-      def parse_type(html_node)
-        https = html_node.content_at('td[5]')
-        https.casecmp('true').zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP
-      end
     end
 
     ProxyFetcher::Configuration.register_provider(:xroxy, XRoxy)

data/lib/proxy_fetcher/utils/proxy_list_validator.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+  # This class validates list of proxies.
+  # Each proxy is validated using <code>ProxyFetcher::ProxyValidator</code>.
+  class ProxyListValidator
+    # @!attribute [r] proxies
+    #   @return [Array<ProxyFetcher::Proxy>] Source array of proxies
+    attr_reader :proxies
+    # @!attribute [r] valid_proxies
+    #   @return [Array<ProxyFetcher::Proxy>] Array of valid proxies after validation
+    attr_reader :valid_proxies
+
+    # @param [Array<ProxyFetcher::Proxy>] *proxies
+    #   Any number of <code>ProxyFetcher::Proxy</code> to validate
+    def initialize(*proxies)
+      @proxies = proxies.flatten
+    end
+
+    # Performs validation
+    #
+    # @return [Array<ProxyFetcher::Proxy>]
+    #   list of valid proxies
+    def validate
+      target_proxies = @proxies.dup
+      target_proxies_lock = Mutex.new
+      connectable_proxies = []
+      connectable_proxies_lock = Mutex.new
+      threads = []
+
+      ProxyFetcher.config.pool_size.times do
+        threads << Thread.new do
+          loop do
+            proxy = target_proxies_lock.synchronize { target_proxies.shift }
+            break unless proxy
+
+            connectable_proxies_lock.synchronize { connectable_proxies << proxy } if proxy.connectable?
+          end
+        end
+      end
+
+      threads.each(&:join)
+
+      @valid_proxies = connectable_proxies
+    end
+  end
+end

data/lib/proxy_fetcher/version.rb

@@ -13,9 +13,9 @@ module ProxyFetcher
     # Major version number
     MAJOR = 0
     # Minor version number
-    MINOR = 7
+    MINOR = 8
     # Smallest version number
-    TINY  = 1
+    TINY  = 0
 
     # Full version number
     STRING = [MAJOR, MINOR, TINY].compact.join('.')

data/spec/support/manager_examples.rb

@@ -35,7 +35,9 @@ RSpec.shared_examples 'a manager' do
 
     manager = ProxyFetcher::Manager.new
 
-    expect { manager.cleanup! }.to change { manager.proxies }.to([])
+    expect do
+      manager.cleanup!
+    end.to change { manager.proxies }.to([])
   end
 
   it "doesn't pollute the output with array of proxies" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.8.0
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-13 00:00:00.000000000 Z
+date: 2018-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
@@ -80,6 +80,7 @@ files:
 - lib/proxy_fetcher/providers/xroxy.rb
 - lib/proxy_fetcher/proxy.rb
 - lib/proxy_fetcher/utils/http_client.rb
+- lib/proxy_fetcher/utils/proxy_list_validator.rb
 - lib/proxy_fetcher/utils/proxy_validator.rb
 - lib/proxy_fetcher/version.rb
 - proxy_fetcher.gemspec