proxy_fetcher 0.13.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '09772e9bc018d8accb01401b2e8b1897804e0a948c67a7eeeef8b1a2e9fcd245'
-  data.tar.gz: 16b49036c2ecdd06e23f53c4fe55982ec7ceb1531e9e223f31590eb83417c4e0
+  metadata.gz: 46d91f40f7c61a9a14aa90530ece0ef42365566ed6055abb141925ac471126d9
+  data.tar.gz: 72f56ef68ee8f46f226bf6de07e5888f485cd533ec2b6b8a64047b8530ec6b3c
 SHA512:
-  metadata.gz: 526c8fdcfb1171c09f2e9bf6a4e13dbbc6e837963ce1c1ef7e2e48ed1ec3052e3dbc6a4d04bb33a0b3f083914149c22af4b4238a6c558825a91e3f26e112a378
-  data.tar.gz: 8dddea87295d1825910e4eecb4a5a16836b06e0070cc07f823a4902d6952550ae3b016bfb2e29d0ce8cc16ba3add12ad28a6ab6fec8eccd8a1656e368d88be5d
+  metadata.gz: a9f59f62fd5c731a1d15430faca81468f4a3368eb0287f5fad405ff3379af55886bd4a7608bba5a90f2117e42bb1d08a46ff6e63fe132c72a253f122164cbc4a
+  data.tar.gz: de8fa38b0895a6faf87c7ddac76b80a331bc6e6beb95a7972d94146d4ecaaf1025a4442bb3639d902e979cb57d1c1b6ba03f33137c56369692a5f2ca0b99b28e

data/CHANGELOG.md

@@ -4,7 +4,12 @@ Reverse Chronological Order:
 
 ## `master`
 
-* Add your description here
+...
+
+## `0.14.0` (2020-05-11)
+
+* Add MTPro provider
+* Add Proxypedia provider
 
 ## `0.13.0` (2020-03-09)
 

data/Gemfile

@@ -6,7 +6,7 @@ gemspec
 
 gem "nokogiri", "~> 1.8"
 gem "oga", "~> 3.2"
-gem "rubocop", "~> 0.74"
+gem "rubocop", "~> 0.80"
 
 group :test do
   gem "coveralls", require: false

data/lib/proxy_fetcher.rb

@@ -33,10 +33,12 @@ module ProxyFetcher
   # ProxyFetcher providers namespace
   module Providers
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/base"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxypedia"
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list"
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl"
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/gather_proxy"
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/mtpro"
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list"
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy"
   end

data/lib/proxy_fetcher/providers/free_proxy_list.rb

@@ -45,27 +45,6 @@ module ProxyFetcher
       #
       def parse_type(html_node)
         https = html_node.content_at("td[6]")
-        # frozen_string_literal: true
-        # FreeProxyList provider class.
-        # Provider URL to fetch proxy list
-        # [NOTE] Doesn't support filtering
-        # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
-        # object.
-        #
-        # @param html_node [Object]
-        #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
-        #
-        # @return [ProxyFetcher::Proxy]
-        #   Proxy object
-        #
-        # Parses HTML node to extract proxy type.
-        #
-        # @param html_node [Object]
-        #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
-        #
-        # @return [String]
-        #   Proxy type
-        #
         https&.casecmp("yes")&.zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP
       end
     end

data/lib/proxy_fetcher/providers/mtpro.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "json"
+
+module ProxyFetcher
+  module Providers
+    # GatherProxy provider class.
+    class MTPro < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://mtpro.xyz/api/?type=socks"
+      end
+
+      def load_proxy_list(filters = {})
+        html = load_html(provider_url, filters)
+        JSON.parse(html)
+      rescue JSON::ParserError
+        []
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(node)
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = node["ip"]
+          proxy.port = Integer(node["port"])
+          proxy.country = node["country"]
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::SOCKS5
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:mtpro, MTPro)
+  end
+end

data/lib/proxy_fetcher/providers/proxypedia.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class Proxypedia < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://proxypedia.org"
+      end
+
+      # [NOTE] Doesn't support filtering
+      def xpath
+        "//main/ul/li[position()>1]"
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.]
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        addr, port = html_node.content_at("a").to_s.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = parse_country(html_node)
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::HTTP
+        end
+      end
+
+      private
+
+      def parse_country(html_node)
+        text = html_node.content.to_s
+        text[/\((.+?)\)/, 1] || "Unknown"
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxypedia, Proxypedia)
+  end
+end

data/lib/proxy_fetcher/version.rb

@@ -13,7 +13,7 @@ module ProxyFetcher
     # Major version number
     MAJOR = 0
     # Minor version number
-    MINOR = 13
+    MINOR = 14
     # Smallest version number
     TINY  = 0
 

data/spec/proxy_fetcher/providers/mtpro_spec.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe ProxyFetcher::Providers::MTPro do
+  before :all do
+    ProxyFetcher.config.provider = :mtpro
+  end
+
+  it_behaves_like "a manager"
+end

data/spec/proxy_fetcher/providers/proxypedia_spec.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe ProxyFetcher::Providers::Proxypedia do
+  before :all do
+    ProxyFetcher.config.provider = :proxypedia
+  end
+
+  it_behaves_like "a manager"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-09 00:00:00.000000000 Z
+date: 2020-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
@@ -97,7 +97,9 @@ files:
 - lib/proxy_fetcher/providers/free_proxy_list_ssl.rb
 - lib/proxy_fetcher/providers/gather_proxy.rb
 - lib/proxy_fetcher/providers/http_tunnel.rb
+- lib/proxy_fetcher/providers/mtpro.rb
 - lib/proxy_fetcher/providers/proxy_list.rb
+- lib/proxy_fetcher/providers/proxypedia.rb
 - lib/proxy_fetcher/providers/xroxy.rb
 - lib/proxy_fetcher/proxy.rb
 - lib/proxy_fetcher/utils/http_client.rb
@@ -116,8 +118,10 @@ files:
 - spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb
 - spec/proxy_fetcher/providers/gather_proxy_spec.rb
 - spec/proxy_fetcher/providers/http_tunnel_spec.rb
+- spec/proxy_fetcher/providers/mtpro_spec.rb
 - spec/proxy_fetcher/providers/multiple_providers_spec.rb
 - spec/proxy_fetcher/providers/proxy_list_spec.rb
+- spec/proxy_fetcher/providers/proxypedia_spec.rb
 - spec/proxy_fetcher/providers/xroxy_spec.rb
 - spec/proxy_fetcher/proxy_spec.rb
 - spec/proxy_fetcher/version_spec.rb