proxy_fetcher 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '09772e9bc018d8accb01401b2e8b1897804e0a948c67a7eeeef8b1a2e9fcd245'
4
- data.tar.gz: 16b49036c2ecdd06e23f53c4fe55982ec7ceb1531e9e223f31590eb83417c4e0
3
+ metadata.gz: 46d91f40f7c61a9a14aa90530ece0ef42365566ed6055abb141925ac471126d9
4
+ data.tar.gz: 72f56ef68ee8f46f226bf6de07e5888f485cd533ec2b6b8a64047b8530ec6b3c
5
5
  SHA512:
6
- metadata.gz: 526c8fdcfb1171c09f2e9bf6a4e13dbbc6e837963ce1c1ef7e2e48ed1ec3052e3dbc6a4d04bb33a0b3f083914149c22af4b4238a6c558825a91e3f26e112a378
7
- data.tar.gz: 8dddea87295d1825910e4eecb4a5a16836b06e0070cc07f823a4902d6952550ae3b016bfb2e29d0ce8cc16ba3add12ad28a6ab6fec8eccd8a1656e368d88be5d
6
+ metadata.gz: a9f59f62fd5c731a1d15430faca81468f4a3368eb0287f5fad405ff3379af55886bd4a7608bba5a90f2117e42bb1d08a46ff6e63fe132c72a253f122164cbc4a
7
+ data.tar.gz: de8fa38b0895a6faf87c7ddac76b80a331bc6e6beb95a7972d94146d4ecaaf1025a4442bb3639d902e979cb57d1c1b6ba03f33137c56369692a5f2ca0b99b28e
@@ -4,7 +4,12 @@ Reverse Chronological Order:
4
4
 
5
5
  ## `master`
6
6
 
7
- * Add your description here
7
+ ...
8
+
9
+ ## `0.14.0` (2020-05-11)
10
+
11
+ * Add MTPro provider
12
+ * Add Proxypedia provider
8
13
 
9
14
  ## `0.13.0` (2020-03-09)
10
15
 
data/Gemfile CHANGED
@@ -6,7 +6,7 @@ gemspec
6
6
 
7
7
  gem "nokogiri", "~> 1.8"
8
8
  gem "oga", "~> 3.2"
9
- gem "rubocop", "~> 0.74"
9
+ gem "rubocop", "~> 0.80"
10
10
 
11
11
  group :test do
12
12
  gem "coveralls", require: false
@@ -33,10 +33,12 @@ module ProxyFetcher
33
33
  # ProxyFetcher providers namespace
34
34
  module Providers
35
35
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/base"
36
+ require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxypedia"
36
37
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list"
37
38
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl"
38
39
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/gather_proxy"
39
40
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel"
41
+ require File.dirname(__FILE__) + "/proxy_fetcher/providers/mtpro"
40
42
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list"
41
43
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy"
42
44
  end
@@ -45,27 +45,6 @@ module ProxyFetcher
45
45
  #
46
46
  def parse_type(html_node)
47
47
  https = html_node.content_at("td[6]")
48
- # frozen_string_literal: true
49
- # FreeProxyList provider class.
50
- # Provider URL to fetch proxy list
51
- # [NOTE] Doesn't support filtering
52
- # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
53
- # object.
54
- #
55
- # @param html_node [Object]
56
- # HTML node from the <code>ProxyFetcher::Document</code> DOM model.
57
- #
58
- # @return [ProxyFetcher::Proxy]
59
- # Proxy object
60
- #
61
- # Parses HTML node to extract proxy type.
62
- #
63
- # @param html_node [Object]
64
- # HTML node from the <code>ProxyFetcher::Document</code> DOM model.
65
- #
66
- # @return [String]
67
- # Proxy type
68
- #
69
48
  https&.casecmp("yes")&.zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP
70
49
  end
71
50
  end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module ProxyFetcher
6
+ module Providers
7
+ # GatherProxy provider class.
8
+ class MTPro < Base
9
+ # Provider URL to fetch proxy list
10
+ def provider_url
11
+ "https://mtpro.xyz/api/?type=socks"
12
+ end
13
+
14
+ def load_proxy_list(filters = {})
15
+ html = load_html(provider_url, filters)
16
+ JSON.parse(html)
17
+ rescue JSON::ParserError
18
+ []
19
+ end
20
+
21
+ # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
22
+ # object.
23
+ #
24
+ # @param node [Object]
25
+ # HTML node from the <code>ProxyFetcher::Document</code> DOM model.
26
+ #
27
+ # @return [ProxyFetcher::Proxy]
28
+ # Proxy object
29
+ #
30
+ def to_proxy(node)
31
+ ProxyFetcher::Proxy.new.tap do |proxy|
32
+ proxy.addr = node["ip"]
33
+ proxy.port = Integer(node["port"])
34
+ proxy.country = node["country"]
35
+ proxy.anonymity = "Unknown"
36
+ proxy.type = ProxyFetcher::Proxy::SOCKS5
37
+ end
38
+ end
39
+ end
40
+
41
+ ProxyFetcher::Configuration.register_provider(:mtpro, MTPro)
42
+ end
43
+ end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ProxyFetcher
4
+ module Providers
5
+ # FreeProxyList provider class.
6
+ class Proxypedia < Base
7
+ # Provider URL to fetch proxy list
8
+ def provider_url
9
+ "https://proxypedia.org"
10
+ end
11
+
12
+ # [NOTE] Doesn't support filtering
13
+ def xpath
14
+ "//main/ul/li[position()>1]"
15
+ end
16
+
17
+ # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
18
+ # object.]
19
+ #
20
+ # @param html_node [Object]
21
+ # HTML node from the <code>ProxyFetcher::Document</code> DOM model.
22
+ #
23
+ # @return [ProxyFetcher::Proxy]
24
+ # Proxy object
25
+ #
26
+ def to_proxy(html_node)
27
+ addr, port = html_node.content_at("a").to_s.split(":")
28
+
29
+ ProxyFetcher::Proxy.new.tap do |proxy|
30
+ proxy.addr = addr
31
+ proxy.port = Integer(port)
32
+ proxy.country = parse_country(html_node)
33
+ proxy.anonymity = "Unknown"
34
+ proxy.type = ProxyFetcher::Proxy::HTTP
35
+ end
36
+ end
37
+
38
+ private
39
+
40
+ def parse_country(html_node)
41
+ text = html_node.content.to_s
42
+ text[/\((.+?)\)/, 1] || "Unknown"
43
+ end
44
+ end
45
+
46
+ ProxyFetcher::Configuration.register_provider(:proxypedia, Proxypedia)
47
+ end
48
+ end
@@ -13,7 +13,7 @@ module ProxyFetcher
13
13
  # Major version number
14
14
  MAJOR = 0
15
15
  # Minor version number
16
- MINOR = 13
16
+ MINOR = 14
17
17
  # Smallest version number
18
18
  TINY = 0
19
19
 
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "spec_helper"
4
+
5
+ describe ProxyFetcher::Providers::MTPro do
6
+ before :all do
7
+ ProxyFetcher.config.provider = :mtpro
8
+ end
9
+
10
+ it_behaves_like "a manager"
11
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "spec_helper"
4
+
5
+ describe ProxyFetcher::Providers::Proxypedia do
6
+ before :all do
7
+ ProxyFetcher.config.provider = :proxypedia
8
+ end
9
+
10
+ it_behaves_like "a manager"
11
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: proxy_fetcher
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.13.0
4
+ version: 0.14.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nikita Bulai
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-03-09 00:00:00.000000000 Z
11
+ date: 2020-05-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: http
@@ -97,7 +97,9 @@ files:
97
97
  - lib/proxy_fetcher/providers/free_proxy_list_ssl.rb
98
98
  - lib/proxy_fetcher/providers/gather_proxy.rb
99
99
  - lib/proxy_fetcher/providers/http_tunnel.rb
100
+ - lib/proxy_fetcher/providers/mtpro.rb
100
101
  - lib/proxy_fetcher/providers/proxy_list.rb
102
+ - lib/proxy_fetcher/providers/proxypedia.rb
101
103
  - lib/proxy_fetcher/providers/xroxy.rb
102
104
  - lib/proxy_fetcher/proxy.rb
103
105
  - lib/proxy_fetcher/utils/http_client.rb
@@ -116,8 +118,10 @@ files:
116
118
  - spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb
117
119
  - spec/proxy_fetcher/providers/gather_proxy_spec.rb
118
120
  - spec/proxy_fetcher/providers/http_tunnel_spec.rb
121
+ - spec/proxy_fetcher/providers/mtpro_spec.rb
119
122
  - spec/proxy_fetcher/providers/multiple_providers_spec.rb
120
123
  - spec/proxy_fetcher/providers/proxy_list_spec.rb
124
+ - spec/proxy_fetcher/providers/proxypedia_spec.rb
121
125
  - spec/proxy_fetcher/providers/xroxy_spec.rb
122
126
  - spec/proxy_fetcher/proxy_spec.rb
123
127
  - spec/proxy_fetcher/version_spec.rb