proxy_fetcher 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -1
- data/Gemfile +1 -1
- data/lib/proxy_fetcher.rb +2 -0
- data/lib/proxy_fetcher/providers/free_proxy_list.rb +0 -21
- data/lib/proxy_fetcher/providers/mtpro.rb +43 -0
- data/lib/proxy_fetcher/providers/proxypedia.rb +48 -0
- data/lib/proxy_fetcher/version.rb +1 -1
- data/spec/proxy_fetcher/providers/mtpro_spec.rb +11 -0
- data/spec/proxy_fetcher/providers/proxypedia_spec.rb +11 -0
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 46d91f40f7c61a9a14aa90530ece0ef42365566ed6055abb141925ac471126d9
|
4
|
+
data.tar.gz: 72f56ef68ee8f46f226bf6de07e5888f485cd533ec2b6b8a64047b8530ec6b3c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a9f59f62fd5c731a1d15430faca81468f4a3368eb0287f5fad405ff3379af55886bd4a7608bba5a90f2117e42bb1d08a46ff6e63fe132c72a253f122164cbc4a
|
7
|
+
data.tar.gz: de8fa38b0895a6faf87c7ddac76b80a331bc6e6beb95a7972d94146d4ecaaf1025a4442bb3639d902e979cb57d1c1b6ba03f33137c56369692a5f2ca0b99b28e
|
data/CHANGELOG.md
CHANGED
data/Gemfile
CHANGED
data/lib/proxy_fetcher.rb
CHANGED
@@ -33,10 +33,12 @@ module ProxyFetcher
|
|
33
33
|
# ProxyFetcher providers namespace
|
34
34
|
module Providers
|
35
35
|
require File.dirname(__FILE__) + "/proxy_fetcher/providers/base"
|
36
|
+
require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxypedia"
|
36
37
|
require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list"
|
37
38
|
require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl"
|
38
39
|
require File.dirname(__FILE__) + "/proxy_fetcher/providers/gather_proxy"
|
39
40
|
require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel"
|
41
|
+
require File.dirname(__FILE__) + "/proxy_fetcher/providers/mtpro"
|
40
42
|
require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list"
|
41
43
|
require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy"
|
42
44
|
end
|
@@ -45,27 +45,6 @@ module ProxyFetcher
|
|
45
45
|
#
|
46
46
|
def parse_type(html_node)
|
47
47
|
https = html_node.content_at("td[6]")
|
48
|
-
# frozen_string_literal: true
|
49
|
-
# FreeProxyList provider class.
|
50
|
-
# Provider URL to fetch proxy list
|
51
|
-
# [NOTE] Doesn't support filtering
|
52
|
-
# Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
|
53
|
-
# object.
|
54
|
-
#
|
55
|
-
# @param html_node [Object]
|
56
|
-
# HTML node from the <code>ProxyFetcher::Document</code> DOM model.
|
57
|
-
#
|
58
|
-
# @return [ProxyFetcher::Proxy]
|
59
|
-
# Proxy object
|
60
|
-
#
|
61
|
-
# Parses HTML node to extract proxy type.
|
62
|
-
#
|
63
|
-
# @param html_node [Object]
|
64
|
-
# HTML node from the <code>ProxyFetcher::Document</code> DOM model.
|
65
|
-
#
|
66
|
-
# @return [String]
|
67
|
-
# Proxy type
|
68
|
-
#
|
69
48
|
https&.casecmp("yes")&.zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP
|
70
49
|
end
|
71
50
|
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "json"
|
4
|
+
|
5
|
+
module ProxyFetcher
|
6
|
+
module Providers
|
7
|
+
# GatherProxy provider class.
|
8
|
+
class MTPro < Base
|
9
|
+
# Provider URL to fetch proxy list
|
10
|
+
def provider_url
|
11
|
+
"https://mtpro.xyz/api/?type=socks"
|
12
|
+
end
|
13
|
+
|
14
|
+
def load_proxy_list(filters = {})
|
15
|
+
html = load_html(provider_url, filters)
|
16
|
+
JSON.parse(html)
|
17
|
+
rescue JSON::ParserError
|
18
|
+
[]
|
19
|
+
end
|
20
|
+
|
21
|
+
# Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
|
22
|
+
# object.
|
23
|
+
#
|
24
|
+
# @param node [Object]
|
25
|
+
# HTML node from the <code>ProxyFetcher::Document</code> DOM model.
|
26
|
+
#
|
27
|
+
# @return [ProxyFetcher::Proxy]
|
28
|
+
# Proxy object
|
29
|
+
#
|
30
|
+
def to_proxy(node)
|
31
|
+
ProxyFetcher::Proxy.new.tap do |proxy|
|
32
|
+
proxy.addr = node["ip"]
|
33
|
+
proxy.port = Integer(node["port"])
|
34
|
+
proxy.country = node["country"]
|
35
|
+
proxy.anonymity = "Unknown"
|
36
|
+
proxy.type = ProxyFetcher::Proxy::SOCKS5
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
ProxyFetcher::Configuration.register_provider(:mtpro, MTPro)
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ProxyFetcher
|
4
|
+
module Providers
|
5
|
+
# FreeProxyList provider class.
|
6
|
+
class Proxypedia < Base
|
7
|
+
# Provider URL to fetch proxy list
|
8
|
+
def provider_url
|
9
|
+
"https://proxypedia.org"
|
10
|
+
end
|
11
|
+
|
12
|
+
# [NOTE] Doesn't support filtering
|
13
|
+
def xpath
|
14
|
+
"//main/ul/li[position()>1]"
|
15
|
+
end
|
16
|
+
|
17
|
+
# Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
|
18
|
+
# object.]
|
19
|
+
#
|
20
|
+
# @param html_node [Object]
|
21
|
+
# HTML node from the <code>ProxyFetcher::Document</code> DOM model.
|
22
|
+
#
|
23
|
+
# @return [ProxyFetcher::Proxy]
|
24
|
+
# Proxy object
|
25
|
+
#
|
26
|
+
def to_proxy(html_node)
|
27
|
+
addr, port = html_node.content_at("a").to_s.split(":")
|
28
|
+
|
29
|
+
ProxyFetcher::Proxy.new.tap do |proxy|
|
30
|
+
proxy.addr = addr
|
31
|
+
proxy.port = Integer(port)
|
32
|
+
proxy.country = parse_country(html_node)
|
33
|
+
proxy.anonymity = "Unknown"
|
34
|
+
proxy.type = ProxyFetcher::Proxy::HTTP
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def parse_country(html_node)
|
41
|
+
text = html_node.content.to_s
|
42
|
+
text[/\((.+?)\)/, 1] || "Unknown"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
ProxyFetcher::Configuration.register_provider(:proxypedia, Proxypedia)
|
47
|
+
end
|
48
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: proxy_fetcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.14.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nikita Bulai
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-05-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: http
|
@@ -97,7 +97,9 @@ files:
|
|
97
97
|
- lib/proxy_fetcher/providers/free_proxy_list_ssl.rb
|
98
98
|
- lib/proxy_fetcher/providers/gather_proxy.rb
|
99
99
|
- lib/proxy_fetcher/providers/http_tunnel.rb
|
100
|
+
- lib/proxy_fetcher/providers/mtpro.rb
|
100
101
|
- lib/proxy_fetcher/providers/proxy_list.rb
|
102
|
+
- lib/proxy_fetcher/providers/proxypedia.rb
|
101
103
|
- lib/proxy_fetcher/providers/xroxy.rb
|
102
104
|
- lib/proxy_fetcher/proxy.rb
|
103
105
|
- lib/proxy_fetcher/utils/http_client.rb
|
@@ -116,8 +118,10 @@ files:
|
|
116
118
|
- spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb
|
117
119
|
- spec/proxy_fetcher/providers/gather_proxy_spec.rb
|
118
120
|
- spec/proxy_fetcher/providers/http_tunnel_spec.rb
|
121
|
+
- spec/proxy_fetcher/providers/mtpro_spec.rb
|
119
122
|
- spec/proxy_fetcher/providers/multiple_providers_spec.rb
|
120
123
|
- spec/proxy_fetcher/providers/proxy_list_spec.rb
|
124
|
+
- spec/proxy_fetcher/providers/proxypedia_spec.rb
|
121
125
|
- spec/proxy_fetcher/providers/xroxy_spec.rb
|
122
126
|
- spec/proxy_fetcher/proxy_spec.rb
|
123
127
|
- spec/proxy_fetcher/version_spec.rb
|