proxy_fetcher 0.10.2 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -1
- data/Gemfile +8 -5
- data/Rakefile +7 -3
- data/gemfiles/nokogiri.gemfile +8 -6
- data/gemfiles/oga.gemfile +8 -6
- data/lib/proxy_fetcher.rb +46 -35
- data/lib/proxy_fetcher/client/client.rb +10 -3
- data/lib/proxy_fetcher/client/request.rb +4 -4
- data/lib/proxy_fetcher/configuration.rb +24 -19
- data/lib/proxy_fetcher/document.rb +0 -9
- data/lib/proxy_fetcher/document/adapters.rb +1 -1
- data/lib/proxy_fetcher/document/adapters/abstract_adapter.rb +3 -12
- data/lib/proxy_fetcher/document/adapters/nokogiri_adapter.rb +1 -1
- data/lib/proxy_fetcher/document/adapters/oga_adapter.rb +1 -1
- data/lib/proxy_fetcher/document/node.rb +2 -2
- data/lib/proxy_fetcher/exceptions.rb +6 -6
- data/lib/proxy_fetcher/manager.rb +42 -9
- data/lib/proxy_fetcher/providers/base.rb +43 -22
- data/lib/proxy_fetcher/providers/free_proxy_list.rb +9 -10
- data/lib/proxy_fetcher/providers/free_proxy_list_socks.rb +58 -0
- data/lib/proxy_fetcher/providers/free_proxy_list_ssl.rb +7 -15
- data/lib/proxy_fetcher/providers/free_proxy_list_us.rb +54 -0
- data/lib/proxy_fetcher/providers/http_tunnel.rb +11 -19
- data/lib/proxy_fetcher/providers/mtpro.rb +43 -0
- data/lib/proxy_fetcher/providers/proxy_list.rb +8 -16
- data/lib/proxy_fetcher/providers/proxypedia.rb +48 -0
- data/lib/proxy_fetcher/providers/proxyscrape_http.rb +65 -0
- data/lib/proxy_fetcher/providers/proxyscrape_socks4.rb +65 -0
- data/lib/proxy_fetcher/providers/proxyscrape_socks5.rb +65 -0
- data/lib/proxy_fetcher/providers/xroxy.rb +9 -17
- data/lib/proxy_fetcher/proxy.rb +16 -4
- data/lib/proxy_fetcher/utils/http_client.rb +7 -12
- data/lib/proxy_fetcher/utils/proxy_list_validator.rb +3 -1
- data/lib/proxy_fetcher/utils/proxy_validator.rb +21 -9
- data/lib/proxy_fetcher/version.rb +3 -3
- data/proxy_fetcher.gemspec +21 -16
- data/spec/fixtures/proxies.txt +14 -0
- data/spec/proxy_fetcher/client/client_spec.rb +72 -57
- data/spec/proxy_fetcher/configuration_spec.rb +11 -11
- data/spec/proxy_fetcher/document/adapters_spec.rb +8 -8
- data/spec/proxy_fetcher/document/node_spec.rb +4 -4
- data/spec/proxy_fetcher/manager_spec.rb +18 -0
- data/spec/proxy_fetcher/providers/base_spec.rb +9 -9
- data/spec/proxy_fetcher/providers/multiple_providers_spec.rb +4 -4
- data/spec/proxy_fetcher/providers/proxy_classes_spec.rb +28 -0
- data/spec/proxy_fetcher/proxy_spec.rb +14 -14
- data/spec/proxy_fetcher/version_spec.rb +2 -0
- data/spec/spec_helper.rb +10 -10
- data/spec/support/manager_examples.rb +21 -21
- metadata +27 -17
- data/lib/proxy_fetcher/providers/gather_proxy.rb +0 -58
- data/spec/proxy_fetcher/providers/free_proxy_list_spec.rb +0 -13
- data/spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb +0 -11
- data/spec/proxy_fetcher/providers/gather_proxy_spec.rb +0 -11
- data/spec/proxy_fetcher/providers/http_tunnel_spec.rb +0 -11
- data/spec/proxy_fetcher/providers/proxy_list_spec.rb +0 -11
- data/spec/proxy_fetcher/providers/xroxy_spec.rb +0 -11
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'json'
|
|
4
|
-
|
|
5
|
-
module ProxyFetcher
|
|
6
|
-
module Providers
|
|
7
|
-
# GatherProxy provider class.
|
|
8
|
-
class GatherProxy < Base
|
|
9
|
-
# Provider URL to fetch proxy list
|
|
10
|
-
def provider_url
|
|
11
|
-
'http://www.gatherproxy.com/'
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
# Fetches HTML content by sending HTTP request to the provider URL and
|
|
15
|
-
# parses the document (built as abstract <code>ProxyFetcher::Document</code>)
|
|
16
|
-
# to return all the proxy entries (HTML nodes).
|
|
17
|
-
#
|
|
18
|
-
# @return [Array<ProxyFetcher::Document::Node>]
|
|
19
|
-
# Collection of extracted HTML nodes with full proxy info
|
|
20
|
-
#
|
|
21
|
-
def load_proxy_list(*)
|
|
22
|
-
doc = load_document(provider_url)
|
|
23
|
-
doc.xpath('//div[@class="proxy-list"]/table/script')
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
# Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
|
|
27
|
-
# object.
|
|
28
|
-
#
|
|
29
|
-
# @param html_node [Object]
|
|
30
|
-
# HTML node from the <code>ProxyFetcher::Document</code> DOM model.
|
|
31
|
-
#
|
|
32
|
-
# @return [ProxyFetcher::Proxy]
|
|
33
|
-
# Proxy object
|
|
34
|
-
#
|
|
35
|
-
def to_proxy(html_node)
|
|
36
|
-
json = parse_json(html_node)
|
|
37
|
-
|
|
38
|
-
ProxyFetcher::Proxy.new.tap do |proxy|
|
|
39
|
-
proxy.addr = json['PROXY_IP']
|
|
40
|
-
proxy.port = json['PROXY_PORT'].to_i(16)
|
|
41
|
-
proxy.anonymity = json['PROXY_TYPE']
|
|
42
|
-
proxy.country = json['PROXY_COUNTRY']
|
|
43
|
-
proxy.response_time = json['PROXY_TIME'].to_i
|
|
44
|
-
proxy.type = ProxyFetcher::Proxy::HTTP
|
|
45
|
-
end
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
private
|
|
49
|
-
|
|
50
|
-
def parse_json(html_node)
|
|
51
|
-
javascript = html_node.content[/{.+}/im]
|
|
52
|
-
JSON.parse(javascript)
|
|
53
|
-
end
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
ProxyFetcher::Configuration.register_provider(:gather_proxy, GatherProxy)
|
|
57
|
-
end
|
|
58
|
-
end
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'spec_helper'
|
|
4
|
-
|
|
5
|
-
describe ProxyFetcher::Providers::FreeProxyList do
|
|
6
|
-
before :all do
|
|
7
|
-
ProxyFetcher.configure do |config|
|
|
8
|
-
config.provider = :free_proxy_list
|
|
9
|
-
end
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
it_behaves_like 'a manager'
|
|
13
|
-
end
|