proxy_fetcher 0.10.2 → 0.15.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -1
- data/Gemfile +8 -5
- data/Rakefile +7 -3
- data/gemfiles/nokogiri.gemfile +8 -6
- data/gemfiles/oga.gemfile +8 -6
- data/lib/proxy_fetcher.rb +46 -35
- data/lib/proxy_fetcher/client/client.rb +10 -3
- data/lib/proxy_fetcher/client/request.rb +4 -4
- data/lib/proxy_fetcher/configuration.rb +24 -19
- data/lib/proxy_fetcher/document.rb +0 -9
- data/lib/proxy_fetcher/document/adapters.rb +1 -1
- data/lib/proxy_fetcher/document/adapters/abstract_adapter.rb +3 -12
- data/lib/proxy_fetcher/document/adapters/nokogiri_adapter.rb +1 -1
- data/lib/proxy_fetcher/document/adapters/oga_adapter.rb +1 -1
- data/lib/proxy_fetcher/document/node.rb +2 -2
- data/lib/proxy_fetcher/exceptions.rb +6 -6
- data/lib/proxy_fetcher/manager.rb +42 -9
- data/lib/proxy_fetcher/providers/base.rb +43 -22
- data/lib/proxy_fetcher/providers/free_proxy_list.rb +9 -10
- data/lib/proxy_fetcher/providers/free_proxy_list_socks.rb +58 -0
- data/lib/proxy_fetcher/providers/free_proxy_list_ssl.rb +7 -15
- data/lib/proxy_fetcher/providers/free_proxy_list_us.rb +54 -0
- data/lib/proxy_fetcher/providers/http_tunnel.rb +11 -19
- data/lib/proxy_fetcher/providers/mtpro.rb +43 -0
- data/lib/proxy_fetcher/providers/proxy_list.rb +8 -16
- data/lib/proxy_fetcher/providers/proxypedia.rb +48 -0
- data/lib/proxy_fetcher/providers/proxyscrape_http.rb +65 -0
- data/lib/proxy_fetcher/providers/proxyscrape_socks4.rb +65 -0
- data/lib/proxy_fetcher/providers/proxyscrape_socks5.rb +65 -0
- data/lib/proxy_fetcher/providers/xroxy.rb +9 -17
- data/lib/proxy_fetcher/proxy.rb +16 -4
- data/lib/proxy_fetcher/utils/http_client.rb +7 -12
- data/lib/proxy_fetcher/utils/proxy_list_validator.rb +3 -1
- data/lib/proxy_fetcher/utils/proxy_validator.rb +21 -9
- data/lib/proxy_fetcher/version.rb +3 -3
- data/proxy_fetcher.gemspec +21 -16
- data/spec/fixtures/proxies.txt +14 -0
- data/spec/proxy_fetcher/client/client_spec.rb +72 -57
- data/spec/proxy_fetcher/configuration_spec.rb +11 -11
- data/spec/proxy_fetcher/document/adapters_spec.rb +8 -8
- data/spec/proxy_fetcher/document/node_spec.rb +4 -4
- data/spec/proxy_fetcher/manager_spec.rb +18 -0
- data/spec/proxy_fetcher/providers/base_spec.rb +9 -9
- data/spec/proxy_fetcher/providers/multiple_providers_spec.rb +4 -4
- data/spec/proxy_fetcher/providers/proxy_classes_spec.rb +28 -0
- data/spec/proxy_fetcher/proxy_spec.rb +14 -14
- data/spec/proxy_fetcher/version_spec.rb +2 -0
- data/spec/spec_helper.rb +10 -10
- data/spec/support/manager_examples.rb +21 -21
- metadata +27 -17
- data/lib/proxy_fetcher/providers/gather_proxy.rb +0 -58
- data/spec/proxy_fetcher/providers/free_proxy_list_spec.rb +0 -13
- data/spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb +0 -11
- data/spec/proxy_fetcher/providers/gather_proxy_spec.rb +0 -11
- data/spec/proxy_fetcher/providers/http_tunnel_spec.rb +0 -11
- data/spec/proxy_fetcher/providers/proxy_list_spec.rb +0 -11
- data/spec/proxy_fetcher/providers/xroxy_spec.rb +0 -11
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'json'
|
|
4
|
-
|
|
5
|
-
module ProxyFetcher
|
|
6
|
-
module Providers
|
|
7
|
-
# GatherProxy provider class.
|
|
8
|
-
class GatherProxy < Base
|
|
9
|
-
# Provider URL to fetch proxy list
|
|
10
|
-
def provider_url
|
|
11
|
-
'http://www.gatherproxy.com/'
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
# Fetches HTML content by sending HTTP request to the provider URL and
|
|
15
|
-
# parses the document (built as abstract <code>ProxyFetcher::Document</code>)
|
|
16
|
-
# to return all the proxy entries (HTML nodes).
|
|
17
|
-
#
|
|
18
|
-
# @return [Array<ProxyFetcher::Document::Node>]
|
|
19
|
-
# Collection of extracted HTML nodes with full proxy info
|
|
20
|
-
#
|
|
21
|
-
def load_proxy_list(*)
|
|
22
|
-
doc = load_document(provider_url)
|
|
23
|
-
doc.xpath('//div[@class="proxy-list"]/table/script')
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
# Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
|
|
27
|
-
# object.
|
|
28
|
-
#
|
|
29
|
-
# @param html_node [Object]
|
|
30
|
-
# HTML node from the <code>ProxyFetcher::Document</code> DOM model.
|
|
31
|
-
#
|
|
32
|
-
# @return [ProxyFetcher::Proxy]
|
|
33
|
-
# Proxy object
|
|
34
|
-
#
|
|
35
|
-
def to_proxy(html_node)
|
|
36
|
-
json = parse_json(html_node)
|
|
37
|
-
|
|
38
|
-
ProxyFetcher::Proxy.new.tap do |proxy|
|
|
39
|
-
proxy.addr = json['PROXY_IP']
|
|
40
|
-
proxy.port = json['PROXY_PORT'].to_i(16)
|
|
41
|
-
proxy.anonymity = json['PROXY_TYPE']
|
|
42
|
-
proxy.country = json['PROXY_COUNTRY']
|
|
43
|
-
proxy.response_time = json['PROXY_TIME'].to_i
|
|
44
|
-
proxy.type = ProxyFetcher::Proxy::HTTP
|
|
45
|
-
end
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
private
|
|
49
|
-
|
|
50
|
-
def parse_json(html_node)
|
|
51
|
-
javascript = html_node.content[/{.+}/im]
|
|
52
|
-
JSON.parse(javascript)
|
|
53
|
-
end
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
ProxyFetcher::Configuration.register_provider(:gather_proxy, GatherProxy)
|
|
57
|
-
end
|
|
58
|
-
end
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'spec_helper'
|
|
4
|
-
|
|
5
|
-
describe ProxyFetcher::Providers::FreeProxyList do
|
|
6
|
-
before :all do
|
|
7
|
-
ProxyFetcher.configure do |config|
|
|
8
|
-
config.provider = :free_proxy_list
|
|
9
|
-
end
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
it_behaves_like 'a manager'
|
|
13
|
-
end
|