http_utilities 1.1.1 → 1.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/http_utilities.gemspec +1 -1
- data/lib/http_utilities/proxies/proxy_checker.rb +16 -15
- data/lib/http_utilities.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1a18294693d9cd95171156f4fccdf461a8c45127
|
|
4
|
+
data.tar.gz: 6d32ede9f59c390b64ebcfcc47ec00901fe6e77f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 51c959c41979b8c1a253c1b5d43e019ac19e692b9fbb7f92b04da37be639099949bd540e2ed2b068f30d9943a2a3f46212401bc416e4a1a44730d1758e553f09
|
|
7
|
+
data.tar.gz: 0fe6a191e9f0e802622d5eb24a2529e0b6c9d1c1cdc1525347570492c1d27aae74baa15e5650566dbc69701397e58d9df47ea4fb8e105002aeea8b69ca98c930
|
data/http_utilities.gemspec
CHANGED
|
@@ -3,7 +3,7 @@ Gem::Specification.new do |s|
|
|
|
3
3
|
s.required_rubygems_version = Gem::Requirement.new(">= 1.3.5") if s.respond_to? :required_rubygems_version=
|
|
4
4
|
|
|
5
5
|
s.name = "http_utilities"
|
|
6
|
-
s.version = "1.1.
|
|
6
|
+
s.version = "1.1.2"
|
|
7
7
|
|
|
8
8
|
s.authors = ["Sebastian Johnsson"]
|
|
9
9
|
s.description = "Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)"
|
|
@@ -11,7 +11,9 @@ module HttpUtilities
|
|
|
11
11
|
attr_accessor :limit, :minimum_successful_attempts, :maximum_failed_attempts
|
|
12
12
|
|
|
13
13
|
def initialize
|
|
14
|
-
self.client = HttpUtilities::Http::Client.new
|
|
14
|
+
self.client = HttpUtilities::Http::Mechanize::Client.new(verbose: false)
|
|
15
|
+
self.client.agent.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
16
|
+
|
|
15
17
|
self.processed_proxies = []
|
|
16
18
|
|
|
17
19
|
self.limit = 1000
|
|
@@ -88,26 +90,25 @@ module HttpUtilities
|
|
|
88
90
|
document = nil
|
|
89
91
|
valid_proxy = false
|
|
90
92
|
|
|
91
|
-
options
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
disable_auth: true
|
|
98
|
-
}
|
|
93
|
+
options = {
|
|
94
|
+
use_proxy: true,
|
|
95
|
+
proxy: proxy.proxy_address,
|
|
96
|
+
proxy_protocol: proxy.protocol,
|
|
97
|
+
timeout: timeout,
|
|
98
|
+
}
|
|
99
99
|
|
|
100
100
|
Rails.logger.info "#{Time.now}: Fetching Google.com with proxy #{proxy.proxy_address}."
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
if (
|
|
105
|
-
|
|
101
|
+
|
|
102
|
+
page = self.client.get_page("https://www.google.com/webhp?hl=en&gws_rd=ssl", options)
|
|
103
|
+
|
|
104
|
+
if (page)
|
|
105
|
+
parser = self.client.get_parser(page)
|
|
106
|
+
title = parser.at_css("head title")
|
|
106
107
|
|
|
107
108
|
if (title && title.content)
|
|
108
109
|
begin
|
|
109
110
|
title = title.content.encode("UTF-8").strip.downcase
|
|
110
|
-
body_content =
|
|
111
|
+
body_content = page.body.to_s.encode("UTF-8").strip.downcase
|
|
111
112
|
|
|
112
113
|
valid_proxy = (title.eql?("google") || !(body_content =~ /google home/i).nil?)
|
|
113
114
|
|
data/lib/http_utilities.rb
CHANGED