http_utilities 1.1.2 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +4 -9
- data/http_utilities.gemspec +10 -17
- data/lib/http_utilities/http/client.rb +62 -8
- data/lib/http_utilities/http/mechanize/client.rb +5 -1
- data/lib/http_utilities/http/proxy_support.rb +10 -0
- data/lib/http_utilities/http/request.rb +2 -3
- data/lib/http_utilities/http/response.rb +15 -4
- data/lib/http_utilities/http/user_agent.rb +1 -1
- data/lib/http_utilities/proxies/mongo/proxy_module.rb +97 -0
- data/lib/http_utilities/proxies/mysql/proxy_module.rb +93 -0
- data/lib/http_utilities/proxies/proxy_checker.rb +34 -66
- data/lib/http_utilities/proxies/proxy_seeder.rb +13 -11
- data/lib/http_utilities.rb +9 -17
- data/spec/http_utilities/client_spec.rb +5 -107
- data/spec/spec_helper.rb +3 -5
- metadata +21 -41
- data/lib/http_utilities/http/adapters/curb.rb +0 -107
- data/lib/http_utilities/http/adapters/net_http.rb +0 -135
- data/lib/http_utilities/http/adapters/open_uri.rb +0 -46
- data/lib/http_utilities/http/cookies.rb +0 -49
- data/lib/http_utilities/http/format.rb +0 -26
- data/lib/http_utilities/http/get.rb +0 -67
- data/lib/http_utilities/http/post.rb +0 -32
- data/lib/http_utilities/proxies/proxy_module.rb +0 -89
data/lib/http_utilities.rb
CHANGED
@@ -1,35 +1,32 @@
|
|
1
1
|
# -*- encoding : utf-8 -*-
|
2
2
|
module HttpUtilities
|
3
|
-
VERSION = "1.1.
|
3
|
+
VERSION = "1.1.5"
|
4
4
|
|
5
5
|
require File.join(File.dirname(__FILE__), 'http_utilities/railtie') if defined?(Rails)
|
6
6
|
|
7
7
|
require File.join(File.dirname(__FILE__), 'http_utilities/http/proxy_support')
|
8
|
-
require File.join(File.dirname(__FILE__), 'http_utilities/http/cookies')
|
9
8
|
require File.join(File.dirname(__FILE__), 'http_utilities/http/user_agent')
|
10
9
|
require File.join(File.dirname(__FILE__), 'http_utilities/http/url')
|
11
|
-
require File.join(File.dirname(__FILE__), 'http_utilities/http/format')
|
12
|
-
require File.join(File.dirname(__FILE__), 'http_utilities/http/get')
|
13
|
-
require File.join(File.dirname(__FILE__), 'http_utilities/http/post')
|
14
10
|
require File.join(File.dirname(__FILE__), 'http_utilities/http/logger')
|
15
11
|
|
16
12
|
require File.join(File.dirname(__FILE__), 'http_utilities/http/request')
|
17
13
|
require File.join(File.dirname(__FILE__), 'http_utilities/http/response')
|
18
14
|
|
19
|
-
require File.join(File.dirname(__FILE__), 'http_utilities/http/adapters/net_http')
|
20
|
-
require File.join(File.dirname(__FILE__), 'http_utilities/http/adapters/open_uri')
|
21
|
-
require File.join(File.dirname(__FILE__), 'http_utilities/http/adapters/curb')
|
22
|
-
|
23
15
|
require File.join(File.dirname(__FILE__), 'http_utilities/http/client')
|
24
16
|
|
25
17
|
require File.join(File.dirname(__FILE__), 'http_utilities/http/mechanize/client')
|
26
18
|
|
27
19
|
if defined?(ActiveRecord)
|
28
|
-
require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_module')
|
29
|
-
require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_checker')
|
20
|
+
require File.join(File.dirname(__FILE__), 'http_utilities/proxies/mysql/proxy_module')
|
30
21
|
require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_seeder')
|
31
22
|
end
|
32
23
|
|
24
|
+
if defined?(Mongoid)
|
25
|
+
require File.join(File.dirname(__FILE__), 'http_utilities/proxies/mongo/proxy_module')
|
26
|
+
end
|
27
|
+
|
28
|
+
require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_checker')
|
29
|
+
|
33
30
|
if defined?(Resque)
|
34
31
|
require File.join(File.dirname(__FILE__), 'http_utilities/jobs/resque/proxies/check_proxies_job')
|
35
32
|
require File.join(File.dirname(__FILE__), 'http_utilities/jobs/resque/proxies/check_proxy_job')
|
@@ -39,9 +36,4 @@ module HttpUtilities
|
|
39
36
|
require File.join(File.dirname(__FILE__), 'http_utilities/jobs/sidekiq/proxies/check_proxies_job')
|
40
37
|
require File.join(File.dirname(__FILE__), 'http_utilities/jobs/sidekiq/proxies/check_proxy_job')
|
41
38
|
end
|
42
|
-
|
43
|
-
if defined?(MultiXml)
|
44
|
-
MultiXml.parser = :nokogiri
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
39
|
+
end
|
@@ -9,42 +9,22 @@ describe HttpUtilities::Http::Client do
|
|
9
9
|
@response = HttpUtilities::Http::Response.new
|
10
10
|
end
|
11
11
|
|
12
|
-
it "should respond to
|
13
|
-
@client.should respond_to(:
|
12
|
+
it "should respond to #get" do
|
13
|
+
@client.should respond_to(:get)
|
14
14
|
end
|
15
15
|
|
16
|
-
it "should respond to
|
17
|
-
@client.should respond_to(:
|
18
|
-
end
|
19
|
-
|
20
|
-
it "should respond to a curb module method" do
|
21
|
-
@client.should respond_to(:post_and_retrieve_content_using_curl)
|
16
|
+
it "should respond to #post" do
|
17
|
+
@client.should respond_to(:post)
|
22
18
|
end
|
23
19
|
|
24
20
|
it "should respond to a proxy module method" do
|
25
21
|
@request.should respond_to(:set_proxy_options)
|
26
22
|
end
|
27
23
|
|
28
|
-
it "should respond to a cookies module method" do
|
29
|
-
@client.should respond_to(:handle_cookies)
|
30
|
-
end
|
31
|
-
|
32
24
|
it "should respond to a user agent module method" do
|
33
25
|
@request.should respond_to(:user_agent)
|
34
26
|
end
|
35
27
|
|
36
|
-
it "should respond to a request module method" do
|
37
|
-
@client.should respond_to(:generate_request_url)
|
38
|
-
end
|
39
|
-
|
40
|
-
it "should respond to a get module method" do
|
41
|
-
@client.should respond_to(:retrieve_raw_content)
|
42
|
-
end
|
43
|
-
|
44
|
-
it "should respond to a post module method" do
|
45
|
-
@client.should respond_to(:post_and_retrieve_content)
|
46
|
-
end
|
47
|
-
|
48
28
|
it "should respond to a format module method" do
|
49
29
|
@response.should respond_to(:as_html)
|
50
30
|
end
|
@@ -59,87 +39,5 @@ describe HttpUtilities::Http::Client do
|
|
59
39
|
it "should have assigned user agent" do
|
60
40
|
@request.user_agent.should_not == nil
|
61
41
|
end
|
62
|
-
|
63
|
-
it "should return a properly formatted request url using supplied parameters" do
|
64
|
-
params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
|
65
|
-
@client.generate_request_url(params).should == "http://www.google.com?q=ruby%20on%20rails&start=0"
|
66
|
-
end
|
67
|
-
|
68
|
-
describe "when retrieving content using Net::Http" do
|
69
|
-
it "should fetch Google results as unparsed HTML" do
|
70
|
-
params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
|
71
|
-
response = @client.retrieve_raw_content(@client.generate_request_url(params), {:method => :net_http})
|
72
|
-
response.body.should be_a(String)
|
73
|
-
end
|
74
|
-
|
75
|
-
it "should fetch Google results as a Nokogiri::HTML::Document" do
|
76
|
-
params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
|
77
|
-
response = @client.retrieve_parsed_html(@client.generate_request_url(params), {:method => :net_http})
|
78
|
-
response.parsed_body.should be_a(Nokogiri::HTML::Document)
|
79
|
-
end
|
80
|
-
|
81
|
-
it "should fetch Google Weather data a Nokogiri::XML::Document" do
|
82
|
-
params = {:url => "http://www.google.com/ig/api", :weather => 90120}
|
83
|
-
response = @client.retrieve_parsed_xml(@client.generate_request_url(params), {:method => :net_http})
|
84
|
-
response.parsed_body.should be_a(Nokogiri::XML::Document)
|
85
|
-
end
|
86
|
-
end
|
87
|
-
|
88
|
-
describe "when retrieving content using a proxy" do
|
89
|
-
it "should have the proxy instance variable properly set" do
|
90
|
-
options = {:method => :net_http, :proxy => "127.0.0.1:80", :response_only => false}
|
91
|
-
params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
|
92
|
-
|
93
|
-
response = @client.retrieve_parsed_html(@client.generate_request_url(params), options)
|
94
|
-
proxy = response.request.proxy
|
95
|
-
|
96
|
-
proxy.should_not be_nil
|
97
|
-
proxy[:host].should == '127.0.0.1'
|
98
|
-
proxy[:port].should == 80
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
describe "when persisting cookies" do
|
103
|
-
it "should have the cookie instance variable properly set" do
|
104
|
-
options = {:method => :net_http, :use_cookies => true, :save_cookies => true, :response_only => false}
|
105
|
-
params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
|
106
|
-
|
107
|
-
response = @client.retrieve_parsed_html(@client.generate_request_url(params), options)
|
108
|
-
cookies = response.request.cookies
|
109
|
-
|
110
|
-
cookies.should_not be_nil
|
111
|
-
end
|
112
|
-
end
|
113
|
-
|
114
|
-
describe "when posting content" do
|
115
|
-
before(:each) do
|
116
|
-
@trackback_url = "http://techcrunch.com/wp-trackback.php?p=314942"
|
117
|
-
@post_data = {
|
118
|
-
:url => "http://www.google.com",
|
119
|
-
:blog_name => "Testing",
|
120
|
-
:title => "Title",
|
121
|
-
:excerpt => "Testing..."
|
122
|
-
}
|
123
|
-
end
|
124
|
-
|
125
|
-
if (!defined?(JRUBY_VERSION))
|
126
|
-
it "should send a trackback to a TechCrunch post using Curb and return the response as a Nokogiri::XML::Document" do
|
127
|
-
options = {:method => :curl}
|
128
|
-
|
129
|
-
response = @client.post_and_retrieve_parsed_xml(@trackback_url, @post_data, options)
|
130
|
-
response.parsed_body.should be_a(Nokogiri::XML::Document)
|
131
|
-
end
|
132
|
-
end
|
133
|
-
|
134
|
-
it "should send a trackback to a TechCrunch post using Net::Http and return the response as a Nokogiri::XML::Document" do
|
135
|
-
options = {:method => :net_http}
|
136
|
-
|
137
|
-
response = @client.post_and_retrieve_parsed_xml(@trackback_url, @post_data, options)
|
138
|
-
response.parsed_body.should be_a(Nokogiri::XML::Document)
|
139
|
-
end
|
140
|
-
end
|
141
|
-
|
142
42
|
end
|
143
|
-
|
144
|
-
end
|
145
|
-
|
43
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -31,11 +31,9 @@ if File.exists?(database_yml)
|
|
31
31
|
ActiveRecord::Base.establish_connection(active_record_configuration)
|
32
32
|
ActiveRecord::Base.logger = Logger.new(File.join(File.dirname(__FILE__), "debug.log"))
|
33
33
|
|
34
|
-
ActiveRecord::
|
35
|
-
|
36
|
-
|
37
|
-
load(File.dirname(__FILE__) + '/models.rb')
|
38
|
-
end
|
34
|
+
ActiveRecord::Migration.verbose = false
|
35
|
+
load(File.dirname(__FILE__) + '/schema.rb')
|
36
|
+
load(File.dirname(__FILE__) + '/models.rb')
|
39
37
|
else
|
40
38
|
raise "Please create #{database_yml} first to configure your database. Take a look at: #{database_yml}.example"
|
41
39
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: http_utilities
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sebastian Johnsson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-08-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: nokogiri
|
@@ -16,64 +16,50 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 1.
|
19
|
+
version: '1.6'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 1.
|
26
|
+
version: '1.6'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: mechanize
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '2.
|
33
|
+
version: '2.7'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '2.
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: multi_xml
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
44
|
-
requirements:
|
45
|
-
- - ">="
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: '0.5'
|
48
|
-
type: :runtime
|
49
|
-
prerelease: false
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
51
|
-
requirements:
|
52
|
-
- - ">="
|
53
|
-
- !ruby/object:Gem::Version
|
54
|
-
version: '0.5'
|
40
|
+
version: '2.7'
|
55
41
|
- !ruby/object:Gem::Dependency
|
56
42
|
name: net-ssh
|
57
43
|
requirement: !ruby/object:Gem::Requirement
|
58
44
|
requirements:
|
59
45
|
- - ">="
|
60
46
|
- !ruby/object:Gem::Version
|
61
|
-
version: '2.
|
47
|
+
version: '2.9'
|
62
48
|
type: :runtime
|
63
49
|
prerelease: false
|
64
50
|
version_requirements: !ruby/object:Gem::Requirement
|
65
51
|
requirements:
|
66
52
|
- - ">="
|
67
53
|
- !ruby/object:Gem::Version
|
68
|
-
version: '2.
|
54
|
+
version: '2.9'
|
69
55
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
56
|
+
name: rails
|
71
57
|
requirement: !ruby/object:Gem::Requirement
|
72
58
|
requirements:
|
73
59
|
- - ">="
|
74
60
|
- !ruby/object:Gem::Version
|
75
61
|
version: '0'
|
76
|
-
type: :
|
62
|
+
type: :development
|
77
63
|
prerelease: false
|
78
64
|
version_requirements: !ruby/object:Gem::Requirement
|
79
65
|
requirements:
|
@@ -81,7 +67,7 @@ dependencies:
|
|
81
67
|
- !ruby/object:Gem::Version
|
82
68
|
version: '0'
|
83
69
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
70
|
+
name: rspec
|
85
71
|
requirement: !ruby/object:Gem::Requirement
|
86
72
|
requirements:
|
87
73
|
- - ">="
|
@@ -95,7 +81,7 @@ dependencies:
|
|
95
81
|
- !ruby/object:Gem::Version
|
96
82
|
version: '0'
|
97
83
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
84
|
+
name: sqlite3
|
99
85
|
requirement: !ruby/object:Gem::Requirement
|
100
86
|
requirements:
|
101
87
|
- - ">="
|
@@ -109,7 +95,7 @@ dependencies:
|
|
109
95
|
- !ruby/object:Gem::Version
|
110
96
|
version: '0'
|
111
97
|
- !ruby/object:Gem::Dependency
|
112
|
-
name:
|
98
|
+
name: mysql2
|
113
99
|
requirement: !ruby/object:Gem::Requirement
|
114
100
|
requirements:
|
115
101
|
- - ">="
|
@@ -123,20 +109,20 @@ dependencies:
|
|
123
109
|
- !ruby/object:Gem::Version
|
124
110
|
version: '0'
|
125
111
|
- !ruby/object:Gem::Dependency
|
126
|
-
name:
|
112
|
+
name: activerecord-import
|
127
113
|
requirement: !ruby/object:Gem::Requirement
|
128
114
|
requirements:
|
129
115
|
- - ">="
|
130
116
|
- !ruby/object:Gem::Version
|
131
|
-
version: 0
|
117
|
+
version: '0'
|
132
118
|
type: :development
|
133
119
|
prerelease: false
|
134
120
|
version_requirements: !ruby/object:Gem::Requirement
|
135
121
|
requirements:
|
136
122
|
- - ">="
|
137
123
|
- !ruby/object:Gem::Version
|
138
|
-
version: 0
|
139
|
-
description: Wrapper for
|
124
|
+
version: '0'
|
125
|
+
description: Wrapper for Faraday with additional functionality
|
140
126
|
email:
|
141
127
|
executables: []
|
142
128
|
extensions: []
|
@@ -156,16 +142,9 @@ files:
|
|
156
142
|
- lib/generators/templates/http_utilities.rb
|
157
143
|
- lib/generators/templates/user_agents.yml
|
158
144
|
- lib/http_utilities.rb
|
159
|
-
- lib/http_utilities/http/adapters/curb.rb
|
160
|
-
- lib/http_utilities/http/adapters/net_http.rb
|
161
|
-
- lib/http_utilities/http/adapters/open_uri.rb
|
162
145
|
- lib/http_utilities/http/client.rb
|
163
|
-
- lib/http_utilities/http/cookies.rb
|
164
|
-
- lib/http_utilities/http/format.rb
|
165
|
-
- lib/http_utilities/http/get.rb
|
166
146
|
- lib/http_utilities/http/logger.rb
|
167
147
|
- lib/http_utilities/http/mechanize/client.rb
|
168
|
-
- lib/http_utilities/http/post.rb
|
169
148
|
- lib/http_utilities/http/proxy_support.rb
|
170
149
|
- lib/http_utilities/http/request.rb
|
171
150
|
- lib/http_utilities/http/response.rb
|
@@ -175,8 +154,9 @@ files:
|
|
175
154
|
- lib/http_utilities/jobs/resque/proxies/check_proxy_job.rb
|
176
155
|
- lib/http_utilities/jobs/sidekiq/proxies/check_proxies_job.rb
|
177
156
|
- lib/http_utilities/jobs/sidekiq/proxies/check_proxy_job.rb
|
157
|
+
- lib/http_utilities/proxies/mongo/proxy_module.rb
|
158
|
+
- lib/http_utilities/proxies/mysql/proxy_module.rb
|
178
159
|
- lib/http_utilities/proxies/proxy_checker.rb
|
179
|
-
- lib/http_utilities/proxies/proxy_module.rb
|
180
160
|
- lib/http_utilities/proxies/proxy_seeder.rb
|
181
161
|
- lib/http_utilities/railtie.rb
|
182
162
|
- lib/tasks/http_utilities_tasks.rake
|
@@ -208,10 +188,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
208
188
|
version: 1.3.5
|
209
189
|
requirements: []
|
210
190
|
rubyforge_project:
|
211
|
-
rubygems_version: 2.
|
191
|
+
rubygems_version: 2.4.8
|
212
192
|
signing_key:
|
213
193
|
specification_version: 2
|
214
|
-
summary: Wrapper for
|
194
|
+
summary: Wrapper for Faraday with additional functionality
|
215
195
|
test_files:
|
216
196
|
- spec/http_utilities/client_spec.rb
|
217
197
|
- spec/http_utilities/mechanize_client_spec.rb
|
@@ -1,107 +0,0 @@
|
|
1
|
-
require 'uri'
|
2
|
-
|
3
|
-
module HttpUtilities
|
4
|
-
module Http
|
5
|
-
module Adapters
|
6
|
-
module Curb
|
7
|
-
|
8
|
-
def post_and_retrieve_content_using_curl(url, data, options = {})
|
9
|
-
request = self.set_curl_options(url, options)
|
10
|
-
response = nil
|
11
|
-
|
12
|
-
if (request.interface && data)
|
13
|
-
if (data.is_a?(Hash))
|
14
|
-
data = data.map { |key, value| Curl::PostField.content(key.to_s, value.to_s) }
|
15
|
-
end
|
16
|
-
|
17
|
-
request.interface.http_post(data)
|
18
|
-
response = request.interface.body_str
|
19
|
-
response = HttpUtilities::Http::Response.new(response, request, options)
|
20
|
-
end
|
21
|
-
|
22
|
-
return response
|
23
|
-
end
|
24
|
-
|
25
|
-
def retrieve_curl_content(url, options = {})
|
26
|
-
request = self.set_curl_options(url, options)
|
27
|
-
response = nil
|
28
|
-
|
29
|
-
begin
|
30
|
-
success = request.interface.perform
|
31
|
-
response = request.interface.body_str
|
32
|
-
response = HttpUtilities::Http::Response.new(response, request, options)
|
33
|
-
|
34
|
-
rescue Exception => e
|
35
|
-
puts "\n\n#{Time.now}: IMPORTANT! Error occurred while trying to retrieve content from url #{url} and parse it. Error: #{e.message}. Error Class: #{e.class}"
|
36
|
-
end
|
37
|
-
|
38
|
-
return response
|
39
|
-
end
|
40
|
-
|
41
|
-
def go_to_url(url, options = {})
|
42
|
-
success = false
|
43
|
-
|
44
|
-
request = self.set_curl_options(url, options)
|
45
|
-
|
46
|
-
success = request.interface.perform
|
47
|
-
success = (success && curl.response_code.eql?(200))
|
48
|
-
|
49
|
-
return success
|
50
|
-
end
|
51
|
-
|
52
|
-
def set_curl_options(url, options = {})
|
53
|
-
options = options.clone()
|
54
|
-
|
55
|
-
request = HttpUtilities::Http::Request.new
|
56
|
-
request.set_proxy_options(options)
|
57
|
-
|
58
|
-
accept_cookies = options.delete(:accept_cookies) { |e| false }
|
59
|
-
timeout = options.delete(:timeout) { |e| 120 }
|
60
|
-
maximum_redirects = options.delete(:maximum_redirects) { |e| 10 }
|
61
|
-
disable_auth = options.delete(:disable_auth) { |e| false }
|
62
|
-
accept_content_type = options.delete(:accept_content_type) { |e| false }
|
63
|
-
content_type = options.delete(:content_type) { |e| false }
|
64
|
-
cookie_file = nil
|
65
|
-
|
66
|
-
curl = Curl::Easy.new(url) do |c|
|
67
|
-
c.headers ||= {}
|
68
|
-
c.headers["User-Agent"] = c.useragent = request.user_agent
|
69
|
-
c.headers["Accept"] = accept_content_type if (accept_content_type)
|
70
|
-
c.headers["Content-Type"] = content_type if (content_type)
|
71
|
-
c.timeout = timeout
|
72
|
-
c.ssl_verify_host = false
|
73
|
-
c.follow_location = true
|
74
|
-
c.max_redirects = maximum_redirects
|
75
|
-
|
76
|
-
if (disable_auth)
|
77
|
-
c.http_auth_types = nil
|
78
|
-
c.proxy_auth_types = nil
|
79
|
-
c.unrestricted_auth = false
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
83
|
-
if (accept_cookies)
|
84
|
-
FileUtils.mkdir_p File.join(Rails.root, "tmp/cookies")
|
85
|
-
identifier = Time.now.to_date.to_s(:db).gsub("-", "_").gsub("\s", "_").gsub(":", "_")
|
86
|
-
cookie_file = File.join(Rails.root, "tmp/cookies", "cookies_#{identifier}.txt")
|
87
|
-
|
88
|
-
curl.enable_cookies = true
|
89
|
-
curl.cookiejar = cookie_file
|
90
|
-
curl.cookiefile = cookie_file
|
91
|
-
end
|
92
|
-
|
93
|
-
if (request.proxy[:host] && request.proxy[:port])
|
94
|
-
curl.proxy_url = ::Proxy.format_proxy_address(request.proxy[:host], request.proxy[:port], false)
|
95
|
-
curl.proxy_type = 5 if (request.proxy[:protocol] && request.proxy[:protocol].downcase.eql?('socks5'))
|
96
|
-
curl.proxypwd = ::Proxy.format_proxy_credentials(request.proxy[:username], request.proxy[:password]) if (request.proxy[:username] && request.proxy[:password])
|
97
|
-
end
|
98
|
-
|
99
|
-
request.interface = curl
|
100
|
-
|
101
|
-
return request
|
102
|
-
end
|
103
|
-
|
104
|
-
end
|
105
|
-
end
|
106
|
-
end
|
107
|
-
end
|
@@ -1,135 +0,0 @@
|
|
1
|
-
require 'open-uri'
|
2
|
-
require 'net/http'
|
3
|
-
require 'uri'
|
4
|
-
|
5
|
-
module HttpUtilities
|
6
|
-
module Http
|
7
|
-
module Adapters
|
8
|
-
module NetHttp
|
9
|
-
|
10
|
-
def post_and_retrieve_content_using_net_http(url, data, options = {})
|
11
|
-
uri = URI.parse(url)
|
12
|
-
request = set_net_http_options(uri, options)
|
13
|
-
response = nil
|
14
|
-
|
15
|
-
opts = options.clone()
|
16
|
-
content_type = opts.delete(:content_type) { |e| nil }
|
17
|
-
|
18
|
-
if (request.interface && data)
|
19
|
-
data = (data.is_a?(Hash)) ? generate_request_params(data) : data
|
20
|
-
|
21
|
-
request.interface.start do |http|
|
22
|
-
headers = {}
|
23
|
-
headers["User-Agent"] = request.user_agent
|
24
|
-
headers["Content-Type"] = content_type if (content_type)
|
25
|
-
|
26
|
-
http.post(uri.request_uri, data, headers) do |response_data|
|
27
|
-
response = response_data
|
28
|
-
end
|
29
|
-
|
30
|
-
response = HttpUtilities::Http::Response.new(response, request, options)
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
return response
|
35
|
-
end
|
36
|
-
|
37
|
-
def set_net_http_options(uri, options = {})
|
38
|
-
request = HttpUtilities::Http::Request.new
|
39
|
-
request.set_proxy_options(options)
|
40
|
-
|
41
|
-
request.interface = Net::HTTP.new(uri.host, uri.port, request.proxy[:host], request.proxy[:port])
|
42
|
-
|
43
|
-
use_ssl = options.fetch(:use_ssl, false)
|
44
|
-
request.interface.use_ssl = use_ssl
|
45
|
-
request.interface.verify_mode = OpenSSL::SSL::VERIFY_NONE if use_ssl
|
46
|
-
|
47
|
-
return request
|
48
|
-
end
|
49
|
-
|
50
|
-
def retrieve_net_http_content(url, options = {})
|
51
|
-
uri = URI.parse(url)
|
52
|
-
request = set_net_http_options(uri, options)
|
53
|
-
return perform_net_http_request(request, uri, options)
|
54
|
-
end
|
55
|
-
|
56
|
-
def perform_net_http_request(request_or_url, uri = nil, options = {}, redirect_count = 0, max_redirects = 5)
|
57
|
-
request = nil
|
58
|
-
response = nil
|
59
|
-
|
60
|
-
if (request_or_url)
|
61
|
-
opts = (options.is_a?(Hash)) ? options.clone() : {}
|
62
|
-
retries = opts.delete(:retries) { |e| 3 }
|
63
|
-
force_encoding = opts.delete(:force_encoding) { |e| false }
|
64
|
-
cookies = opts.delete(:cookies) { |e| nil }
|
65
|
-
timeout = opts.delete(:timeout) { |e| 30 }
|
66
|
-
|
67
|
-
if (request_or_url.is_a?(String))
|
68
|
-
uri = URI.parse(request_or_url)
|
69
|
-
request = self.set_net_http_options(uri, options)
|
70
|
-
else
|
71
|
-
request = request_or_url
|
72
|
-
end
|
73
|
-
|
74
|
-
if (uri && uri.respond_to?(:request_uri) && uri.request_uri)
|
75
|
-
headers = {"User-Agent" => request.user_agent}
|
76
|
-
headers = set_cookies(headers, cookies)
|
77
|
-
request_uri = uri.request_uri
|
78
|
-
http_request = Net::HTTP::Get.new(request_uri, headers)
|
79
|
-
|
80
|
-
begin
|
81
|
-
request.interface.start do |http|
|
82
|
-
http.read_timeout = timeout
|
83
|
-
response = http.request(http_request)
|
84
|
-
end
|
85
|
-
|
86
|
-
rescue Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::ENETUNREACH, Errno::ECONNRESET, Timeout::Error, Net::HTTPUnauthorized, Net::HTTPForbidden => error
|
87
|
-
log(:error, "[HttpUtilities::Http::Client] - Error occurred while trying to fetch url '#{uri.request_uri}'. Error Class: #{error.class.name}. Error Message: #{error.message}")
|
88
|
-
retries -= 1
|
89
|
-
retry if (retries > 0)
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
if (response)
|
95
|
-
location = response['location']
|
96
|
-
|
97
|
-
if (!(response.code.to_s =~ /^30\d{1}/i).nil? && location && location.present?)
|
98
|
-
location = location.strip.downcase
|
99
|
-
redirect_count += 1
|
100
|
-
|
101
|
-
if (redirect_count < max_redirects)
|
102
|
-
request.cookies = handle_cookies(response)
|
103
|
-
location = "http://#{uri.host}/#{location.gsub(/^\//i, "")}" if (uri && (location =~ /^http(s)?/i).nil?)
|
104
|
-
|
105
|
-
log(:info, "[HttpUtilities::Http::Client] - Redirecting to location: #{location}.")
|
106
|
-
|
107
|
-
options = options.merge(:cookies => request.cookies) if request.cookies
|
108
|
-
response = perform_net_http_request(location, uri, options, redirect_count, max_redirects)
|
109
|
-
end
|
110
|
-
end
|
111
|
-
|
112
|
-
request.cookies = handle_cookies(response)
|
113
|
-
response = set_response(response)
|
114
|
-
response = HttpUtilities::Http::Response.new(response, request, options)
|
115
|
-
end
|
116
|
-
|
117
|
-
return response
|
118
|
-
end
|
119
|
-
|
120
|
-
def set_response(response)
|
121
|
-
if (response.is_a?(String))
|
122
|
-
response = response
|
123
|
-
elsif (response.is_a?(Net::HTTPResponse))
|
124
|
-
response = response.body
|
125
|
-
elsif (response.is_a?(HttpUtilities::Http::Response))
|
126
|
-
response = response.body
|
127
|
-
end
|
128
|
-
|
129
|
-
return response
|
130
|
-
end
|
131
|
-
|
132
|
-
end
|
133
|
-
end
|
134
|
-
end
|
135
|
-
end
|
@@ -1,46 +0,0 @@
|
|
1
|
-
require 'open-uri'
|
2
|
-
require 'uri'
|
3
|
-
|
4
|
-
module HttpUtilities
|
5
|
-
module Http
|
6
|
-
module Adapters
|
7
|
-
module OpenUri
|
8
|
-
|
9
|
-
def retrieve_open_uri_content(url, options = {}, retries = 0, max_retries = 3)
|
10
|
-
response = nil
|
11
|
-
|
12
|
-
options = options.clone()
|
13
|
-
request = HttpUtilities::Http::Request.new
|
14
|
-
request.set_proxy_options(options)
|
15
|
-
|
16
|
-
open_uri_options = {"UserAgent" => request.user_agent}
|
17
|
-
open_uri_options[:read_timeout] = options.delete(:timeout) { |e| 120 }
|
18
|
-
|
19
|
-
if (request.proxy[:host] && request.proxy[:port])
|
20
|
-
proxy_address = Proxy.format_proxy_address(request.proxy[:host], request.proxy[:port], true)
|
21
|
-
open_uri_options[:proxy] = proxy_address
|
22
|
-
|
23
|
-
if (request.proxy[:username] && request.proxy[:password])
|
24
|
-
open_uri_options[:proxy_http_basic_authentication] = [proxy_address, request.proxy[:username], request.proxy[:password]]
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
connection = nil
|
29
|
-
while (connection.nil? && retries < max_retries)
|
30
|
-
connection = open(url, open_uri_options)
|
31
|
-
retries += 1
|
32
|
-
end
|
33
|
-
|
34
|
-
if (connection)
|
35
|
-
connection.rewind
|
36
|
-
response = connection.readlines.join("\n")
|
37
|
-
response = HttpUtilities::Http::Response.new(response, request)
|
38
|
-
end
|
39
|
-
|
40
|
-
return response
|
41
|
-
end
|
42
|
-
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|