http_utilities 1.1.2 → 1.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,35 +1,32 @@
1
1
  # -*- encoding : utf-8 -*-
2
2
  module HttpUtilities
3
- VERSION = "1.1.2"
3
+ VERSION = "1.1.5"
4
4
 
5
5
  require File.join(File.dirname(__FILE__), 'http_utilities/railtie') if defined?(Rails)
6
6
 
7
7
  require File.join(File.dirname(__FILE__), 'http_utilities/http/proxy_support')
8
- require File.join(File.dirname(__FILE__), 'http_utilities/http/cookies')
9
8
  require File.join(File.dirname(__FILE__), 'http_utilities/http/user_agent')
10
9
  require File.join(File.dirname(__FILE__), 'http_utilities/http/url')
11
- require File.join(File.dirname(__FILE__), 'http_utilities/http/format')
12
- require File.join(File.dirname(__FILE__), 'http_utilities/http/get')
13
- require File.join(File.dirname(__FILE__), 'http_utilities/http/post')
14
10
  require File.join(File.dirname(__FILE__), 'http_utilities/http/logger')
15
11
 
16
12
  require File.join(File.dirname(__FILE__), 'http_utilities/http/request')
17
13
  require File.join(File.dirname(__FILE__), 'http_utilities/http/response')
18
14
 
19
- require File.join(File.dirname(__FILE__), 'http_utilities/http/adapters/net_http')
20
- require File.join(File.dirname(__FILE__), 'http_utilities/http/adapters/open_uri')
21
- require File.join(File.dirname(__FILE__), 'http_utilities/http/adapters/curb')
22
-
23
15
  require File.join(File.dirname(__FILE__), 'http_utilities/http/client')
24
16
 
25
17
  require File.join(File.dirname(__FILE__), 'http_utilities/http/mechanize/client')
26
18
 
27
19
  if defined?(ActiveRecord)
28
- require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_module')
29
- require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_checker')
20
+ require File.join(File.dirname(__FILE__), 'http_utilities/proxies/mysql/proxy_module')
30
21
  require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_seeder')
31
22
  end
32
23
 
24
+ if defined?(Mongoid)
25
+ require File.join(File.dirname(__FILE__), 'http_utilities/proxies/mongo/proxy_module')
26
+ end
27
+
28
+ require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_checker')
29
+
33
30
  if defined?(Resque)
34
31
  require File.join(File.dirname(__FILE__), 'http_utilities/jobs/resque/proxies/check_proxies_job')
35
32
  require File.join(File.dirname(__FILE__), 'http_utilities/jobs/resque/proxies/check_proxy_job')
@@ -39,9 +36,4 @@ module HttpUtilities
39
36
  require File.join(File.dirname(__FILE__), 'http_utilities/jobs/sidekiq/proxies/check_proxies_job')
40
37
  require File.join(File.dirname(__FILE__), 'http_utilities/jobs/sidekiq/proxies/check_proxy_job')
41
38
  end
42
-
43
- if defined?(MultiXml)
44
- MultiXml.parser = :nokogiri
45
- end
46
- end
47
-
39
+ end
@@ -9,42 +9,22 @@ describe HttpUtilities::Http::Client do
9
9
  @response = HttpUtilities::Http::Response.new
10
10
  end
11
11
 
12
- it "should respond to a net http module method" do
13
- @client.should respond_to(:post_and_retrieve_content_using_net_http)
12
+ it "should respond to #get" do
13
+ @client.should respond_to(:get)
14
14
  end
15
15
 
16
- it "should respond to a open uri module method" do
17
- @client.should respond_to(:retrieve_open_uri_content)
18
- end
19
-
20
- it "should respond to a curb module method" do
21
- @client.should respond_to(:post_and_retrieve_content_using_curl)
16
+ it "should respond to #post" do
17
+ @client.should respond_to(:post)
22
18
  end
23
19
 
24
20
  it "should respond to a proxy module method" do
25
21
  @request.should respond_to(:set_proxy_options)
26
22
  end
27
23
 
28
- it "should respond to a cookies module method" do
29
- @client.should respond_to(:handle_cookies)
30
- end
31
-
32
24
  it "should respond to a user agent module method" do
33
25
  @request.should respond_to(:user_agent)
34
26
  end
35
27
 
36
- it "should respond to a request module method" do
37
- @client.should respond_to(:generate_request_url)
38
- end
39
-
40
- it "should respond to a get module method" do
41
- @client.should respond_to(:retrieve_raw_content)
42
- end
43
-
44
- it "should respond to a post module method" do
45
- @client.should respond_to(:post_and_retrieve_content)
46
- end
47
-
48
28
  it "should respond to a format module method" do
49
29
  @response.should respond_to(:as_html)
50
30
  end
@@ -59,87 +39,5 @@ describe HttpUtilities::Http::Client do
59
39
  it "should have assigned user agent" do
60
40
  @request.user_agent.should_not == nil
61
41
  end
62
-
63
- it "should return a properly formatted request url using supplied parameters" do
64
- params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
65
- @client.generate_request_url(params).should == "http://www.google.com?q=ruby%20on%20rails&start=0"
66
- end
67
-
68
- describe "when retrieving content using Net::Http" do
69
- it "should fetch Google results as unparsed HTML" do
70
- params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
71
- response = @client.retrieve_raw_content(@client.generate_request_url(params), {:method => :net_http})
72
- response.body.should be_a(String)
73
- end
74
-
75
- it "should fetch Google results as a Nokogiri::HTML::Document" do
76
- params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
77
- response = @client.retrieve_parsed_html(@client.generate_request_url(params), {:method => :net_http})
78
- response.parsed_body.should be_a(Nokogiri::HTML::Document)
79
- end
80
-
81
- it "should fetch Google Weather data a Nokogiri::XML::Document" do
82
- params = {:url => "http://www.google.com/ig/api", :weather => 90120}
83
- response = @client.retrieve_parsed_xml(@client.generate_request_url(params), {:method => :net_http})
84
- response.parsed_body.should be_a(Nokogiri::XML::Document)
85
- end
86
- end
87
-
88
- describe "when retrieving content using a proxy" do
89
- it "should have the proxy instance variable properly set" do
90
- options = {:method => :net_http, :proxy => "127.0.0.1:80", :response_only => false}
91
- params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
92
-
93
- response = @client.retrieve_parsed_html(@client.generate_request_url(params), options)
94
- proxy = response.request.proxy
95
-
96
- proxy.should_not be_nil
97
- proxy[:host].should == '127.0.0.1'
98
- proxy[:port].should == 80
99
- end
100
- end
101
-
102
- describe "when persisting cookies" do
103
- it "should have the cookie instance variable properly set" do
104
- options = {:method => :net_http, :use_cookies => true, :save_cookies => true, :response_only => false}
105
- params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
106
-
107
- response = @client.retrieve_parsed_html(@client.generate_request_url(params), options)
108
- cookies = response.request.cookies
109
-
110
- cookies.should_not be_nil
111
- end
112
- end
113
-
114
- describe "when posting content" do
115
- before(:each) do
116
- @trackback_url = "http://techcrunch.com/wp-trackback.php?p=314942"
117
- @post_data = {
118
- :url => "http://www.google.com",
119
- :blog_name => "Testing",
120
- :title => "Title",
121
- :excerpt => "Testing..."
122
- }
123
- end
124
-
125
- if (!defined?(JRUBY_VERSION))
126
- it "should send a trackback to a TechCrunch post using Curb and return the response as a Nokogiri::XML::Document" do
127
- options = {:method => :curl}
128
-
129
- response = @client.post_and_retrieve_parsed_xml(@trackback_url, @post_data, options)
130
- response.parsed_body.should be_a(Nokogiri::XML::Document)
131
- end
132
- end
133
-
134
- it "should send a trackback to a TechCrunch post using Net::Http and return the response as a Nokogiri::XML::Document" do
135
- options = {:method => :net_http}
136
-
137
- response = @client.post_and_retrieve_parsed_xml(@trackback_url, @post_data, options)
138
- response.parsed_body.should be_a(Nokogiri::XML::Document)
139
- end
140
- end
141
-
142
42
  end
143
-
144
- end
145
-
43
+ end
data/spec/spec_helper.rb CHANGED
@@ -31,11 +31,9 @@ if File.exists?(database_yml)
31
31
  ActiveRecord::Base.establish_connection(active_record_configuration)
32
32
  ActiveRecord::Base.logger = Logger.new(File.join(File.dirname(__FILE__), "debug.log"))
33
33
 
34
- ActiveRecord::Base.silence do
35
- ActiveRecord::Migration.verbose = false
36
- load(File.dirname(__FILE__) + '/schema.rb')
37
- load(File.dirname(__FILE__) + '/models.rb')
38
- end
34
+ ActiveRecord::Migration.verbose = false
35
+ load(File.dirname(__FILE__) + '/schema.rb')
36
+ load(File.dirname(__FILE__) + '/models.rb')
39
37
  else
40
38
  raise "Please create #{database_yml} first to configure your database. Take a look at: #{database_yml}.example"
41
39
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: http_utilities
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.2
4
+ version: 1.1.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sebastian Johnsson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-02-07 00:00:00.000000000 Z
11
+ date: 2015-08-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: nokogiri
@@ -16,64 +16,50 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 1.5.5
19
+ version: '1.6'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: 1.5.5
26
+ version: '1.6'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: mechanize
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: '2.5'
33
+ version: '2.7'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: '2.5'
41
- - !ruby/object:Gem::Dependency
42
- name: multi_xml
43
- requirement: !ruby/object:Gem::Requirement
44
- requirements:
45
- - - ">="
46
- - !ruby/object:Gem::Version
47
- version: '0.5'
48
- type: :runtime
49
- prerelease: false
50
- version_requirements: !ruby/object:Gem::Requirement
51
- requirements:
52
- - - ">="
53
- - !ruby/object:Gem::Version
54
- version: '0.5'
40
+ version: '2.7'
55
41
  - !ruby/object:Gem::Dependency
56
42
  name: net-ssh
57
43
  requirement: !ruby/object:Gem::Requirement
58
44
  requirements:
59
45
  - - ">="
60
46
  - !ruby/object:Gem::Version
61
- version: '2.8'
47
+ version: '2.9'
62
48
  type: :runtime
63
49
  prerelease: false
64
50
  version_requirements: !ruby/object:Gem::Requirement
65
51
  requirements:
66
52
  - - ">="
67
53
  - !ruby/object:Gem::Version
68
- version: '2.8'
54
+ version: '2.9'
69
55
  - !ruby/object:Gem::Dependency
70
- name: activerecord-import
56
+ name: rails
71
57
  requirement: !ruby/object:Gem::Requirement
72
58
  requirements:
73
59
  - - ">="
74
60
  - !ruby/object:Gem::Version
75
61
  version: '0'
76
- type: :runtime
62
+ type: :development
77
63
  prerelease: false
78
64
  version_requirements: !ruby/object:Gem::Requirement
79
65
  requirements:
@@ -81,7 +67,7 @@ dependencies:
81
67
  - !ruby/object:Gem::Version
82
68
  version: '0'
83
69
  - !ruby/object:Gem::Dependency
84
- name: rails
70
+ name: rspec
85
71
  requirement: !ruby/object:Gem::Requirement
86
72
  requirements:
87
73
  - - ">="
@@ -95,7 +81,7 @@ dependencies:
95
81
  - !ruby/object:Gem::Version
96
82
  version: '0'
97
83
  - !ruby/object:Gem::Dependency
98
- name: rspec
84
+ name: sqlite3
99
85
  requirement: !ruby/object:Gem::Requirement
100
86
  requirements:
101
87
  - - ">="
@@ -109,7 +95,7 @@ dependencies:
109
95
  - !ruby/object:Gem::Version
110
96
  version: '0'
111
97
  - !ruby/object:Gem::Dependency
112
- name: sqlite3
98
+ name: mysql2
113
99
  requirement: !ruby/object:Gem::Requirement
114
100
  requirements:
115
101
  - - ">="
@@ -123,20 +109,20 @@ dependencies:
123
109
  - !ruby/object:Gem::Version
124
110
  version: '0'
125
111
  - !ruby/object:Gem::Dependency
126
- name: mysql2
112
+ name: activerecord-import
127
113
  requirement: !ruby/object:Gem::Requirement
128
114
  requirements:
129
115
  - - ">="
130
116
  - !ruby/object:Gem::Version
131
- version: 0.3.11
117
+ version: '0'
132
118
  type: :development
133
119
  prerelease: false
134
120
  version_requirements: !ruby/object:Gem::Requirement
135
121
  requirements:
136
122
  - - ">="
137
123
  - !ruby/object:Gem::Version
138
- version: 0.3.11
139
- description: Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)
124
+ version: '0'
125
+ description: Wrapper for Faraday with additional functionality
140
126
  email:
141
127
  executables: []
142
128
  extensions: []
@@ -156,16 +142,9 @@ files:
156
142
  - lib/generators/templates/http_utilities.rb
157
143
  - lib/generators/templates/user_agents.yml
158
144
  - lib/http_utilities.rb
159
- - lib/http_utilities/http/adapters/curb.rb
160
- - lib/http_utilities/http/adapters/net_http.rb
161
- - lib/http_utilities/http/adapters/open_uri.rb
162
145
  - lib/http_utilities/http/client.rb
163
- - lib/http_utilities/http/cookies.rb
164
- - lib/http_utilities/http/format.rb
165
- - lib/http_utilities/http/get.rb
166
146
  - lib/http_utilities/http/logger.rb
167
147
  - lib/http_utilities/http/mechanize/client.rb
168
- - lib/http_utilities/http/post.rb
169
148
  - lib/http_utilities/http/proxy_support.rb
170
149
  - lib/http_utilities/http/request.rb
171
150
  - lib/http_utilities/http/response.rb
@@ -175,8 +154,9 @@ files:
175
154
  - lib/http_utilities/jobs/resque/proxies/check_proxy_job.rb
176
155
  - lib/http_utilities/jobs/sidekiq/proxies/check_proxies_job.rb
177
156
  - lib/http_utilities/jobs/sidekiq/proxies/check_proxy_job.rb
157
+ - lib/http_utilities/proxies/mongo/proxy_module.rb
158
+ - lib/http_utilities/proxies/mysql/proxy_module.rb
178
159
  - lib/http_utilities/proxies/proxy_checker.rb
179
- - lib/http_utilities/proxies/proxy_module.rb
180
160
  - lib/http_utilities/proxies/proxy_seeder.rb
181
161
  - lib/http_utilities/railtie.rb
182
162
  - lib/tasks/http_utilities_tasks.rake
@@ -208,10 +188,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
208
188
  version: 1.3.5
209
189
  requirements: []
210
190
  rubyforge_project:
211
- rubygems_version: 2.2.2
191
+ rubygems_version: 2.4.8
212
192
  signing_key:
213
193
  specification_version: 2
214
- summary: Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)
194
+ summary: Wrapper for Faraday with additional functionality
215
195
  test_files:
216
196
  - spec/http_utilities/client_spec.rb
217
197
  - spec/http_utilities/mechanize_client_spec.rb
@@ -1,107 +0,0 @@
1
- require 'uri'
2
-
3
- module HttpUtilities
4
- module Http
5
- module Adapters
6
- module Curb
7
-
8
- def post_and_retrieve_content_using_curl(url, data, options = {})
9
- request = self.set_curl_options(url, options)
10
- response = nil
11
-
12
- if (request.interface && data)
13
- if (data.is_a?(Hash))
14
- data = data.map { |key, value| Curl::PostField.content(key.to_s, value.to_s) }
15
- end
16
-
17
- request.interface.http_post(data)
18
- response = request.interface.body_str
19
- response = HttpUtilities::Http::Response.new(response, request, options)
20
- end
21
-
22
- return response
23
- end
24
-
25
- def retrieve_curl_content(url, options = {})
26
- request = self.set_curl_options(url, options)
27
- response = nil
28
-
29
- begin
30
- success = request.interface.perform
31
- response = request.interface.body_str
32
- response = HttpUtilities::Http::Response.new(response, request, options)
33
-
34
- rescue Exception => e
35
- puts "\n\n#{Time.now}: IMPORTANT! Error occurred while trying to retrieve content from url #{url} and parse it. Error: #{e.message}. Error Class: #{e.class}"
36
- end
37
-
38
- return response
39
- end
40
-
41
- def go_to_url(url, options = {})
42
- success = false
43
-
44
- request = self.set_curl_options(url, options)
45
-
46
- success = request.interface.perform
47
- success = (success && curl.response_code.eql?(200))
48
-
49
- return success
50
- end
51
-
52
- def set_curl_options(url, options = {})
53
- options = options.clone()
54
-
55
- request = HttpUtilities::Http::Request.new
56
- request.set_proxy_options(options)
57
-
58
- accept_cookies = options.delete(:accept_cookies) { |e| false }
59
- timeout = options.delete(:timeout) { |e| 120 }
60
- maximum_redirects = options.delete(:maximum_redirects) { |e| 10 }
61
- disable_auth = options.delete(:disable_auth) { |e| false }
62
- accept_content_type = options.delete(:accept_content_type) { |e| false }
63
- content_type = options.delete(:content_type) { |e| false }
64
- cookie_file = nil
65
-
66
- curl = Curl::Easy.new(url) do |c|
67
- c.headers ||= {}
68
- c.headers["User-Agent"] = c.useragent = request.user_agent
69
- c.headers["Accept"] = accept_content_type if (accept_content_type)
70
- c.headers["Content-Type"] = content_type if (content_type)
71
- c.timeout = timeout
72
- c.ssl_verify_host = false
73
- c.follow_location = true
74
- c.max_redirects = maximum_redirects
75
-
76
- if (disable_auth)
77
- c.http_auth_types = nil
78
- c.proxy_auth_types = nil
79
- c.unrestricted_auth = false
80
- end
81
- end
82
-
83
- if (accept_cookies)
84
- FileUtils.mkdir_p File.join(Rails.root, "tmp/cookies")
85
- identifier = Time.now.to_date.to_s(:db).gsub("-", "_").gsub("\s", "_").gsub(":", "_")
86
- cookie_file = File.join(Rails.root, "tmp/cookies", "cookies_#{identifier}.txt")
87
-
88
- curl.enable_cookies = true
89
- curl.cookiejar = cookie_file
90
- curl.cookiefile = cookie_file
91
- end
92
-
93
- if (request.proxy[:host] && request.proxy[:port])
94
- curl.proxy_url = ::Proxy.format_proxy_address(request.proxy[:host], request.proxy[:port], false)
95
- curl.proxy_type = 5 if (request.proxy[:protocol] && request.proxy[:protocol].downcase.eql?('socks5'))
96
- curl.proxypwd = ::Proxy.format_proxy_credentials(request.proxy[:username], request.proxy[:password]) if (request.proxy[:username] && request.proxy[:password])
97
- end
98
-
99
- request.interface = curl
100
-
101
- return request
102
- end
103
-
104
- end
105
- end
106
- end
107
- end
@@ -1,135 +0,0 @@
1
- require 'open-uri'
2
- require 'net/http'
3
- require 'uri'
4
-
5
- module HttpUtilities
6
- module Http
7
- module Adapters
8
- module NetHttp
9
-
10
- def post_and_retrieve_content_using_net_http(url, data, options = {})
11
- uri = URI.parse(url)
12
- request = set_net_http_options(uri, options)
13
- response = nil
14
-
15
- opts = options.clone()
16
- content_type = opts.delete(:content_type) { |e| nil }
17
-
18
- if (request.interface && data)
19
- data = (data.is_a?(Hash)) ? generate_request_params(data) : data
20
-
21
- request.interface.start do |http|
22
- headers = {}
23
- headers["User-Agent"] = request.user_agent
24
- headers["Content-Type"] = content_type if (content_type)
25
-
26
- http.post(uri.request_uri, data, headers) do |response_data|
27
- response = response_data
28
- end
29
-
30
- response = HttpUtilities::Http::Response.new(response, request, options)
31
- end
32
- end
33
-
34
- return response
35
- end
36
-
37
- def set_net_http_options(uri, options = {})
38
- request = HttpUtilities::Http::Request.new
39
- request.set_proxy_options(options)
40
-
41
- request.interface = Net::HTTP.new(uri.host, uri.port, request.proxy[:host], request.proxy[:port])
42
-
43
- use_ssl = options.fetch(:use_ssl, false)
44
- request.interface.use_ssl = use_ssl
45
- request.interface.verify_mode = OpenSSL::SSL::VERIFY_NONE if use_ssl
46
-
47
- return request
48
- end
49
-
50
- def retrieve_net_http_content(url, options = {})
51
- uri = URI.parse(url)
52
- request = set_net_http_options(uri, options)
53
- return perform_net_http_request(request, uri, options)
54
- end
55
-
56
- def perform_net_http_request(request_or_url, uri = nil, options = {}, redirect_count = 0, max_redirects = 5)
57
- request = nil
58
- response = nil
59
-
60
- if (request_or_url)
61
- opts = (options.is_a?(Hash)) ? options.clone() : {}
62
- retries = opts.delete(:retries) { |e| 3 }
63
- force_encoding = opts.delete(:force_encoding) { |e| false }
64
- cookies = opts.delete(:cookies) { |e| nil }
65
- timeout = opts.delete(:timeout) { |e| 30 }
66
-
67
- if (request_or_url.is_a?(String))
68
- uri = URI.parse(request_or_url)
69
- request = self.set_net_http_options(uri, options)
70
- else
71
- request = request_or_url
72
- end
73
-
74
- if (uri && uri.respond_to?(:request_uri) && uri.request_uri)
75
- headers = {"User-Agent" => request.user_agent}
76
- headers = set_cookies(headers, cookies)
77
- request_uri = uri.request_uri
78
- http_request = Net::HTTP::Get.new(request_uri, headers)
79
-
80
- begin
81
- request.interface.start do |http|
82
- http.read_timeout = timeout
83
- response = http.request(http_request)
84
- end
85
-
86
- rescue Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::ENETUNREACH, Errno::ECONNRESET, Timeout::Error, Net::HTTPUnauthorized, Net::HTTPForbidden => error
87
- log(:error, "[HttpUtilities::Http::Client] - Error occurred while trying to fetch url '#{uri.request_uri}'. Error Class: #{error.class.name}. Error Message: #{error.message}")
88
- retries -= 1
89
- retry if (retries > 0)
90
- end
91
- end
92
- end
93
-
94
- if (response)
95
- location = response['location']
96
-
97
- if (!(response.code.to_s =~ /^30\d{1}/i).nil? && location && location.present?)
98
- location = location.strip.downcase
99
- redirect_count += 1
100
-
101
- if (redirect_count < max_redirects)
102
- request.cookies = handle_cookies(response)
103
- location = "http://#{uri.host}/#{location.gsub(/^\//i, "")}" if (uri && (location =~ /^http(s)?/i).nil?)
104
-
105
- log(:info, "[HttpUtilities::Http::Client] - Redirecting to location: #{location}.")
106
-
107
- options = options.merge(:cookies => request.cookies) if request.cookies
108
- response = perform_net_http_request(location, uri, options, redirect_count, max_redirects)
109
- end
110
- end
111
-
112
- request.cookies = handle_cookies(response)
113
- response = set_response(response)
114
- response = HttpUtilities::Http::Response.new(response, request, options)
115
- end
116
-
117
- return response
118
- end
119
-
120
- def set_response(response)
121
- if (response.is_a?(String))
122
- response = response
123
- elsif (response.is_a?(Net::HTTPResponse))
124
- response = response.body
125
- elsif (response.is_a?(HttpUtilities::Http::Response))
126
- response = response.body
127
- end
128
-
129
- return response
130
- end
131
-
132
- end
133
- end
134
- end
135
- end
@@ -1,46 +0,0 @@
1
- require 'open-uri'
2
- require 'uri'
3
-
4
- module HttpUtilities
5
- module Http
6
- module Adapters
7
- module OpenUri
8
-
9
- def retrieve_open_uri_content(url, options = {}, retries = 0, max_retries = 3)
10
- response = nil
11
-
12
- options = options.clone()
13
- request = HttpUtilities::Http::Request.new
14
- request.set_proxy_options(options)
15
-
16
- open_uri_options = {"UserAgent" => request.user_agent}
17
- open_uri_options[:read_timeout] = options.delete(:timeout) { |e| 120 }
18
-
19
- if (request.proxy[:host] && request.proxy[:port])
20
- proxy_address = Proxy.format_proxy_address(request.proxy[:host], request.proxy[:port], true)
21
- open_uri_options[:proxy] = proxy_address
22
-
23
- if (request.proxy[:username] && request.proxy[:password])
24
- open_uri_options[:proxy_http_basic_authentication] = [proxy_address, request.proxy[:username], request.proxy[:password]]
25
- end
26
- end
27
-
28
- connection = nil
29
- while (connection.nil? && retries < max_retries)
30
- connection = open(url, open_uri_options)
31
- retries += 1
32
- end
33
-
34
- if (connection)
35
- connection.rewind
36
- response = connection.readlines.join("\n")
37
- response = HttpUtilities::Http::Response.new(response, request)
38
- end
39
-
40
- return response
41
- end
42
-
43
- end
44
- end
45
- end
46
- end