right_http_connection 1.2.4 → 1.3.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -57,3 +57,6 @@ Initial public release
57
57
  * fixed a bug: <NoMethodError: You have a nil object when you didn't expect it!
58
58
  The error occurred while evaluating nil.body_stream>
59
59
 
60
+ == 1.2.5 (not released yet)
61
+
62
+ - ActiveSupport dependency removal
@@ -4,4 +4,5 @@ README.txt
4
4
  Rakefile
5
5
  lib/net_fix.rb
6
6
  lib/right_http_connection.rb
7
+ lib/support.rb
7
8
  setup.rb
data/Rakefile CHANGED
@@ -1,4 +1,6 @@
1
1
  require 'rubygems'
2
+ require 'bundler'
3
+ Bundler::GemHelper.install_tasks
2
4
  require 'rake'
3
5
  require 'rake/clean'
4
6
  require 'rake/testtask'
@@ -6,67 +8,25 @@ require 'rake/packagetask'
6
8
  require 'rake/gempackagetask'
7
9
  require 'rake/rdoctask'
8
10
  require 'rake/contrib/rubyforgepublisher'
11
+ require 'rspec/core/rake_task'
12
+ require 'cucumber/rake/task'
9
13
  require 'fileutils'
10
- require 'hoe'
11
14
  include FileUtils
12
15
  require File.join(File.dirname(__FILE__), 'lib', 'right_http_connection')
13
16
 
14
- AUTHOR = 'RightScale' # can also be an array of Authors
15
- EMAIL = "rubygems@rightscale.com"
16
- DESCRIPTION = "RightScale's robust HTTP/S connection module"
17
- GEM_NAME = 'right_http_connection' # what ppl will type to install your gem
18
- RUBYFORGE_PROJECT = 'rightscale' # The unix name for your project
19
- HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
20
- DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
21
-
22
- NAME = "right_http_connection"
23
- REV = nil # UNCOMMENT IF REQUIRED: File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil
24
- VERS = RightHttpConnection::VERSION::STRING + (REV ? ".#{REV}" : "")
25
- CLEAN.include ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']
26
- RDOC_OPTS = ['--quiet', '--title', 'right_http_connection documentation',
27
- "--opname", "index.html",
28
- "--line-numbers",
29
- "--main", "README",
30
- "--inline-source"]
31
-
32
- # Suppress Hoe's self-inclusion as a dependency for our Gem. This also keeps
33
- # Rake & rubyforge out of the dependency list. Users must manually install
34
- # these gems to run tests, etc.
35
- # TRB 2/19/09: also do this for the extra_dev_deps array present in newer hoes.
36
- # Older versions of RubyGems will try to install developer-dependencies as
37
- # required runtime dependencies....
38
- class Hoe
39
- def extra_deps
40
- @extra_deps.reject do |x|
41
- Array(x).first == 'hoe'
42
- end
43
- end
44
- def extra_dev_deps
45
- @extra_dev_deps.reject do |x|
46
- Array(x).first == 'hoe'
47
- end
48
- end
49
- end
17
+ Bundler::GemHelper.install_tasks
18
+
19
+ # == Gem == #
50
20
 
51
- # Generate all the Rake tasks
52
- # Run 'rake -T' to see list of generated tasks (from gem root directory)
53
- hoe = Hoe.new(GEM_NAME, VERS) do |p|
54
- p.author = AUTHOR
55
- p.description = DESCRIPTION
56
- p.email = EMAIL
57
- p.summary = DESCRIPTION
58
- p.url = HOMEPATH
59
- p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
60
- p.test_globs = ["test/**/test_*.rb"]
61
- p.clean_globs = CLEAN #An array of file patterns to delete on clean.
62
- p.remote_rdoc_dir = "right_http_gem_doc"
63
-
64
- # == Optional
65
- p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
66
- #p.extra_deps = [] # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
67
- #p.spec_extras = {} # A hash of extra values to set in the gemspec.
21
+ gemtask = Rake::GemPackageTask.new(Gem::Specification.load("right_http_connection.gemspec")) do |package|
22
+ package.package_dir = ENV['PACKAGE_DIR'] || 'pkg'
23
+ package.need_zip = true
24
+ package.need_tar = true
68
25
  end
69
26
 
27
+ directory gemtask.package_dir
28
+
29
+ CLEAN.include(gemtask.package_dir)
70
30
 
71
31
  desc 'Generate website files'
72
32
  task :website_generate do
@@ -101,3 +61,29 @@ task :check_version do
101
61
  exit
102
62
  end
103
63
  end
64
+
65
+ task :default => 'spec'
66
+
67
+ # == Unit Tests == #
68
+
69
+ desc "Run unit tests"
70
+ RSpec::Core::RakeTask.new
71
+
72
+ namespace :spec do
73
+ desc "Run unit tests with RCov"
74
+ RSpec::Core::RakeTask.new(:rcov) do |t|
75
+ t.rcov = true
76
+ t.rcov_opts = %q[--exclude "spec"]
77
+ end
78
+
79
+ desc "Print Specdoc for unit tests"
80
+ RSpec::Core::RakeTask.new(:doc) do |t|
81
+ t.rspec_opts = ["--format", "documentation"]
82
+ end
83
+ end
84
+
85
+ # == Functional tests == #
86
+ desc "Run functional tests"
87
+ Cucumber::Rake::Task.new do |t|
88
+ t.cucumber_opts = %w{--color --format pretty}
89
+ end
@@ -27,20 +27,10 @@ require "time"
27
27
  require "logger"
28
28
 
29
29
  $:.unshift(File.dirname(__FILE__))
30
+ require 'version'
31
+ require 'support'
30
32
  require "net_fix"
31
33
 
32
-
33
- module RightHttpConnection #:nodoc:
34
- module VERSION #:nodoc:
35
- MAJOR = 1
36
- MINOR = 2
37
- TINY = 4
38
-
39
- STRING = [MAJOR, MINOR, TINY].join('.')
40
- end
41
- end
42
-
43
-
44
34
  module Rightscale
45
35
 
46
36
  =begin rdoc
@@ -78,13 +68,13 @@ them.
78
68
  class HttpConnection
79
69
 
80
70
  # Number of times to retry the request after encountering the first error
81
- HTTP_CONNECTION_RETRY_COUNT = 3
71
+ HTTP_CONNECTION_RETRY_COUNT = 3 unless defined?(HTTP_CONNECTION_RETRY_COUNT)
82
72
  # Throw a Timeout::Error if a connection isn't established within this number of seconds
83
- HTTP_CONNECTION_OPEN_TIMEOUT = 5
73
+ HTTP_CONNECTION_OPEN_TIMEOUT = 5 unless defined?(HTTP_CONNECTION_OPEN_TIMEOUT)
84
74
  # Throw a Timeout::Error if no data have been read on this connnection within this number of seconds
85
- HTTP_CONNECTION_READ_TIMEOUT = 120
75
+ HTTP_CONNECTION_READ_TIMEOUT = 120 unless defined?(HTTP_CONNECTION_READ_TIMEOUT)
86
76
  # Length of the post-error probationary period during which all requests will fail
87
- HTTP_CONNECTION_RETRY_DELAY = 15
77
+ HTTP_CONNECTION_RETRY_DELAY = 15 unless defined?(HTTP_CONNECTION_RETRY_DELAY)
88
78
 
89
79
  #--------------------
90
80
  # class methods
@@ -100,13 +90,19 @@ them.
100
90
  #
101
91
  # :user_agent => 'www.HostName.com' # String to report as HTTP User agent
102
92
  # :ca_file => 'path_to_file' # Path to a CA certification file in PEM format. The file can contain several CA certificates. If this parameter isn't set, HTTPS certs won't be verified.
93
+ # :fail_if_ca_mismatch => Boolean # If ca_file is set and the server certificate doesn't verify, a log line is generated regardless, but normally right_http_connection continues on past the failure. If this is set, fail to connect in that case. Defaults to false.
103
94
  # :logger => Logger object # If omitted, HttpConnection logs to STDOUT
104
95
  # :exception => Exception to raise # The type of exception to raise
105
96
  # # if a request repeatedly fails. RuntimeError is raised if this parameter is omitted.
97
+ # :proxy_host => 'hostname' # hostname of HTTP proxy host to use, default none.
98
+ # :proxy_port => port # port of HTTP proxy host to use, default none.
99
+ # :proxy_username => 'username' # username to use for proxy authentication, default none.
100
+ # :proxy_password => 'password' # password to use for proxy authentication, default none.
106
101
  # :http_connection_retry_count # by default == Rightscale::HttpConnection::HTTP_CONNECTION_RETRY_COUNT
107
102
  # :http_connection_open_timeout # by default == Rightscale::HttpConnection::HTTP_CONNECTION_OPEN_TIMEOUT
108
103
  # :http_connection_read_timeout # by default == Rightscale::HttpConnection::HTTP_CONNECTION_READ_TIMEOUT
109
104
  # :http_connection_retry_delay # by default == Rightscale::HttpConnection::HTTP_CONNECTION_RETRY_DELAY
105
+ # :raise_on_timeout # do not perform a retry if timeout is received (false by default)
110
106
  def self.params
111
107
  @@params
112
108
  end
@@ -127,28 +123,42 @@ them.
127
123
  # Params hash:
128
124
  # :user_agent => 'www.HostName.com' # String to report as HTTP User agent
129
125
  # :ca_file => 'path_to_file' # A path of a CA certification file in PEM format. The file can contain several CA certificates.
126
+ # :fail_if_ca_mismatch => Boolean # If ca_file is set and the server certificate doesn't verify, a log line is generated regardless, but normally right_http_connection continues on past the failure. If this is set, fail to connect in that case. Defaults to false.
130
127
  # :logger => Logger object # If omitted, HttpConnection logs to STDOUT
131
128
  # :exception => Exception to raise # The type of exception to raise if a request repeatedly fails. RuntimeError is raised if this parameter is omitted.
129
+ # :proxy_host => 'hostname' # hostname of HTTP proxy host to use, default none.
130
+ # :proxy_port => port # port of HTTP proxy host to use, default none.
131
+ # :proxy_username => 'username' # username to use for proxy authentication, default none.
132
+ # :proxy_password => 'password' # password to use for proxy authentication, default none.
132
133
  # :http_connection_retry_count # by default == Rightscale::HttpConnection.params[:http_connection_retry_count]
133
134
  # :http_connection_open_timeout # by default == Rightscale::HttpConnection.params[:http_connection_open_timeout]
134
135
  # :http_connection_read_timeout # by default == Rightscale::HttpConnection.params[:http_connection_read_timeout]
135
136
  # :http_connection_retry_delay # by default == Rightscale::HttpConnection.params[:http_connection_retry_delay]
136
- #
137
+ # :raise_on_timeout # do not perform a retry if timeout is received (false by default)
137
138
  def initialize(params={})
138
139
  @params = params
139
140
  @params[:http_connection_retry_count] ||= @@params[:http_connection_retry_count]
140
141
  @params[:http_connection_open_timeout] ||= @@params[:http_connection_open_timeout]
141
142
  @params[:http_connection_read_timeout] ||= @@params[:http_connection_read_timeout]
142
143
  @params[:http_connection_retry_delay] ||= @@params[:http_connection_retry_delay]
144
+ @params[:proxy_host] ||= @@params[:proxy_host]
145
+ @params[:proxy_port] ||= @@params[:proxy_port]
146
+ @params[:proxy_username] ||= @@params[:proxy_username]
147
+ @params[:proxy_password] ||= @@params[:proxy_password]
143
148
  @http = nil
144
149
  @server = nil
145
150
  @logger = get_param(:logger) ||
146
151
  (RAILS_DEFAULT_LOGGER if defined?(RAILS_DEFAULT_LOGGER)) ||
147
152
  Logger.new(STDOUT)
153
+ #--------------
154
+ # Retry state - Keep track of errors on a per-server basis
155
+ #--------------
156
+ @state = {} # retry state indexed by server: consecutive error count, error time, and error
157
+ @eof = {}
148
158
  end
149
159
 
150
- def get_param(name)
151
- @params[name] || @@params[name]
160
+ def get_param(name, custom_options={})
161
+ custom_options [name] || @params[name] || @@params[name]
152
162
  end
153
163
 
154
164
  # Query for the maximum size (in bytes) of a single read from the underlying
@@ -180,35 +190,30 @@ them.
180
190
  end
181
191
 
182
192
  private
183
- #--------------
184
- # Retry state - Keep track of errors on a per-server basis
185
- #--------------
186
- @@state = {} # retry state indexed by server: consecutive error count, error time, and error
187
- @@eof = {}
188
193
 
189
194
  # number of consecutive errors seen for server, 0 all is ok
190
195
  def error_count
191
- @@state[@server] ? @@state[@server][:count] : 0
196
+ @state[@server] ? @state[@server][:count] : 0
192
197
  end
193
198
 
194
199
  # time of last error for server, nil if all is ok
195
200
  def error_time
196
- @@state[@server] && @@state[@server][:time]
201
+ @state[@server] && @state[@server][:time]
197
202
  end
198
203
 
199
204
  # message for last error for server, "" if all is ok
200
205
  def error_message
201
- @@state[@server] ? @@state[@server][:message] : ""
206
+ @state[@server] ? @state[@server][:message] : ""
202
207
  end
203
208
 
204
209
  # add an error for a server
205
210
  def error_add(message)
206
- @@state[@server] = { :count => error_count+1, :time => Time.now, :message => message }
211
+ @state[@server] = { :count => error_count+1, :time => Time.now, :message => message }
207
212
  end
208
213
 
209
214
  # reset the error state for a server (i.e. a request succeeded)
210
215
  def error_reset
211
- @@state.delete(@server)
216
+ @state.delete(@server)
212
217
  end
213
218
 
214
219
  # Error message stuff...
@@ -224,25 +229,25 @@ them.
224
229
  # Returns the number of seconds to wait before new conection retry:
225
230
  # 0.5, 1, 2, 4, 8
226
231
  def add_eof
227
- (@@eof[@server] ||= []).unshift Time.now
228
- 0.25 * 2 ** @@eof[@server].size
232
+ (@eof[@server] ||= []).unshift Time.now
233
+ 0.25 * 2 ** @eof[@server].size
229
234
  end
230
235
 
231
236
  # Returns first EOF timestamp or nul if have no EOFs being tracked.
232
237
  def eof_time
233
- @@eof[@server] && @@eof[@server].last
238
+ @eof[@server] && @eof[@server].last
234
239
  end
235
240
 
236
241
  # Returns true if we are receiving EOFs during last @params[:http_connection_retry_delay] seconds
237
242
  # and there were no successful response from server
238
243
  def raise_on_eof_exception?
239
- @@eof[@server].blank? ? false : ( (Time.now.to_i-@params[:http_connection_retry_delay]) > @@eof[@server].last.to_i )
244
+ @eof[@server].nil? ? false : ( (Time.now.to_i-@params[:http_connection_retry_delay]) > @eof[@server].last.to_i )
240
245
  end
241
246
 
242
247
  # Reset a list of EOFs for this server.
243
248
  # This is being called when we have got an successful response from server.
244
249
  def eof_reset
245
- @@eof.delete(@server)
250
+ @eof.delete(@server)
246
251
  end
247
252
 
248
253
  # Detects if an object is 'streamable' - can we read from it, and can we know the size?
@@ -281,14 +286,22 @@ them.
281
286
  # close the previous if exists
282
287
  finish
283
288
  # create new connection
284
- @server = request_params[:server]
285
- @port = request_params[:port]
286
- @protocol = request_params[:protocol]
289
+ @server = request_params[:server]
290
+ @port = request_params[:port]
291
+ @protocol = request_params[:protocol]
292
+ @proxy_host = request_params[:proxy_host]
293
+ @proxy_port = request_params[:proxy_port]
294
+ @proxy_username = request_params[:proxy_username]
295
+ @proxy_password = request_params[:proxy_password]
287
296
 
288
297
  @logger.info("Opening new #{@protocol.upcase} connection to #@server:#@port")
289
- @http = Net::HTTP.new(@server, @port)
290
- @http.open_timeout = @params[:http_connection_open_timeout]
291
- @http.read_timeout = @params[:http_connection_read_timeout]
298
+ @logger.info("Connecting to proxy #{@proxy_host}:#{@proxy_port} with username" +
299
+ " #{@proxy_username}") unless @proxy_host.nil?
300
+
301
+ @http = Net::HTTP.new(@server, @port, @proxy_host, @proxy_port, @proxy_username,
302
+ @proxy_password)
303
+ @http.open_timeout = get_param(:http_connection_open_timeout, request_params)
304
+ @http.read_timeout = get_param(:http_connection_read_timeout, request_params)
292
305
 
293
306
  if @protocol == 'https'
294
307
  verifyCallbackProc = Proc.new{ |ok, x509_store_ctx|
@@ -296,7 +309,11 @@ them.
296
309
  msg = x509_store_ctx.error_string
297
310
  #debugger
298
311
  @logger.warn("##### #{@server} certificate verify failed: #{msg}") unless code == 0
299
- true
312
+ if request_params[:fail_if_ca_mismatch] && code != 0
313
+ false
314
+ else
315
+ true
316
+ end
300
317
  }
301
318
  @http.use_ssl = true
302
319
  ca_file = get_param(:ca_file)
@@ -304,6 +321,8 @@ them.
304
321
  @http.verify_mode = OpenSSL::SSL::VERIFY_PEER
305
322
  @http.verify_callback = verifyCallbackProc
306
323
  @http.ca_file = ca_file
324
+ else
325
+ @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
307
326
  end
308
327
  end
309
328
  # open connection
@@ -320,45 +339,64 @@ them.
320
339
  :port => '80' # Port of HTTP server
321
340
  :protocol => 'https' # http and https are supported on any port
322
341
  :request => 'requeststring' # Fully-formed HTTP request to make
342
+ :proxy_host => 'hostname' # hostname of HTTP proxy host to use, default none.
343
+ :proxy_port => port # port of HTTP proxy host to use, default none.
344
+ :proxy_username => 'username' # username to use for proxy authentication, default none.
345
+ :proxy_password => 'password' # password to use for proxy authentication, default none.
346
+
347
+ :raise_on_timeout # do not perform a retry if timeout is received (false by default)
348
+ :http_connection_retry_count
349
+ :http_connection_open_timeout
350
+ :http_connection_read_timeout
351
+ :http_connection_retry_delay
352
+ :user_agent
353
+ :exception
323
354
 
324
355
  Raises RuntimeError, Interrupt, and params[:exception] (if specified in new).
325
356
 
326
357
  =end
327
358
  def request(request_params, &block)
359
+ current_params = @params.merge(request_params)
360
+ exception = get_param(:exception, current_params) || RuntimeError
361
+
328
362
  # We save the offset here so that if we need to retry, we can return the file pointer to its initial position
329
- mypos = get_fileptr_offset(request_params)
363
+ mypos = get_fileptr_offset(current_params)
330
364
  loop do
365
+ current_params[:protocol] ||= (current_params[:port] == 443 ? 'https' : 'http')
366
+ # (re)open connection to server if none exists or params has changed
367
+ same_server_as_before = @server == current_params[:server] &&
368
+ @port == current_params[:port] &&
369
+ @protocol == current_params[:protocol]
370
+
331
371
  # if we are inside a delay between retries: no requests this time!
332
- if error_count > @params[:http_connection_retry_count] &&
333
- error_time + @params[:http_connection_retry_delay] > Time.now
372
+ # (skip this step if the endpoint has changed)
373
+ if error_count > current_params[:http_connection_retry_count] &&
374
+ error_time + current_params[:http_connection_retry_delay] > Time.now &&
375
+ same_server_as_before
376
+
334
377
  # store the message (otherwise it will be lost after error_reset and
335
378
  # we will raise an exception with an empty text)
336
379
  banana_message_text = banana_message
337
380
  @logger.warn("#{err_header} re-raising same error: #{banana_message_text} " +
338
381
  "-- error count: #{error_count}, error age: #{Time.now.to_i - error_time.to_i}")
339
- exception = get_param(:exception) || RuntimeError
340
382
  raise exception.new(banana_message_text)
341
383
  end
342
384
 
343
385
  # try to connect server(if connection does not exist) and get response data
344
386
  begin
345
- request_params[:protocol] ||= (request_params[:port] == 443 ? 'https' : 'http')
346
-
347
- request = request_params[:request]
348
- request['User-Agent'] = get_param(:user_agent) || ''
349
-
350
- # (re)open connection to server if none exists or params has changed
387
+ request = current_params[:request]
388
+ request['User-Agent'] = get_param(:user_agent, current_params) || ''
351
389
  unless @http &&
352
390
  @http.started? &&
353
- @server == request_params[:server] &&
354
- @port == request_params[:port] &&
355
- @protocol == request_params[:protocol]
356
- start(request_params)
391
+ same_server_as_before
392
+ start(current_params)
357
393
  end
358
394
 
359
395
  # Detect if the body is a streamable object like a file or socket. If so, stream that
360
396
  # bad boy.
361
397
  setup_streaming(request)
398
+ # update READ_TIMEOUT value (it can be passed with request_params hash)
399
+ @http.read_timeout = get_param(:http_connection_read_timeout, current_params)
362
400
  response = @http.request(request, &block)
363
401
 
364
402
  error_reset
@@ -381,7 +419,6 @@ them.
381
419
 
382
420
  # if we have waited long enough - raise an exception...
383
421
  if raise_on_eof_exception?
384
- exception = get_param(:exception) || RuntimeError
385
422
  @logger.warn("#{err_header} raising #{exception} due to permanent EOF being received from #{@server}, error age: #{Time.now.to_i - eof_time.to_i}")
386
423
  raise exception.new("Permanent EOF is being received from #{@server}.")
387
424
  else
@@ -392,13 +429,20 @@ them.
392
429
  end
393
430
  rescue Exception => e # See comment at bottom for the list of errors seen...
394
431
  @http = nil
432
+ timeout_exception = e.is_a?(Errno::ETIMEDOUT) || e.is_a?(Timeout::Error)
433
+ # Omit retries if it was explicitly requested
434
+ if current_params[:raise_on_timeout] && timeout_exception
435
+ # #6481:
436
+ # ... When creating a resource in EC2 (instance, volume, snapshot, etc) it is undetermined what happened if the call times out.
437
+ # The resource may or may not have been created in EC2. Retrying the call may cause multiple resources to be created...
438
+ raise e
439
+ end
395
440
  # if ctrl+c is pressed - we have to reraise exception to terminate proggy
396
- if e.is_a?(Interrupt) && !( e.is_a?(Errno::ETIMEDOUT) || e.is_a?(Timeout::Error))
441
+ if e.is_a?(Interrupt) && !timeout_exception
397
442
  @logger.debug( "#{err_header} request to server #{@server} interrupted by ctrl-c")
398
443
  raise
399
444
  elsif e.is_a?(ArgumentError) && e.message.include?('wrong number of arguments (5 for 4)')
400
445
  # seems our net_fix patch was overriden...
401
- exception = get_param(:exception) || RuntimeError
402
446
  raise exception.new('incompatible Net::HTTP monkey-patch')
403
447
  end
404
448
  # oops - we got a banana: log it
@@ -414,7 +458,7 @@ them.
414
458
 
415
459
  def finish(reason = '')
416
460
  if @http && @http.started?
417
- reason = ", reason: '#{reason}'" unless reason.blank?
461
+ reason = ", reason: '#{reason}'" unless reason.empty?
418
462
  @logger.info("Closing #{@http.use_ssl? ? 'HTTPS' : 'HTTP'} connection to #{@http.address}:#{@http.port}#{reason}")
419
463
  @http.finish
420
464
  end