right_http_connection 1.2.4 → 1.3.0

data/History.txt

@@ -57,3 +57,6 @@ Initial public release
   * fixed a bug: <NoMethodError: You have a nil object when you didn't expect it!
                   The error occurred while evaluating nil.body_stream>
 
+== 1.2.5 (not released yet)
+
+- ActiveSupport dependency removal 

data/Manifest.txt

@@ -4,4 +4,5 @@ README.txt
 Rakefile
 lib/net_fix.rb
 lib/right_http_connection.rb
+lib/support.rb
 setup.rb

data/Rakefile

@@ -1,4 +1,6 @@
 require 'rubygems'
+require 'bundler'
+Bundler::GemHelper.install_tasks
 require 'rake'
 require 'rake/clean'
 require 'rake/testtask'
@@ -6,67 +8,25 @@ require 'rake/packagetask'
 require 'rake/gempackagetask'
 require 'rake/rdoctask'
 require 'rake/contrib/rubyforgepublisher'
+require 'rspec/core/rake_task'
+require 'cucumber/rake/task'
 require 'fileutils'
-require 'hoe'
 include FileUtils
 require File.join(File.dirname(__FILE__), 'lib', 'right_http_connection')
 
-AUTHOR = 'RightScale'  # can also be an array of Authors
-EMAIL = "rubygems@rightscale.com"
-DESCRIPTION = "RightScale's robust HTTP/S connection module"
-GEM_NAME = 'right_http_connection' # what ppl will type to install your gem
-RUBYFORGE_PROJECT = 'rightscale' # The unix name for your project
-HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
-DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
-
-NAME = "right_http_connection"
-REV = nil # UNCOMMENT IF REQUIRED: File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil
-VERS = RightHttpConnection::VERSION::STRING + (REV ? ".#{REV}" : "")
-CLEAN.include ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']
-RDOC_OPTS = ['--quiet', '--title', 'right_http_connection documentation',
-    "--opname", "index.html",
-    "--line-numbers", 
-    "--main", "README",
-    "--inline-source"]
-
-# Suppress Hoe's self-inclusion as a dependency for our Gem. This also keeps
-# Rake & rubyforge out of the dependency list. Users must manually install
-# these gems to run tests, etc.
-# TRB 2/19/09: also do this for the extra_dev_deps array present in newer hoes.
-# Older versions of RubyGems will try to install developer-dependencies as
-# required runtime dependencies....
-class Hoe
-  def extra_deps
-    @extra_deps.reject do |x|
-      Array(x).first == 'hoe'
-    end
-  end
-  def extra_dev_deps
-    @extra_dev_deps.reject do |x|
-      Array(x).first == 'hoe'
-    end
-  end
-end
+Bundler::GemHelper.install_tasks
+
+# == Gem == #
 
-# Generate all the Rake tasks
-# Run 'rake -T' to see list of generated tasks (from gem root directory)
-hoe = Hoe.new(GEM_NAME, VERS) do |p|
-  p.author = AUTHOR 
-  p.description = DESCRIPTION
-  p.email = EMAIL
-  p.summary = DESCRIPTION
-  p.url = HOMEPATH
-  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
-  p.test_globs = ["test/**/test_*.rb"]
-  p.clean_globs = CLEAN  #An array of file patterns to delete on clean.
-  p.remote_rdoc_dir = "right_http_gem_doc"
-  
-  # == Optional
-  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
-  #p.extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
-  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+gemtask = Rake::GemPackageTask.new(Gem::Specification.load("right_http_connection.gemspec")) do |package|
+  package.package_dir = ENV['PACKAGE_DIR'] || 'pkg'
+  package.need_zip = true
+  package.need_tar = true
 end
 
+directory gemtask.package_dir
+
+CLEAN.include(gemtask.package_dir)
 
 desc 'Generate website files'
 task :website_generate do
@@ -101,3 +61,29 @@ task :check_version do
     exit
   end
 end
+
+task :default => 'spec'
+
+# == Unit Tests == #
+
+desc "Run unit tests"
+RSpec::Core::RakeTask.new
+
+namespace :spec do
+  desc "Run unit tests with RCov"
+  RSpec::Core::RakeTask.new(:rcov) do |t|
+    t.rcov = true
+    t.rcov_opts = %q[--exclude "spec"]
+  end
+
+  desc "Print Specdoc for unit tests"
+  RSpec::Core::RakeTask.new(:doc) do |t|
+    t.rspec_opts = ["--format", "documentation"]
+  end
+end
+
+# == Functional tests == #
+desc "Run functional tests"
+Cucumber::Rake::Task.new do |t|
+  t.cucumber_opts = %w{--color --format pretty}
+end

data/lib/right_http_connection.rb

@@ -27,20 +27,10 @@ require "time"
 require "logger"
 
 $:.unshift(File.dirname(__FILE__))
+require 'version'
+require 'support'
 require "net_fix"
 
-
-module RightHttpConnection #:nodoc:
-  module VERSION #:nodoc:
-    MAJOR = 1
-    MINOR = 2
-    TINY  = 4
-
-    STRING = [MAJOR, MINOR, TINY].join('.')
-  end
-end
-
-
 module Rightscale
 
 =begin rdoc
@@ -78,13 +68,13 @@ them.
   class HttpConnection
 
     # Number of times to retry the request after encountering the first error
-    HTTP_CONNECTION_RETRY_COUNT   = 3
+    HTTP_CONNECTION_RETRY_COUNT   = 3   unless defined?(HTTP_CONNECTION_RETRY_COUNT)
     # Throw a Timeout::Error if a connection isn't established within this number of seconds
-    HTTP_CONNECTION_OPEN_TIMEOUT  = 5
+    HTTP_CONNECTION_OPEN_TIMEOUT  = 5   unless defined?(HTTP_CONNECTION_OPEN_TIMEOUT)
     # Throw a Timeout::Error if no data have been read on this connnection within this number of seconds
-    HTTP_CONNECTION_READ_TIMEOUT  = 120
+    HTTP_CONNECTION_READ_TIMEOUT  = 120 unless defined?(HTTP_CONNECTION_READ_TIMEOUT)
     # Length of the post-error probationary period during which all requests will fail
-    HTTP_CONNECTION_RETRY_DELAY   = 15
+    HTTP_CONNECTION_RETRY_DELAY   = 15  unless defined?(HTTP_CONNECTION_RETRY_DELAY)
 
     #--------------------
     # class methods
@@ -100,13 +90,19 @@ them.
     #
     #  :user_agent => 'www.HostName.com'    # String to report as HTTP User agent
     #  :ca_file    => 'path_to_file'        # Path to a CA certification file in PEM format. The file can contain several CA certificates.  If this parameter isn't set, HTTPS certs won't be verified.
+    #  :fail_if_ca_mismatch => Boolean      # If ca_file is set and the server certificate doesn't verify, a log line is generated regardless, but normally right_http_connection continues on past the failure.  If this is set, fail to connect in that case.  Defaults to false.
     #  :logger     => Logger object         # If omitted, HttpConnection logs to STDOUT
     #  :exception  => Exception to raise    # The type of exception to raise
     #                                       # if a request repeatedly fails. RuntimeError is raised if this parameter is omitted.
+    #  :proxy_host => 'hostname'            # hostname of HTTP proxy host to use, default none.
+    #  :proxy_port => port                  # port of HTTP proxy host to use, default none.
+    #  :proxy_username => 'username'        # username to use for proxy authentication, default none.
+    #  :proxy_password => 'password'        # password to use for proxy authentication, default none.
     #  :http_connection_retry_count         # by default == Rightscale::HttpConnection::HTTP_CONNECTION_RETRY_COUNT
     #  :http_connection_open_timeout        # by default == Rightscale::HttpConnection::HTTP_CONNECTION_OPEN_TIMEOUT
     #  :http_connection_read_timeout        # by default == Rightscale::HttpConnection::HTTP_CONNECTION_READ_TIMEOUT
     #  :http_connection_retry_delay         # by default == Rightscale::HttpConnection::HTTP_CONNECTION_RETRY_DELAY
+    #  :raise_on_timeout                    # do not perform a retry if timeout is received (false by default)
     def self.params
       @@params
     end
@@ -127,28 +123,42 @@ them.
      # Params hash:
      #  :user_agent => 'www.HostName.com'    # String to report as HTTP User agent
      #  :ca_file    => 'path_to_file'        # A path of a CA certification file in PEM format. The file can contain several CA certificates.
+     #  :fail_if_ca_mismatch => Boolean      # If ca_file is set and the server certificate doesn't verify, a log line is generated regardless, but normally right_http_connection continues on past the failure.  If this is set, fail to connect in that case.  Defaults to false.
      #  :logger     => Logger object         # If omitted, HttpConnection logs to STDOUT
      #  :exception  => Exception to raise    # The type of exception to raise if a request repeatedly fails. RuntimeError is raised if this parameter is omitted.
+     #  :proxy_host => 'hostname'            # hostname of HTTP proxy host to use, default none.
+     #  :proxy_port => port                  # port of HTTP proxy host to use, default none.
+     #  :proxy_username => 'username'        # username to use for proxy authentication, default none.
+     #  :proxy_password => 'password'        # password to use for proxy authentication, default none.
      #  :http_connection_retry_count         # by default == Rightscale::HttpConnection.params[:http_connection_retry_count]
      #  :http_connection_open_timeout        # by default == Rightscale::HttpConnection.params[:http_connection_open_timeout]
      #  :http_connection_read_timeout        # by default == Rightscale::HttpConnection.params[:http_connection_read_timeout]
      #  :http_connection_retry_delay         # by default == Rightscale::HttpConnection.params[:http_connection_retry_delay]
-     #
+     #  :raise_on_timeout                    # do not perform a retry if timeout is received (false by default)
     def initialize(params={})
       @params = params
       @params[:http_connection_retry_count]  ||= @@params[:http_connection_retry_count]
       @params[:http_connection_open_timeout] ||= @@params[:http_connection_open_timeout]
       @params[:http_connection_read_timeout] ||= @@params[:http_connection_read_timeout]
       @params[:http_connection_retry_delay]  ||= @@params[:http_connection_retry_delay]
+      @params[:proxy_host] ||= @@params[:proxy_host]
+      @params[:proxy_port] ||= @@params[:proxy_port]
+      @params[:proxy_username] ||= @@params[:proxy_username]
+      @params[:proxy_password] ||= @@params[:proxy_password]
       @http   = nil
       @server = nil
       @logger = get_param(:logger) ||
                 (RAILS_DEFAULT_LOGGER if defined?(RAILS_DEFAULT_LOGGER)) ||
                 Logger.new(STDOUT)
+      #--------------
+      # Retry state - Keep track of errors on a per-server basis
+      #--------------
+      @state = {}  # retry state indexed by server: consecutive error count, error time, and error
+      @eof   = {}
     end
 
-    def get_param(name)
-      @params[name] || @@params[name]
+    def get_param(name, custom_options={})
+      custom_options [name] || @params[name] || @@params[name]
     end
 
     # Query for the maximum size (in bytes) of a single read from the underlying
@@ -180,35 +190,30 @@ them.
     end
 
   private
-    #--------------
-    # Retry state - Keep track of errors on a per-server basis
-    #--------------
-    @@state = {}  # retry state indexed by server: consecutive error count, error time, and error
-    @@eof   = {}
 
     # number of consecutive errors seen for server, 0 all is ok
     def error_count
-      @@state[@server] ? @@state[@server][:count] : 0
+      @state[@server] ? @state[@server][:count] : 0
     end
 
     # time of last error for server, nil if all is ok
     def error_time
-      @@state[@server] && @@state[@server][:time]
+      @state[@server] && @state[@server][:time]
     end
 
     # message for last error for server, "" if all is ok
     def error_message
-      @@state[@server] ? @@state[@server][:message] : ""
+      @state[@server] ? @state[@server][:message] : ""
     end
 
     # add an error for a server
     def error_add(message)
-      @@state[@server] = { :count => error_count+1, :time => Time.now, :message => message }
+      @state[@server] = { :count => error_count+1, :time => Time.now, :message => message }
     end
 
     # reset the error state for a server (i.e. a request succeeded)
     def error_reset
-      @@state.delete(@server)
+      @state.delete(@server)
     end
 
     # Error message stuff...
@@ -224,25 +229,25 @@ them.
       # Returns the number of seconds to wait before new conection retry:
       #  0.5, 1, 2, 4, 8
     def add_eof
-      (@@eof[@server] ||= []).unshift Time.now
-      0.25 * 2 ** @@eof[@server].size
+      (@eof[@server] ||= []).unshift Time.now
+      0.25 * 2 ** @eof[@server].size
     end
 
       # Returns first EOF timestamp or nul if have no EOFs being tracked.
     def eof_time
-      @@eof[@server] && @@eof[@server].last
+      @eof[@server] && @eof[@server].last
     end
 
       # Returns true if we are receiving EOFs during last @params[:http_connection_retry_delay] seconds
       # and there were no successful response from server
     def raise_on_eof_exception?
-      @@eof[@server].blank? ? false : ( (Time.now.to_i-@params[:http_connection_retry_delay]) > @@eof[@server].last.to_i )
+      @eof[@server].nil? ? false : ( (Time.now.to_i-@params[:http_connection_retry_delay]) > @eof[@server].last.to_i )
     end
 
       # Reset a list of EOFs for this server.
       # This is being called when we have got an successful response from server.
     def eof_reset
-      @@eof.delete(@server)
+      @eof.delete(@server)
     end
 
     # Detects if an object is 'streamable' - can we read from it, and can we know the size?
@@ -281,14 +286,22 @@ them.
       # close the previous if exists
       finish
       # create new connection
-      @server   = request_params[:server]
-      @port     = request_params[:port]
-      @protocol = request_params[:protocol]
+      @server         = request_params[:server]
+      @port           = request_params[:port]
+      @protocol       = request_params[:protocol]
+      @proxy_host     = request_params[:proxy_host]
+      @proxy_port     = request_params[:proxy_port]
+      @proxy_username = request_params[:proxy_username]
+      @proxy_password = request_params[:proxy_password]
 
       @logger.info("Opening new #{@protocol.upcase} connection to #@server:#@port")
-      @http = Net::HTTP.new(@server, @port)
-      @http.open_timeout = @params[:http_connection_open_timeout]
-      @http.read_timeout = @params[:http_connection_read_timeout]
+      @logger.info("Connecting to proxy #{@proxy_host}:#{@proxy_port} with username" +
+                   " #{@proxy_username}") unless @proxy_host.nil?
+
+      @http = Net::HTTP.new(@server, @port, @proxy_host, @proxy_port, @proxy_username,
+                            @proxy_password)
+      @http.open_timeout = get_param(:http_connection_open_timeout, request_params)
+      @http.read_timeout = get_param(:http_connection_read_timeout, request_params)
 
       if @protocol == 'https'
         verifyCallbackProc = Proc.new{ |ok, x509_store_ctx|
@@ -296,7 +309,11 @@ them.
           msg = x509_store_ctx.error_string
             #debugger
           @logger.warn("##### #{@server} certificate verify failed: #{msg}") unless code == 0
-          true
+          if request_params[:fail_if_ca_mismatch] && code != 0
+            false
+          else
+            true
+          end
         }
         @http.use_ssl = true
         ca_file = get_param(:ca_file)
@@ -304,6 +321,8 @@ them.
           @http.verify_mode     = OpenSSL::SSL::VERIFY_PEER
           @http.verify_callback = verifyCallbackProc
           @http.ca_file         = ca_file
+        else
+          @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
         end
       end
       # open connection
@@ -320,45 +339,64 @@ them.
      :port     => '80'                 # Port of HTTP server
      :protocol => 'https'              # http and https are supported on any port
      :request  => 'requeststring'      # Fully-formed HTTP request to make
+     :proxy_host => 'hostname'         # hostname of HTTP proxy host to use, default none.
+     :proxy_port => port               # port of HTTP proxy host to use, default none.
+     :proxy_username => 'username'     # username to use for proxy authentication, default none.
+     :proxy_password => 'password'     # password to use for proxy authentication, default none.
+
+     :raise_on_timeout                 # do not perform a retry if timeout is received (false by default)
+     :http_connection_retry_count
+     :http_connection_open_timeout
+     :http_connection_read_timeout
+     :http_connection_retry_delay
+     :user_agent
+     :exception
 
     Raises RuntimeError, Interrupt, and params[:exception] (if specified in new).
 
 =end
     def request(request_params, &block)
+      current_params = @params.merge(request_params)
+      exception = get_param(:exception, current_params) || RuntimeError
+
       # We save the offset here so that if we need to retry, we can return the file pointer to its initial position
-      mypos = get_fileptr_offset(request_params)
+      mypos = get_fileptr_offset(current_params)
       loop do
+        current_params[:protocol] ||= (current_params[:port] == 443 ? 'https' : 'http')
+        # (re)open connection to server if none exists or params has changed
+        same_server_as_before = @server   == current_params[:server] &&
+                                @port     == current_params[:port]   &&
+                                @protocol == current_params[:protocol]
+
         # if we are inside a delay between retries: no requests this time!
-        if error_count > @params[:http_connection_retry_count] &&
-           error_time + @params[:http_connection_retry_delay] > Time.now
+        # (skip this step if the endpoint has changed)
+        if error_count > current_params[:http_connection_retry_count]            &&
+           error_time  + current_params[:http_connection_retry_delay] > Time.now &&
+           same_server_as_before
+
           # store the message (otherwise it will be lost after error_reset and
           # we will raise an exception with an empty text)
           banana_message_text = banana_message
           @logger.warn("#{err_header} re-raising same error: #{banana_message_text} " +
                       "-- error count: #{error_count}, error age: #{Time.now.to_i - error_time.to_i}")
-          exception = get_param(:exception) || RuntimeError
           raise exception.new(banana_message_text)
         end
 
         # try to connect server(if connection does not exist) and get response data
         begin
-          request_params[:protocol] ||= (request_params[:port] == 443 ? 'https' : 'http')
-
-          request = request_params[:request]
-          request['User-Agent'] = get_param(:user_agent) || ''
-
-          # (re)open connection to server if none exists or params has changed
+          request = current_params[:request]
+          request['User-Agent'] = get_param(:user_agent, current_params) || ''
           unless @http          &&
                  @http.started? &&
-                 @server   == request_params[:server] &&
-                 @port     == request_params[:port]   &&
-                 @protocol == request_params[:protocol]
-            start(request_params)
+                 same_server_as_before
+            start(current_params)
           end
 
           # Detect if the body is a streamable object like a file or socket.  If so, stream that
           # bad boy.
           setup_streaming(request)
+          # update READ_TIMEOUT value (it can be passed with request_params hash)
+          @http.read_timeout = get_param(:http_connection_read_timeout, current_params)
           response = @http.request(request, &block)
 
           error_reset
@@ -381,7 +419,6 @@ them.
 
             # if we have waited long enough - raise an exception...
           if raise_on_eof_exception?
-            exception = get_param(:exception) || RuntimeError
             @logger.warn("#{err_header} raising #{exception} due to permanent EOF being received from #{@server}, error age: #{Time.now.to_i - eof_time.to_i}")
             raise exception.new("Permanent EOF is being received from #{@server}.")
           else
@@ -392,13 +429,20 @@ them.
           end
         rescue Exception => e  # See comment at bottom for the list of errors seen...
           @http = nil
+          timeout_exception = e.is_a?(Errno::ETIMEDOUT) || e.is_a?(Timeout::Error)
+          # Omit retries if it was explicitly requested
+          if current_params[:raise_on_timeout] && timeout_exception
+            # #6481:
+            # ... When creating a resource in EC2 (instance, volume, snapshot, etc) it is undetermined what happened if the call times out.
+            # The resource may or may not have been created in EC2. Retrying the call may cause multiple resources to be created...
+            raise e
+          end
           # if ctrl+c is pressed - we have to reraise exception to terminate proggy
-          if e.is_a?(Interrupt) && !( e.is_a?(Errno::ETIMEDOUT) || e.is_a?(Timeout::Error))
+          if e.is_a?(Interrupt) && !timeout_exception
             @logger.debug( "#{err_header} request to server #{@server} interrupted by ctrl-c")
             raise
           elsif e.is_a?(ArgumentError) && e.message.include?('wrong number of arguments (5 for 4)')
             # seems our net_fix patch was overriden...
-            exception = get_param(:exception) || RuntimeError
             raise exception.new('incompatible Net::HTTP monkey-patch')
           end
           # oops - we got a banana: log it
@@ -414,7 +458,7 @@ them.
 
     def finish(reason = '')
       if @http && @http.started?
-        reason = ", reason: '#{reason}'" unless reason.blank?
+        reason = ", reason: '#{reason}'" unless reason.empty?
         @logger.info("Closing #{@http.use_ssl? ? 'HTTPS' : 'HTTP'} connection to #{@http.address}:#{@http.port}#{reason}")
         @http.finish
       end