net-http-persistent 1.0.1 → 1.1

data/History.txt

@@ -1,9 +1,23 @@
+=== 1.1
+
+* Minor Enhancements
+  * Proxy support, see Net::HTTP::Persistent::new,
+    Net::HTTP::Persistent#proxy_from_env
+  * Added +name+ parameter to Net::HTTP::Persistent::new for separation of
+    connection pools.
+  * Added Net::HTTP::Persistent#shutdown so you can clean up after yourself
+  * Net::HTTP::Persistent now suppresses "peer certificate won't be verified
+    in this SSL session" warning.
+
+* Bug Fixes
+  * Net::HTTP::Persistent retries requests in accordance with RFC 2616.
+
 === 1.0.1 / 2010-05-05
 
 * Minor Enhancements
   * Added #debug_output
   * Now uses Hoe minitest plugin
-* Bug fixes
+* Bug Fixes
   * Tests pass on 1.9
 
 === 1.0.0 / 2010-05-04

data/lib/net/http/persistent.rb

@@ -12,6 +12,13 @@ require 'uri'
 # Multiple Net::HTTP::Persistent objects will share the same set of
 # connections.
 #
+# For each thread you start a new connection will be created.  A
+# Net::HTTP::Persistent connection will not be shared across threads.
+#
+# You can shut down the HTTP connections when done by calling #shutdown.  You
+# should name your Net::HTTP::Persistent object if you intend to call this
+# method.
+#
 # Example:
 #
 #   uri = URI.parse 'http://example.com/awesome/web/service'
@@ -29,7 +36,7 @@ class Net::HTTP::Persistent
   ##
   # The version of Net::HTTP::Persistent use are using
 
-  VERSION = '1.0.1'
+  VERSION = '1.1'
 
   ##
   # Error class for errors raised by Net::HTTP::Persistent.  Various
@@ -49,6 +56,11 @@ class Net::HTTP::Persistent
 
   attr_accessor :ca_file
 
+  ##
+  # Where this instance's connections live in the thread local variables
+
+  attr_reader :connection_key # :nodoc:
+
   ##
   # Sends debug_output to this IO via Net::HTTP#set_debug_output.
   #
@@ -63,33 +75,97 @@ class Net::HTTP::Persistent
   attr_reader :headers
 
   ##
-  # The value sent in the Keep-Alive header.  Defaults to 30 seconds
+  # The value sent in the Keep-Alive header.  Defaults to 30.  Not needed for
+  # HTTP/1.1 servers.
+  #
+  # This may not work correctly for HTTP/1.0 servers
+  #
+  # This method may be removed in a future version as RFC 2616 does not
+  # require this header.
 
   attr_accessor :keep_alive
 
+  ##
+  # A name for this connection.  Allows you to keep your connections apart
+  # from everybody else's.
+
+  attr_reader :name
+
   ##
   # This client's SSL private key
 
   attr_accessor :private_key
 
+  ##
+  # The URL through which requests will be proxied
+
+  attr_reader :proxy_uri
+
+  ##
+  # Where this instance's request counts live in the thread local variables
+
+  attr_reader :request_key # :nodoc:
+
   ##
   # SSL verification callback.  Used when ca_file is set.
 
   attr_accessor :verify_callback
 
   ##
-  # HTTPS verify mode.  Set to OpenSSL::SSL::VERIFY_NONE to ignore certificate
-  # problems.
+  # HTTPS verify mode.  Defaults to OpenSSL::SSL::VERIFY_NONE which ignores
+  # certificate problems.
   #
   # You can use +verify_mode+ to override any default values.
 
   attr_accessor :verify_mode
 
-  def initialize # :nodoc:
+  ##
+  # Creates a new Net::HTTP::Persistent.
+  #
+  # Set +name+ to keep your connections apart from everybody else's.  Not
+  # required currently, but highly recommended.  Your library name should be
+  # good enough.  This parameter will be required in a future version.
+  #
+  # +proxy+ may be set to a URI::HTTP or :ENV to pick up proxy options from
+  # the environment.  See proxy_from_env for details.
+  #
+  # In order to use a URI for the proxy you'll need to do some extra work
+  # beyond URI.parse:
+  #
+  #   proxy = URI.parse 'http://proxy.example'
+  #   proxy.user     = 'AzureDiamond'
+  #   proxy.password = 'hunter2'
+
+  def initialize name = nil, proxy = nil
+    @name = name
+
+    @proxy_uri = case proxy
+                 when :ENV      then proxy_from_env
+                 when URI::HTTP then proxy
+                 when nil       then # ignore
+                 else raise ArgumentError, 'proxy must be :ENV or a URI::HTTP'
+                 end
+
+    if @proxy_uri then
+      @proxy_args = [
+        @proxy_uri.host,
+        @proxy_uri.port,
+        @proxy_uri.user,
+        @proxy_uri.password,
+      ]
+
+      @proxy_connection_id = [nil, *@proxy_args].join ':'
+    end
+
     @debug_output = nil
     @headers      = {}
     @keep_alive   = 30
 
+    key = ['net_http_persistent', name, 'connections'].compact.join '_'
+    @connection_key = key.intern
+    key = ['net_http_persistent', name, 'requests'].compact.join '_'
+    @request_key    = key.intern
+
     @certificate     = nil
     @ca_file         = nil
     @private_key     = nil
@@ -101,12 +177,18 @@ class Net::HTTP::Persistent
   # Creates a new connection for +uri+
 
   def connection_for uri
-    Thread.current[:net_http_persistent_connections] ||= {}
-    connections = Thread.current[:net_http_persistent_connections]
+    Thread.current[@connection_key] ||= {}
+    connections = Thread.current[@connection_key]
 
-    connection_id = [uri.host, uri.port].join ':'
+    net_http_args = [uri.host, uri.port]
+    connection_id = net_http_args.join ':'
 
-    connections[connection_id] ||= Net::HTTP.new uri.host, uri.port
+    if @proxy_uri then
+      connection_id << @proxy_connection_id
+      net_http_args.concat @proxy_args
+    end
+
+    connections[connection_id] ||= Net::HTTP.new(*net_http_args)
     connection = connections[connection_id]
 
     unless connection.started? then
@@ -128,16 +210,68 @@ class Net::HTTP::Persistent
 
   def error_message connection
     requests =
-      Thread.current[:net_http_persistent_requests][connection.object_id]
+      Thread.current[@request_key][connection.object_id]
 
     "after #{requests} requests on #{connection.object_id}"
   end
 
+  ##
+  # URI::escape wrapper
+
+  def escape str
+    URI.escape str if str
+  end
+
+  ##
+  # Is +req+ idempotent according to RFC 2616?
+
+  def idempotent? req
+    case req
+    when Net::HTTP::Delete, Net::HTTP::Get, Net::HTTP::Head,
+         Net::HTTP::Options, Net::HTTP::Put, Net::HTTP::Trace then
+      true
+    end
+  end
+
+  ##
+  # Adds "http://" to the String +uri+ if it is missing.
+
+  def normalize_uri uri
+    (uri =~ /^https?:/) ? uri : "http://#{uri}"
+  end
+
+  ##
+  # Creates a URI for an HTTP proxy server from ENV variables.
+  #
+  # If +HTTP_PROXY+ is set a proxy will be returned.
+  #
+  # If +HTTP_PROXY_USER+ or +HTTP_PROXY_PASS+ are set the URI is given the
+  # indicated user and password unless HTTP_PROXY contains either of these in
+  # the URI.
+  #
+  # For Windows users lowercase ENV variables are preferred over uppercase ENV
+  # variables.
+
+  def proxy_from_env
+    env_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY']
+
+    return nil if env_proxy.nil? or env_proxy.empty?
+
+    uri = URI.parse normalize_uri env_proxy
+
+    unless uri.user or uri.password then
+      uri.user     = escape ENV['http_proxy_user'] || ENV['HTTP_PROXY_USER']
+      uri.password = escape ENV['http_proxy_pass'] || ENV['HTTP_PROXY_PASS']
+    end
+
+    uri
+  end
+
   ##
   # Finishes then restarts the Net::HTTP +connection+
 
   def reset connection
-    Thread.current[:net_http_persistent_requests].delete connection.object_id
+    Thread.current[@request_key].delete connection.object_id
 
     begin
       connection.finish
@@ -156,9 +290,12 @@ class Net::HTTP::Persistent
   # against +uri+.
   #
   # +req+ must be a Net::HTTPRequest subclass (see Net::HTTP for a list).
+  #
+  # If there is an error and the request is idempontent according to RFC 2616
+  # it will be retried automatically.
 
   def request uri, req = nil
-    Thread.current[:net_http_persistent_requests] ||= Hash.new 0
+    Thread.current[@request_key] ||= Hash.new 0
     retried      = false
     bad_response = false
 
@@ -175,7 +312,7 @@ class Net::HTTP::Persistent
     connection_id = connection.object_id
 
     begin
-      count = Thread.current[:net_http_persistent_requests][connection_id] += 1
+      count = Thread.current[@request_key][connection_id] += 1
       response = connection.request req
 
     rescue Net::HTTPBadResponse => e
@@ -183,7 +320,8 @@ class Net::HTTP::Persistent
 
       reset connection
 
-      raise Error, "too many bad responses #{message}" if bad_response
+      raise Error, "too many bad responses #{message}" if
+        bad_response or not idempotent? req
 
       bad_response = true
       retry
@@ -194,7 +332,8 @@ class Net::HTTP::Persistent
 
       reset connection
 
-      raise Error, "too many connection resets #{due_to} #{message}" if retried
+      raise Error, "too many connection resets #{due_to} #{message}" if
+        retried or not idempotent? req
 
       retried = true
       retry
@@ -203,6 +342,21 @@ class Net::HTTP::Persistent
     response
   end
 
+  ##
+  # Shuts down all connections in this thread.
+  #
+  # If you've used Net::HTTP::Persistent across multiple threads you must call
+  # this in each thread.
+
+  def shutdown
+    Thread.current[@connection_key].each do |_, connection|
+      connection.finish
+    end
+
+    Thread.current[@connection_key] = nil
+    Thread.current[@request_key]    = nil
+  end
+
   ##
   # Enables SSL on +connection+
 
@@ -210,6 +364,9 @@ class Net::HTTP::Persistent
     require 'net/https'
     connection.use_ssl = true
 
+    # suppress warning but allow override
+    connection.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @verify_mode
+
     if @ca_file then
       connection.ca_file = @ca_file
       connection.verify_mode = OpenSSL::SSL::VERIFY_PEER

data/test/test_net_http_persistent.rb

@@ -3,22 +3,35 @@ require 'net/http/persistent'
 require 'openssl'
 require 'stringio'
 
+class Net::HTTP
+  def connect
+  end
+end
+
 class TestNetHttpPersistent < MiniTest::Unit::TestCase
 
   def setup
     @http = Net::HTTP::Persistent.new
     @uri  = URI.parse 'http://example.com/path'
+
+    ENV.delete 'http_proxy'
+    ENV.delete 'HTTP_PROXY'
+    ENV.delete 'http_proxy_user'
+    ENV.delete 'HTTP_PROXY_USER'
+    ENV.delete 'http_proxy_pass'
+    ENV.delete 'HTTP_PROXY_PASS'
   end
 
   def teardown
-    Thread.current[:net_http_persistent_connections] = nil
-    Thread.current[:net_http_persistent_requests] = nil
+    Thread.current.keys.each do |key|
+      Thread.current[key] = nil
+    end
   end
 
   def connection
     c = Object.new
     # Net::HTTP
-    def c.finish; @finish = true end
+    def c.finish; @finished = true end
     def c.request(req) @req = req; :response end
     def c.reset; @reset = true end
     def c.start; end
@@ -27,22 +40,48 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     def c.req() @req; end
     def c.reset?; @reset end
     def c.started?; true end
+    def c.finished?; @finished end
     conns["#{@uri.host}:#{@uri.port}"] = c
     c
   end
 
   def conns
-    Thread.current[:net_http_persistent_connections] ||= {}
+    Thread.current[@http.connection_key] ||= {}
   end
 
   def reqs
-    Thread.current[:net_http_persistent_requests] ||= {}
+    Thread.current[@http.request_key] ||= {}
+  end
+
+  def test_initialize
+    assert_nil @http.proxy_uri
+  end
+
+  def test_initialize_name
+    http = Net::HTTP::Persistent.new 'name'
+    assert_equal 'name', http.name
+  end
+
+  def test_initialize_env
+    ENV['HTTP_PROXY'] = 'proxy.example'
+    http = Net::HTTP::Persistent.new nil, :ENV
+
+    assert_equal URI.parse('http://proxy.example'), http.proxy_uri
+  end
+
+  def test_initialize_uri
+    proxy_uri = URI.parse 'http://proxy.example'
+
+    http = Net::HTTP::Persistent.new nil, proxy_uri
+
+    assert_equal proxy_uri, http.proxy_uri
   end
 
   def test_connection_for
     c = @http.connection_for @uri
 
     assert c.started?
+    refute c.proxy?
 
     assert_includes conns.keys, 'example.com:80'
     assert_same c, conns['example.com:80']
@@ -73,6 +112,34 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     assert_same c, conns['example.com:80']
   end
 
+  def test_connection_for_name
+    http = Net::HTTP::Persistent.new 'name'
+    uri = URI.parse 'http://example/'
+
+    c = http.connection_for uri
+
+    assert c.started?
+
+    refute_includes conns.keys, 'example:80'
+  end
+
+  def test_connection_for_proxy
+    uri = URI.parse 'http://proxy.example'
+    uri.user     = 'johndoe'
+    uri.password = 'muffins'
+
+    http = Net::HTTP::Persistent.new nil, uri
+
+    c = http.connection_for @uri
+
+    assert c.started?
+    assert c.proxy?
+
+    assert_includes conns.keys,
+                    'example.com:80:proxy.example:80:johndoe:muffins'
+    assert_same c, conns['example.com:80:proxy.example:80:johndoe:muffins']
+  end
+
   def test_connection_for_refused
     cached = Object.new
     def cached.address; 'example.com' end
@@ -95,6 +162,69 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     assert_equal "after 5 requests on #{c.object_id}", @http.error_message(c)
   end
 
+  def test_escape
+    assert_nil @http.escape nil
+
+    assert_equal '%20', @http.escape(' ')
+  end
+
+  def test_idempotent_eh
+    assert @http.idempotent? Net::HTTP::Delete.new '/'
+    assert @http.idempotent? Net::HTTP::Get.new '/'
+    assert @http.idempotent? Net::HTTP::Head.new '/'
+    assert @http.idempotent? Net::HTTP::Options.new '/'
+    assert @http.idempotent? Net::HTTP::Put.new '/'
+    assert @http.idempotent? Net::HTTP::Trace.new '/'
+
+    refute @http.idempotent? Net::HTTP::Post.new '/'
+  end
+
+  def test_normalize_uri
+    assert_equal 'http://example',  @http.normalize_uri('example')
+    assert_equal 'http://example',  @http.normalize_uri('http://example')
+    assert_equal 'https://example', @http.normalize_uri('https://example')
+  end
+
+  def test_proxy_from_env
+    ENV['HTTP_PROXY']      = 'proxy.example'
+    ENV['HTTP_PROXY_USER'] = 'johndoe'
+    ENV['HTTP_PROXY_PASS'] = 'muffins'
+
+    uri = @http.proxy_from_env
+
+    expected = URI.parse 'http://proxy.example'
+    expected.user     = 'johndoe'
+    expected.password = 'muffins'
+
+    assert_equal expected, uri
+  end
+
+  def test_proxy_from_env_lower
+    ENV['http_proxy']      = 'proxy.example'
+    ENV['http_proxy_user'] = 'johndoe'
+    ENV['http_proxy_pass'] = 'muffins'
+
+    uri = @http.proxy_from_env
+
+    expected = URI.parse 'http://proxy.example'
+    expected.user     = 'johndoe'
+    expected.password = 'muffins'
+
+    assert_equal expected, uri
+  end
+
+  def test_proxy_from_env_nil
+    uri = @http.proxy_from_env
+
+    assert_nil uri
+
+    ENV['HTTP_PROXY'] = ''
+
+    uri = @http.proxy_from_env
+
+    assert_nil uri
+  end
+
   def test_reset
     c = Object.new
     def c.finish; @finished = true end
@@ -184,6 +314,25 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     assert_match %r%too many bad responses%, e.message
   end
 
+  def test_request_bad_response_unsafe
+    c = connection
+    def c.request(*a)
+      if instance_variable_defined? :@request then
+        raise 'POST must not be retried'
+      else
+        @request = true
+        raise Net::HTTPBadResponse
+      end
+    end
+
+    e = assert_raises Net::HTTP::Persistent::Error do
+      @http.request @uri, Net::HTTP::Post.new(@uri.path)
+    end
+
+    assert_equal 0, reqs[c.object_id]
+    assert_match %r%too many bad responses%, e.message
+  end
+
   def test_request_reset
     c = connection
     def c.request(*a) raise Errno::ECONNRESET end
@@ -196,6 +345,25 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     assert_match %r%too many connection resets%, e.message
   end
 
+  def test_request_reset_unsafe
+    c = connection
+    def c.request(*a)
+      if instance_variable_defined? :@request then
+        raise 'POST must not be retried'
+      else
+        @request = true
+        raise Errno::ECONNRESET
+      end
+    end
+
+    e = assert_raises Net::HTTP::Persistent::Error do
+      @http.request @uri, Net::HTTP::Post.new(@uri.path)
+    end
+
+    assert_equal 0, reqs[c.object_id]
+    assert_match %r%too many connection resets%, e.message
+  end
+
   def test_request_post
     c = connection
 
@@ -209,6 +377,24 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     assert_same post, req
   end
 
+  def test_shutdown
+    c = connection
+    cs = conns
+    rs = reqs
+
+    orig = @http
+    @http = Net::HTTP::Persistent.new 'name'
+    c2 = connection
+
+    orig.shutdown
+
+    assert c.finished?
+    refute c2.finished?
+
+    refute_same cs, conns
+    refute_same rs, reqs
+  end
+
   def test_ssl
     @http.verify_callback = :callback
     c = Net::HTTP.new 'localhost', 80
@@ -216,7 +402,7 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     @http.ssl c
 
     assert c.use_ssl?
-    assert_nil c.verify_mode
+    assert_equal OpenSSL::SSL::VERIFY_NONE, c.verify_mode
     assert_nil c.verify_callback
   end
 

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: net-http-persistent
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 13
   prerelease: false
   segments: 
   - 1
-  - 0
   - 1
-  version: 1.0.1
+  version: "1.1"
 platform: ruby
 authors: 
 - Eric Hodel
@@ -36,7 +35,7 @@ cert_chain:
   x52qPcexcYZR7w==
   -----END CERTIFICATE-----
 
-date: 2010-05-05 00:00:00 -07:00
+date: 2010-05-18 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency