net-http-persistent 2.1 → 2.2

data/History.txt

@@ -1,3 +1,12 @@
+=== 2.2 / 2011-10-24
+
+* Minor Enhancement
+  * Added timeouts for idle connections which are set through #idle_timeout.
+    The default timeout is 5 seconds.  Reducing the idle timeout is preferred
+    over setting #retry_change_requests to true if you wish to avoid the "too
+    many connection resets" error when POSTing data.
+  * Documented tunables and settings in one place in Net::HTTP::Persistent
+
 === 2.1 / 2011-09-19
 
 * Minor Enhancement

data/README.txt

@@ -13,7 +13,7 @@ Creating a new HTTP connection for every request involves an extra TCP
 round-trip and causes TCP congestion avoidance negotiation to start over.
 
 Net::HTTP supports persistent connections with some API methods but does not
-handle reconnection gracefully.  net-http-persistent supports reconnection
+handle reconnection gracefully.  Net::HTTP::Persistent supports reconnection
 and retry according to RFC 2616.
 
 == FEATURES/PROBLEMS:

data/lib/net/http/persistent.rb

@@ -28,22 +28,128 @@ end
 #
 # Example:
 #
-#   uri = URI.parse 'http://example.com/awesome/web/service'
-#   http = Net::HTTP::Persistent.new
-#   stuff = http.request uri # performs a GET
+#   require 'net/http/persistent'
 #
-#   # perform a POST
+#   uri = URI 'http://example.com/awesome/web/service'
+#
+#   http = Net::HTTP::Persistent.new 'my_app_name'
+#
+#   # perform a GET
+#   response = http.request uri
+#
+#   # create a POST
 #   post_uri = uri + 'create'
 #   post = Net::HTTP::Post.new post_uri.path
 #   post.set_form_data 'some' => 'cool data'
-#   http.request post_uri, post # URI is always required
+#
+#   # perform the POST, the URI is always required
+#   response http.request post_uri, post
+#
+# == SSL
+#
+# SSL connections are automatically created depending upon the scheme of the
+# URI.  SSL connections are automatically verified against the default
+# certificate store for your computer.  You can override this by changing
+# verify_mode or by specifying an alternate cert_store.
+#
+# Here are the SSL settings, see the individual methods for documentation:
+#
+# #certificate        :: This client's certificate
+# #ca_file            :: The certificate-authority
+# #cert_store         :: An SSL certificate store
+# #private_key        :: The client's SSL private key
+# #reuse_ssl_sessions :: Reuse a previously opened SSL session for a new
+#                        connection
+# #verify_callback    :: For server certificate verification
+# #verify_mode        :: How connections should be verified
+#
+# == Proxies
+#
+# A proxy can be used either by providing a URI as the second argument of
+# ::new or by giving <code>:ENV</code> as the second argument which will
+# consult environment variables.  See proxy_from_env for details.
+#
+# == Tuning
+#
+# === Segregation
+#
+# By providing an application name to ::new you can separate your connections
+# from the connections of other applications.
+#
+# === Idle Timeout
+#
+# If a connection hasn't been used for 5 seconds it will automatically be
+# reset upon the next use to avoid attempting to send to a closed connection.
+# This can be adjusted through idle_timeout.
+#
+# Reducing this value may help avoid the "too many connection resets" error
+# when sending non-idempotent requests while increasing this value will cause
+# fewer round-trips.
+#
+# === Read Timeout
+#
+# The amount of time allowed between reading two chunks from the socket.  Set
+# through #read_timeout
+#
+# === Open Timeout
+#
+# The amount of time to wait for a connection to be opened.  Set through
+# #open_timeout.
+#
+# === Socket Options
+#
+# Socket options may be set on newly-created connections.  See #socket_options
+# for details.
+#
+# === Non-Idempotent Requests
+#
+# By default non-idempotent requests will not be retried per RFC 2616.  By
+# setting retry_change_requests to true requests will automatically be retried
+# once.
+#
+# Only do this when you know that retrying a POST is safe for your
+# application and will not create duplicate resources.
+#
+# The recommended way to handle non-idempotent requests is the following:
+#
+#   require 'net/http/persistent'
+#
+#   uri = URI 'http://example.com/awesome/web/service'
+#   post_uri = uri + 'create'
+#
+#   http = Net::HTTP::Persistent.new 'my_app_name'
+#
+#   post = Net::HTTP::Post.new post_uri.path
+#   # ... fill in POST request
+#
+#   begin
+#     response = http.request post_uri, post
+#   rescue Net::HTTP::Persistent::Error
+#
+#     # POST failed, make a new request to verify the server did not process
+#     # the request
+#     exists_uri = uri + '...'
+#     response = http.get exists_uri
+#
+#     # Retry if it failed
+#     retry if response.code == '404'
+#   end
+#
+# The method of determining if the resource was created or not is unique to
+# the particular service you are using.  Of course, you will want to add
+# protection from infinite looping.
 
 class Net::HTTP::Persistent
 
   ##
-  # The version of Net::HTTP::Persistent use are using
+  # The beginning of Time
+
+  EPOCH = Time.at 0 # :nodoc:
+
+  ##
+  # The version of Net::HTTP::Persistent you are using
 
-  VERSION = '2.1'
+  VERSION = '2.2'
 
   ##
   # Error class for errors raised by Net::HTTP::Persistent.  Various
@@ -93,6 +199,12 @@ class Net::HTTP::Persistent
 
   attr_reader :http_versions
 
+  ##
+  # Maximum time an unused connection can remain idle before being
+  # automatically closed.
+
+  attr_accessor :idle_timeout
+
   ##
   # The value sent in the Keep-Alive header.  Defaults to 30.  Not needed for
   # HTTP/1.1 servers.
@@ -155,6 +267,11 @@ class Net::HTTP::Persistent
 
   attr_reader :socket_options
 
+  ##
+  # Where this instance's last-use times live in the thread local variables
+
+  attr_reader :timeout_key # :nodoc:
+
   ##
   # SSL verification callback.  Used when ca_file is set.
 
@@ -195,10 +312,10 @@ class Net::HTTP::Persistent
   # +proxy+ may be set to a URI::HTTP or :ENV to pick up proxy options from
   # the environment.  See proxy_from_env for details.
   #
-  # In order to use a URI for the proxy you'll need to do some extra work
-  # beyond URI.parse:
+  # In order to use a URI for the proxy you may need to do some extra work
+  # beyond URI parsing if the proxy requires a password:
   #
-  #   proxy = URI.parse 'http://proxy.example'
+  #   proxy = URI 'http://proxy.example'
   #   proxy.user     = 'AzureDiamond'
   #   proxy.password = 'hunter2'
 
@@ -229,15 +346,16 @@ class Net::HTTP::Persistent
     @keep_alive     = 30
     @open_timeout   = nil
     @read_timeout   = nil
+    @idle_timeout   = 5
     @socket_options = []
 
     @socket_options << [Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1] if
       Socket.const_defined? :TCP_NODELAY
 
-    key = ['net_http_persistent', name, 'connections'].compact.join '_'
-    @connection_key = key.intern
-    key = ['net_http_persistent', name, 'requests'].compact.join '_'
-    @request_key    = key.intern
+    key = ['net_http_persistent', name].compact
+    @connection_key = [key, 'connections'].join('_').intern
+    @request_key    = [key, 'requests'].join('_').intern
+    @timeout_key    = [key, 'timeouts'].join('_').intern
 
     @certificate        = nil
     @ca_file            = nil
@@ -256,6 +374,7 @@ class Net::HTTP::Persistent
   def connection_for uri
     Thread.current[@connection_key] ||= {}
     Thread.current[@request_key]    ||= Hash.new 0
+    Thread.current[@timeout_key]    ||= Hash.new EPOCH
 
     connections = Thread.current[@connection_key]
 
@@ -267,10 +386,15 @@ class Net::HTTP::Persistent
       net_http_args.concat @proxy_args
     end
 
+    connection = connections[connection_id] 
+
     unless connection = connections[connection_id] then
       connections[connection_id] = http_class.new(*net_http_args)
       connection = connections[connection_id]
       ssl connection if uri.scheme.downcase == 'https'
+    else
+      last_used = Thread.current[@timeout_key][connection.object_id]
+      reset connection unless last_used > max_age
     end
 
     unless connection.started? then
@@ -353,6 +477,13 @@ class Net::HTTP::Persistent
     retry_change_requests or idempotent?(req)
   end
 
+  ##
+  # If a connection hasn't been used since max_age it will be reset and reused
+
+  def max_age
+    Time.now - @idle_timeout
+  end
+
   ##
   # Adds "http://" to the String +uri+ if it is missing.
 
@@ -394,7 +525,7 @@ class Net::HTTP::Persistent
 
     return nil if env_proxy.nil? or env_proxy.empty?
 
-    uri = URI.parse normalize_uri env_proxy
+    uri = URI normalize_uri env_proxy
 
     unless uri.user or uri.password then
       uri.user     = escape ENV['http_proxy_user'] || ENV['HTTP_PROXY_USER']
@@ -409,6 +540,7 @@ class Net::HTTP::Persistent
 
   def reset connection
     Thread.current[@request_key].delete connection.object_id
+    Thread.current[@timeout_key].delete connection.object_id
 
     finish connection
 
@@ -450,7 +582,6 @@ class Net::HTTP::Persistent
     begin
       Thread.current[@request_key][connection_id] += 1
       response = connection.request req, &block
-
     rescue Net::HTTPBadResponse => e
       message = error_message connection
 
@@ -478,6 +609,8 @@ class Net::HTTP::Persistent
 
       retried = true
       retry
+    ensure
+      Thread.current[@timeout_key][connection_id] = Time.now
     end
 
     @http_versions["#{uri.host}:#{uri.port}"] ||= response.http_version

data/test/test_net_http_persistent.rb

@@ -120,6 +120,10 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     Thread.current[@http.request_key] ||= {}
   end
 
+  def touts
+    Thread.current[@http.timeout_key] ||= Hash.new Net::HTTP::Persistent::EPOCH
+  end
+
   def test_initialize
     assert_nil @http.proxy_uri
   end
@@ -186,9 +190,9 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     cached = basic_connection
     cached.start
     if Socket.const_defined? :TCP_NODELAY then
-      def (cached.instance_variable_get(:@socket).io).setsockopt(*a)
-        raise IOError, 'closed stream'
-      end
+      io = Object.new
+      def io.setsockopt(*a) raise IOError, 'closed stream' end
+      cached.instance_variable_set :@socket, Net::BufferedIO.new(io)
     end
     conns['example.com:80'] = cached
 
@@ -200,7 +204,9 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     assert_same c, conns['example.com:80']
 
     socket = c.instance_variable_get :@socket
-    assert_nil socket.io.instance_variable_get(:@setsockopt)
+
+    refute_includes socket.io.instance_variables, :@setsockopt
+    refute_includes socket.io.instance_variables, '@setsockopt'
   end
 
   def test_connection_for_debug_output
@@ -325,6 +331,21 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     assert c.use_ssl?
   end
 
+  def test_connection_for_timeout
+    cached = basic_connection
+    cached.start
+    reqs[cached.object_id] = 10
+    touts[cached.object_id] = Time.now - 6
+    conns['example.com:80'] = cached
+
+    c = @http.connection_for @uri
+
+    assert c.started?
+    assert_nil reqs[c.object_id]
+
+    assert_same cached, c
+  end
+
   def test_error_message
     c = basic_connection
     reqs[c.object_id] = 5
@@ -387,6 +408,10 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     assert_equal 'https://example', @http.normalize_uri('https://example')
   end
 
+  def test_max_age
+    assert_in_delta Time.now - 5, @http.max_age
+  end
+
   def test_pipeline
     skip 'net-http-pipeline not installed' unless defined?(Net::HTTP::Pipeline)
 
@@ -449,7 +474,8 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
   def test_reset
     c = basic_connection
     c.start
-    reqs[c.object_id] = 5
+    touts[c.object_id] = Time.now
+    reqs[c.object_id]  = 5
 
     @http.reset c
 
@@ -457,10 +483,12 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     assert c.finished?
     assert c.reset?
     assert_nil reqs[c.object_id]
+    assert_equal Net::HTTP::Persistent::EPOCH, touts[c.object_id]
   end
 
   def test_reset_io_error
     c = basic_connection
+    touts[c.object_id] = Time.now
     reqs[c.object_id] = 5
 
     @http.reset c
@@ -471,6 +499,7 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
 
   def test_reset_host_down
     c = basic_connection
+    touts[c.object_id] = Time.now
     def c.start; raise Errno::EHOSTDOWN end
     reqs[c.object_id] = 5
 
@@ -483,6 +512,7 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
 
   def test_reset_refused
     c = basic_connection
+    touts[c.object_id] = Time.now
     def c.start; raise Errno::ECONNREFUSED end
     reqs[c.object_id] = 5
 
@@ -521,6 +551,8 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
     assert_equal '30',         req['keep-alive']
     assert_match %r%test ua%,  req['user-agent']
 
+    assert_in_delta Time.now, touts[c.object_id]
+
     assert_equal 1, reqs[c.object_id]
   end
 
@@ -608,6 +640,8 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
 
   def test_request_invalid_retry
     c = basic_connection
+    touts[c.object_id] = Time.now
+
     def c.request(*a)
       if defined? @called then
         Net::HTTPResponse.allocate
@@ -637,6 +671,7 @@ class TestNetHttpPersistent < MiniTest::Unit::TestCase
 
   def test_request_reset_retry
     c = basic_connection
+    touts[c.object_id] = Time.now
     def c.request(*a)
       if defined? @called then
         Net::HTTPResponse.allocate

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: net-http-persistent
 version: !ruby/object:Gem::Version 
-  hash: 1
+  hash: 7
   prerelease: 
   segments: 
   - 2
-  - 1
-  version: "2.1"
+  - 2
+  version: "2.2"
 platform: ruby
 authors: 
 - Eric Hodel
@@ -35,7 +35,7 @@ cert_chain:
   x52qPcexcYZR7w==
   -----END CERTIFICATE-----
 
-date: 2011-09-20 00:00:00 Z
+date: 2011-10-25 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: minitest
@@ -45,11 +45,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 5
+        hash: 15
         segments: 
         - 2
-        - 3
-        version: "2.3"
+        - 6
+        version: "2.6"
   type: :development
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -76,7 +76,7 @@ description: |-
   round-trip and causes TCP congestion avoidance negotiation to start over.
   
   Net::HTTP supports persistent connections with some API methods but does not
-  handle reconnection gracefully.  net-http-persistent supports reconnection
+  handle reconnection gracefully.  Net::HTTP::Persistent supports reconnection
   and retry according to RFC 2616.
 email: 
 - drbrain@segment7.net