httpclient 2.2.1 → 2.2.2

data/lib/httpclient.rb

@@ -11,6 +11,7 @@ require 'stringio'
 require 'digest/sha1'
 
 # Extra library
+require 'httpclient/version'
 require 'httpclient/util'
 require 'httpclient/ssl_config'
 require 'httpclient/connection'
@@ -229,7 +230,6 @@ require 'httpclient/cookie'
 #   ruby -rhttpclient -e 'p HTTPClient.head(ARGV.shift).header["last-modified"]' http://dev.ctor.org/
 #
 class HTTPClient
-  VERSION = '2.2.1'
   RUBY_VERSION_STRING = "ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
   /: (\S+) (\S+)/ =~ %q$Id$
   LIB_NAME = "(#{$1}/#{$2}, #{RUBY_VERSION_STRING})"
@@ -335,6 +335,8 @@ class HTTPClient
   attr_proxy(:send_timeout, true)
   # Response receiving timeout in sec.
   attr_proxy(:receive_timeout, true)
+  # Reuse the same connection within this timeout in sec. from last used.
+  attr_proxy(:keep_alive_timeout, true)
   # Negotiation retry count for authentication.  5 by default.
   attr_proxy(:protocol_retry_count, true)
   # if your ruby is older than 2005-09-06, do not set socket_sync = false to
@@ -1060,7 +1062,7 @@ private
     do_get_header(req, res, sess)
     conn.push(res)
     sess.get_body do |part|
-      force_binary(part)
+      set_encoding(part, res.body_encoding)
       if block
         block.call(res, part)
       else
@@ -1096,7 +1098,7 @@ private
     do_get_header(req, res, sess)
     conn.push(res)
     sess.get_body do |part|
-      force_binary(part)
+      set_encoding(part, res.body_encoding)
       pipew.write(part)
     end
     pipew.close
@@ -1109,9 +1111,7 @@ private
 
   def do_get_header(req, res, sess)
     res.http_version, res.status, res.reason, headers = sess.get_header
-    headers.each do |key, value|
-      res.header.add(key, value)
-    end
+    res.header.set_headers(headers)
     if @cookie_manager
       res.header['set-cookie'].each do |cookie|
         @cookie_manager.parse(cookie, req.header.request_uri)
@@ -1125,4 +1125,8 @@ private
     @debug_dev << "\n\n= Dummy Response\n\n"
     @debug_dev << res
   end
+
+  def set_encoding(str, encoding)
+    str.force_encoding(encoding) if encoding
+  end
 end

data/lib/httpclient/http.rb

@@ -7,6 +7,7 @@
 
 
 require 'time'
+require 'open-uri' # for encoding
 
 
 # A namespace module for HTTP Message definitions used by HTTPClient.
@@ -122,6 +123,8 @@ module HTTP
       attr_accessor :body_charset # :nodoc:
       # Used for dumping response.
       attr_accessor :body_date # :nodoc:
+      # Used for keeping content encoding.
+      attr_reader :body_encoding # :nodoc:
 
       # HTTP response status code to reason phrase mapping definition.
       STATUS_CODE_MAP = {
@@ -166,6 +169,7 @@ module HTTP
         @body_type = nil
         @body_charset = nil
         @body_date = nil
+        @body_encoding = nil
 
         @is_request = nil
         @header_item = []
@@ -205,14 +209,31 @@ module HTTP
       end
 
       # Returns 'Content-Type' header value.
-      def contenttype
+      def content_type
         self['Content-Type'][0]
       end
 
       # Sets 'Content-Type' header value.  Overrides if already exists.
-      def contenttype=(contenttype)
+      def content_type=(content_type)
         delete('Content-Type')
-        self['Content-Type'] = contenttype
+        self['Content-Type'] = content_type
+      end
+
+      alias contenttype content_type
+      alias contenttype= content_type=
+
+      if defined?(Encoding::ASCII_8BIT)
+        def set_body_encoding
+          if type = self.content_type
+            OpenURI::Meta.init(o = '')
+            o.meta_add_field('content-type', type)
+            @body_encoding = o.encoding
+          end
+        end
+      else
+        def set_body_encoding
+          @body_encoding = nil
+        end
       end
 
       # Sets byte size of message body.
@@ -285,6 +306,13 @@ module HTTP
         get(key).collect { |item| item[1] }
       end
 
+      def set_headers(headers)
+        headers.each do |key, value|
+          add(key, value)
+        end
+        set_body_encoding
+      end
+
       def create_query_uri()
         if @request_method == 'CONNECT'
           return "#{@request_uri.host}:#{@request_uri.port}"
@@ -787,7 +815,7 @@ module HTTP
       end
 
       def escape_query(query) # :nodoc:
-        query.sort_by { |attr, value| attr.to_s }.collect { |attr, value|
+        query.collect { |attr, value|
           if value.respond_to?(:read)
             value = value.read
           end
@@ -796,12 +824,14 @@ module HTTP
       end
 
       # from CGI.escape
-      def escape(str) # :nodoc:
-        if defined?(Encoding::ASCII_8BIT)
+      if defined?(Encoding::ASCII_8BIT)
+        def escape(str) # :nodoc:
           str.dup.force_encoding(Encoding::ASCII_8BIT).gsub(/([^ a-zA-Z0-9_.-]+)/) {
             '%' + $1.unpack('H2' * $1.bytesize).join('%').upcase
           }.tr(' ', '+')
-        else
+        end
+      else
+        def escape(str) # :nodoc:
           str.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
             '%' + $1.unpack('H2' * $1.bytesize).join('%').upcase
           }.tr(' ', '+')
@@ -915,14 +945,21 @@ module HTTP
       @http_header.reason_phrase = reason
     end
 
+    # Returns 'Content-Type' header value.
+    def content_type
+      @http_header.content_type
+    end
+
     # Sets 'Content-Type' header value.  Overrides if already exists.
-    def contenttype
-      @http_header.contenttype
+    def content_type=(content_type)
+      @http_header.content_type = content_type
     end
+    alias contenttype content_type
+    alias contenttype= content_type=
 
-    # Returns 'Content-Type' header value.
-    def contenttype=(contenttype)
-      @http_header.contenttype = contenttype
+    # Returns content encoding
+    def body_encoding
+      @http_header.body_encoding
     end
 
     # Returns a content of message body.  A String or an IO.

data/lib/httpclient/session.rb

@@ -102,6 +102,7 @@ class HTTPClient
     attr_accessor :connect_retry
     attr_accessor :send_timeout
     attr_accessor :receive_timeout
+    attr_accessor :keep_alive_timeout
     attr_accessor :read_block_size
     attr_accessor :protocol_retry_count
 
@@ -129,7 +130,8 @@ class HTTPClient
       @connect_timeout = 60
       @connect_retry = 1
       @send_timeout = 120
-      @receive_timeout = 60       # For each read_block_size bytes
+      @receive_timeout = 60        # For each read_block_size bytes
+      @keep_alive_timeout = 15     # '15' is from Apache 2 default
       @read_block_size = 1024 * 16 # follows net/http change in 1.8.7
       @protocol_retry_count = 5
 
@@ -139,8 +141,9 @@ class HTTPClient
       @transparent_gzip_decompression = false
       @socket_local = Site.new
 
-      @sess_pool = []
+      @sess_pool = {}
       @sess_pool_mutex = Mutex.new
+      @sess_pool_last_checked = Time.now
     end
 
     def proxy=(proxy)
@@ -172,14 +175,15 @@ class HTTPClient
       close_all
     end
 
+    # assert: sess.last_used must not be nil
     def keep(sess)
       add_cached_session(sess)
     end
 
     def invalidate(site)
       @sess_pool_mutex.synchronize do
-        @sess_pool.each do |sess|
-          if sess.dest == site
+        if pool = @sess_pool[site]
+          pool.each do |sess|
             sess.invalidate
           end
         end
@@ -189,11 +193,12 @@ class HTTPClient
   private
 
     def open(uri, via_proxy = false)
+      site = Site.new(uri)
       sess = nil
-      if cached = get_cached_session(uri)
+      if cached = get_cached_session(site)
         sess = cached
       else
-        sess = Session.new(@client, Site.new(uri), @agent_name, @from)
+        sess = Session.new(@client, site, @agent_name, @from)
         sess.proxy = via_proxy ? @proxy : nil
         sess.socket_sync = @socket_sync
         sess.requested_version = @protocol_version if @protocol_version
@@ -214,8 +219,10 @@ class HTTPClient
 
     def close_all
       @sess_pool_mutex.synchronize do
-        @sess_pool.each do |sess|
-          sess.close
+        @sess_pool.each do |site, pool|
+          pool.each do |sess|
+            sess.close
+          end
         end
       end
       @sess_pool.clear
@@ -223,7 +230,7 @@ class HTTPClient
 
     # This method might not work as you expected...
     def close(dest)
-      if cached = get_cached_session(dest)
+      if cached = get_cached_session(Site.new(dest))
         cached.close
         true
       else
@@ -231,27 +238,44 @@ class HTTPClient
       end
     end
 
-    def get_cached_session(uri)
-      cached = nil
+    def get_cached_session(site)
       @sess_pool_mutex.synchronize do
-        new_pool = []
-        @sess_pool.each do |s|
-          if s.invalidated?
-            s.close # close & remove from the pool
-          elsif !cached && s.dest.match(uri)
-            cached = s
-          else
-            new_pool << s
+        now = Time.now
+        if now > @sess_pool_last_checked + @keep_alive_timeout
+          scrub_cached_session(now)
+          @sess_pool_last_checked = now
+        end
+        if pool = @sess_pool[site]
+          pool.each_with_index do |sess, idx|
+            if valid_session?(sess, now)
+              return pool.slice!(idx)
+            end
           end
         end
-        @sess_pool = new_pool
       end
-      cached
+      nil
+    end
+
+    def scrub_cached_session(now)
+      @sess_pool.each do |site, pool|
+        pool.replace(pool.select { |sess|
+          if valid_session?(sess, now)
+            true
+          else
+            sess.close # close & remove from the pool
+            false
+          end
+        })
+      end
+    end
+
+    def valid_session?(sess, now)
+      !sess.invalidated? and (now <= sess.last_used + @keep_alive_timeout)
     end
 
     def add_cached_session(sess)
       @sess_pool_mutex.synchronize do
-        @sess_pool << sess
+        (@sess_pool[sess.dest] ||= []).unshift(sess)
       end
     end
   end
@@ -523,6 +547,7 @@ class HTTPClient
     attr_accessor :test_loopback_http_response
 
     attr_accessor :transparent_gzip_decompression
+    attr_reader :last_used
 
     def initialize(client, dest, agent_name, from)
       @client = client
@@ -561,6 +586,7 @@ class HTTPClient
       @readbuf = nil
 
       @transparent_gzip_decompression = false
+      @last_used = nil
     end
 
     # Send a request to the server
@@ -594,6 +620,7 @@ class HTTPClient
       @state = :META if @state == :WAIT
       @next_connection = nil
       @requests.push(req)
+      @last_used = Time.now
     end
 
     def close

data/lib/httpclient/util.rb

@@ -116,11 +116,6 @@ class HTTPClient
     def https?(uri)
       uri.scheme.downcase == 'https'
     end
-
-    def force_binary(str)
-      str.force_encoding(Encoding::ASCII_8BIT) if defined?(Encoding::ASCII_8BIT)
-      str
-    end
   end
 
 

data/lib/httpclient/version.rb

@@ -0,0 +1,3 @@
+class HTTPClient
+  VERSION = '2.2.2'
+end

data/test/test_httpclient.rb

@@ -1356,6 +1356,19 @@ EOS
     end
   end
 
+  def test_body_param_order
+    ary = ('b'..'d').map { |k| ['key2', k] } << ['key1', 'a'] << ['key3', 'z']
+    assert_equal("key2=b&key2=c&key2=d&key1=a&key3=z", HTTP::Message.escape_query(ary))
+  end
+
+  if RUBY_VERSION > "1.9"
+    def test_charset
+      body = @client.get(serverurl + 'charset').body
+      assert_equal(Encoding::EUC_JP, body.encoding)
+      assert_equal('あいうえお'.encode(Encoding::EUC_JP), body)
+    end
+  end
+
 private
 
   def check_query_get(query)
@@ -1379,7 +1392,7 @@ private
       :DocumentRoot => File.dirname(File.expand_path(__FILE__))
     )
     @serverport = @server.config[:Port]
-    [:hello, :sleep, :servlet_redirect, :redirect1, :redirect2, :redirect3, :redirect_self, :relative_redirect, :chunked, :largebody, :status, :compressed].each do |sym|
+    [:hello, :sleep, :servlet_redirect, :redirect1, :redirect2, :redirect3, :redirect_self, :relative_redirect, :chunked, :largebody, :status, :compressed, :charset].each do |sym|
       @server.mount(
 	"/#{sym}",
 	WEBrick::HTTPServlet::ProcHandler.new(method("do_#{sym}").to_proc)
@@ -1456,6 +1469,7 @@ private
   end
 
   def do_compressed(req, res)
+    res['content-type'] = 'application/octet-stream'
     if req.query['enc'] == 'gzip'
       res['content-encoding'] = 'gzip'
       res.body = GZIP_CONTENT
@@ -1465,6 +1479,15 @@ private
     end
   end
 
+  def do_charset(req, res)
+    if RUBY_VERSION > "1.9"
+      res.body = 'あいうえお'.encode("euc-jp")
+      res['Content-Type'] = 'text/plain; charset=euc-jp'
+    else
+      res.body = 'this endpoint is for 1.9 or later'
+    end
+  end
+
   def do_status(req, res)
     res.status = req.query['status'].to_i
   end
@@ -1485,6 +1508,7 @@ private
     end
 
     def do_POST(req, res)
+      res["content-type"] = "text/plain" # iso-8859-1, not US-ASCII
       res.body = 'post,' + req.body.to_s
       res["x-query"] = body_response(req)
     end

data/test/test_ssl.rb

@@ -99,16 +99,25 @@ class TestSSL < Test::Unit::TestCase
     assert_equal("hello", @client.get_content(@url))
     assert(@verify_callback_called)
     #
-    cfg.verify_depth = 1
+unless ENV['TRAVIS']
+# On travis environment, verify_depth seems to not work properly.
+# Ubuntu 10.04 + OpenSSL 0.9.8k issue?
+    cfg.verify_depth = 1 # 2 required: root-sub
     @verify_callback_called = false
     begin
       @client.get(@url)
-      assert(false)
+      assert(false, "verify_depth is not supported? #{OpenSSL::OPENSSL_VERSION}")
     rescue OpenSSL::SSL::SSLError => ssle
       assert_match(/certificate verify failed/, ssle.message)
       assert(@verify_callback_called)
     end
     #
+    cfg.verify_depth = 2 # 2 required: root-sub
+    @verify_callback_called = false
+    @client.get(@url)
+    assert(@verify_callback_called)
+    #
+end
     cfg.verify_depth = nil
     cfg.cert_store = OpenSSL::X509::Store.new
     cfg.verify_mode = OpenSSL::SSL::VERIFY_PEER

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: httpclient
 version: !ruby/object:Gem::Version
-  version: 2.2.1
+  version: 2.2.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-06-02 00:00:00.000000000 Z
+date: 2011-10-17 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: nahi@ruby-lang.org
@@ -17,56 +17,57 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/oauthclient.rb
-- lib/http-access2.rb
-- lib/http-access2/cookie.rb
-- lib/http-access2/http.rb
-- lib/hexdump.rb
+- lib/httpclient/auth.rb
+- lib/httpclient/ssl_config.rb
+- lib/httpclient/http.rb
 - lib/httpclient/cacert_sha1.p7s
-- lib/httpclient/cookie.rb
-- lib/httpclient/session.rb
 - lib/httpclient/timeout.rb
-- lib/httpclient/util.rb
-- lib/httpclient/cacert.p7s
-- lib/httpclient/ssl_config.rb
+- lib/httpclient/version.rb
 - lib/httpclient/connection.rb
-- lib/httpclient/auth.rb
-- lib/httpclient/http.rb
+- lib/httpclient/session.rb
+- lib/httpclient/cacert.p7s
+- lib/httpclient/cookie.rb
+- lib/httpclient/util.rb
+- lib/oauthclient.rb
 - lib/httpclient.rb
-- sample/async.rb
-- sample/howto.rb
-- sample/cookie.rb
-- sample/oauth_buzz.rb
-- sample/wcat.rb
-- sample/oauth_twitter.rb
+- lib/http-access2.rb
+- lib/http-access2/http.rb
+- lib/http-access2/cookie.rb
+- lib/hexdump.rb
+- sample/auth.rb
 - sample/dav.rb
-- sample/thread.rb
 - sample/stream.rb
-- sample/ssl/0key.pem
+- sample/async.rb
+- sample/wcat.rb
+- sample/ssl/1000cert.pem
+- sample/ssl/1000key.pem
 - sample/ssl/ssl_client.rb
+- sample/ssl/0key.pem
 - sample/ssl/htdocs/index.html
-- sample/ssl/webrick_httpsd.rb
 - sample/ssl/0cert.pem
-- sample/ssl/1000cert.pem
-- sample/ssl/1000key.pem
+- sample/ssl/webrick_httpsd.rb
+- sample/thread.rb
 - sample/oauth_friendfeed.rb
-- sample/auth.rb
+- sample/oauth_twitter.rb
+- sample/cookie.rb
+- sample/howto.rb
+- sample/oauth_buzz.rb
+- test/client.cert
+- test/test_http-access2.rb
+- test/server.cert
+- test/htdigest
+- test/test_ssl.rb
 - test/test_cookie.rb
+- test/runner.rb
 - test/htpasswd
-- test/test_httpclient.rb
 - test/test_auth.rb
-- test/runner.rb
-- test/helper.rb
-- test/subca.cert
-- test/server.key
 - test/client.key
+- test/helper.rb
 - test/ca.cert
-- test/htdigest
-- test/test_http-access2.rb
-- test/server.cert
-- test/test_ssl.rb
-- test/client.cert
 - test/sslsvr.rb
+- test/subca.cert
+- test/server.key
+- test/test_httpclient.rb
 homepage: http://github.com/nahi/httpclient
 licenses: []
 post_install_message: 
@@ -87,7 +88,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.4
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: gives something like the functionality of libwww-perl (LWP) in Ruby