httpclient 2.1.5.2 → 2.1.6

data/lib/hexdump.rb

@@ -0,0 +1,36 @@
+# This was written by Arai-san and published at
+# http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-list/31987
+
+
+module HexDump
+  def encode(str)
+    offset = 0
+    result = []
+    while raw = str.slice(offset, 16) and raw.length > 0
+      # data field
+      data = ''
+      for v in raw.unpack('N* a*')
+	if v.kind_of? Integer
+	  data << sprintf("%08x ", v)
+	else
+	  v.each_byte {|c| data << sprintf("%02x", c) }
+	end
+      end
+      # text field
+      text = raw.tr("\000-\037\177-\377", ".")
+      result << sprintf("%08x  %-36s  %s", offset, data, text)
+      offset += 16
+      # omit duplicate line
+      if /^(#{ Regexp.quote(raw) })+/n =~ str[offset .. -1]
+	result << sprintf("%08x  ...", offset)
+	offset += $&.length
+	# should print at the end
+	if offset == str.length
+	  result << sprintf("%08x  %-36s  %s", offset-16, data, text)
+	end
+      end
+    end
+    result
+  end
+  module_function :encode
+end

data/lib/httpclient.rb

@@ -19,7 +19,7 @@ require 'httpclient/http'
 require 'httpclient/auth'
 require 'httpclient/cookie'
 
-
+# :main:HTTPClient
 # The HTTPClient class provides several methods for accessing Web resources
 # via HTTP.
 #
@@ -98,6 +98,17 @@ require 'httpclient/cookie'
 #       res = clnt.post(uri, body)
 #     end
 #
+# 3. Do multipart wth custom body.
+#
+#     File.open('/tmp/post_data') do |file|
+#       body = [{ 'Content-Type' => 'application/atom+xml; charset=UTF-8',
+#                 :content => '<entry>...</entry>' },
+#               { 'Content-Type' => 'video/mp4',
+#                 'Content-Transfer-Encoding' => 'binary',
+#                 :content => file }]
+#       res = clnt.post(uri, body)
+#     end
+#
 # === Accessing via SSL
 #
 # Ruby needs to be compiled with OpenSSL.
@@ -199,9 +210,9 @@ require 'httpclient/cookie'
 #   ruby -rhttpclient -e 'p HTTPClient.head(ARGV.shift).header["last-modified"]' http://dev.ctor.org/
 #
 class HTTPClient
-  VERSION = '2.1.5'
+  VERSION = '2.1.6'
   RUBY_VERSION_STRING = "ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
-  /: (\S+) (\S+)/ =~ %q$Id: httpclient.rb 280 2009-06-02 15:44:28Z nahi $
+  /: (\S+) (\S+)/ =~ %q$Id$
   LIB_NAME = "(#{$1}/#{$2}, #{RUBY_VERSION_STRING})"
 
   include Util
@@ -317,6 +328,8 @@ class HTTPClient
   # An array of response HTTP String (not a HTTP message body) which is used
   # for loopback test.  See test/* to see how to use it.
   attr_proxy(:test_loopback_http_response)
+  # Decompress a compressed (with gzip or deflate) content body transparently. false by default.
+  attr_proxy(:transparent_gzip_decompression, true)
 
   # Default extheader for PROPFIND request.
   PROPFIND_DEFAULT_EXTHEADER = { 'Depth' => '0' }
@@ -336,7 +349,7 @@ class HTTPClient
   # You can use a keyword argument style Hash.  Keys are :proxy, :agent_name
   # and :from.
   #
-  #   HTTPClient.new(:agent_name = 'MyAgent/0.1')
+  #   HTTPClient.new(:agent_name => 'MyAgent/0.1')
   def initialize(*args)
     proxy, agent_name, from = keyword_argument(args, :proxy, :agent_name, :from)
     @proxy = nil        # assigned later.
@@ -522,17 +535,25 @@ class HTTPClient
   # Posts a content.
   #
   # uri:: a String or an URI object which represents an URL of web resource.
-  # body:: a Hash or an Array of body part.
-  #        e.g. { "a" => "b" } => 'a=b'.
+  # body:: a Hash or an Array of body part. e.g.
+  #          { "a" => "b" } => 'a=b'
   #        Give an array to pass multiple value like
-  #        [["a", "b"], ["a", "c"]] => 'a=b&a=c'.
+  #          [["a", "b"], ["a", "c"]] => 'a=b&a=c'
   #        When you pass a File as a value, it will be posted as a
-  #        multipart/form-data.  e.g. { 'upload' => file }
-  # extheader:: a Hash or an Array of extra headers.  e.g.
-  #             { 'Accept' => '*/*' } or
-  #             [['Accept', 'image/jpeg'], ['Accept', 'image/png']].
+  #        multipart/form-data.  e.g.
+  #          { 'upload' => file }
+  #        You can also send custom multipart by passing an array of hashes.
+  #        Each part must have a :content attribute which can be a file, all
+  #        other keys will become headers.
+  #          [{ 'Content-Type' => 'text/plain', :content => "some text" },
+  #           { 'Content-Type' => 'video/mp4', :content => File.new('video.mp4') }]
+  #          => <Two parts with custom Content-Type header>
+  # extheader:: a Hash or an Array of extra headers. e.g.
+  #               { 'Accept' => '*/*' }
+  #             or
+  #               [['Accept', 'image/jpeg'], ['Accept', 'image/png']].
   # &block:: Give a block to get chunked message-body of response like
-  #          post_content(uri) { |chunked_body| ... }.
+  #            post_content(uri) { |chunked_body| ... }.
   #          Size of each chunk may not be the same.
   #
   # post_content follows HTTP redirect status (see HTTP::Status.redirect?)
@@ -569,14 +590,14 @@ class HTTPClient
   # in HTTP header.
   def default_redirect_uri_callback(uri, res)
     newuri = URI.parse(res.header['location'][0])
-    if https?(uri) && !https?(newuri)
-      raise BadResponseError.new("redirecting to non-https resource")
-    end
     unless newuri.is_a?(URI::HTTP)
       newuri = uri + newuri
       STDERR.puts("could be a relative URI in location header which is not recommended")
       STDERR.puts("'The field value consists of a single absolute URI' in HTTP spec")
     end
+    if https?(uri) && !https?(newuri)
+      raise BadResponseError.new("redirecting to non-https resource")
+    end
     puts "redirect to: #{newuri}" if $DEBUG
     newuri
   end
@@ -634,13 +655,21 @@ class HTTPClient
   #         e.g. { "a" => "b" } => 'http://host/part?a=b'
   #         Give an array to pass multiple value like
   #         [["a", "b"], ["a", "c"]] => 'http://host/part?a=b&a=c'
-  # body:: a Hash or an Array of body part.
-  #        e.g. { "a" => "b" } => 'a=b'.
+  # body:: a Hash or an Array of body part. e.g.
+  #          { "a" => "b" }
+  #          => 'a=b'
   #        Give an array to pass multiple value like
-  #        [["a", "b"], ["a", "c"]] => 'a=b&a=c'.
+  #          [["a", "b"], ["a", "c"]]
+  #          => 'a=b&a=c'.
   #        When the given method is 'POST' and the given body contains a file
-  #        as a value, it will be posted as a multipart/form-data.
-  #        e.g. { 'upload' => file }
+  #        as a value, it will be posted as a multipart/form-data. e.g.
+  #          { 'upload' => file }
+  #        You can also send custom multipart by passing an array of hashes.
+  #        Each part must have a :content attribute which can be a file, all
+  #        other keys will become headers.
+  #          [{ 'Content-Type' => 'text/plain', :content => "some text" },
+  #           { 'Content-Type' => 'video/mp4', :content => File.new('video.mp4') }]
+  #          => <Two parts with custom Content-Type header>
   #        See HTTP::Message.file? for actual condition of 'a file'.
   # extheader:: a Hash or an Array of extra headers.  e.g.
   #             { 'Accept' => '*/*' } or
@@ -747,6 +776,10 @@ private
   end
 
   class KeepAliveDisconnected < StandardError # :nodoc:
+    attr_reader :sess
+    def initialize(sess = nil)
+      @sess = sess
+    end
   end
 
   def do_request(method, uri, query, body, extheader, &block)
@@ -777,22 +810,26 @@ private
   def do_request_async(method, uri, query, body, extheader)
     conn = Connection.new
     t = Thread.new(conn) { |tconn|
-      if HTTP::Message.file?(body)
-        pos = body.pos rescue nil
-      end
-      retry_count = @session_manager.protocol_retry_count
-      proxy = no_proxy?(uri) ? nil : @proxy
-      while retry_count > 0
-        body.pos = pos if pos
-        req = create_request(method, uri, query, body, extheader)
-        begin
-          protect_keep_alive_disconnected do
-            do_get_stream(req, proxy, tconn)
+      begin
+        if HTTP::Message.file?(body)
+          pos = body.pos rescue nil
+        end
+        retry_count = @session_manager.protocol_retry_count
+        proxy = no_proxy?(uri) ? nil : @proxy
+        while retry_count > 0
+          body.pos = pos if pos
+          req = create_request(method, uri, query, body, extheader)
+          begin
+            protect_keep_alive_disconnected do
+              do_get_stream(req, proxy, tconn)
+            end
+            break
+          rescue RetryableResponse
+            retry_count -= 1
           end
-          break
-        rescue RetryableResponse
-          retry_count -= 1
         end
+      rescue Exception
+        conn.push $!
       end
     }
     conn.async_thread = t
@@ -846,7 +883,10 @@ private
   def protect_keep_alive_disconnected
     begin
       yield
-    rescue KeepAliveDisconnected
+    rescue KeepAliveDisconnected => e
+      if e.sess
+        @session_manager.invalidate(e.sess.dest)
+      end
       yield
     end
   end
@@ -860,7 +900,7 @@ private
     end
     boundary = nil
     if body
-      dummy, content_type = extheader.find { |key, value|
+      _, content_type = extheader.find { |key, value|
         key.downcase == 'content-type'
       }
       if content_type
@@ -932,10 +972,6 @@ private
     false
   end
 
-  def https?(uri)
-    uri.scheme.downcase == 'https'
-  end
-
   # !! CAUTION !!
   #   Method 'do_get*' runs under MT conditon. Be careful to change.
   def do_get_block(req, proxy, conn, &block)
@@ -956,12 +992,15 @@ private
     do_get_header(req, res, sess)
     conn.push(res)
     sess.get_body do |part|
+      force_binary(part)
       if block
         block.call(res, part)
       else
         content << part
       end
     end
+    # there could be a race condition but it's OK to cache unreusable
+    # connection because we do retry for that case.
     @session_manager.keep(sess) unless sess.closed?
     commands = @request_filter.collect { |filter|
       filter.filter_response(req, res)
@@ -989,18 +1028,19 @@ private
     do_get_header(req, res, sess)
     conn.push(res)
     sess.get_body do |part|
-      pipew.syswrite(part)
+      force_binary(part)
+      pipew.write(part)
     end
     pipew.close
     @session_manager.keep(sess) unless sess.closed?
-    commands = @request_filter.collect { |filter|
+    _ = @request_filter.collect { |filter|
       filter.filter_response(req, res)
     }
     # ignore commands (not retryable in async mode)
   end
 
   def do_get_header(req, res, sess)
-    res.version, res.status, res.reason, headers = sess.get_header
+    res.http_version, res.status, res.reason, headers = sess.get_header
     headers.each do |key, value|
       res.header.add(key, value)
     end

data/lib/httpclient/auth.rb

@@ -63,22 +63,25 @@ class HTTPClient
   #
   # WWWAuth has sub filters (BasicAuth, DigestAuth, NegotiateAuth and
   # SSPINegotiateAuth) and delegates some operations to it.
-  # NegotiateAuth requires 'ruby/ntlm' module.
-  # SSPINegotiateAuth requires 'win32/sspi' module.
+  # NegotiateAuth requires 'ruby/ntlm' module (rubyntlm gem).
+  # SSPINegotiateAuth requires 'win32/sspi' module (rubysspi gem).
   class WWWAuth < AuthFilterBase
     attr_reader :basic_auth
     attr_reader :digest_auth
     attr_reader :negotiate_auth
     attr_reader :sspi_negotiate_auth
+    attr_reader :oauth
 
     # Creates new WWWAuth.
     def initialize
       @basic_auth = BasicAuth.new
       @digest_auth = DigestAuth.new
       @negotiate_auth = NegotiateAuth.new
+      @ntlm_auth = NegotiateAuth.new('NTLM')
       @sspi_negotiate_auth = SSPINegotiateAuth.new
+      @oauth = OAuth.new
       # sort authenticators by priority
-      @authenticator = [@negotiate_auth, @sspi_negotiate_auth, @digest_auth, @basic_auth]
+      @authenticator = [@oauth, @negotiate_auth, @ntlm_auth, @sspi_negotiate_auth, @digest_auth, @basic_auth]
     end
 
     # Resets challenge state.  See sub filters for more details.
@@ -151,9 +154,10 @@ class HTTPClient
     def initialize
       @basic_auth = BasicAuth.new
       @negotiate_auth = NegotiateAuth.new
+      @ntlm_auth = NegotiateAuth.new('NTLM')
       @sspi_negotiate_auth = SSPINegotiateAuth.new
       # sort authenticators by priority
-      @authenticator = [@negotiate_auth, @sspi_negotiate_auth, @basic_auth]
+      @authenticator = [@negotiate_auth, @ntlm_auth, @sspi_negotiate_auth, @basic_auth]
     end
 
     # Resets challenge state.  See sub filters for more details.
@@ -207,6 +211,8 @@ class HTTPClient
   # Authentication filter for handling BasicAuth negotiation.
   # Used in WWWAuth and ProxyAuth.
   class BasicAuth
+    include HTTPClient::Util
+
     # Authentication scheme.
     attr_reader :scheme
 
@@ -251,8 +257,8 @@ class HTTPClient
     end
 
     # Challenge handler: remember URL for response.
-    def challenge(uri, param_str)
-      @challengeable[uri] = true
+    def challenge(uri, param_str = nil)
+      @challengeable[urify(uri)] = true
       true
     end
   end
@@ -301,8 +307,7 @@ class HTTPClient
         Util.uri_part_of(target_uri, uri)
       }
       return nil unless user
-      uri = req.header.request_uri
-      calc_cred(req.header.request_method, uri, user, passwd, param)
+      calc_cred(req, user, passwd, param)
     end
 
     # Challenge handler: remember URL and challenge token for response.
@@ -317,9 +322,11 @@ class HTTPClient
     # http://tools.assembla.com/breakout/wiki/DigestForSoap
     # Thanks!
     # supported algorithm: MD5 only for now
-    def calc_cred(method, uri, user, passwd, param)
+    def calc_cred(req, user, passwd, param)
+      method = req.header.request_method
+      path = req.header.create_query_uri
       a_1 = "#{user}:#{param['realm']}:#{passwd}"
-      a_2 = "#{method}:#{uri.path}"
+      a_2 = "#{method}:#{path}"
       nonce = param['nonce']
       cnonce = generate_cnonce()
       @nonce_count += 1
@@ -334,12 +341,12 @@ class HTTPClient
       header << "username=\"#{user}\""
       header << "realm=\"#{param['realm']}\""
       header << "nonce=\"#{nonce}\""
-      header << "uri=\"#{uri.path}\""
+      header << "uri=\"#{path}\""
       header << "cnonce=\"#{cnonce}\""
       header << "nc=#{'%08x' % @nonce_count}"
-      header << "qop=\"#{param['qop']}\""
+      header << "qop=#{param['qop']}"
       header << "response=\"#{Digest::MD5.hexdigest(message_digest.join(":"))}\""
-      header << "algorithm=\"MD5\""
+      header << "algorithm=MD5"
       header << "opaque=\"#{param['opaque']}\"" if param.key?('opaque')
       header.join(", ")
     end
@@ -376,11 +383,11 @@ class HTTPClient
     attr_reader :ntlm_opt
 
     # Creates new NegotiateAuth filter.
-    def initialize
+    def initialize(scheme = "Negotiate")
       @auth = {}
       @auth_default = nil
       @challenge = {}
-      @scheme = "Negotiate"
+      @scheme = scheme
       @ntlm_opt = {
         :ntlmv2 => true
       }
@@ -518,5 +525,243 @@ class HTTPClient
     end
   end
 
+  # Authentication filter for handling OAuth negotiation.
+  # Used in WWWAuth.
+  #
+  # CAUTION: This impl only support '#7 Accessing Protected Resources' in OAuth
+  # Core 1.0 spec for now. You need to obtain Access token and Access secret by
+  # yourself.
+  #
+  # CAUTION: This impl does NOT support OAuth Request Body Hash spec for now.
+  # http://oauth.googlecode.com/svn/spec/ext/body_hash/1.0/oauth-bodyhash.html
+  #
+  class OAuth
+    include HTTPClient::Util
+
+    # Authentication scheme.
+    attr_reader :scheme
+
+    class Config
+      include HTTPClient::Util
+
+      attr_accessor :http_method
+      attr_accessor :realm
+      attr_accessor :consumer_key
+      attr_accessor :consumer_secret
+      attr_accessor :token
+      attr_accessor :secret
+      attr_accessor :signature_method
+      attr_accessor :version
+      attr_accessor :callback
+      attr_accessor :verifier
+
+      # for OAuth Session 1.0 (draft)
+      attr_accessor :session_handle
+
+      attr_reader :signature_handler
+
+      attr_accessor :debug_timestamp
+      attr_accessor :debug_nonce
+
+      def initialize(*args)
+        @http_method,
+          @realm,
+          @consumer_key,
+          @consumer_secret,
+          @token,
+          @secret,
+          @signature_method,
+          @version,
+          @callback,
+          @verifier =
+        keyword_argument(args,
+          :http_method,
+          :realm,
+          :consumer_key,
+          :consumer_secret,
+          :token,
+          :secret,
+          :signature_method,
+          :version,
+          :callback,
+          :verifier
+        )
+        @http_method ||= :post
+        @session_handle = nil
+        @signature_handler = {}
+      end
+    end
+
+    def self.escape(str) # :nodoc:
+      if str.respond_to?(:force_encoding)
+        str.dup.force_encoding('BINARY').gsub(/([^a-zA-Z0-9_.~-]+)/) {
+          '%' + $1.unpack('H2' * $1.bytesize).join('%').upcase
+        }
+      else
+        str.gsub(/([^a-zA-Z0-9_.~-]+)/n) {
+          '%' + $1.unpack('H2' * $1.bytesize).join('%').upcase
+        }
+      end
+    end
+
+    def escape(str)
+      self.class.escape(str)
+    end
+
+    # Creates new DigestAuth filter.
+    def initialize
+      @config = nil # common config
+      @auth = {} # configs for each site
+      @challengeable = {}
+      @nonce_count = 0
+      @signature_handler = {
+        'HMAC-SHA1' => method(:sign_hmac_sha1)
+      }
+      @scheme = "OAuth"
+    end
+
+    # Resets challenge state.  Do not send '*Authorization' header until the
+    # server sends '*Authentication' again.
+    def reset_challenge
+      @challengeable.clear
+    end
+
+    # Set authentication credential.
+    # You cannot set OAuth config via WWWAuth#set_auth. Use OAuth#config=
+    def set(uri, user, passwd)
+      # not supported
+    end
+
+    # Set authentication credential.
+    def set_config(uri, config)
+      if uri.nil?
+        @config = config
+      else
+        uri = Util.uri_dirname(urify(uri))
+        @auth[uri] = config
+      end
+    end
+
+    # Get authentication credential.
+    def get_config(uri = nil)
+      if uri.nil?
+        @config
+      else
+        uri = urify(uri)
+        Util.hash_find_value(@auth) { |cand_uri, cred|
+          Util.uri_part_of(uri, cand_uri)
+        }
+      end
+    end
+
+    # Response handler: returns credential.
+    # It sends cred only when a given uri is;
+    # * child page of challengeable(got *Authenticate before) uri and,
+    # * child page of defined credential
+    def get(req)
+      target_uri = req.header.request_uri
+      return nil unless @challengeable[nil] or @challengeable.find { |uri, ok|
+        Util.uri_part_of(target_uri, uri) and ok
+      }
+      config = get_config(target_uri) || @config
+      return nil unless config
+      calc_cred(req, config)
+    end
+
+    # Challenge handler: remember URL for response.
+    def challenge(uri, param_str = nil)
+      if uri.nil?
+        @challengeable[nil] = true
+      else
+        @challengeable[urify(uri)] = true
+      end
+      true
+    end
+
+  private
+
+    def calc_cred(req, config)
+      header = {}
+      header['oauth_consumer_key'] = config.consumer_key
+      header['oauth_token'] = config.token
+      header['oauth_signature_method'] = config.signature_method
+      header['oauth_timestamp'] = config.debug_timestamp || Time.now.to_i.to_s
+      header['oauth_nonce'] = config.debug_nonce || generate_nonce()
+      header['oauth_version'] = config.version if config.version
+      header['oauth_callback'] = config.callback if config.callback
+      header['oauth_verifier'] = config.verifier if config.verifier
+      header['oauth_session_handle'] = config.session_handle if config.session_handle
+      signature = sign(config, header, req)
+      header['oauth_signature'] = signature
+      # no need to do but we should sort for easier to test.
+      str = header.sort_by { |k, v| k }.map { |k, v| encode_header(k, v) }.join(', ')
+      if config.realm
+        str = %Q(realm="#{config.realm}", ) + str
+      end
+      str
+    end
+
+    def generate_nonce
+      @nonce_count += 1
+      now = "%012d" % Time.now.to_i
+      pk = Digest::MD5.hexdigest([@nonce_count.to_s, now, self.__id__, Process.pid, rand(65535)].join)[0, 32]
+      [now + ':' + pk].pack('m*').chop
+    end
+
+    def encode_header(k, v)
+      %Q(#{escape(k.to_s)}="#{escape(v.to_s)}")
+    end
+
+    def encode_param(params)
+      params.map { |k, v|
+        [v].flatten.map { |vv|
+          %Q(#{escape(k.to_s)}=#{escape(vv.to_s)})
+        }
+      }.flatten
+    end
+
+    def sign(config, header, req)
+      base_string = create_base_string(config, header, req)
+      if handler = config.signature_handler[config.signature_method] || @signature_handler[config.signature_method.to_s]
+        handler.call(config, base_string)
+      else
+        raise ConfigurationError.new("Unknown OAuth signature method: #{config.signature_method}")
+      end
+    end
+
+    def create_base_string(config, header, req)
+      params = encode_param(header)
+      query = req.header.request_query
+      if query and HTTP::Message.multiparam_query?(query)
+        params += encode_param(query)
+      end
+      # captures HTTP Message body only for 'application/x-www-form-urlencoded'
+      if req.header.contenttype == 'application/x-www-form-urlencoded' and req.body.size
+        params += encode_param(HTTP::Message.parse(req.body.content))
+      end
+      uri = req.header.request_uri
+      if uri.query
+        params += encode_param(HTTP::Message.parse(uri.query))
+      end
+      if uri.port == uri.default_port
+        request_url = "#{uri.scheme.downcase}://#{uri.host}#{uri.path}"
+      else
+        request_url = "#{uri.scheme.downcase}://#{uri.host}:#{uri.port}#{uri.path}"
+      end
+      [req.header.request_method.upcase, request_url, params.sort.join('&')].map { |e|
+        escape(e)
+      }.join('&')
+    end
+
+    def sign_hmac_sha1(config, base_string)
+      unless SSLEnabled
+        raise ConfigurationError.new("openssl required for OAuth implementation")
+      end
+      key = [escape(config.consumer_secret.to_s), escape(config.secret.to_s)].join('&')
+      digester = OpenSSL::Digest::SHA1.new
+      [OpenSSL::HMAC.digest(digester, key, base_string)].pack('m*').chomp
+    end
+  end
+
 
 end