httpclient 2.5.3.3 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f9393154ead7b6adecbda232894fb4fd0f6c0c4
-  data.tar.gz: 93aba8378632669c8c99d5c91a222a97b59fb28a
+  metadata.gz: a6c0658e8a69a8fc6b9bd591912d99dfdb01aeb0
+  data.tar.gz: e513505c66b946b6c3dd2fb3b60a3f9b9252b76c
 SHA512:
-  metadata.gz: 20aaefda3dadda680b3fe25bc54fc0f7f099654d2b59177f1bedba3bc9442d03f1c1953971c316da32e3fe8a11419c0f5d0fabf5267cf40d353625eb07e14661
-  data.tar.gz: 11be25379bf090278ac260ab235d1ac4bddad1bb03298d614e5716c455c8816ff6f4e189cb5df5e9e2c15574c5cf996c5311f78d2cf672d0f04e05efb74b4e7e
+  metadata.gz: aca638628efee8a54e2a6747db133232be76f355b892f89f4816790180955019aa1edfc9d3426b97988e6fa51fbc19eb2b7735c4c6ab8b570bb166678beccfd6
+  data.tar.gz: 02ec73e1b3ae1edf0dd0fe5f6e070de3f8b4a309066e1ce8f2b945ed4d53d532a2bfd513374ce1dfd793862221b57e172c21e4cc4166b32b811409d773e74af7

data/bin/httpclient

@@ -12,12 +12,18 @@
 require 'httpclient'
 
 METHODS = ['head', 'get', 'post', 'put', 'delete', 'options', 'propfind', 'proppatch', 'trace']
-if ARGV.size >= 2 && METHODS.include?(ARGV[0])
+method = ARGV.shift
+url = ARGV.shift
+if method && url
   client = HTTPClient.new
-  client.debug_dev = STDERR
-  $DEBUG = true
-  require 'pp'
-  pp client.send(*ARGV)
+  if method == 'download'
+    print client.get_content(url)
+  else
+    client.debug_dev = STDERR
+    $DEBUG = true
+    require 'pp'
+    pp client.send(*ARGV)
+  end
   exit
 end
 

data/bin/jsonclient

@@ -1,98 +1,52 @@
 #!/usr/bin/env ruby
 
-# httpclient shell command.
+# jsonclient shell command.
 #
-# Usage: 1) % httpclient get https://www.google.co.jp/ q=ruby
-# Usage: 2) % httpclient
+# Usage: 1) % jsonclient post https://www.example.com/ content.json
+# Usage: 2) % jsonclient
 #
 # For 1) it issues a GET request to the given URI and shows the wiredump and
 # the parsed result.  For 2) it invokes irb shell with the binding that has a
-# HTTPClient as 'self'.  You can call HTTPClient instance methods like;
-# > get "https://www.google.co.jp/", :q => :ruby
-require 'httpclient'
-require 'json'
+# JSONClient as 'self'.  You can call JSONClient instance methods like;
+# > post "https://www.example.com/resource", {'hello' => 'world'}
+require 'jsonclient'
  
-module HTTP
-  class Message
-    # Returns JSON object of message body
-    alias original_content content
-    def content
-      if JSONClient::CONTENT_TYPE_JSON_REGEX =~ content_type
-        JSON.parse(original_content)
-      else
-        original_content
-      end
-    end
+method = ARGV.shift
+url = ARGV.shift
+body = []
+if ['post', 'put'].include?(method)
+  if ARGV.size == 1 && File.exist?(ARGV[0])
+    body << File.read(ARGV[0])
+  else
+    body << ARGF.read
   end
 end
- 
- 
-# JSONClient provides JSON related methods in addition to HTTPClient.
-class JSONClient < HTTPClient
-  CONTENT_TYPE_JSON_REGEX = /(application|text)\/(x-)?json/i
- 
-  attr_accessor :content_type_json
- 
-  class JSONRequestHeaderFilter
-    attr_accessor :replace
- 
-    def initialize(client)
-      @client = client
-      @replace = false
-    end
- 
-    def filter_request(req)
-      req.header['content-type'] = @client.content_type_json if @replace
-    end
- 
-    def filter_response(req, res)
-      @replace = false
-    end
-  end
- 
-  def initialize(*args)
-    super
-    @header_filter = JSONRequestHeaderFilter.new(self)
-    @request_filter << @header_filter
-    @content_type_json = 'application/json; charset=utf-8'
-  end
- 
-  def post(uri, *args, &block)
-    @header_filter.replace = true
-    request(:post, uri, jsonify(argument_to_hash(args, :body, :header, :follow_redirect)), &block)
-  end
- 
-  def put(uri, *args, &block)
-    @header_filter.replace = true
-    request(:put, uri, jsonify(argument_to_hash(args, :body, :header)), &block)
-  end
- 
-private
- 
-  def jsonify(hash)
-    if hash[:body] && hash[:body].is_a?(Hash)
-      hash[:body] = JSON.generate(hash[:body])
+if method && url
+  require 'pp'
+  client = JSONClient.new
+  client.debug_dev = STDERR if $DEBUG
+  res = client.send(method, url, *body)
+  STDERR.puts('RESPONSE HEADER: ')
+  PP.pp(res.headers, STDERR)
+  if res.ok?
+    begin
+      puts JSON.pretty_generate(res.content)
+    rescue JSON::GeneratorError
+      puts res.content
     end
-    hash
+    exit 0
+  else
+    STDERR.puts res.content
+    exit 1
   end
 end
 
-METHODS = ['head', 'get', 'post', 'put', 'delete', 'options', 'propfind', 'proppatch', 'trace']
-if ARGV.size >= 2 && METHODS.include?(ARGV[0])
-  client = JSONClient.new
-  client.debug_dev = STDERR
-  $DEBUG = true
-  require 'pp'
-  pp client.send(*ARGV)
-  exit
-end
-
 require 'irb'
 require 'irb/completion'
 
 class Runner
   def initialize
-    @httpclient = HTTPClient.new
+    @httpclient = JSONClient.new
   end
 
   def method_missing(msg, *a, &b)
@@ -124,7 +78,7 @@ class Runner
   end
 
   def to_s
-    'HTTPClient'
+    'JSONClient'
   end
 end
 

data/lib/httpclient.rb

@@ -308,7 +308,7 @@ class HTTPClient
 
   # HTTPClient::SSLConfig:: SSL configurator.
   attr_reader :ssl_config
-  # WebAgent::CookieManager:: Cookies configurator.
+  # HTTPClient::CookieManager:: Cookies configurator.
   attr_accessor :cookie_manager
   # An array of response HTTP message body String which is used for loop-back
   # test.  See test/* to see how to use it.  If you want to do loop-back test
@@ -415,7 +415,7 @@ class HTTPClient
     @session_manager.agent_name = agent_name || DEFAULT_AGENT_NAME
     @session_manager.from = from
     @session_manager.ssl_config = @ssl_config = SSLConfig.new(self)
-    @cookie_manager = WebAgent::CookieManager.new
+    @cookie_manager = CookieManager.new
     @follow_redirect_count = 10
     load_environment
     self.proxy = proxy if proxy
@@ -576,7 +576,6 @@ class HTTPClient
 
   # Try to save Cookies to the file specified in set_cookie_store.  Unexpected
   # error will be raised if you don't call set_cookie_store first.
-  # (interface mismatch between WebAgent::CookieManager implementation)
   def save_cookie_store
     @cookie_manager.save_cookies
   end
@@ -927,6 +926,11 @@ class HTTPClient
 private
 
   class RetryableResponse < StandardError # :nodoc:
+    attr_reader :res
+
+    def initialize(res = nil)
+      @res = res
+    end
   end
 
   class KeepAliveDisconnected < StandardError # :nodoc:
@@ -947,24 +951,37 @@ private
   end
 
   def do_request(method, uri, query, body, header, &block)
-    conn = Connection.new
     res = nil
     if HTTP::Message.file?(body)
       pos = body.pos rescue nil
     end
     retry_count = @session_manager.protocol_retry_count
     proxy = no_proxy?(uri) ? nil : @proxy
+    previous_request = previous_response = nil
     while retry_count > 0
       body.pos = pos if pos
       req = create_request(method, uri, query, body, header)
+      if previous_request
+        # to remember IO positions to read
+        req.http_body.positions = previous_request.http_body.positions
+      end
       begin
         protect_keep_alive_disconnected do
-          do_get_block(req, proxy, conn, &block)
+          # TODO: remove Connection.new
+          # We want to delete Connection usage in do_get_block but Newrelic gem depends on it.
+          # https://github.com/newrelic/rpm/blob/master/lib/new_relic/agent/instrumentation/httpclient.rb#L34-L36
+          conn = Connection.new
+          res = do_get_block(req, proxy, conn, &block)
+          # Webmock's do_get_block returns ConditionVariable
+          if !res.respond_to?(:previous)
+            res = conn.pop
+          end
         end
-        res = conn.pop
+        res.previous = previous_response
         break
-      rescue RetryableResponse
-        res = conn.pop
+      rescue RetryableResponse => e
+        previous_request = req
+        previous_response = res = e.res
         retry_count -= 1
       end
     end
@@ -1029,15 +1046,21 @@ private
       pos = body.pos rescue nil
     end
     retry_number = 0
+    previous = nil
+    request_query = query
     while retry_number < @follow_redirect_count
       body.pos = pos if pos
-      res = do_request(method, uri, query, body, header, &filtered_block)
+      res = do_request(method, uri, request_query, body, header, &filtered_block)
+      res.previous = previous
       if res.redirect?
         if res.header['location'].empty?
           raise BadResponseError.new("Missing Location header for redirect", res)
         end
         method = :get if res.see_other? # See RFC2616 10.3.4
         uri = urify(@redirect_uri_callback.call(uri, res))
+        # To avoid duped query parameter. 'location' must include query part.
+        request_query = nil
+        previous = res
         retry_number += 1
       else
         return res
@@ -1058,10 +1081,13 @@ private
     begin
       yield
     rescue KeepAliveDisconnected => e
-      if e.sess
-        @session_manager.invalidate(e.sess.dest)
+      # Force to create new connection
+      Thread.current[:HTTPClient_AcquireNewConnection] = true
+      begin
+        yield
+      ensure
+        Thread.current[:HTTPClient_AcquireNewConnection] = false
       end
-      yield
     end
   end
 
@@ -1101,8 +1127,11 @@ private
     header.each do |key, value|
       req.header.add(key.to_s, value)
     end
-    if @cookie_manager && cookie = @cookie_manager.find(uri)
-      req.header.add('Cookie', cookie)
+    if @cookie_manager
+      cookie_value = @cookie_manager.cookie_value(uri)
+      if cookie_value
+        req.header.add('Cookie', cookie_value)
+      end
     end
     req
   end
@@ -1152,8 +1181,9 @@ private
     end
     if str = @test_loopback_response.shift
       dump_dummy_request_response(req.http_body.dump, str) if @debug_dev
-      conn.push(HTTP::Message.new_response(str, req.header))
-      return
+      res = HTTP::Message.new_response(str, req.header)
+      conn.push(res)
+      return res
     end
     content = block ? nil : ''
     res = HTTP::Message.new_response(content, req.header)
@@ -1178,8 +1208,9 @@ private
       filter.filter_response(req, res)
     }
     if commands.find { |command| command == :retry }
-      raise RetryableResponse.new
+      raise RetryableResponse.new(res)
     end
+    res
   end
 
   def do_get_stream(req, proxy, conn)
@@ -1209,6 +1240,7 @@ private
       filter.filter_response(req, res)
     }
     # ignore commands (not retryable in async mode)
+    res
   end
 
   def do_get_header(req, res, sess)

data/lib/httpclient/auth.rb

@@ -95,6 +95,13 @@ class HTTPClient
       @authenticator.each do |auth|
         next unless auth.set? # hasn't be set, don't use it
         if cred = auth.get(req)
+          if cred == :skip
+            # some authenticator (NTLM and Negotiate) does not
+            # need to send extra header after authorization. In such case
+            # it should block other authenticators to respond and :skip is
+            # the marker for such case.
+            return
+          end
           req.header.set('Authorization', auth.scheme + " " + cred)
           return
         end
@@ -179,6 +186,13 @@ class HTTPClient
       @authenticator.each do |auth|
         next unless auth.set? # hasn't be set, don't use it
         if cred = auth.get(req)
+          if cred == :skip
+            # some authenticator (NTLM and Negotiate) does not
+            # need to send extra header after authorization. In such case
+            # it should block other authenticators to respond and :skip is
+            # the marker for such case.
+            return
+          end
           req.header.set('Proxy-Authorization', auth.scheme + " " + cred)
           return
         end
@@ -208,36 +222,43 @@ class HTTPClient
     end
   end
 
-  # Authentication filter for handling BasicAuth negotiation.
-  # Used in WWWAuth and ProxyAuth.
-  class BasicAuth
+  # Authentication filter base class.
+  class AuthBase
     include HTTPClient::Util
-    include Mutex_m
 
     # Authentication scheme.
     attr_reader :scheme
 
+    def initialize(scheme)
+      @scheme = scheme
+      @challenge = {}
+    end
+
+    # Resets challenge state.  Do not send '*Authorization' header until the
+    # server sends '*Authentication' again.
+    def reset_challenge
+      synchronize do
+        @challenge.clear
+      end
+    end
+  end
+
+  # Authentication filter for handling BasicAuth negotiation.
+  # Used in WWWAuth and ProxyAuth.
+  class BasicAuth < AuthBase
+    include Mutex_m
+
     # Send Authorization Header without receiving 401
     attr_accessor :force_auth
 
     # Creates new BasicAuth filter.
     def initialize
-      super
+      super('Basic')
       @cred = nil
       @auth = {}
-      @challenge = {}
-      @scheme = "Basic"
       @force_auth = false
     end
 
-    # Resets challenge state.  Do not send '*Authorization' header until the
-    # server sends '*Authentication' again.
-    def reset_challenge
-      synchronize {
-        @challenge.clear
-      }
-    end
-
     # Set authentication credential.
     # uri == nil for generic purpose (allow to use user/password for any URL).
     def set(uri, user, passwd)
@@ -283,7 +304,6 @@ class HTTPClient
   end
 
   class ProxyBasicAuth < BasicAuth
-
     def set(uri, user, passwd)
       synchronize do
         @cred = ["#{user}:#{passwd}"].pack('m').tr("\n", '')
@@ -308,27 +328,14 @@ class HTTPClient
 
   # Authentication filter for handling DigestAuth negotiation.
   # Used in WWWAuth.
-  class DigestAuth
+  class DigestAuth < AuthBase
     include Mutex_m
 
-    # Authentication scheme.
-    attr_reader :scheme
-
     # Creates new DigestAuth filter.
     def initialize
-      super
+      super('Digest')
       @auth = {}
-      @challenge = {}
       @nonce_count = 0
-      @scheme = "Digest"
-    end
-
-    # Resets challenge state.  Do not send '*Authorization' header until the
-    # server sends '*Authentication' again.
-    def reset_challenge
-      synchronize do
-        @challenge.clear
-      end
     end
 
     # Set authentication credential.
@@ -483,41 +490,28 @@ class HTTPClient
         true
       }
     end
-
   end
 
   # Authentication filter for handling Negotiate/NTLM negotiation.
   # Used in WWWAuth and ProxyAuth.
   #
   # NegotiateAuth depends on 'ruby/ntlm' module.
-  class NegotiateAuth
+  class NegotiateAuth < AuthBase
     include Mutex_m
 
-    # Authentication scheme.
-    attr_reader :scheme
     # NTLM opt for ruby/ntlm.  {:ntlmv2 => true} by default.
     attr_reader :ntlm_opt
 
     # Creates new NegotiateAuth filter.
     def initialize(scheme = "Negotiate")
-      super()
+      super(scheme)
       @auth = {}
       @auth_default = nil
-      @challenge = {}
-      @scheme = scheme
       @ntlm_opt = {
         :ntlmv2 => true
       }
     end
 
-    # Resets challenge state.  Do not send '*Authorization' header until the
-    # server sends '*Authentication' again.
-    def reset_challenge
-      synchronize do
-        @challenge.clear
-      end
-    end
-
     # Set authentication credential.
     # uri == nil for generic purpose (allow to use user/password for any URL).
     def set(uri, user, passwd)
@@ -561,16 +555,19 @@ class HTTPClient
         when :init
           t1 = Net::NTLM::Message::Type1.new
           t1.domain = domain if domain
-          return t1.encode64
+          t1.encode64
         when :response
           t2 = Net::NTLM::Message.decode64(authphrase)
           param = {:user => user, :password => passwd}
           param[:domain] = domain if domain
           t3 = t2.response(param, @ntlm_opt.dup)
-          @challenge.delete(domain_uri)
-          return t3.encode64
+          @challenge[target_uri][:state] = :done
+          t3.encode64
+        when :done
+          :skip
+        else
+          nil
         end
-        nil
       }
     end
 
@@ -596,25 +593,12 @@ class HTTPClient
   # Used in ProxyAuth.
   #
   # SSPINegotiateAuth depends on 'win32/sspi' module.
-  class SSPINegotiateAuth
+  class SSPINegotiateAuth < AuthBase
     include Mutex_m
 
-    # Authentication scheme.
-    attr_reader :scheme
-
     # Creates new SSPINegotiateAuth filter.
     def initialize
-      super
-      @challenge = {}
-      @scheme = "Negotiate"
-    end
-
-    # Resets challenge state.  Do not send '*Authorization' header until the
-    # server sends '*Authentication' again.
-    def reset_challenge
-      synchronize do
-        @challenge.clear
-      end
+      super('Negotiate')
     end
 
     # Set authentication credential.
@@ -646,21 +630,24 @@ class HTTPClient
         when :init
           if defined?(Win32::SSPI)
             authenticator = param[:authenticator] = Win32::SSPI::NegotiateAuth.new
-            return authenticator.get_initial_token(@scheme)
+            authenticator.get_initial_token(@scheme)
           else # use GSSAPI
             authenticator = param[:authenticator] = GSSAPI::Simple.new(domain_uri.host, 'HTTP')
             # Base64 encode the context token
-            return [authenticator.init_context].pack('m').gsub(/\n/,'')
+            [authenticator.init_context].pack('m').gsub(/\n/,'')
           end
         when :response
-          @challenge.delete(domain_uri)
+          @challenge[target_uri][:state] = :done
           if defined?(Win32::SSPI)
-            return authenticator.complete_authentication(authphrase)
+            authenticator.complete_authentication(authphrase)
           else # use GSSAPI
-            return authenticator.init_context(authphrase.unpack('m').pop)
+            authenticator.init_context(authphrase.unpack('m').pop)
           end
+        when :done
+          :skip
+        else
+          nil
         end
-        nil
       }
     end
 
@@ -692,13 +679,9 @@ class HTTPClient
   # CAUTION: This impl does NOT support OAuth Request Body Hash spec for now.
   # http://oauth.googlecode.com/svn/spec/ext/body_hash/1.0/oauth-bodyhash.html
   #
-  class OAuth
-    include HTTPClient::Util
+  class OAuth < AuthBase
     include Mutex_m
 
-    # Authentication scheme.
-    attr_reader :scheme
-
     class Config
       include HTTPClient::Util
 
@@ -768,23 +751,13 @@ class HTTPClient
 
     # Creates new DigestAuth filter.
     def initialize
-      super
+      super('OAuth')
       @config = nil # common config
       @auth = {} # configs for each site
-      @challenge = {}
       @nonce_count = 0
       @signature_handler = {
         'HMAC-SHA1' => method(:sign_hmac_sha1)
       }
-      @scheme = "OAuth"
-    end
-
-    # Resets challenge state.  Do not send '*Authorization' header until the
-    # server sends '*Authentication' again.
-    def reset_challenge
-      synchronize do
-        @challenge.clear
-      end
     end
 
     # Set authentication credential.