httpclient 2.1.5 → 2.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +85 -0
  3. data/bin/httpclient +77 -0
  4. data/bin/jsonclient +85 -0
  5. data/lib/hexdump.rb +50 -0
  6. data/lib/http-access2.rb +6 -4
  7. data/lib/httpclient/auth.rb +575 -173
  8. data/lib/httpclient/cacert.pem +3952 -0
  9. data/lib/httpclient/cacert1024.pem +3866 -0
  10. data/lib/httpclient/connection.rb +6 -2
  11. data/lib/httpclient/cookie.rb +162 -504
  12. data/lib/httpclient/http.rb +334 -119
  13. data/lib/httpclient/include_client.rb +85 -0
  14. data/lib/httpclient/jruby_ssl_socket.rb +588 -0
  15. data/lib/httpclient/session.rb +385 -288
  16. data/lib/httpclient/ssl_config.rb +195 -155
  17. data/lib/httpclient/ssl_socket.rb +150 -0
  18. data/lib/httpclient/timeout.rb +14 -10
  19. data/lib/httpclient/util.rb +142 -6
  20. data/lib/httpclient/version.rb +3 -0
  21. data/lib/httpclient/webagent-cookie.rb +459 -0
  22. data/lib/httpclient.rb +509 -202
  23. data/lib/jsonclient.rb +63 -0
  24. data/lib/oauthclient.rb +111 -0
  25. data/sample/async.rb +8 -0
  26. data/sample/auth.rb +11 -0
  27. data/sample/cookie.rb +18 -0
  28. data/sample/dav.rb +103 -0
  29. data/sample/howto.rb +49 -0
  30. data/sample/jsonclient.rb +67 -0
  31. data/sample/oauth_buzz.rb +57 -0
  32. data/sample/oauth_friendfeed.rb +59 -0
  33. data/sample/oauth_twitter.rb +61 -0
  34. data/sample/ssl/0cert.pem +22 -0
  35. data/sample/ssl/0key.pem +30 -0
  36. data/sample/ssl/1000cert.pem +19 -0
  37. data/sample/ssl/1000key.pem +18 -0
  38. data/sample/ssl/htdocs/index.html +10 -0
  39. data/sample/ssl/ssl_client.rb +22 -0
  40. data/sample/ssl/webrick_httpsd.rb +29 -0
  41. data/sample/stream.rb +21 -0
  42. data/sample/thread.rb +27 -0
  43. data/sample/wcat.rb +21 -0
  44. data/test/ca-chain.pem +44 -0
  45. data/test/ca.cert +23 -0
  46. data/test/client-pass.key +18 -0
  47. data/test/client.cert +19 -0
  48. data/test/client.key +15 -0
  49. data/test/helper.rb +131 -0
  50. data/test/htdigest +1 -0
  51. data/test/htpasswd +2 -0
  52. data/test/jruby_ssl_socket/test_pemutils.rb +32 -0
  53. data/test/runner.rb +2 -0
  54. data/test/server.cert +19 -0
  55. data/test/server.key +15 -0
  56. data/test/sslsvr.rb +65 -0
  57. data/test/subca.cert +21 -0
  58. data/test/test_auth.rb +492 -0
  59. data/test/test_cookie.rb +309 -0
  60. data/test/test_hexdump.rb +14 -0
  61. data/test/test_http-access2.rb +508 -0
  62. data/test/test_httpclient.rb +2145 -0
  63. data/test/test_include_client.rb +52 -0
  64. data/test/test_jsonclient.rb +80 -0
  65. data/test/test_ssl.rb +559 -0
  66. data/test/test_webagent-cookie.rb +465 -0
  67. metadata +85 -44
  68. data/lib/httpclient/auth.rb.orig +0 -513
  69. data/lib/httpclient/cacert.p7s +0 -1579
  70. data/lib/httpclient.rb.orig +0 -1020
  71. data/lib/tags +0 -908
data/lib/httpclient.rb CHANGED
@@ -1,16 +1,16 @@
1
1
  # HTTPClient - HTTP client library.
2
- # Copyright (C) 2000-2009 NAKAMURA, Hiroshi <nahi@ruby-lang.org>.
2
+ # Copyright (C) 2000-2015 NAKAMURA, Hiroshi <nahi@ruby-lang.org>.
3
3
  #
4
4
  # This program is copyrighted free software by NAKAMURA, Hiroshi. You can
5
5
  # redistribute it and/or modify it under the same terms of Ruby's license;
6
6
  # either the dual license version in 2003, or any later version.
7
7
 
8
8
 
9
- require 'uri'
10
9
  require 'stringio'
11
10
  require 'digest/sha1'
12
11
 
13
12
  # Extra library
13
+ require 'httpclient/version'
14
14
  require 'httpclient/util'
15
15
  require 'httpclient/ssl_config'
16
16
  require 'httpclient/connection'
@@ -19,7 +19,7 @@ require 'httpclient/http'
19
19
  require 'httpclient/auth'
20
20
  require 'httpclient/cookie'
21
21
 
22
-
22
+ # :main:HTTPClient
23
23
  # The HTTPClient class provides several methods for accessing Web resources
24
24
  # via HTTP.
25
25
  #
@@ -50,13 +50,23 @@ require 'httpclient/cookie'
50
50
  #
51
51
  # === How to retrieve web resources
52
52
  #
53
- # See get_content.
53
+ # See get and get_content.
54
+ #
55
+ # 1. Get content of specified URL. It returns HTTP::Message object and
56
+ # calling 'body' method of it returns a content String.
54
57
  #
55
- # 1. Get content of specified URL. It returns a String of whole result.
58
+ # puts clnt.get('http://dev.ctor.org/').body
59
+ #
60
+ # 2. For getting content directly, use get_content. It follows redirect
61
+ # response and returns a String of whole result.
56
62
  #
57
63
  # puts clnt.get_content('http://dev.ctor.org/')
58
64
  #
59
- # 2. Get content as chunks of String. It yields chunks of String.
65
+ # 3. You can pass :follow_redirect option to follow redirect response in get.
66
+ #
67
+ # puts clnt.get('http://dev.ctor.org/', :follow_redirect => true)
68
+ #
69
+ # 4. Get content as chunks of String. It yields chunks of String.
60
70
  #
61
71
  # clnt.get_content('http://dev.ctor.org/') do |chunk|
62
72
  # puts chunk
@@ -79,7 +89,11 @@ require 'httpclient/cookie'
79
89
  # p res.status
80
90
  # p res.contenttype
81
91
  # p res.header['X-Custom']
82
- # puts res.content
92
+ # puts res.body
93
+ #
94
+ # You can also use keyword argument style.
95
+ #
96
+ # res = clnt.get(uri, :query => { :keyword => 'ruby', :lang => 'en' })
83
97
  #
84
98
  # === How to POST
85
99
  #
@@ -90,6 +104,10 @@ require 'httpclient/cookie'
90
104
  # body = { 'keyword' => 'ruby', 'lang' => 'en' }
91
105
  # res = clnt.post(uri, body)
92
106
  #
107
+ # Keyword argument style.
108
+ #
109
+ # res = clnt.post(uri, :body => ...)
110
+ #
93
111
  # 2. Do multipart file upload with POST. No need to set extra header by
94
112
  # yourself from httpclient/2.1.4.
95
113
  #
@@ -98,6 +116,17 @@ require 'httpclient/cookie'
98
116
  # res = clnt.post(uri, body)
99
117
  # end
100
118
  #
119
+ # 3. Do multipart with custom body.
120
+ #
121
+ # File.open('/tmp/post_data') do |file|
122
+ # body = [{ 'Content-Type' => 'application/atom+xml; charset=UTF-8',
123
+ # :content => '<entry>...</entry>' },
124
+ # { 'Content-Type' => 'video/mp4',
125
+ # 'Content-Transfer-Encoding' => 'binary',
126
+ # :content => file }]
127
+ # res = clnt.post(uri, body)
128
+ # end
129
+ #
101
130
  # === Accessing via SSL
102
131
  #
103
132
  # Ruby needs to be compiled with OpenSSL.
@@ -106,7 +135,7 @@ require 'httpclient/cookie'
106
135
  # Just pass an URL which starts with 'https://'.
107
136
  #
108
137
  # https_url = 'https://www.rsa.com'
109
- # clnt.get_content(https_url)
138
+ # clnt.get(https_url)
110
139
  #
111
140
  # 2. Getting peer certificate from response.
112
141
  #
@@ -118,22 +147,33 @@ require 'httpclient/cookie'
118
147
  # user_cert_file = 'cert.pem'
119
148
  # user_key_file = 'privkey.pem'
120
149
  # clnt.ssl_config.set_client_cert_file(user_cert_file, user_key_file)
121
- # clnt.get_content(https_url)
150
+ # clnt.get(https_url)
151
+ #
152
+ # 4. Revocation check. On JRuby you can set following options to let
153
+ # HTTPClient to perform revocation check with CRL and OCSP:
154
+ #
155
+ # -J-Dcom.sun.security.enableCRLDP=true -J-Dcom.sun.net.ssl.checkRevocation=true
156
+ # ex. jruby -J-Dcom.sun.security.enableCRLDP=true -J-Dcom.sun.net.ssl.checkRevocation=true app.rb
157
+ # Revoked cert example: https://test-sspev.verisign.com:2443/test-SSPEV-revoked-verisign.html
158
+ #
159
+ # On other platform you can download CRL by yourself and set it via
160
+ # SSLConfig#add_crl.
122
161
  #
123
162
  # === Handling Cookies
124
163
  #
125
164
  # 1. Using volatile Cookies. Nothing to do. HTTPClient handles Cookies.
126
165
  #
127
166
  # clnt = HTTPClient.new
128
- # clnt.get_content(url1) # receives Cookies.
129
- # clnt.get_content(url2) # sends Cookies if needed.
167
+ # res = clnt.get(url1) # receives Cookies.
168
+ # res = clnt.get(url2) # sends Cookies if needed.
169
+ # p res.cookies
130
170
  #
131
171
  # 2. Saving non volatile Cookies to a specified file. Need to set a file at
132
172
  # first and invoke save method at last.
133
173
  #
134
174
  # clnt = HTTPClient.new
135
175
  # clnt.set_cookie_store('/home/nahi/cookie.dat')
136
- # clnt.get_content(url)
176
+ # clnt.get(url)
137
177
  # ...
138
178
  # clnt.save_cookie_store
139
179
  #
@@ -152,7 +192,7 @@ require 'httpclient/cookie'
152
192
  # user = 'user'
153
193
  # password = 'user'
154
194
  # clnt.set_auth(domain, user, password)
155
- # p clnt.get_content('http://dev.ctor.org/http-access2/login').status
195
+ # p clnt.get('http://dev.ctor.org/http-access2/login').status
156
196
  #
157
197
  # 2. Authentication with Proxy server. Supports BasicAuth and NTLM
158
198
  # (requires win32/sspi)
@@ -161,17 +201,17 @@ require 'httpclient/cookie'
161
201
  # user = 'proxy'
162
202
  # password = 'proxy'
163
203
  # clnt.set_proxy_auth(user, password)
164
- # p clnt.get_content(url)
204
+ # p clnt.get(url)
165
205
  #
166
206
  # === Invoking HTTP methods with custom header
167
207
  #
168
- # Pass a Hash or an Array for extheader argument.
208
+ # Pass a Hash or an Array for header argument.
169
209
  #
170
- # extheader = { 'Accept' => '*/*' }
171
- # clnt.get_content(uri, query, extheader)
210
+ # header = { 'Accept' => 'text/html' }
211
+ # clnt.get(uri, query, header)
172
212
  #
173
- # extheader = [['Accept', 'image/jpeg'], ['Accept', 'image/png']]
174
- # clnt.get_content(uri, query, extheader)
213
+ # header = [['Accept', 'image/jpeg'], ['Accept', 'image/png']]
214
+ # clnt.get_content(uri, query, header)
175
215
  #
176
216
  # === Invoking HTTP methods asynchronously
177
217
  #
@@ -189,7 +229,7 @@ require 'httpclient/cookie'
189
229
  # puts '.'
190
230
  # res = connection.pop
191
231
  # p res.status
192
- # p res.content.read # res.content is an IO for the res of async method.
232
+ # p res.body.read # res.body is an IO for the res of async method.
193
233
  #
194
234
  # === Shortcut methods
195
235
  #
@@ -199,10 +239,8 @@ require 'httpclient/cookie'
199
239
  # ruby -rhttpclient -e 'p HTTPClient.head(ARGV.shift).header["last-modified"]' http://dev.ctor.org/
200
240
  #
201
241
  class HTTPClient
202
- VERSION = '2.1.5'
203
- RUBY_VERSION_STRING = "ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
204
- /: (\S+) (\S+)/ =~ %q$Id: httpclient.rb 280 2009-06-02 15:44:28Z nahi $
205
- LIB_NAME = "(#{$1}/#{$2}, #{RUBY_VERSION_STRING})"
242
+ RUBY_VERSION_STRING = "ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE})"
243
+ LIB_NAME = "(#{VERSION}, #{RUBY_VERSION_STRING})"
206
244
 
207
245
  include Util
208
246
 
@@ -271,7 +309,6 @@ class HTTPClient
271
309
  if assignable
272
310
  aname = name + '='
273
311
  define_method(aname) { |rhs|
274
- reset_all
275
312
  @session_manager.__send__(aname, rhs)
276
313
  }
277
314
  end
@@ -280,7 +317,7 @@ class HTTPClient
280
317
 
281
318
  # HTTPClient::SSLConfig:: SSL configurator.
282
319
  attr_reader :ssl_config
283
- # WebAgent::CookieManager:: Cookies configurator.
320
+ # HTTPClient::CookieManager:: Cookies configurator.
284
321
  attr_accessor :cookie_manager
285
322
  # An array of response HTTP message body String which is used for loop-back
286
323
  # test. See test/* to see how to use it. If you want to do loop-back test
@@ -296,6 +333,10 @@ class HTTPClient
296
333
  # How many times get_content and post_content follows HTTP redirect.
297
334
  # 10 by default.
298
335
  attr_accessor :follow_redirect_count
336
+ # Base url of resources.
337
+ attr_accessor :base_url
338
+ # Default request header.
339
+ attr_accessor :default_header
299
340
 
300
341
  # Set HTTP version as a String:: 'HTTP/1.0' or 'HTTP/1.1'
301
342
  attr_proxy(:protocol_version, true)
@@ -305,11 +346,17 @@ class HTTPClient
305
346
  attr_proxy(:send_timeout, true)
306
347
  # Response receiving timeout in sec.
307
348
  attr_proxy(:receive_timeout, true)
349
+ # Reuse the same connection within this timeout in sec. from last used.
350
+ attr_proxy(:keep_alive_timeout, true)
351
+ # Size of reading block for non-chunked response.
352
+ attr_proxy(:read_block_size, true)
308
353
  # Negotiation retry count for authentication. 5 by default.
309
354
  attr_proxy(:protocol_retry_count, true)
310
355
  # if your ruby is older than 2005-09-06, do not set socket_sync = false to
311
356
  # avoid an SSL socket blocking bug in openssl/buffering.rb.
312
357
  attr_proxy(:socket_sync, true)
358
+ # Enables TCP keepalive; no timing settings exist at present
359
+ attr_proxy(:tcp_keepalive, true)
313
360
  # User-Agent header in HTTP request.
314
361
  attr_proxy(:agent_name, true)
315
362
  # From header in HTTP request.
@@ -317,44 +364,107 @@ class HTTPClient
317
364
  # An array of response HTTP String (not a HTTP message body) which is used
318
365
  # for loopback test. See test/* to see how to use it.
319
366
  attr_proxy(:test_loopback_http_response)
320
-
321
- # Default extheader for PROPFIND request.
367
+ # Decompress a compressed (with gzip or deflate) content body transparently. false by default.
368
+ attr_proxy(:transparent_gzip_decompression, true)
369
+ # Raise BadResponseError if response size does not match with Content-Length header in response. false by default.
370
+ # TODO: enable by default
371
+ attr_proxy(:strict_response_size_check, true)
372
+ # Local socket address. Set HTTPClient#socket_local.host and HTTPClient#socket_local.port to specify local binding hostname and port of TCP socket.
373
+ attr_proxy(:socket_local, true)
374
+
375
+ # Default header for PROPFIND request.
322
376
  PROPFIND_DEFAULT_EXTHEADER = { 'Depth' => '0' }
323
377
 
378
+ # Default User-Agent header
379
+ DEFAULT_AGENT_NAME = 'HTTPClient/1.0'
380
+
324
381
  # Creates a HTTPClient instance which manages sessions, cookies, etc.
325
382
  #
326
- # HTTPClient.new takes 3 optional arguments for proxy url string,
327
- # User-Agent String and From header String. User-Agent and From are embedded
328
- # in HTTP request Header if given. No User-Agent and From header added
329
- # without setting it explicitly.
383
+ # HTTPClient.new takes optional arguments as a Hash.
384
+ # * :proxy - proxy url string
385
+ # * :agent_name - User-Agent String
386
+ # * :from - from header String
387
+ # * :base_url - base URL of resources
388
+ # * :default_header - header Hash all HTTP requests should have
389
+ # * :force_basic_auth - flag for sending Authorization header w/o gettin 401 first
390
+ # User-Agent and From are embedded in HTTP request Header if given.
391
+ # From header is not set without setting it explicitly.
330
392
  #
331
393
  # proxy = 'http://myproxy:8080'
332
394
  # agent_name = 'MyAgent/0.1'
333
395
  # from = 'from@example.com'
334
396
  # HTTPClient.new(proxy, agent_name, from)
335
397
  #
336
- # You can use a keyword argument style Hash. Keys are :proxy, :agent_name
337
- # and :from.
398
+ # After you set :base_url, all resources you pass to get, post and other
399
+ # methods are recognized to be prefixed with base_url. Say base_url is
400
+ # 'https://api.example.com/v1/, get('users') is the same as
401
+ # get('https://api.example.com/v1/users') internally. You can also pass
402
+ # full URL from 'http://' even after setting base_url.
403
+ #
404
+ # The expected base_url and path behavior is the following. Please take
405
+ # care of '/' in base_url and path.
406
+ #
407
+ # The last '/' is important for base_url:
408
+ # 1. http://foo/bar/baz/ + path -> http://foo/bar/baz/path
409
+ # 2. http://foo/bar/baz + path -> http://foo/bar/path
410
+ # Relative path handling:
411
+ # 3. http://foo/bar/baz/ + ../path -> http://foo/bar/path
412
+ # 4. http://foo/bar/baz + ../path -> http://foo/path
413
+ # 5. http://foo/bar/baz/ + ./path -> http://foo/bar/baz/path
414
+ # 6. http://foo/bar/baz + ./path -> http://foo/bar/path
415
+ # The leading '/' of path means absolute path:
416
+ # 7. http://foo/bar/baz/ + /path -> http://foo/path
417
+ # 8. http://foo/bar/baz + /path -> http://foo/path
418
+ #
419
+ # :default_header is for providing default headers Hash that all HTTP
420
+ # requests should have, such as custom 'Authorization' header in API.
421
+ # You can override :default_header with :header Hash parameter in HTTP
422
+ # request methods.
338
423
  #
339
- # HTTPClient.new(:agent_name = 'MyAgent/0.1')
340
- def initialize(*args)
341
- proxy, agent_name, from = keyword_argument(args, :proxy, :agent_name, :from)
424
+ # :force_basic_auth turns on/off the BasicAuth force flag. Generally
425
+ # HTTP client must send Authorization header after it gets 401 error
426
+ # from server from security reason. But in some situation (e.g. API
427
+ # client) you might want to send Authorization from the beginning.
428
+ def initialize(*args, &block)
429
+ proxy, agent_name, from, base_url, default_header, force_basic_auth =
430
+ keyword_argument(args, :proxy, :agent_name, :from, :base_url, :default_header, :force_basic_auth)
342
431
  @proxy = nil # assigned later.
343
432
  @no_proxy = nil
433
+ @no_proxy_regexps = []
434
+ @base_url = base_url
435
+ @default_header = default_header || {}
344
436
  @www_auth = WWWAuth.new
345
437
  @proxy_auth = ProxyAuth.new
438
+ @www_auth.basic_auth.force_auth = @proxy_auth.basic_auth.force_auth = force_basic_auth
346
439
  @request_filter = [@proxy_auth, @www_auth]
347
440
  @debug_dev = nil
348
441
  @redirect_uri_callback = method(:default_redirect_uri_callback)
349
442
  @test_loopback_response = []
350
443
  @session_manager = SessionManager.new(self)
351
- @session_manager.agent_name = agent_name
444
+ @session_manager.agent_name = agent_name || DEFAULT_AGENT_NAME
352
445
  @session_manager.from = from
353
446
  @session_manager.ssl_config = @ssl_config = SSLConfig.new(self)
354
- @cookie_manager = WebAgent::CookieManager.new
447
+ @cookie_manager = CookieManager.new
355
448
  @follow_redirect_count = 10
356
449
  load_environment
357
450
  self.proxy = proxy if proxy
451
+ keep_webmock_compat
452
+ instance_eval(&block) if block
453
+ end
454
+
455
+ # webmock 1.6.2 depends on HTTP::Message#body.content to work.
456
+ # let's keep it work iif webmock is loaded for a while.
457
+ def keep_webmock_compat
458
+ if respond_to?(:do_get_block_with_webmock)
459
+ ::HTTP::Message.module_eval do
460
+ def body
461
+ def (o = self.content).content
462
+ self
463
+ end
464
+ o
465
+ end
466
+ end
467
+ end
358
468
  end
359
469
 
360
470
  # Returns debug device if exists. See debug_dev=.
@@ -388,7 +498,7 @@ class HTTPClient
388
498
  #
389
499
  # Calling this method resets all existing sessions.
390
500
  def proxy=(proxy)
391
- if proxy.nil?
501
+ if proxy.nil? || proxy.to_s.empty?
392
502
  @proxy = nil
393
503
  @proxy_auth.reset_challenge
394
504
  else
@@ -425,6 +535,17 @@ class HTTPClient
425
535
  # Calling this method resets all existing sessions.
426
536
  def no_proxy=(no_proxy)
427
537
  @no_proxy = no_proxy
538
+ @no_proxy_regexps.clear
539
+ if @no_proxy
540
+ @no_proxy.scan(/([^\s:,]+)(?::(\d+))?/) do |host, port|
541
+ if host[0] == ?.
542
+ regexp = /#{Regexp.quote(host)}\z/i
543
+ else
544
+ regexp = /(\A|\.)#{Regexp.quote(host)}\z/i
545
+ end
546
+ @no_proxy_regexps << [regexp, port]
547
+ end
548
+ end
428
549
  reset_all
429
550
  end
430
551
 
@@ -442,14 +563,14 @@ class HTTPClient
442
563
  #
443
564
  # Calling this method resets all existing sessions.
444
565
  def set_auth(domain, user, passwd)
445
- uri = urify(domain)
566
+ uri = to_resource_url(domain)
446
567
  @www_auth.set_auth(uri, user, passwd)
447
568
  reset_all
448
569
  end
449
570
 
450
571
  # Deprecated. Use set_auth instead.
451
572
  def set_basic_auth(domain, user, passwd)
452
- uri = urify(domain)
573
+ uri = to_resource_url(domain)
453
574
  @www_auth.basic_auth.set(uri, user, passwd)
454
575
  reset_all
455
576
  end
@@ -464,6 +585,14 @@ class HTTPClient
464
585
  reset_all
465
586
  end
466
587
 
588
+ # Turn on/off the BasicAuth force flag. Generally HTTP client must
589
+ # send Authorization header after it gets 401 error from server from
590
+ # security reason. But in some situation (e.g. API client) you might
591
+ # want to send Authorization from the beginning.
592
+ def force_basic_auth=(force_basic_auth)
593
+ @www_auth.basic_auth.force_auth = @proxy_auth.basic_auth.force_auth = force_basic_auth
594
+ end
595
+
467
596
  # Sets the filename where non-volatile Cookies be saved by calling
468
597
  # save_cookie_store.
469
598
  # This method tries to load and managing Cookies from the specified file.
@@ -477,11 +606,17 @@ class HTTPClient
477
606
 
478
607
  # Try to save Cookies to the file specified in set_cookie_store. Unexpected
479
608
  # error will be raised if you don't call set_cookie_store first.
480
- # (interface mismatch between WebAgent::CookieManager implementation)
481
609
  def save_cookie_store
482
610
  @cookie_manager.save_cookies
483
611
  end
484
612
 
613
+ # Returns stored cookies.
614
+ def cookies
615
+ if @cookie_manager
616
+ @cookie_manager.cookies
617
+ end
618
+ end
619
+
485
620
  # Sets callback proc when HTTP redirect status is returned for get_content
486
621
  # and post_content. default_redirect_uri_callback is used by default.
487
622
  #
@@ -501,9 +636,9 @@ class HTTPClient
501
636
  # e.g. { "a" => "b" } => 'http://host/part?a=b'.
502
637
  # Give an array to pass multiple value like
503
638
  # [["a", "b"], ["a", "c"]] => 'http://host/part?a=b&a=c'.
504
- # extheader:: a Hash or an Array of extra headers. e.g.
505
- # { 'Accept' => '*/*' } or
506
- # [['Accept', 'image/jpeg'], ['Accept', 'image/png']].
639
+ # header:: a Hash or an Array of extra headers. e.g.
640
+ # { 'Accept' => 'text/html' } or
641
+ # [['Accept', 'image/jpeg'], ['Accept', 'image/png']].
507
642
  # &block:: Give a block to get chunked message-body of response like
508
643
  # get_content(uri) { |chunked_body| ... }.
509
644
  # Size of each chunk may not be the same.
@@ -515,34 +650,51 @@ class HTTPClient
515
650
  # If you need to get full HTTP response including HTTP status and headers,
516
651
  # use get method. get returns HTTP::Message as a response and you need to
517
652
  # follow HTTP redirect by yourself if you need.
518
- def get_content(uri, query = nil, extheader = {}, &block)
519
- follow_redirect(:get, uri, query, nil, extheader, &block).content
653
+ def get_content(uri, *args, &block)
654
+ query, header = keyword_argument(args, :query, :header)
655
+ success_content(follow_redirect(:get, uri, query, nil, header || {}, &block))
520
656
  end
521
657
 
522
658
  # Posts a content.
523
659
  #
524
660
  # uri:: a String or an URI object which represents an URL of web resource.
525
- # body:: a Hash or an Array of body part.
526
- # e.g. { "a" => "b" } => 'a=b'.
661
+ # body:: a Hash or an Array of body part. e.g.
662
+ # { "a" => "b" } => 'a=b'
527
663
  # Give an array to pass multiple value like
528
- # [["a", "b"], ["a", "c"]] => 'a=b&a=c'.
664
+ # [["a", "b"], ["a", "c"]] => 'a=b&a=c'
529
665
  # When you pass a File as a value, it will be posted as a
530
- # multipart/form-data. e.g. { 'upload' => file }
531
- # extheader:: a Hash or an Array of extra headers. e.g.
532
- # { 'Accept' => '*/*' } or
533
- # [['Accept', 'image/jpeg'], ['Accept', 'image/png']].
666
+ # multipart/form-data. e.g.
667
+ # { 'upload' => file }
668
+ # You can also send custom multipart by passing an array of hashes.
669
+ # Each part must have a :content attribute which can be a file, all
670
+ # other keys will become headers.
671
+ # [{ 'Content-Type' => 'text/plain', :content => "some text" },
672
+ # { 'Content-Type' => 'video/mp4', :content => File.new('video.mp4') }]
673
+ # => <Two parts with custom Content-Type header>
674
+ # header:: a Hash or an Array of extra headers. e.g.
675
+ # { 'Accept' => 'text/html' }
676
+ # or
677
+ # [['Accept', 'image/jpeg'], ['Accept', 'image/png']].
534
678
  # &block:: Give a block to get chunked message-body of response like
535
- # post_content(uri) { |chunked_body| ... }.
679
+ # post_content(uri) { |chunked_body| ... }.
536
680
  # Size of each chunk may not be the same.
537
681
  #
538
682
  # post_content follows HTTP redirect status (see HTTP::Status.redirect?)
539
683
  # internally and try to post the content to redirected URL. See
540
684
  # redirect_uri_callback= how HTTP redirection is handled.
685
+ # Bear in mind that you should not depend on post_content because it sends
686
+ # the same POST method to the new location which is prohibited in HTTP spec.
541
687
  #
542
688
  # If you need to get full HTTP response including HTTP status and headers,
543
689
  # use post method.
544
- def post_content(uri, body = nil, extheader = {}, &block)
545
- follow_redirect(:post, uri, nil, body, extheader, &block).content
690
+ def post_content(uri, *args, &block)
691
+ if hashy_argument_has_keys(args, :query, :body)
692
+ query, body, header = keyword_argument(args, :query, :body, :header)
693
+ else
694
+ query = nil
695
+ body, header = keyword_argument(args, :body, :header)
696
+ end
697
+ success_content(follow_redirect(:post, uri, query, body, header || {}, &block))
546
698
  end
547
699
 
548
700
  # A method for redirect uri callback. How to use:
@@ -551,11 +703,11 @@ class HTTPClient
551
703
  # Location: ../foo/
552
704
  # in HTTP header. (raises BadResponseError instead)
553
705
  def strict_redirect_uri_callback(uri, res)
554
- newuri = URI.parse(res.header['location'][0])
706
+ newuri = urify(res.header['location'][0])
555
707
  if https?(uri) && !https?(newuri)
556
708
  raise BadResponseError.new("redirecting to non-https resource")
557
709
  end
558
- unless newuri.is_a?(URI::HTTP)
710
+ if !http?(newuri) && !https?(newuri)
559
711
  raise BadResponseError.new("unexpected location: #{newuri}", res)
560
712
  end
561
713
  puts "redirect to: #{newuri}" if $DEBUG
@@ -568,62 +720,85 @@ class HTTPClient
568
720
  # Location: ../foo/
569
721
  # in HTTP header.
570
722
  def default_redirect_uri_callback(uri, res)
571
- newuri = URI.parse(res.header['location'][0])
723
+ newuri = urify(res.header['location'][0])
724
+ if !http?(newuri) && !https?(newuri)
725
+ warn("#{newuri}: a relative URI in location header which is not recommended")
726
+ warn("'The field value consists of a single absolute URI' in HTTP spec")
727
+ newuri = uri + newuri
728
+ end
572
729
  if https?(uri) && !https?(newuri)
573
730
  raise BadResponseError.new("redirecting to non-https resource")
574
731
  end
575
- unless newuri.is_a?(URI::HTTP)
576
- newuri = uri + newuri
577
- STDERR.puts("could be a relative URI in location header which is not recommended")
578
- STDERR.puts("'The field value consists of a single absolute URI' in HTTP spec")
579
- end
580
732
  puts "redirect to: #{newuri}" if $DEBUG
581
733
  newuri
582
734
  end
583
735
 
584
736
  # Sends HEAD request to the specified URL. See request for arguments.
585
- def head(uri, query = nil, extheader = {})
586
- request(:head, uri, query, nil, extheader)
737
+ def head(uri, *args)
738
+ request(:head, uri, argument_to_hash(args, :query, :header, :follow_redirect))
587
739
  end
588
740
 
589
741
  # Sends GET request to the specified URL. See request for arguments.
590
- def get(uri, query = nil, extheader = {}, &block)
591
- request(:get, uri, query, nil, extheader, &block)
742
+ def get(uri, *args, &block)
743
+ request(:get, uri, argument_to_hash(args, :query, :header, :follow_redirect), &block)
744
+ end
745
+
746
+ # Sends PATCH request to the specified URL. See request for arguments.
747
+ def patch(uri, *args, &block)
748
+ if hashy_argument_has_keys(args, :query, :body)
749
+ new_args = args[0]
750
+ else
751
+ new_args = argument_to_hash(args, :body, :header)
752
+ end
753
+ request(:patch, uri, new_args, &block)
592
754
  end
593
755
 
594
756
  # Sends POST request to the specified URL. See request for arguments.
595
- def post(uri, body = '', extheader = {}, &block)
596
- request(:post, uri, nil, body, extheader, &block)
757
+ # You should not depend on :follow_redirect => true for POST method. It
758
+ # sends the same POST method to the new location which is prohibited in HTTP spec.
759
+ def post(uri, *args, &block)
760
+ if hashy_argument_has_keys(args, :query, :body)
761
+ new_args = args[0]
762
+ else
763
+ new_args = argument_to_hash(args, :body, :header, :follow_redirect)
764
+ end
765
+ request(:post, uri, new_args, &block)
597
766
  end
598
767
 
599
768
  # Sends PUT request to the specified URL. See request for arguments.
600
- def put(uri, body = '', extheader = {}, &block)
601
- request(:put, uri, nil, body, extheader, &block)
769
+ def put(uri, *args, &block)
770
+ if hashy_argument_has_keys(args, :query, :body)
771
+ new_args = args[0]
772
+ else
773
+ new_args = argument_to_hash(args, :body, :header)
774
+ end
775
+ request(:put, uri, new_args, &block)
602
776
  end
603
777
 
604
778
  # Sends DELETE request to the specified URL. See request for arguments.
605
- def delete(uri, extheader = {}, &block)
606
- request(:delete, uri, nil, nil, extheader, &block)
779
+ def delete(uri, *args, &block)
780
+ request(:delete, uri, argument_to_hash(args, :body, :header, :query), &block)
607
781
  end
608
782
 
609
783
  # Sends OPTIONS request to the specified URL. See request for arguments.
610
- def options(uri, extheader = {}, &block)
611
- request(:options, uri, nil, nil, extheader, &block)
784
+ def options(uri, *args, &block)
785
+ new_args = argument_to_hash(args, :header, :query, :body)
786
+ request(:options, uri, new_args, &block)
612
787
  end
613
788
 
614
789
  # Sends PROPFIND request to the specified URL. See request for arguments.
615
- def propfind(uri, extheader = PROPFIND_DEFAULT_EXTHEADER, &block)
616
- request(:propfind, uri, nil, nil, extheader, &block)
790
+ def propfind(uri, *args, &block)
791
+ request(:propfind, uri, argument_to_hash(args, :header), &block)
617
792
  end
618
793
 
619
794
  # Sends PROPPATCH request to the specified URL. See request for arguments.
620
- def proppatch(uri, body = nil, extheader = {}, &block)
621
- request(:proppatch, uri, nil, body, extheader, &block)
795
+ def proppatch(uri, *args, &block)
796
+ request(:proppatch, uri, argument_to_hash(args, :body, :header), &block)
622
797
  end
623
798
 
624
799
  # Sends TRACE request to the specified URL. See request for arguments.
625
- def trace(uri, query = nil, body = nil, extheader = {}, &block)
626
- request('TRACE', uri, query, body, extheader, &block)
800
+ def trace(uri, *args, &block)
801
+ request('TRACE', uri, argument_to_hash(args, :query, :header), &block)
627
802
  end
628
803
 
629
804
  # Sends a request to the specified URL.
@@ -634,17 +809,25 @@ class HTTPClient
634
809
  # e.g. { "a" => "b" } => 'http://host/part?a=b'
635
810
  # Give an array to pass multiple value like
636
811
  # [["a", "b"], ["a", "c"]] => 'http://host/part?a=b&a=c'
637
- # body:: a Hash or an Array of body part.
638
- # e.g. { "a" => "b" } => 'a=b'.
812
+ # body:: a Hash or an Array of body part. e.g.
813
+ # { "a" => "b" }
814
+ # => 'a=b'
639
815
  # Give an array to pass multiple value like
640
- # [["a", "b"], ["a", "c"]] => 'a=b&a=c'.
816
+ # [["a", "b"], ["a", "c"]]
817
+ # => 'a=b&a=c'.
641
818
  # When the given method is 'POST' and the given body contains a file
642
- # as a value, it will be posted as a multipart/form-data.
643
- # e.g. { 'upload' => file }
819
+ # as a value, it will be posted as a multipart/form-data. e.g.
820
+ # { 'upload' => file }
821
+ # You can also send custom multipart by passing an array of hashes.
822
+ # Each part must have a :content attribute which can be a file, all
823
+ # other keys will become headers.
824
+ # [{ 'Content-Type' => 'text/plain', :content => "some text" },
825
+ # { 'Content-Type' => 'video/mp4', :content => File.new('video.mp4') }]
826
+ # => <Two parts with custom Content-Type header>
644
827
  # See HTTP::Message.file? for actual condition of 'a file'.
645
- # extheader:: a Hash or an Array of extra headers. e.g.
646
- # { 'Accept' => '*/*' } or
647
- # [['Accept', 'image/jpeg'], ['Accept', 'image/png']].
828
+ # header:: a Hash or an Array of extra headers. e.g.
829
+ # { 'Accept' => 'text/html' } or
830
+ # [['Accept', 'image/jpeg'], ['Accept', 'image/png']].
648
831
  # &block:: Give a block to get chunked message-body of response like
649
832
  # get(uri) { |chunked_body| ... }.
650
833
  # Size of each chunk may not be the same.
@@ -653,86 +836,130 @@ class HTTPClient
653
836
  # a HTTP request message body.
654
837
  #
655
838
  # When you pass an IO as a body, HTTPClient sends it as a HTTP request with
656
- # chunked encoding (Transfer-Encoding: chunked in HTTP header). Bear in mind
657
- # that some server application does not support chunked request. At least
658
- # cgi.rb does not support it.
659
- def request(method, uri, query = nil, body = nil, extheader = {}, &block)
660
- uri = urify(uri)
839
+ # chunked encoding (Transfer-Encoding: chunked in HTTP header) if IO does not
840
+ # respond to :size. Bear in mind that some server application does not support
841
+ # chunked request. At least cgi.rb does not support it.
842
+ def request(method, uri, *args, &block)
843
+ query, body, header, follow_redirect = keyword_argument(args, :query, :body, :header, :follow_redirect)
844
+ if method == :propfind
845
+ header ||= PROPFIND_DEFAULT_EXTHEADER
846
+ else
847
+ header ||= {}
848
+ end
849
+ uri = to_resource_url(uri)
661
850
  if block
662
- filtered_block = proc { |res, str|
663
- block.call(str)
664
- }
851
+ filtered_block = adapt_block(&block)
852
+ end
853
+ if follow_redirect
854
+ follow_redirect(method, uri, query, body, header, &block)
855
+ else
856
+ do_request(method, uri, query, body, header, &filtered_block)
665
857
  end
666
- do_request(method, uri, query, body, extheader, &filtered_block)
667
858
  end
668
859
 
669
860
  # Sends HEAD request in async style. See request_async for arguments.
670
861
  # It immediately returns a HTTPClient::Connection instance as a result.
671
- def head_async(uri, query = nil, extheader = {})
672
- request_async(:head, uri, query, nil, extheader)
862
+ def head_async(uri, *args)
863
+ request_async2(:head, uri, argument_to_hash(args, :query, :header))
673
864
  end
674
865
 
675
866
  # Sends GET request in async style. See request_async for arguments.
676
867
  # It immediately returns a HTTPClient::Connection instance as a result.
677
- def get_async(uri, query = nil, extheader = {})
678
- request_async(:get, uri, query, nil, extheader)
868
+ def get_async(uri, *args)
869
+ request_async2(:get, uri, argument_to_hash(args, :query, :header))
870
+ end
871
+
872
+ # Sends PATCH request in async style. See request_async2 for arguments.
873
+ # It immediately returns a HTTPClient::Connection instance as a result.
874
+ def patch_async(uri, *args)
875
+ if hashy_argument_has_keys(args, :query, :body)
876
+ new_args = args[0]
877
+ else
878
+ new_args = argument_to_hash(args, :body, :header)
879
+ end
880
+ request_async2(:patch, uri, new_args)
679
881
  end
680
882
 
681
883
  # Sends POST request in async style. See request_async for arguments.
682
884
  # It immediately returns a HTTPClient::Connection instance as a result.
683
- def post_async(uri, body = nil, extheader = {})
684
- request_async(:post, uri, nil, body, extheader)
885
+ def post_async(uri, *args)
886
+ if hashy_argument_has_keys(args, :query, :body)
887
+ new_args = args[0]
888
+ else
889
+ new_args = argument_to_hash(args, :body, :header)
890
+ end
891
+ request_async2(:post, uri, new_args)
685
892
  end
686
893
 
687
- # Sends PUT request in async style. See request_async for arguments.
894
+ # Sends PUT request in async style. See request_async2 for arguments.
688
895
  # It immediately returns a HTTPClient::Connection instance as a result.
689
- def put_async(uri, body = nil, extheader = {})
690
- request_async(:put, uri, nil, body, extheader)
896
+ def put_async(uri, *args)
897
+ if hashy_argument_has_keys(args, :query, :body)
898
+ new_args = args[0]
899
+ else
900
+ new_args = argument_to_hash(args, :body, :header)
901
+ end
902
+ request_async2(:put, uri, new_args)
691
903
  end
692
904
 
693
- # Sends DELETE request in async style. See request_async for arguments.
905
+ # Sends DELETE request in async style. See request_async2 for arguments.
694
906
  # It immediately returns a HTTPClient::Connection instance as a result.
695
- def delete_async(uri, extheader = {})
696
- request_async(:delete, uri, nil, nil, extheader)
907
+ def delete_async(uri, *args)
908
+ request_async2(:delete, uri, argument_to_hash(args, :body, :header, :query))
697
909
  end
698
910
 
699
- # Sends OPTIONS request in async style. See request_async for arguments.
911
+ # Sends OPTIONS request in async style. See request_async2 for arguments.
700
912
  # It immediately returns a HTTPClient::Connection instance as a result.
701
- def options_async(uri, extheader = {})
702
- request_async(:options, uri, nil, nil, extheader)
913
+ def options_async(uri, *args)
914
+ request_async2(:options, uri, argument_to_hash(args, :header, :query, :body))
703
915
  end
704
916
 
705
- # Sends PROPFIND request in async style. See request_async for arguments.
917
+ # Sends PROPFIND request in async style. See request_async2 for arguments.
706
918
  # It immediately returns a HTTPClient::Connection instance as a result.
707
- def propfind_async(uri, extheader = PROPFIND_DEFAULT_EXTHEADER)
708
- request_async(:propfind, uri, nil, nil, extheader)
919
+ def propfind_async(uri, *args)
920
+ request_async2(:propfind, uri, argument_to_hash(args, :body, :header))
709
921
  end
710
922
 
711
- # Sends PROPPATCH request in async style. See request_async for arguments.
923
+ # Sends PROPPATCH request in async style. See request_async2 for arguments.
712
924
  # It immediately returns a HTTPClient::Connection instance as a result.
713
- def proppatch_async(uri, body = nil, extheader = {})
714
- request_async(:proppatch, uri, nil, body, extheader)
925
+ def proppatch_async(uri, *args)
926
+ request_async2(:proppatch, uri, argument_to_hash(args, :body, :header))
715
927
  end
716
928
 
717
- # Sends TRACE request in async style. See request_async for arguments.
929
+ # Sends TRACE request in async style. See request_async2 for arguments.
718
930
  # It immediately returns a HTTPClient::Connection instance as a result.
719
- def trace_async(uri, query = nil, body = nil, extheader = {})
720
- request_async(:trace, uri, query, body, extheader)
931
+ def trace_async(uri, *args)
932
+ request_async2(:trace, uri, argument_to_hash(args, :query, :header))
721
933
  end
722
934
 
723
935
  # Sends a request in async style. request method creates new Thread for
724
936
  # HTTP connection and returns a HTTPClient::Connection instance immediately.
725
937
  #
726
938
  # Arguments definition is the same as request.
727
- def request_async(method, uri, query = nil, body = nil, extheader = {})
728
- uri = urify(uri)
729
- do_request_async(method, uri, query, body, extheader)
939
+ def request_async(method, uri, query = nil, body = nil, header = {})
940
+ uri = to_resource_url(uri)
941
+ do_request_async(method, uri, query, body, header)
942
+ end
943
+
944
+ # new method that has same signature as 'request'
945
+ def request_async2(method, uri, *args)
946
+ query, body, header = keyword_argument(args, :query, :body, :header)
947
+ if [:post, :put].include?(method)
948
+ body ||= ''
949
+ end
950
+ if method == :propfind
951
+ header ||= PROPFIND_DEFAULT_EXTHEADER
952
+ else
953
+ header ||= {}
954
+ end
955
+ uri = to_resource_url(uri)
956
+ do_request_async(method, uri, query, body, header)
730
957
  end
731
958
 
732
959
  # Resets internal session for the given URL. Keep-alive connection for the
733
960
  # site (host-port pair) is disconnected if exists.
734
961
  def reset(uri)
735
- uri = urify(uri)
962
+ uri = to_resource_url(uri)
736
963
  @session_manager.reset(uri)
737
964
  end
738
965
 
@@ -744,55 +971,91 @@ class HTTPClient
744
971
  private
745
972
 
746
973
  class RetryableResponse < StandardError # :nodoc:
974
+ attr_reader :res
975
+
976
+ def initialize(res = nil)
977
+ @res = res
978
+ end
747
979
  end
748
980
 
749
981
  class KeepAliveDisconnected < StandardError # :nodoc:
982
+ attr_reader :sess
983
+ attr_reader :cause
984
+ def initialize(sess = nil, cause = nil)
985
+ super("#{self.class.name}: #{cause ? cause.message : nil}")
986
+ @sess = sess
987
+ @cause = cause
988
+ end
750
989
  end
751
990
 
752
- def do_request(method, uri, query, body, extheader, &block)
753
- conn = Connection.new
991
+ def hashy_argument_has_keys(args, *key)
992
+ # if the given arg is a single Hash and...
993
+ args.size == 1 and Hash === args[0] and
994
+ # it has any one of the key
995
+ key.all? { |e| args[0].key?(e) }
996
+ end
997
+
998
+ def do_request(method, uri, query, body, header, &block)
754
999
  res = nil
755
1000
  if HTTP::Message.file?(body)
756
1001
  pos = body.pos rescue nil
757
1002
  end
758
1003
  retry_count = @session_manager.protocol_retry_count
759
1004
  proxy = no_proxy?(uri) ? nil : @proxy
1005
+ previous_request = previous_response = nil
760
1006
  while retry_count > 0
761
1007
  body.pos = pos if pos
762
- req = create_request(method, uri, query, body, extheader)
1008
+ req = create_request(method, uri, query, body, header)
1009
+ if previous_request
1010
+ # to remember IO positions to read
1011
+ req.http_body.positions = previous_request.http_body.positions
1012
+ end
763
1013
  begin
764
1014
  protect_keep_alive_disconnected do
765
- do_get_block(req, proxy, conn, &block)
1015
+ # TODO: remove Connection.new
1016
+ # We want to delete Connection usage in do_get_block but Newrelic gem depends on it.
1017
+ # https://github.com/newrelic/rpm/blob/master/lib/new_relic/agent/instrumentation/httpclient.rb#L34-L36
1018
+ conn = Connection.new
1019
+ res = do_get_block(req, proxy, conn, &block)
1020
+ # Webmock's do_get_block returns ConditionVariable
1021
+ if !res.respond_to?(:previous)
1022
+ res = conn.pop
1023
+ end
766
1024
  end
767
- res = conn.pop
1025
+ res.previous = previous_response
768
1026
  break
769
- rescue RetryableResponse
770
- res = conn.pop
1027
+ rescue RetryableResponse => e
1028
+ previous_request = req
1029
+ previous_response = res = e.res
771
1030
  retry_count -= 1
772
1031
  end
773
1032
  end
774
1033
  res
775
1034
  end
776
1035
 
777
- def do_request_async(method, uri, query, body, extheader)
1036
+ def do_request_async(method, uri, query, body, header)
778
1037
  conn = Connection.new
779
1038
  t = Thread.new(conn) { |tconn|
780
- if HTTP::Message.file?(body)
781
- pos = body.pos rescue nil
782
- end
783
- retry_count = @session_manager.protocol_retry_count
784
- proxy = no_proxy?(uri) ? nil : @proxy
785
- while retry_count > 0
786
- body.pos = pos if pos
787
- req = create_request(method, uri, query, body, extheader)
788
- begin
789
- protect_keep_alive_disconnected do
790
- do_get_stream(req, proxy, tconn)
1039
+ begin
1040
+ if HTTP::Message.file?(body)
1041
+ pos = body.pos rescue nil
1042
+ end
1043
+ retry_count = @session_manager.protocol_retry_count
1044
+ proxy = no_proxy?(uri) ? nil : @proxy
1045
+ while retry_count > 0
1046
+ body.pos = pos if pos
1047
+ req = create_request(method, uri, query, body, header)
1048
+ begin
1049
+ protect_keep_alive_disconnected do
1050
+ do_get_stream(req, proxy, tconn)
1051
+ end
1052
+ break
1053
+ rescue RetryableResponse
1054
+ retry_count -= 1
791
1055
  end
792
- break
793
- rescue RetryableResponse
794
- retry_count -= 1
795
1056
  end
1057
+ rescue Exception => e
1058
+ conn.push e
796
1059
  end
797
1060
  }
798
1061
  conn.async_thread = t
@@ -817,51 +1080,79 @@ private
817
1080
  ENV[name.downcase] || ENV[name.upcase]
818
1081
  end
819
1082
 
820
- def follow_redirect(method, uri, query, body, extheader, &block)
821
- uri = urify(uri)
1083
+ def adapt_block(&block)
1084
+ return block if block.arity == 2
1085
+ proc { |r, str| block.call(str) }
1086
+ end
1087
+
1088
+ def follow_redirect(method, uri, query, body, header, &block)
1089
+ uri = to_resource_url(uri)
822
1090
  if block
1091
+ b = adapt_block(&block)
823
1092
  filtered_block = proc { |r, str|
824
- block.call(str) if HTTP::Status.successful?(r.status)
1093
+ b.call(r, str) if r.ok?
825
1094
  }
826
1095
  end
827
1096
  if HTTP::Message.file?(body)
828
1097
  pos = body.pos rescue nil
829
1098
  end
830
1099
  retry_number = 0
1100
+ previous = nil
1101
+ request_query = query
831
1102
  while retry_number < @follow_redirect_count
832
1103
  body.pos = pos if pos
833
- res = do_request(method, uri, query, body, extheader, &filtered_block)
834
- if HTTP::Status.successful?(res.status)
835
- return res
836
- elsif HTTP::Status.redirect?(res.status)
1104
+ res = do_request(method, uri, request_query, body, header, &filtered_block)
1105
+ res.previous = previous
1106
+ if res.redirect?
1107
+ if res.header['location'].empty?
1108
+ raise BadResponseError.new("Missing Location header for redirect", res)
1109
+ end
1110
+ method = :get if res.see_other? # See RFC2616 10.3.4
837
1111
  uri = urify(@redirect_uri_callback.call(uri, res))
1112
+ # To avoid duped query parameter. 'location' must include query part.
1113
+ request_query = nil
1114
+ previous = res
838
1115
  retry_number += 1
839
1116
  else
840
- raise BadResponseError.new("unexpected response: #{res.header.inspect}", res)
1117
+ return res
841
1118
  end
842
1119
  end
843
1120
  raise BadResponseError.new("retry count exceeded", res)
844
1121
  end
845
1122
 
1123
+ def success_content(res)
1124
+ if res.ok?
1125
+ return res.content
1126
+ else
1127
+ raise BadResponseError.new("unexpected response: #{res.header.inspect}", res)
1128
+ end
1129
+ end
1130
+
846
1131
  def protect_keep_alive_disconnected
847
1132
  begin
848
1133
  yield
849
1134
  rescue KeepAliveDisconnected
850
- yield
1135
+ # Force to create new connection
1136
+ Thread.current[:HTTPClient_AcquireNewConnection] = true
1137
+ begin
1138
+ yield
1139
+ ensure
1140
+ Thread.current[:HTTPClient_AcquireNewConnection] = false
1141
+ end
851
1142
  end
852
1143
  end
853
1144
 
854
- def create_request(method, uri, query, body, extheader)
1145
+ def create_request(method, uri, query, body, header)
855
1146
  method = method.to_s.upcase
856
- if extheader.is_a?(Hash)
857
- extheader = extheader.to_a
1147
+ if header.is_a?(Hash)
1148
+ header = @default_header.merge(header).to_a
858
1149
  else
859
- extheader = extheader.dup
1150
+ header = @default_header.to_a + header.dup
860
1151
  end
861
1152
  boundary = nil
862
1153
  if body
863
- dummy, content_type = extheader.find { |key, value|
864
- key.downcase == 'content-type'
1154
+ _, content_type = header.find { |key, value|
1155
+ key.to_s.downcase == 'content-type'
865
1156
  }
866
1157
  if content_type
867
1158
  if /\Amultipart/ =~ content_type
@@ -870,25 +1161,28 @@ private
870
1161
  else
871
1162
  boundary = create_boundary
872
1163
  content_type = "#{content_type}; boundary=#{boundary}"
873
- extheader = override_header(extheader, 'Content-Type', content_type)
1164
+ header = override_header(header, 'content-type', content_type)
874
1165
  end
875
1166
  end
876
- elsif method == 'POST'
1167
+ else
877
1168
  if file_in_form_data?(body)
878
1169
  boundary = create_boundary
879
1170
  content_type = "multipart/form-data; boundary=#{boundary}"
880
1171
  else
881
1172
  content_type = 'application/x-www-form-urlencoded'
882
1173
  end
883
- extheader << ['Content-Type', content_type]
1174
+ header << ['Content-Type', content_type]
884
1175
  end
885
1176
  end
886
1177
  req = HTTP::Message.new_request(method, uri, query, body, boundary)
887
- extheader.each do |key, value|
888
- req.header.add(key, value)
1178
+ header.each do |key, value|
1179
+ req.header.add(key.to_s, value)
889
1180
  end
890
- if @cookie_manager && cookie = @cookie_manager.find(uri)
891
- req.header.add('Cookie', cookie)
1181
+ if @cookie_manager
1182
+ cookie_value = @cookie_manager.cookie_value(uri)
1183
+ if cookie_value
1184
+ req.header.add('Cookie', cookie_value)
1185
+ end
892
1186
  end
893
1187
  req
894
1188
  end
@@ -902,10 +1196,10 @@ private
902
1196
  body.any? { |k, v| HTTP::Message.file?(v) }
903
1197
  end
904
1198
 
905
- def override_header(extheader, key, value)
1199
+ def override_header(header, key, value)
906
1200
  result = []
907
- extheader.each do |k, v|
908
- if k.downcase == key.downcase
1201
+ header.each do |k, v|
1202
+ if k.to_s.downcase == key
909
1203
  result << [key, value]
910
1204
  else
911
1205
  result << [k, v]
@@ -920,22 +1214,16 @@ private
920
1214
  if !@proxy or NO_PROXY_HOSTS.include?(uri.host)
921
1215
  return true
922
1216
  end
923
- unless @no_proxy
924
- return false
925
- end
926
- @no_proxy.scan(/([^:,]+)(?::(\d+))?/) do |host, port|
927
- if /(\A|\.)#{Regexp.quote(host)}\z/i =~ uri.host &&
928
- (!port || uri.port == port.to_i)
929
- return true
1217
+ @no_proxy_regexps.each do |regexp, port|
1218
+ if !port || uri.port == port.to_i
1219
+ if regexp =~ uri.host
1220
+ return true
1221
+ end
930
1222
  end
931
1223
  end
932
1224
  false
933
1225
  end
934
1226
 
935
- def https?(uri)
936
- uri.scheme.downcase == 'https'
937
- end
938
-
939
1227
  # !! CAUTION !!
940
1228
  # Method 'do_get*' runs under MT conditon. Be careful to change.
941
1229
  def do_get_block(req, proxy, conn, &block)
@@ -943,12 +1231,13 @@ private
943
1231
  filter.filter_request(req)
944
1232
  end
945
1233
  if str = @test_loopback_response.shift
946
- dump_dummy_request_response(req.body.dump, str) if @debug_dev
947
- conn.push(HTTP::Message.new_response(str))
948
- return
1234
+ dump_dummy_request_response(req.http_body.dump, str) if @debug_dev
1235
+ res = HTTP::Message.new_response(str, req.header)
1236
+ conn.push(res)
1237
+ return res
949
1238
  end
950
1239
  content = block ? nil : ''
951
- res = HTTP::Message.new_response(content)
1240
+ res = HTTP::Message.new_response(content, req.header)
952
1241
  @debug_dev << "= Request\n\n" if @debug_dev
953
1242
  sess = @session_manager.query(req, proxy)
954
1243
  res.peer_cert = sess.ssl_peer_cert
@@ -956,19 +1245,23 @@ private
956
1245
  do_get_header(req, res, sess)
957
1246
  conn.push(res)
958
1247
  sess.get_body do |part|
1248
+ set_encoding(part, res.body_encoding)
959
1249
  if block
960
- block.call(res, part)
1250
+ block.call(res, part.dup)
961
1251
  else
962
1252
  content << part
963
1253
  end
964
1254
  end
1255
+ # there could be a race condition but it's OK to cache unreusable
1256
+ # connection because we do retry for that case.
965
1257
  @session_manager.keep(sess) unless sess.closed?
966
1258
  commands = @request_filter.collect { |filter|
967
1259
  filter.filter_response(req, res)
968
1260
  }
969
1261
  if commands.find { |command| command == :retry }
970
- raise RetryableResponse.new
1262
+ raise RetryableResponse.new(res)
971
1263
  end
1264
+ res
972
1265
  end
973
1266
 
974
1267
  def do_get_stream(req, proxy, conn)
@@ -976,12 +1269,13 @@ private
976
1269
  filter.filter_request(req)
977
1270
  end
978
1271
  if str = @test_loopback_response.shift
979
- dump_dummy_request_response(req.body.dump, str) if @debug_dev
980
- conn.push(HTTP::Message.new_response(StringIO.new(str)))
1272
+ dump_dummy_request_response(req.http_body.dump, str) if @debug_dev
1273
+ conn.push(HTTP::Message.new_response(StringIO.new(str), req.header))
981
1274
  return
982
1275
  end
983
1276
  piper, pipew = IO.pipe
984
- res = HTTP::Message.new_response(piper)
1277
+ pipew.binmode
1278
+ res = HTTP::Message.new_response(piper, req.header)
985
1279
  @debug_dev << "= Request\n\n" if @debug_dev
986
1280
  sess = @session_manager.query(req, proxy)
987
1281
  res.peer_cert = sess.ssl_peer_cert
@@ -989,21 +1283,21 @@ private
989
1283
  do_get_header(req, res, sess)
990
1284
  conn.push(res)
991
1285
  sess.get_body do |part|
992
- pipew.syswrite(part)
1286
+ set_encoding(part, res.body_encoding)
1287
+ pipew.write(part)
993
1288
  end
994
1289
  pipew.close
995
1290
  @session_manager.keep(sess) unless sess.closed?
996
- commands = @request_filter.collect { |filter|
1291
+ _ = @request_filter.collect { |filter|
997
1292
  filter.filter_response(req, res)
998
1293
  }
999
1294
  # ignore commands (not retryable in async mode)
1295
+ res
1000
1296
  end
1001
1297
 
1002
1298
  def do_get_header(req, res, sess)
1003
- res.version, res.status, res.reason, headers = sess.get_header
1004
- headers.each do |key, value|
1005
- res.header.add(key, value)
1006
- end
1299
+ res.http_version, res.status, res.reason, headers = sess.get_header
1300
+ res.header.set_headers(headers)
1007
1301
  if @cookie_manager
1008
1302
  res.header['set-cookie'].each do |cookie|
1009
1303
  @cookie_manager.parse(cookie, req.header.request_uri)
@@ -1017,4 +1311,17 @@ private
1017
1311
  @debug_dev << "\n\n= Dummy Response\n\n"
1018
1312
  @debug_dev << res
1019
1313
  end
1314
+
1315
+ def set_encoding(str, encoding)
1316
+ str.force_encoding(encoding) if encoding
1317
+ end
1318
+
1319
+ def to_resource_url(uri)
1320
+ u = urify(uri)
1321
+ if @base_url && u.scheme.nil? && u.host.nil?
1322
+ urify(@base_url) + uri
1323
+ else
1324
+ u
1325
+ end
1326
+ end
1020
1327
  end