httpclient 2.1.5 → 2.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +85 -0
- data/bin/httpclient +77 -0
- data/bin/jsonclient +85 -0
- data/lib/hexdump.rb +50 -0
- data/lib/http-access2.rb +6 -4
- data/lib/httpclient/auth.rb +575 -173
- data/lib/httpclient/cacert.pem +3952 -0
- data/lib/httpclient/cacert1024.pem +3866 -0
- data/lib/httpclient/connection.rb +6 -2
- data/lib/httpclient/cookie.rb +162 -504
- data/lib/httpclient/http.rb +334 -119
- data/lib/httpclient/include_client.rb +85 -0
- data/lib/httpclient/jruby_ssl_socket.rb +588 -0
- data/lib/httpclient/session.rb +385 -288
- data/lib/httpclient/ssl_config.rb +195 -155
- data/lib/httpclient/ssl_socket.rb +150 -0
- data/lib/httpclient/timeout.rb +14 -10
- data/lib/httpclient/util.rb +142 -6
- data/lib/httpclient/version.rb +3 -0
- data/lib/httpclient/webagent-cookie.rb +459 -0
- data/lib/httpclient.rb +509 -202
- data/lib/jsonclient.rb +63 -0
- data/lib/oauthclient.rb +111 -0
- data/sample/async.rb +8 -0
- data/sample/auth.rb +11 -0
- data/sample/cookie.rb +18 -0
- data/sample/dav.rb +103 -0
- data/sample/howto.rb +49 -0
- data/sample/jsonclient.rb +67 -0
- data/sample/oauth_buzz.rb +57 -0
- data/sample/oauth_friendfeed.rb +59 -0
- data/sample/oauth_twitter.rb +61 -0
- data/sample/ssl/0cert.pem +22 -0
- data/sample/ssl/0key.pem +30 -0
- data/sample/ssl/1000cert.pem +19 -0
- data/sample/ssl/1000key.pem +18 -0
- data/sample/ssl/htdocs/index.html +10 -0
- data/sample/ssl/ssl_client.rb +22 -0
- data/sample/ssl/webrick_httpsd.rb +29 -0
- data/sample/stream.rb +21 -0
- data/sample/thread.rb +27 -0
- data/sample/wcat.rb +21 -0
- data/test/ca-chain.pem +44 -0
- data/test/ca.cert +23 -0
- data/test/client-pass.key +18 -0
- data/test/client.cert +19 -0
- data/test/client.key +15 -0
- data/test/helper.rb +131 -0
- data/test/htdigest +1 -0
- data/test/htpasswd +2 -0
- data/test/jruby_ssl_socket/test_pemutils.rb +32 -0
- data/test/runner.rb +2 -0
- data/test/server.cert +19 -0
- data/test/server.key +15 -0
- data/test/sslsvr.rb +65 -0
- data/test/subca.cert +21 -0
- data/test/test_auth.rb +492 -0
- data/test/test_cookie.rb +309 -0
- data/test/test_hexdump.rb +14 -0
- data/test/test_http-access2.rb +508 -0
- data/test/test_httpclient.rb +2145 -0
- data/test/test_include_client.rb +52 -0
- data/test/test_jsonclient.rb +80 -0
- data/test/test_ssl.rb +559 -0
- data/test/test_webagent-cookie.rb +465 -0
- metadata +85 -44
- data/lib/httpclient/auth.rb.orig +0 -513
- data/lib/httpclient/cacert.p7s +0 -1579
- data/lib/httpclient.rb.orig +0 -1020
- data/lib/tags +0 -908
data/lib/httpclient.rb
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
# HTTPClient - HTTP client library.
|
2
|
-
# Copyright (C) 2000-
|
2
|
+
# Copyright (C) 2000-2015 NAKAMURA, Hiroshi <nahi@ruby-lang.org>.
|
3
3
|
#
|
4
4
|
# This program is copyrighted free software by NAKAMURA, Hiroshi. You can
|
5
5
|
# redistribute it and/or modify it under the same terms of Ruby's license;
|
6
6
|
# either the dual license version in 2003, or any later version.
|
7
7
|
|
8
8
|
|
9
|
-
require 'uri'
|
10
9
|
require 'stringio'
|
11
10
|
require 'digest/sha1'
|
12
11
|
|
13
12
|
# Extra library
|
13
|
+
require 'httpclient/version'
|
14
14
|
require 'httpclient/util'
|
15
15
|
require 'httpclient/ssl_config'
|
16
16
|
require 'httpclient/connection'
|
@@ -19,7 +19,7 @@ require 'httpclient/http'
|
|
19
19
|
require 'httpclient/auth'
|
20
20
|
require 'httpclient/cookie'
|
21
21
|
|
22
|
-
|
22
|
+
# :main:HTTPClient
|
23
23
|
# The HTTPClient class provides several methods for accessing Web resources
|
24
24
|
# via HTTP.
|
25
25
|
#
|
@@ -50,13 +50,23 @@ require 'httpclient/cookie'
|
|
50
50
|
#
|
51
51
|
# === How to retrieve web resources
|
52
52
|
#
|
53
|
-
# See get_content.
|
53
|
+
# See get and get_content.
|
54
|
+
#
|
55
|
+
# 1. Get content of specified URL. It returns HTTP::Message object and
|
56
|
+
# calling 'body' method of it returns a content String.
|
54
57
|
#
|
55
|
-
#
|
58
|
+
# puts clnt.get('http://dev.ctor.org/').body
|
59
|
+
#
|
60
|
+
# 2. For getting content directly, use get_content. It follows redirect
|
61
|
+
# response and returns a String of whole result.
|
56
62
|
#
|
57
63
|
# puts clnt.get_content('http://dev.ctor.org/')
|
58
64
|
#
|
59
|
-
#
|
65
|
+
# 3. You can pass :follow_redirect option to follow redirect response in get.
|
66
|
+
#
|
67
|
+
# puts clnt.get('http://dev.ctor.org/', :follow_redirect => true)
|
68
|
+
#
|
69
|
+
# 4. Get content as chunks of String. It yields chunks of String.
|
60
70
|
#
|
61
71
|
# clnt.get_content('http://dev.ctor.org/') do |chunk|
|
62
72
|
# puts chunk
|
@@ -79,7 +89,11 @@ require 'httpclient/cookie'
|
|
79
89
|
# p res.status
|
80
90
|
# p res.contenttype
|
81
91
|
# p res.header['X-Custom']
|
82
|
-
# puts res.
|
92
|
+
# puts res.body
|
93
|
+
#
|
94
|
+
# You can also use keyword argument style.
|
95
|
+
#
|
96
|
+
# res = clnt.get(uri, :query => { :keyword => 'ruby', :lang => 'en' })
|
83
97
|
#
|
84
98
|
# === How to POST
|
85
99
|
#
|
@@ -90,6 +104,10 @@ require 'httpclient/cookie'
|
|
90
104
|
# body = { 'keyword' => 'ruby', 'lang' => 'en' }
|
91
105
|
# res = clnt.post(uri, body)
|
92
106
|
#
|
107
|
+
# Keyword argument style.
|
108
|
+
#
|
109
|
+
# res = clnt.post(uri, :body => ...)
|
110
|
+
#
|
93
111
|
# 2. Do multipart file upload with POST. No need to set extra header by
|
94
112
|
# yourself from httpclient/2.1.4.
|
95
113
|
#
|
@@ -98,6 +116,17 @@ require 'httpclient/cookie'
|
|
98
116
|
# res = clnt.post(uri, body)
|
99
117
|
# end
|
100
118
|
#
|
119
|
+
# 3. Do multipart with custom body.
|
120
|
+
#
|
121
|
+
# File.open('/tmp/post_data') do |file|
|
122
|
+
# body = [{ 'Content-Type' => 'application/atom+xml; charset=UTF-8',
|
123
|
+
# :content => '<entry>...</entry>' },
|
124
|
+
# { 'Content-Type' => 'video/mp4',
|
125
|
+
# 'Content-Transfer-Encoding' => 'binary',
|
126
|
+
# :content => file }]
|
127
|
+
# res = clnt.post(uri, body)
|
128
|
+
# end
|
129
|
+
#
|
101
130
|
# === Accessing via SSL
|
102
131
|
#
|
103
132
|
# Ruby needs to be compiled with OpenSSL.
|
@@ -106,7 +135,7 @@ require 'httpclient/cookie'
|
|
106
135
|
# Just pass an URL which starts with 'https://'.
|
107
136
|
#
|
108
137
|
# https_url = 'https://www.rsa.com'
|
109
|
-
# clnt.
|
138
|
+
# clnt.get(https_url)
|
110
139
|
#
|
111
140
|
# 2. Getting peer certificate from response.
|
112
141
|
#
|
@@ -118,22 +147,33 @@ require 'httpclient/cookie'
|
|
118
147
|
# user_cert_file = 'cert.pem'
|
119
148
|
# user_key_file = 'privkey.pem'
|
120
149
|
# clnt.ssl_config.set_client_cert_file(user_cert_file, user_key_file)
|
121
|
-
# clnt.
|
150
|
+
# clnt.get(https_url)
|
151
|
+
#
|
152
|
+
# 4. Revocation check. On JRuby you can set following options to let
|
153
|
+
# HTTPClient to perform revocation check with CRL and OCSP:
|
154
|
+
#
|
155
|
+
# -J-Dcom.sun.security.enableCRLDP=true -J-Dcom.sun.net.ssl.checkRevocation=true
|
156
|
+
# ex. jruby -J-Dcom.sun.security.enableCRLDP=true -J-Dcom.sun.net.ssl.checkRevocation=true app.rb
|
157
|
+
# Revoked cert example: https://test-sspev.verisign.com:2443/test-SSPEV-revoked-verisign.html
|
158
|
+
#
|
159
|
+
# On other platform you can download CRL by yourself and set it via
|
160
|
+
# SSLConfig#add_crl.
|
122
161
|
#
|
123
162
|
# === Handling Cookies
|
124
163
|
#
|
125
164
|
# 1. Using volatile Cookies. Nothing to do. HTTPClient handles Cookies.
|
126
165
|
#
|
127
166
|
# clnt = HTTPClient.new
|
128
|
-
# clnt.
|
129
|
-
# clnt.
|
167
|
+
# res = clnt.get(url1) # receives Cookies.
|
168
|
+
# res = clnt.get(url2) # sends Cookies if needed.
|
169
|
+
# p res.cookies
|
130
170
|
#
|
131
171
|
# 2. Saving non volatile Cookies to a specified file. Need to set a file at
|
132
172
|
# first and invoke save method at last.
|
133
173
|
#
|
134
174
|
# clnt = HTTPClient.new
|
135
175
|
# clnt.set_cookie_store('/home/nahi/cookie.dat')
|
136
|
-
# clnt.
|
176
|
+
# clnt.get(url)
|
137
177
|
# ...
|
138
178
|
# clnt.save_cookie_store
|
139
179
|
#
|
@@ -152,7 +192,7 @@ require 'httpclient/cookie'
|
|
152
192
|
# user = 'user'
|
153
193
|
# password = 'user'
|
154
194
|
# clnt.set_auth(domain, user, password)
|
155
|
-
# p clnt.
|
195
|
+
# p clnt.get('http://dev.ctor.org/http-access2/login').status
|
156
196
|
#
|
157
197
|
# 2. Authentication with Proxy server. Supports BasicAuth and NTLM
|
158
198
|
# (requires win32/sspi)
|
@@ -161,17 +201,17 @@ require 'httpclient/cookie'
|
|
161
201
|
# user = 'proxy'
|
162
202
|
# password = 'proxy'
|
163
203
|
# clnt.set_proxy_auth(user, password)
|
164
|
-
# p clnt.
|
204
|
+
# p clnt.get(url)
|
165
205
|
#
|
166
206
|
# === Invoking HTTP methods with custom header
|
167
207
|
#
|
168
|
-
# Pass a Hash or an Array for
|
208
|
+
# Pass a Hash or an Array for header argument.
|
169
209
|
#
|
170
|
-
#
|
171
|
-
# clnt.
|
210
|
+
# header = { 'Accept' => 'text/html' }
|
211
|
+
# clnt.get(uri, query, header)
|
172
212
|
#
|
173
|
-
#
|
174
|
-
# clnt.get_content(uri, query,
|
213
|
+
# header = [['Accept', 'image/jpeg'], ['Accept', 'image/png']]
|
214
|
+
# clnt.get_content(uri, query, header)
|
175
215
|
#
|
176
216
|
# === Invoking HTTP methods asynchronously
|
177
217
|
#
|
@@ -189,7 +229,7 @@ require 'httpclient/cookie'
|
|
189
229
|
# puts '.'
|
190
230
|
# res = connection.pop
|
191
231
|
# p res.status
|
192
|
-
# p res.
|
232
|
+
# p res.body.read # res.body is an IO for the res of async method.
|
193
233
|
#
|
194
234
|
# === Shortcut methods
|
195
235
|
#
|
@@ -199,10 +239,8 @@ require 'httpclient/cookie'
|
|
199
239
|
# ruby -rhttpclient -e 'p HTTPClient.head(ARGV.shift).header["last-modified"]' http://dev.ctor.org/
|
200
240
|
#
|
201
241
|
class HTTPClient
|
202
|
-
|
203
|
-
|
204
|
-
/: (\S+) (\S+)/ =~ %q$Id: httpclient.rb 280 2009-06-02 15:44:28Z nahi $
|
205
|
-
LIB_NAME = "(#{$1}/#{$2}, #{RUBY_VERSION_STRING})"
|
242
|
+
RUBY_VERSION_STRING = "ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE})"
|
243
|
+
LIB_NAME = "(#{VERSION}, #{RUBY_VERSION_STRING})"
|
206
244
|
|
207
245
|
include Util
|
208
246
|
|
@@ -271,7 +309,6 @@ class HTTPClient
|
|
271
309
|
if assignable
|
272
310
|
aname = name + '='
|
273
311
|
define_method(aname) { |rhs|
|
274
|
-
reset_all
|
275
312
|
@session_manager.__send__(aname, rhs)
|
276
313
|
}
|
277
314
|
end
|
@@ -280,7 +317,7 @@ class HTTPClient
|
|
280
317
|
|
281
318
|
# HTTPClient::SSLConfig:: SSL configurator.
|
282
319
|
attr_reader :ssl_config
|
283
|
-
#
|
320
|
+
# HTTPClient::CookieManager:: Cookies configurator.
|
284
321
|
attr_accessor :cookie_manager
|
285
322
|
# An array of response HTTP message body String which is used for loop-back
|
286
323
|
# test. See test/* to see how to use it. If you want to do loop-back test
|
@@ -296,6 +333,10 @@ class HTTPClient
|
|
296
333
|
# How many times get_content and post_content follows HTTP redirect.
|
297
334
|
# 10 by default.
|
298
335
|
attr_accessor :follow_redirect_count
|
336
|
+
# Base url of resources.
|
337
|
+
attr_accessor :base_url
|
338
|
+
# Default request header.
|
339
|
+
attr_accessor :default_header
|
299
340
|
|
300
341
|
# Set HTTP version as a String:: 'HTTP/1.0' or 'HTTP/1.1'
|
301
342
|
attr_proxy(:protocol_version, true)
|
@@ -305,11 +346,17 @@ class HTTPClient
|
|
305
346
|
attr_proxy(:send_timeout, true)
|
306
347
|
# Response receiving timeout in sec.
|
307
348
|
attr_proxy(:receive_timeout, true)
|
349
|
+
# Reuse the same connection within this timeout in sec. from last used.
|
350
|
+
attr_proxy(:keep_alive_timeout, true)
|
351
|
+
# Size of reading block for non-chunked response.
|
352
|
+
attr_proxy(:read_block_size, true)
|
308
353
|
# Negotiation retry count for authentication. 5 by default.
|
309
354
|
attr_proxy(:protocol_retry_count, true)
|
310
355
|
# if your ruby is older than 2005-09-06, do not set socket_sync = false to
|
311
356
|
# avoid an SSL socket blocking bug in openssl/buffering.rb.
|
312
357
|
attr_proxy(:socket_sync, true)
|
358
|
+
# Enables TCP keepalive; no timing settings exist at present
|
359
|
+
attr_proxy(:tcp_keepalive, true)
|
313
360
|
# User-Agent header in HTTP request.
|
314
361
|
attr_proxy(:agent_name, true)
|
315
362
|
# From header in HTTP request.
|
@@ -317,44 +364,107 @@ class HTTPClient
|
|
317
364
|
# An array of response HTTP String (not a HTTP message body) which is used
|
318
365
|
# for loopback test. See test/* to see how to use it.
|
319
366
|
attr_proxy(:test_loopback_http_response)
|
320
|
-
|
321
|
-
|
367
|
+
# Decompress a compressed (with gzip or deflate) content body transparently. false by default.
|
368
|
+
attr_proxy(:transparent_gzip_decompression, true)
|
369
|
+
# Raise BadResponseError if response size does not match with Content-Length header in response. false by default.
|
370
|
+
# TODO: enable by default
|
371
|
+
attr_proxy(:strict_response_size_check, true)
|
372
|
+
# Local socket address. Set HTTPClient#socket_local.host and HTTPClient#socket_local.port to specify local binding hostname and port of TCP socket.
|
373
|
+
attr_proxy(:socket_local, true)
|
374
|
+
|
375
|
+
# Default header for PROPFIND request.
|
322
376
|
PROPFIND_DEFAULT_EXTHEADER = { 'Depth' => '0' }
|
323
377
|
|
378
|
+
# Default User-Agent header
|
379
|
+
DEFAULT_AGENT_NAME = 'HTTPClient/1.0'
|
380
|
+
|
324
381
|
# Creates a HTTPClient instance which manages sessions, cookies, etc.
|
325
382
|
#
|
326
|
-
# HTTPClient.new takes
|
327
|
-
#
|
328
|
-
#
|
329
|
-
#
|
383
|
+
# HTTPClient.new takes optional arguments as a Hash.
|
384
|
+
# * :proxy - proxy url string
|
385
|
+
# * :agent_name - User-Agent String
|
386
|
+
# * :from - from header String
|
387
|
+
# * :base_url - base URL of resources
|
388
|
+
# * :default_header - header Hash all HTTP requests should have
|
389
|
+
# * :force_basic_auth - flag for sending Authorization header w/o gettin 401 first
|
390
|
+
# User-Agent and From are embedded in HTTP request Header if given.
|
391
|
+
# From header is not set without setting it explicitly.
|
330
392
|
#
|
331
393
|
# proxy = 'http://myproxy:8080'
|
332
394
|
# agent_name = 'MyAgent/0.1'
|
333
395
|
# from = 'from@example.com'
|
334
396
|
# HTTPClient.new(proxy, agent_name, from)
|
335
397
|
#
|
336
|
-
#
|
337
|
-
#
|
398
|
+
# After you set :base_url, all resources you pass to get, post and other
|
399
|
+
# methods are recognized to be prefixed with base_url. Say base_url is
|
400
|
+
# 'https://api.example.com/v1/, get('users') is the same as
|
401
|
+
# get('https://api.example.com/v1/users') internally. You can also pass
|
402
|
+
# full URL from 'http://' even after setting base_url.
|
403
|
+
#
|
404
|
+
# The expected base_url and path behavior is the following. Please take
|
405
|
+
# care of '/' in base_url and path.
|
406
|
+
#
|
407
|
+
# The last '/' is important for base_url:
|
408
|
+
# 1. http://foo/bar/baz/ + path -> http://foo/bar/baz/path
|
409
|
+
# 2. http://foo/bar/baz + path -> http://foo/bar/path
|
410
|
+
# Relative path handling:
|
411
|
+
# 3. http://foo/bar/baz/ + ../path -> http://foo/bar/path
|
412
|
+
# 4. http://foo/bar/baz + ../path -> http://foo/path
|
413
|
+
# 5. http://foo/bar/baz/ + ./path -> http://foo/bar/baz/path
|
414
|
+
# 6. http://foo/bar/baz + ./path -> http://foo/bar/path
|
415
|
+
# The leading '/' of path means absolute path:
|
416
|
+
# 7. http://foo/bar/baz/ + /path -> http://foo/path
|
417
|
+
# 8. http://foo/bar/baz + /path -> http://foo/path
|
418
|
+
#
|
419
|
+
# :default_header is for providing default headers Hash that all HTTP
|
420
|
+
# requests should have, such as custom 'Authorization' header in API.
|
421
|
+
# You can override :default_header with :header Hash parameter in HTTP
|
422
|
+
# request methods.
|
338
423
|
#
|
339
|
-
#
|
340
|
-
|
341
|
-
|
424
|
+
# :force_basic_auth turns on/off the BasicAuth force flag. Generally
|
425
|
+
# HTTP client must send Authorization header after it gets 401 error
|
426
|
+
# from server from security reason. But in some situation (e.g. API
|
427
|
+
# client) you might want to send Authorization from the beginning.
|
428
|
+
def initialize(*args, &block)
|
429
|
+
proxy, agent_name, from, base_url, default_header, force_basic_auth =
|
430
|
+
keyword_argument(args, :proxy, :agent_name, :from, :base_url, :default_header, :force_basic_auth)
|
342
431
|
@proxy = nil # assigned later.
|
343
432
|
@no_proxy = nil
|
433
|
+
@no_proxy_regexps = []
|
434
|
+
@base_url = base_url
|
435
|
+
@default_header = default_header || {}
|
344
436
|
@www_auth = WWWAuth.new
|
345
437
|
@proxy_auth = ProxyAuth.new
|
438
|
+
@www_auth.basic_auth.force_auth = @proxy_auth.basic_auth.force_auth = force_basic_auth
|
346
439
|
@request_filter = [@proxy_auth, @www_auth]
|
347
440
|
@debug_dev = nil
|
348
441
|
@redirect_uri_callback = method(:default_redirect_uri_callback)
|
349
442
|
@test_loopback_response = []
|
350
443
|
@session_manager = SessionManager.new(self)
|
351
|
-
@session_manager.agent_name = agent_name
|
444
|
+
@session_manager.agent_name = agent_name || DEFAULT_AGENT_NAME
|
352
445
|
@session_manager.from = from
|
353
446
|
@session_manager.ssl_config = @ssl_config = SSLConfig.new(self)
|
354
|
-
@cookie_manager =
|
447
|
+
@cookie_manager = CookieManager.new
|
355
448
|
@follow_redirect_count = 10
|
356
449
|
load_environment
|
357
450
|
self.proxy = proxy if proxy
|
451
|
+
keep_webmock_compat
|
452
|
+
instance_eval(&block) if block
|
453
|
+
end
|
454
|
+
|
455
|
+
# webmock 1.6.2 depends on HTTP::Message#body.content to work.
|
456
|
+
# let's keep it work iif webmock is loaded for a while.
|
457
|
+
def keep_webmock_compat
|
458
|
+
if respond_to?(:do_get_block_with_webmock)
|
459
|
+
::HTTP::Message.module_eval do
|
460
|
+
def body
|
461
|
+
def (o = self.content).content
|
462
|
+
self
|
463
|
+
end
|
464
|
+
o
|
465
|
+
end
|
466
|
+
end
|
467
|
+
end
|
358
468
|
end
|
359
469
|
|
360
470
|
# Returns debug device if exists. See debug_dev=.
|
@@ -388,7 +498,7 @@ class HTTPClient
|
|
388
498
|
#
|
389
499
|
# Calling this method resets all existing sessions.
|
390
500
|
def proxy=(proxy)
|
391
|
-
if proxy.nil?
|
501
|
+
if proxy.nil? || proxy.to_s.empty?
|
392
502
|
@proxy = nil
|
393
503
|
@proxy_auth.reset_challenge
|
394
504
|
else
|
@@ -425,6 +535,17 @@ class HTTPClient
|
|
425
535
|
# Calling this method resets all existing sessions.
|
426
536
|
def no_proxy=(no_proxy)
|
427
537
|
@no_proxy = no_proxy
|
538
|
+
@no_proxy_regexps.clear
|
539
|
+
if @no_proxy
|
540
|
+
@no_proxy.scan(/([^\s:,]+)(?::(\d+))?/) do |host, port|
|
541
|
+
if host[0] == ?.
|
542
|
+
regexp = /#{Regexp.quote(host)}\z/i
|
543
|
+
else
|
544
|
+
regexp = /(\A|\.)#{Regexp.quote(host)}\z/i
|
545
|
+
end
|
546
|
+
@no_proxy_regexps << [regexp, port]
|
547
|
+
end
|
548
|
+
end
|
428
549
|
reset_all
|
429
550
|
end
|
430
551
|
|
@@ -442,14 +563,14 @@ class HTTPClient
|
|
442
563
|
#
|
443
564
|
# Calling this method resets all existing sessions.
|
444
565
|
def set_auth(domain, user, passwd)
|
445
|
-
uri =
|
566
|
+
uri = to_resource_url(domain)
|
446
567
|
@www_auth.set_auth(uri, user, passwd)
|
447
568
|
reset_all
|
448
569
|
end
|
449
570
|
|
450
571
|
# Deprecated. Use set_auth instead.
|
451
572
|
def set_basic_auth(domain, user, passwd)
|
452
|
-
uri =
|
573
|
+
uri = to_resource_url(domain)
|
453
574
|
@www_auth.basic_auth.set(uri, user, passwd)
|
454
575
|
reset_all
|
455
576
|
end
|
@@ -464,6 +585,14 @@ class HTTPClient
|
|
464
585
|
reset_all
|
465
586
|
end
|
466
587
|
|
588
|
+
# Turn on/off the BasicAuth force flag. Generally HTTP client must
|
589
|
+
# send Authorization header after it gets 401 error from server from
|
590
|
+
# security reason. But in some situation (e.g. API client) you might
|
591
|
+
# want to send Authorization from the beginning.
|
592
|
+
def force_basic_auth=(force_basic_auth)
|
593
|
+
@www_auth.basic_auth.force_auth = @proxy_auth.basic_auth.force_auth = force_basic_auth
|
594
|
+
end
|
595
|
+
|
467
596
|
# Sets the filename where non-volatile Cookies be saved by calling
|
468
597
|
# save_cookie_store.
|
469
598
|
# This method tries to load and managing Cookies from the specified file.
|
@@ -477,11 +606,17 @@ class HTTPClient
|
|
477
606
|
|
478
607
|
# Try to save Cookies to the file specified in set_cookie_store. Unexpected
|
479
608
|
# error will be raised if you don't call set_cookie_store first.
|
480
|
-
# (interface mismatch between WebAgent::CookieManager implementation)
|
481
609
|
def save_cookie_store
|
482
610
|
@cookie_manager.save_cookies
|
483
611
|
end
|
484
612
|
|
613
|
+
# Returns stored cookies.
|
614
|
+
def cookies
|
615
|
+
if @cookie_manager
|
616
|
+
@cookie_manager.cookies
|
617
|
+
end
|
618
|
+
end
|
619
|
+
|
485
620
|
# Sets callback proc when HTTP redirect status is returned for get_content
|
486
621
|
# and post_content. default_redirect_uri_callback is used by default.
|
487
622
|
#
|
@@ -501,9 +636,9 @@ class HTTPClient
|
|
501
636
|
# e.g. { "a" => "b" } => 'http://host/part?a=b'.
|
502
637
|
# Give an array to pass multiple value like
|
503
638
|
# [["a", "b"], ["a", "c"]] => 'http://host/part?a=b&a=c'.
|
504
|
-
#
|
505
|
-
#
|
506
|
-
#
|
639
|
+
# header:: a Hash or an Array of extra headers. e.g.
|
640
|
+
# { 'Accept' => 'text/html' } or
|
641
|
+
# [['Accept', 'image/jpeg'], ['Accept', 'image/png']].
|
507
642
|
# &block:: Give a block to get chunked message-body of response like
|
508
643
|
# get_content(uri) { |chunked_body| ... }.
|
509
644
|
# Size of each chunk may not be the same.
|
@@ -515,34 +650,51 @@ class HTTPClient
|
|
515
650
|
# If you need to get full HTTP response including HTTP status and headers,
|
516
651
|
# use get method. get returns HTTP::Message as a response and you need to
|
517
652
|
# follow HTTP redirect by yourself if you need.
|
518
|
-
def get_content(uri,
|
519
|
-
|
653
|
+
def get_content(uri, *args, &block)
|
654
|
+
query, header = keyword_argument(args, :query, :header)
|
655
|
+
success_content(follow_redirect(:get, uri, query, nil, header || {}, &block))
|
520
656
|
end
|
521
657
|
|
522
658
|
# Posts a content.
|
523
659
|
#
|
524
660
|
# uri:: a String or an URI object which represents an URL of web resource.
|
525
|
-
# body:: a Hash or an Array of body part.
|
526
|
-
#
|
661
|
+
# body:: a Hash or an Array of body part. e.g.
|
662
|
+
# { "a" => "b" } => 'a=b'
|
527
663
|
# Give an array to pass multiple value like
|
528
|
-
#
|
664
|
+
# [["a", "b"], ["a", "c"]] => 'a=b&a=c'
|
529
665
|
# When you pass a File as a value, it will be posted as a
|
530
|
-
# multipart/form-data. e.g.
|
531
|
-
#
|
532
|
-
#
|
533
|
-
#
|
666
|
+
# multipart/form-data. e.g.
|
667
|
+
# { 'upload' => file }
|
668
|
+
# You can also send custom multipart by passing an array of hashes.
|
669
|
+
# Each part must have a :content attribute which can be a file, all
|
670
|
+
# other keys will become headers.
|
671
|
+
# [{ 'Content-Type' => 'text/plain', :content => "some text" },
|
672
|
+
# { 'Content-Type' => 'video/mp4', :content => File.new('video.mp4') }]
|
673
|
+
# => <Two parts with custom Content-Type header>
|
674
|
+
# header:: a Hash or an Array of extra headers. e.g.
|
675
|
+
# { 'Accept' => 'text/html' }
|
676
|
+
# or
|
677
|
+
# [['Accept', 'image/jpeg'], ['Accept', 'image/png']].
|
534
678
|
# &block:: Give a block to get chunked message-body of response like
|
535
|
-
#
|
679
|
+
# post_content(uri) { |chunked_body| ... }.
|
536
680
|
# Size of each chunk may not be the same.
|
537
681
|
#
|
538
682
|
# post_content follows HTTP redirect status (see HTTP::Status.redirect?)
|
539
683
|
# internally and try to post the content to redirected URL. See
|
540
684
|
# redirect_uri_callback= how HTTP redirection is handled.
|
685
|
+
# Bear in mind that you should not depend on post_content because it sends
|
686
|
+
# the same POST method to the new location which is prohibited in HTTP spec.
|
541
687
|
#
|
542
688
|
# If you need to get full HTTP response including HTTP status and headers,
|
543
689
|
# use post method.
|
544
|
-
def post_content(uri,
|
545
|
-
|
690
|
+
def post_content(uri, *args, &block)
|
691
|
+
if hashy_argument_has_keys(args, :query, :body)
|
692
|
+
query, body, header = keyword_argument(args, :query, :body, :header)
|
693
|
+
else
|
694
|
+
query = nil
|
695
|
+
body, header = keyword_argument(args, :body, :header)
|
696
|
+
end
|
697
|
+
success_content(follow_redirect(:post, uri, query, body, header || {}, &block))
|
546
698
|
end
|
547
699
|
|
548
700
|
# A method for redirect uri callback. How to use:
|
@@ -551,11 +703,11 @@ class HTTPClient
|
|
551
703
|
# Location: ../foo/
|
552
704
|
# in HTTP header. (raises BadResponseError instead)
|
553
705
|
def strict_redirect_uri_callback(uri, res)
|
554
|
-
newuri =
|
706
|
+
newuri = urify(res.header['location'][0])
|
555
707
|
if https?(uri) && !https?(newuri)
|
556
708
|
raise BadResponseError.new("redirecting to non-https resource")
|
557
709
|
end
|
558
|
-
|
710
|
+
if !http?(newuri) && !https?(newuri)
|
559
711
|
raise BadResponseError.new("unexpected location: #{newuri}", res)
|
560
712
|
end
|
561
713
|
puts "redirect to: #{newuri}" if $DEBUG
|
@@ -568,62 +720,85 @@ class HTTPClient
|
|
568
720
|
# Location: ../foo/
|
569
721
|
# in HTTP header.
|
570
722
|
def default_redirect_uri_callback(uri, res)
|
571
|
-
newuri =
|
723
|
+
newuri = urify(res.header['location'][0])
|
724
|
+
if !http?(newuri) && !https?(newuri)
|
725
|
+
warn("#{newuri}: a relative URI in location header which is not recommended")
|
726
|
+
warn("'The field value consists of a single absolute URI' in HTTP spec")
|
727
|
+
newuri = uri + newuri
|
728
|
+
end
|
572
729
|
if https?(uri) && !https?(newuri)
|
573
730
|
raise BadResponseError.new("redirecting to non-https resource")
|
574
731
|
end
|
575
|
-
unless newuri.is_a?(URI::HTTP)
|
576
|
-
newuri = uri + newuri
|
577
|
-
STDERR.puts("could be a relative URI in location header which is not recommended")
|
578
|
-
STDERR.puts("'The field value consists of a single absolute URI' in HTTP spec")
|
579
|
-
end
|
580
732
|
puts "redirect to: #{newuri}" if $DEBUG
|
581
733
|
newuri
|
582
734
|
end
|
583
735
|
|
584
736
|
# Sends HEAD request to the specified URL. See request for arguments.
|
585
|
-
def head(uri,
|
586
|
-
request(:head, uri, query,
|
737
|
+
def head(uri, *args)
|
738
|
+
request(:head, uri, argument_to_hash(args, :query, :header, :follow_redirect))
|
587
739
|
end
|
588
740
|
|
589
741
|
# Sends GET request to the specified URL. See request for arguments.
|
590
|
-
def get(uri,
|
591
|
-
request(:get, uri, query,
|
742
|
+
def get(uri, *args, &block)
|
743
|
+
request(:get, uri, argument_to_hash(args, :query, :header, :follow_redirect), &block)
|
744
|
+
end
|
745
|
+
|
746
|
+
# Sends PATCH request to the specified URL. See request for arguments.
|
747
|
+
def patch(uri, *args, &block)
|
748
|
+
if hashy_argument_has_keys(args, :query, :body)
|
749
|
+
new_args = args[0]
|
750
|
+
else
|
751
|
+
new_args = argument_to_hash(args, :body, :header)
|
752
|
+
end
|
753
|
+
request(:patch, uri, new_args, &block)
|
592
754
|
end
|
593
755
|
|
594
756
|
# Sends POST request to the specified URL. See request for arguments.
|
595
|
-
|
596
|
-
|
757
|
+
# You should not depend on :follow_redirect => true for POST method. It
|
758
|
+
# sends the same POST method to the new location which is prohibited in HTTP spec.
|
759
|
+
def post(uri, *args, &block)
|
760
|
+
if hashy_argument_has_keys(args, :query, :body)
|
761
|
+
new_args = args[0]
|
762
|
+
else
|
763
|
+
new_args = argument_to_hash(args, :body, :header, :follow_redirect)
|
764
|
+
end
|
765
|
+
request(:post, uri, new_args, &block)
|
597
766
|
end
|
598
767
|
|
599
768
|
# Sends PUT request to the specified URL. See request for arguments.
|
600
|
-
def put(uri,
|
601
|
-
|
769
|
+
def put(uri, *args, &block)
|
770
|
+
if hashy_argument_has_keys(args, :query, :body)
|
771
|
+
new_args = args[0]
|
772
|
+
else
|
773
|
+
new_args = argument_to_hash(args, :body, :header)
|
774
|
+
end
|
775
|
+
request(:put, uri, new_args, &block)
|
602
776
|
end
|
603
777
|
|
604
778
|
# Sends DELETE request to the specified URL. See request for arguments.
|
605
|
-
def delete(uri,
|
606
|
-
request(:delete, uri,
|
779
|
+
def delete(uri, *args, &block)
|
780
|
+
request(:delete, uri, argument_to_hash(args, :body, :header, :query), &block)
|
607
781
|
end
|
608
782
|
|
609
783
|
# Sends OPTIONS request to the specified URL. See request for arguments.
|
610
|
-
def options(uri,
|
611
|
-
|
784
|
+
def options(uri, *args, &block)
|
785
|
+
new_args = argument_to_hash(args, :header, :query, :body)
|
786
|
+
request(:options, uri, new_args, &block)
|
612
787
|
end
|
613
788
|
|
614
789
|
# Sends PROPFIND request to the specified URL. See request for arguments.
|
615
|
-
def propfind(uri,
|
616
|
-
request(:propfind, uri,
|
790
|
+
def propfind(uri, *args, &block)
|
791
|
+
request(:propfind, uri, argument_to_hash(args, :header), &block)
|
617
792
|
end
|
618
793
|
|
619
794
|
# Sends PROPPATCH request to the specified URL. See request for arguments.
|
620
|
-
def proppatch(uri,
|
621
|
-
request(:proppatch, uri,
|
795
|
+
def proppatch(uri, *args, &block)
|
796
|
+
request(:proppatch, uri, argument_to_hash(args, :body, :header), &block)
|
622
797
|
end
|
623
798
|
|
624
799
|
# Sends TRACE request to the specified URL. See request for arguments.
|
625
|
-
def trace(uri,
|
626
|
-
request('TRACE', uri,
|
800
|
+
def trace(uri, *args, &block)
|
801
|
+
request('TRACE', uri, argument_to_hash(args, :query, :header), &block)
|
627
802
|
end
|
628
803
|
|
629
804
|
# Sends a request to the specified URL.
|
@@ -634,17 +809,25 @@ class HTTPClient
|
|
634
809
|
# e.g. { "a" => "b" } => 'http://host/part?a=b'
|
635
810
|
# Give an array to pass multiple value like
|
636
811
|
# [["a", "b"], ["a", "c"]] => 'http://host/part?a=b&a=c'
|
637
|
-
# body:: a Hash or an Array of body part.
|
638
|
-
#
|
812
|
+
# body:: a Hash or an Array of body part. e.g.
|
813
|
+
# { "a" => "b" }
|
814
|
+
# => 'a=b'
|
639
815
|
# Give an array to pass multiple value like
|
640
|
-
#
|
816
|
+
# [["a", "b"], ["a", "c"]]
|
817
|
+
# => 'a=b&a=c'.
|
641
818
|
# When the given method is 'POST' and the given body contains a file
|
642
|
-
# as a value, it will be posted as a multipart/form-data.
|
643
|
-
#
|
819
|
+
# as a value, it will be posted as a multipart/form-data. e.g.
|
820
|
+
# { 'upload' => file }
|
821
|
+
# You can also send custom multipart by passing an array of hashes.
|
822
|
+
# Each part must have a :content attribute which can be a file, all
|
823
|
+
# other keys will become headers.
|
824
|
+
# [{ 'Content-Type' => 'text/plain', :content => "some text" },
|
825
|
+
# { 'Content-Type' => 'video/mp4', :content => File.new('video.mp4') }]
|
826
|
+
# => <Two parts with custom Content-Type header>
|
644
827
|
# See HTTP::Message.file? for actual condition of 'a file'.
|
645
|
-
#
|
646
|
-
#
|
647
|
-
#
|
828
|
+
# header:: a Hash or an Array of extra headers. e.g.
|
829
|
+
# { 'Accept' => 'text/html' } or
|
830
|
+
# [['Accept', 'image/jpeg'], ['Accept', 'image/png']].
|
648
831
|
# &block:: Give a block to get chunked message-body of response like
|
649
832
|
# get(uri) { |chunked_body| ... }.
|
650
833
|
# Size of each chunk may not be the same.
|
@@ -653,86 +836,130 @@ class HTTPClient
|
|
653
836
|
# a HTTP request message body.
|
654
837
|
#
|
655
838
|
# When you pass an IO as a body, HTTPClient sends it as a HTTP request with
|
656
|
-
# chunked encoding (Transfer-Encoding: chunked in HTTP header)
|
657
|
-
# that some server application does not support
|
658
|
-
# cgi.rb does not support it.
|
659
|
-
def request(method, uri,
|
660
|
-
|
839
|
+
# chunked encoding (Transfer-Encoding: chunked in HTTP header) if IO does not
|
840
|
+
# respond to :size. Bear in mind that some server application does not support
|
841
|
+
# chunked request. At least cgi.rb does not support it.
|
842
|
+
def request(method, uri, *args, &block)
|
843
|
+
query, body, header, follow_redirect = keyword_argument(args, :query, :body, :header, :follow_redirect)
|
844
|
+
if method == :propfind
|
845
|
+
header ||= PROPFIND_DEFAULT_EXTHEADER
|
846
|
+
else
|
847
|
+
header ||= {}
|
848
|
+
end
|
849
|
+
uri = to_resource_url(uri)
|
661
850
|
if block
|
662
|
-
filtered_block =
|
663
|
-
|
664
|
-
|
851
|
+
filtered_block = adapt_block(&block)
|
852
|
+
end
|
853
|
+
if follow_redirect
|
854
|
+
follow_redirect(method, uri, query, body, header, &block)
|
855
|
+
else
|
856
|
+
do_request(method, uri, query, body, header, &filtered_block)
|
665
857
|
end
|
666
|
-
do_request(method, uri, query, body, extheader, &filtered_block)
|
667
858
|
end
|
668
859
|
|
669
860
|
# Sends HEAD request in async style. See request_async for arguments.
|
670
861
|
# It immediately returns a HTTPClient::Connection instance as a result.
|
671
|
-
def head_async(uri,
|
672
|
-
|
862
|
+
def head_async(uri, *args)
|
863
|
+
request_async2(:head, uri, argument_to_hash(args, :query, :header))
|
673
864
|
end
|
674
865
|
|
675
866
|
# Sends GET request in async style. See request_async for arguments.
|
676
867
|
# It immediately returns a HTTPClient::Connection instance as a result.
|
677
|
-
def get_async(uri,
|
678
|
-
|
868
|
+
def get_async(uri, *args)
|
869
|
+
request_async2(:get, uri, argument_to_hash(args, :query, :header))
|
870
|
+
end
|
871
|
+
|
872
|
+
# Sends PATCH request in async style. See request_async2 for arguments.
|
873
|
+
# It immediately returns a HTTPClient::Connection instance as a result.
|
874
|
+
def patch_async(uri, *args)
|
875
|
+
if hashy_argument_has_keys(args, :query, :body)
|
876
|
+
new_args = args[0]
|
877
|
+
else
|
878
|
+
new_args = argument_to_hash(args, :body, :header)
|
879
|
+
end
|
880
|
+
request_async2(:patch, uri, new_args)
|
679
881
|
end
|
680
882
|
|
681
883
|
# Sends POST request in async style. See request_async for arguments.
|
682
884
|
# It immediately returns a HTTPClient::Connection instance as a result.
|
683
|
-
def post_async(uri,
|
684
|
-
|
885
|
+
def post_async(uri, *args)
|
886
|
+
if hashy_argument_has_keys(args, :query, :body)
|
887
|
+
new_args = args[0]
|
888
|
+
else
|
889
|
+
new_args = argument_to_hash(args, :body, :header)
|
890
|
+
end
|
891
|
+
request_async2(:post, uri, new_args)
|
685
892
|
end
|
686
893
|
|
687
|
-
# Sends PUT request in async style. See
|
894
|
+
# Sends PUT request in async style. See request_async2 for arguments.
|
688
895
|
# It immediately returns a HTTPClient::Connection instance as a result.
|
689
|
-
def put_async(uri,
|
690
|
-
|
896
|
+
def put_async(uri, *args)
|
897
|
+
if hashy_argument_has_keys(args, :query, :body)
|
898
|
+
new_args = args[0]
|
899
|
+
else
|
900
|
+
new_args = argument_to_hash(args, :body, :header)
|
901
|
+
end
|
902
|
+
request_async2(:put, uri, new_args)
|
691
903
|
end
|
692
904
|
|
693
|
-
# Sends DELETE request in async style. See
|
905
|
+
# Sends DELETE request in async style. See request_async2 for arguments.
|
694
906
|
# It immediately returns a HTTPClient::Connection instance as a result.
|
695
|
-
def delete_async(uri,
|
696
|
-
|
907
|
+
def delete_async(uri, *args)
|
908
|
+
request_async2(:delete, uri, argument_to_hash(args, :body, :header, :query))
|
697
909
|
end
|
698
910
|
|
699
|
-
# Sends OPTIONS request in async style. See
|
911
|
+
# Sends OPTIONS request in async style. See request_async2 for arguments.
|
700
912
|
# It immediately returns a HTTPClient::Connection instance as a result.
|
701
|
-
def options_async(uri,
|
702
|
-
|
913
|
+
def options_async(uri, *args)
|
914
|
+
request_async2(:options, uri, argument_to_hash(args, :header, :query, :body))
|
703
915
|
end
|
704
916
|
|
705
|
-
# Sends PROPFIND request in async style. See
|
917
|
+
# Sends PROPFIND request in async style. See request_async2 for arguments.
|
706
918
|
# It immediately returns a HTTPClient::Connection instance as a result.
|
707
|
-
def propfind_async(uri,
|
708
|
-
|
919
|
+
def propfind_async(uri, *args)
|
920
|
+
request_async2(:propfind, uri, argument_to_hash(args, :body, :header))
|
709
921
|
end
|
710
922
|
|
711
|
-
# Sends PROPPATCH request in async style. See
|
923
|
+
# Sends PROPPATCH request in async style. See request_async2 for arguments.
|
712
924
|
# It immediately returns a HTTPClient::Connection instance as a result.
|
713
|
-
def proppatch_async(uri,
|
714
|
-
|
925
|
+
def proppatch_async(uri, *args)
|
926
|
+
request_async2(:proppatch, uri, argument_to_hash(args, :body, :header))
|
715
927
|
end
|
716
928
|
|
717
|
-
# Sends TRACE request in async style. See
|
929
|
+
# Sends TRACE request in async style. See request_async2 for arguments.
|
718
930
|
# It immediately returns a HTTPClient::Connection instance as a result.
|
719
|
-
def trace_async(uri,
|
720
|
-
|
931
|
+
def trace_async(uri, *args)
|
932
|
+
request_async2(:trace, uri, argument_to_hash(args, :query, :header))
|
721
933
|
end
|
722
934
|
|
723
935
|
# Sends a request in async style. request method creates new Thread for
|
724
936
|
# HTTP connection and returns a HTTPClient::Connection instance immediately.
|
725
937
|
#
|
726
938
|
# Arguments definition is the same as request.
|
727
|
-
def request_async(method, uri, query = nil, body = nil,
|
728
|
-
uri =
|
729
|
-
do_request_async(method, uri, query, body,
|
939
|
+
def request_async(method, uri, query = nil, body = nil, header = {})
|
940
|
+
uri = to_resource_url(uri)
|
941
|
+
do_request_async(method, uri, query, body, header)
|
942
|
+
end
|
943
|
+
|
944
|
+
# new method that has same signature as 'request'
|
945
|
+
def request_async2(method, uri, *args)
|
946
|
+
query, body, header = keyword_argument(args, :query, :body, :header)
|
947
|
+
if [:post, :put].include?(method)
|
948
|
+
body ||= ''
|
949
|
+
end
|
950
|
+
if method == :propfind
|
951
|
+
header ||= PROPFIND_DEFAULT_EXTHEADER
|
952
|
+
else
|
953
|
+
header ||= {}
|
954
|
+
end
|
955
|
+
uri = to_resource_url(uri)
|
956
|
+
do_request_async(method, uri, query, body, header)
|
730
957
|
end
|
731
958
|
|
732
959
|
# Resets internal session for the given URL. Keep-alive connection for the
|
733
960
|
# site (host-port pair) is disconnected if exists.
|
734
961
|
def reset(uri)
|
735
|
-
uri =
|
962
|
+
uri = to_resource_url(uri)
|
736
963
|
@session_manager.reset(uri)
|
737
964
|
end
|
738
965
|
|
@@ -744,55 +971,91 @@ class HTTPClient
|
|
744
971
|
private
|
745
972
|
|
746
973
|
class RetryableResponse < StandardError # :nodoc:
|
974
|
+
attr_reader :res
|
975
|
+
|
976
|
+
def initialize(res = nil)
|
977
|
+
@res = res
|
978
|
+
end
|
747
979
|
end
|
748
980
|
|
749
981
|
class KeepAliveDisconnected < StandardError # :nodoc:
|
982
|
+
attr_reader :sess
|
983
|
+
attr_reader :cause
|
984
|
+
def initialize(sess = nil, cause = nil)
|
985
|
+
super("#{self.class.name}: #{cause ? cause.message : nil}")
|
986
|
+
@sess = sess
|
987
|
+
@cause = cause
|
988
|
+
end
|
750
989
|
end
|
751
990
|
|
752
|
-
def
|
753
|
-
|
991
|
+
def hashy_argument_has_keys(args, *key)
|
992
|
+
# if the given arg is a single Hash and...
|
993
|
+
args.size == 1 and Hash === args[0] and
|
994
|
+
# it has any one of the key
|
995
|
+
key.all? { |e| args[0].key?(e) }
|
996
|
+
end
|
997
|
+
|
998
|
+
def do_request(method, uri, query, body, header, &block)
|
754
999
|
res = nil
|
755
1000
|
if HTTP::Message.file?(body)
|
756
1001
|
pos = body.pos rescue nil
|
757
1002
|
end
|
758
1003
|
retry_count = @session_manager.protocol_retry_count
|
759
1004
|
proxy = no_proxy?(uri) ? nil : @proxy
|
1005
|
+
previous_request = previous_response = nil
|
760
1006
|
while retry_count > 0
|
761
1007
|
body.pos = pos if pos
|
762
|
-
req = create_request(method, uri, query, body,
|
1008
|
+
req = create_request(method, uri, query, body, header)
|
1009
|
+
if previous_request
|
1010
|
+
# to remember IO positions to read
|
1011
|
+
req.http_body.positions = previous_request.http_body.positions
|
1012
|
+
end
|
763
1013
|
begin
|
764
1014
|
protect_keep_alive_disconnected do
|
765
|
-
|
1015
|
+
# TODO: remove Connection.new
|
1016
|
+
# We want to delete Connection usage in do_get_block but Newrelic gem depends on it.
|
1017
|
+
# https://github.com/newrelic/rpm/blob/master/lib/new_relic/agent/instrumentation/httpclient.rb#L34-L36
|
1018
|
+
conn = Connection.new
|
1019
|
+
res = do_get_block(req, proxy, conn, &block)
|
1020
|
+
# Webmock's do_get_block returns ConditionVariable
|
1021
|
+
if !res.respond_to?(:previous)
|
1022
|
+
res = conn.pop
|
1023
|
+
end
|
766
1024
|
end
|
767
|
-
res =
|
1025
|
+
res.previous = previous_response
|
768
1026
|
break
|
769
|
-
rescue RetryableResponse
|
770
|
-
|
1027
|
+
rescue RetryableResponse => e
|
1028
|
+
previous_request = req
|
1029
|
+
previous_response = res = e.res
|
771
1030
|
retry_count -= 1
|
772
1031
|
end
|
773
1032
|
end
|
774
1033
|
res
|
775
1034
|
end
|
776
1035
|
|
777
|
-
def do_request_async(method, uri, query, body,
|
1036
|
+
def do_request_async(method, uri, query, body, header)
|
778
1037
|
conn = Connection.new
|
779
1038
|
t = Thread.new(conn) { |tconn|
|
780
|
-
|
781
|
-
|
782
|
-
|
783
|
-
|
784
|
-
|
785
|
-
|
786
|
-
|
787
|
-
|
788
|
-
|
789
|
-
|
790
|
-
|
1039
|
+
begin
|
1040
|
+
if HTTP::Message.file?(body)
|
1041
|
+
pos = body.pos rescue nil
|
1042
|
+
end
|
1043
|
+
retry_count = @session_manager.protocol_retry_count
|
1044
|
+
proxy = no_proxy?(uri) ? nil : @proxy
|
1045
|
+
while retry_count > 0
|
1046
|
+
body.pos = pos if pos
|
1047
|
+
req = create_request(method, uri, query, body, header)
|
1048
|
+
begin
|
1049
|
+
protect_keep_alive_disconnected do
|
1050
|
+
do_get_stream(req, proxy, tconn)
|
1051
|
+
end
|
1052
|
+
break
|
1053
|
+
rescue RetryableResponse
|
1054
|
+
retry_count -= 1
|
791
1055
|
end
|
792
|
-
break
|
793
|
-
rescue RetryableResponse
|
794
|
-
retry_count -= 1
|
795
1056
|
end
|
1057
|
+
rescue Exception => e
|
1058
|
+
conn.push e
|
796
1059
|
end
|
797
1060
|
}
|
798
1061
|
conn.async_thread = t
|
@@ -817,51 +1080,79 @@ private
|
|
817
1080
|
ENV[name.downcase] || ENV[name.upcase]
|
818
1081
|
end
|
819
1082
|
|
820
|
-
def
|
821
|
-
|
1083
|
+
def adapt_block(&block)
|
1084
|
+
return block if block.arity == 2
|
1085
|
+
proc { |r, str| block.call(str) }
|
1086
|
+
end
|
1087
|
+
|
1088
|
+
def follow_redirect(method, uri, query, body, header, &block)
|
1089
|
+
uri = to_resource_url(uri)
|
822
1090
|
if block
|
1091
|
+
b = adapt_block(&block)
|
823
1092
|
filtered_block = proc { |r, str|
|
824
|
-
|
1093
|
+
b.call(r, str) if r.ok?
|
825
1094
|
}
|
826
1095
|
end
|
827
1096
|
if HTTP::Message.file?(body)
|
828
1097
|
pos = body.pos rescue nil
|
829
1098
|
end
|
830
1099
|
retry_number = 0
|
1100
|
+
previous = nil
|
1101
|
+
request_query = query
|
831
1102
|
while retry_number < @follow_redirect_count
|
832
1103
|
body.pos = pos if pos
|
833
|
-
res = do_request(method, uri,
|
834
|
-
|
835
|
-
|
836
|
-
|
1104
|
+
res = do_request(method, uri, request_query, body, header, &filtered_block)
|
1105
|
+
res.previous = previous
|
1106
|
+
if res.redirect?
|
1107
|
+
if res.header['location'].empty?
|
1108
|
+
raise BadResponseError.new("Missing Location header for redirect", res)
|
1109
|
+
end
|
1110
|
+
method = :get if res.see_other? # See RFC2616 10.3.4
|
837
1111
|
uri = urify(@redirect_uri_callback.call(uri, res))
|
1112
|
+
# To avoid duped query parameter. 'location' must include query part.
|
1113
|
+
request_query = nil
|
1114
|
+
previous = res
|
838
1115
|
retry_number += 1
|
839
1116
|
else
|
840
|
-
|
1117
|
+
return res
|
841
1118
|
end
|
842
1119
|
end
|
843
1120
|
raise BadResponseError.new("retry count exceeded", res)
|
844
1121
|
end
|
845
1122
|
|
1123
|
+
def success_content(res)
|
1124
|
+
if res.ok?
|
1125
|
+
return res.content
|
1126
|
+
else
|
1127
|
+
raise BadResponseError.new("unexpected response: #{res.header.inspect}", res)
|
1128
|
+
end
|
1129
|
+
end
|
1130
|
+
|
846
1131
|
def protect_keep_alive_disconnected
|
847
1132
|
begin
|
848
1133
|
yield
|
849
1134
|
rescue KeepAliveDisconnected
|
850
|
-
|
1135
|
+
# Force to create new connection
|
1136
|
+
Thread.current[:HTTPClient_AcquireNewConnection] = true
|
1137
|
+
begin
|
1138
|
+
yield
|
1139
|
+
ensure
|
1140
|
+
Thread.current[:HTTPClient_AcquireNewConnection] = false
|
1141
|
+
end
|
851
1142
|
end
|
852
1143
|
end
|
853
1144
|
|
854
|
-
def create_request(method, uri, query, body,
|
1145
|
+
def create_request(method, uri, query, body, header)
|
855
1146
|
method = method.to_s.upcase
|
856
|
-
if
|
857
|
-
|
1147
|
+
if header.is_a?(Hash)
|
1148
|
+
header = @default_header.merge(header).to_a
|
858
1149
|
else
|
859
|
-
|
1150
|
+
header = @default_header.to_a + header.dup
|
860
1151
|
end
|
861
1152
|
boundary = nil
|
862
1153
|
if body
|
863
|
-
|
864
|
-
key.downcase == 'content-type'
|
1154
|
+
_, content_type = header.find { |key, value|
|
1155
|
+
key.to_s.downcase == 'content-type'
|
865
1156
|
}
|
866
1157
|
if content_type
|
867
1158
|
if /\Amultipart/ =~ content_type
|
@@ -870,25 +1161,28 @@ private
|
|
870
1161
|
else
|
871
1162
|
boundary = create_boundary
|
872
1163
|
content_type = "#{content_type}; boundary=#{boundary}"
|
873
|
-
|
1164
|
+
header = override_header(header, 'content-type', content_type)
|
874
1165
|
end
|
875
1166
|
end
|
876
|
-
|
1167
|
+
else
|
877
1168
|
if file_in_form_data?(body)
|
878
1169
|
boundary = create_boundary
|
879
1170
|
content_type = "multipart/form-data; boundary=#{boundary}"
|
880
1171
|
else
|
881
1172
|
content_type = 'application/x-www-form-urlencoded'
|
882
1173
|
end
|
883
|
-
|
1174
|
+
header << ['Content-Type', content_type]
|
884
1175
|
end
|
885
1176
|
end
|
886
1177
|
req = HTTP::Message.new_request(method, uri, query, body, boundary)
|
887
|
-
|
888
|
-
req.header.add(key, value)
|
1178
|
+
header.each do |key, value|
|
1179
|
+
req.header.add(key.to_s, value)
|
889
1180
|
end
|
890
|
-
if @cookie_manager
|
891
|
-
|
1181
|
+
if @cookie_manager
|
1182
|
+
cookie_value = @cookie_manager.cookie_value(uri)
|
1183
|
+
if cookie_value
|
1184
|
+
req.header.add('Cookie', cookie_value)
|
1185
|
+
end
|
892
1186
|
end
|
893
1187
|
req
|
894
1188
|
end
|
@@ -902,10 +1196,10 @@ private
|
|
902
1196
|
body.any? { |k, v| HTTP::Message.file?(v) }
|
903
1197
|
end
|
904
1198
|
|
905
|
-
def override_header(
|
1199
|
+
def override_header(header, key, value)
|
906
1200
|
result = []
|
907
|
-
|
908
|
-
if k.downcase == key
|
1201
|
+
header.each do |k, v|
|
1202
|
+
if k.to_s.downcase == key
|
909
1203
|
result << [key, value]
|
910
1204
|
else
|
911
1205
|
result << [k, v]
|
@@ -920,22 +1214,16 @@ private
|
|
920
1214
|
if !@proxy or NO_PROXY_HOSTS.include?(uri.host)
|
921
1215
|
return true
|
922
1216
|
end
|
923
|
-
|
924
|
-
|
925
|
-
|
926
|
-
|
927
|
-
|
928
|
-
(!port || uri.port == port.to_i)
|
929
|
-
return true
|
1217
|
+
@no_proxy_regexps.each do |regexp, port|
|
1218
|
+
if !port || uri.port == port.to_i
|
1219
|
+
if regexp =~ uri.host
|
1220
|
+
return true
|
1221
|
+
end
|
930
1222
|
end
|
931
1223
|
end
|
932
1224
|
false
|
933
1225
|
end
|
934
1226
|
|
935
|
-
def https?(uri)
|
936
|
-
uri.scheme.downcase == 'https'
|
937
|
-
end
|
938
|
-
|
939
1227
|
# !! CAUTION !!
|
940
1228
|
# Method 'do_get*' runs under MT conditon. Be careful to change.
|
941
1229
|
def do_get_block(req, proxy, conn, &block)
|
@@ -943,12 +1231,13 @@ private
|
|
943
1231
|
filter.filter_request(req)
|
944
1232
|
end
|
945
1233
|
if str = @test_loopback_response.shift
|
946
|
-
dump_dummy_request_response(req.
|
947
|
-
|
948
|
-
|
1234
|
+
dump_dummy_request_response(req.http_body.dump, str) if @debug_dev
|
1235
|
+
res = HTTP::Message.new_response(str, req.header)
|
1236
|
+
conn.push(res)
|
1237
|
+
return res
|
949
1238
|
end
|
950
1239
|
content = block ? nil : ''
|
951
|
-
res = HTTP::Message.new_response(content)
|
1240
|
+
res = HTTP::Message.new_response(content, req.header)
|
952
1241
|
@debug_dev << "= Request\n\n" if @debug_dev
|
953
1242
|
sess = @session_manager.query(req, proxy)
|
954
1243
|
res.peer_cert = sess.ssl_peer_cert
|
@@ -956,19 +1245,23 @@ private
|
|
956
1245
|
do_get_header(req, res, sess)
|
957
1246
|
conn.push(res)
|
958
1247
|
sess.get_body do |part|
|
1248
|
+
set_encoding(part, res.body_encoding)
|
959
1249
|
if block
|
960
|
-
block.call(res, part)
|
1250
|
+
block.call(res, part.dup)
|
961
1251
|
else
|
962
1252
|
content << part
|
963
1253
|
end
|
964
1254
|
end
|
1255
|
+
# there could be a race condition but it's OK to cache unreusable
|
1256
|
+
# connection because we do retry for that case.
|
965
1257
|
@session_manager.keep(sess) unless sess.closed?
|
966
1258
|
commands = @request_filter.collect { |filter|
|
967
1259
|
filter.filter_response(req, res)
|
968
1260
|
}
|
969
1261
|
if commands.find { |command| command == :retry }
|
970
|
-
raise RetryableResponse.new
|
1262
|
+
raise RetryableResponse.new(res)
|
971
1263
|
end
|
1264
|
+
res
|
972
1265
|
end
|
973
1266
|
|
974
1267
|
def do_get_stream(req, proxy, conn)
|
@@ -976,12 +1269,13 @@ private
|
|
976
1269
|
filter.filter_request(req)
|
977
1270
|
end
|
978
1271
|
if str = @test_loopback_response.shift
|
979
|
-
dump_dummy_request_response(req.
|
980
|
-
conn.push(HTTP::Message.new_response(StringIO.new(str)))
|
1272
|
+
dump_dummy_request_response(req.http_body.dump, str) if @debug_dev
|
1273
|
+
conn.push(HTTP::Message.new_response(StringIO.new(str), req.header))
|
981
1274
|
return
|
982
1275
|
end
|
983
1276
|
piper, pipew = IO.pipe
|
984
|
-
|
1277
|
+
pipew.binmode
|
1278
|
+
res = HTTP::Message.new_response(piper, req.header)
|
985
1279
|
@debug_dev << "= Request\n\n" if @debug_dev
|
986
1280
|
sess = @session_manager.query(req, proxy)
|
987
1281
|
res.peer_cert = sess.ssl_peer_cert
|
@@ -989,21 +1283,21 @@ private
|
|
989
1283
|
do_get_header(req, res, sess)
|
990
1284
|
conn.push(res)
|
991
1285
|
sess.get_body do |part|
|
992
|
-
|
1286
|
+
set_encoding(part, res.body_encoding)
|
1287
|
+
pipew.write(part)
|
993
1288
|
end
|
994
1289
|
pipew.close
|
995
1290
|
@session_manager.keep(sess) unless sess.closed?
|
996
|
-
|
1291
|
+
_ = @request_filter.collect { |filter|
|
997
1292
|
filter.filter_response(req, res)
|
998
1293
|
}
|
999
1294
|
# ignore commands (not retryable in async mode)
|
1295
|
+
res
|
1000
1296
|
end
|
1001
1297
|
|
1002
1298
|
def do_get_header(req, res, sess)
|
1003
|
-
res.
|
1004
|
-
headers
|
1005
|
-
res.header.add(key, value)
|
1006
|
-
end
|
1299
|
+
res.http_version, res.status, res.reason, headers = sess.get_header
|
1300
|
+
res.header.set_headers(headers)
|
1007
1301
|
if @cookie_manager
|
1008
1302
|
res.header['set-cookie'].each do |cookie|
|
1009
1303
|
@cookie_manager.parse(cookie, req.header.request_uri)
|
@@ -1017,4 +1311,17 @@ private
|
|
1017
1311
|
@debug_dev << "\n\n= Dummy Response\n\n"
|
1018
1312
|
@debug_dev << res
|
1019
1313
|
end
|
1314
|
+
|
1315
|
+
def set_encoding(str, encoding)
|
1316
|
+
str.force_encoding(encoding) if encoding
|
1317
|
+
end
|
1318
|
+
|
1319
|
+
def to_resource_url(uri)
|
1320
|
+
u = urify(uri)
|
1321
|
+
if @base_url && u.scheme.nil? && u.host.nil?
|
1322
|
+
urify(@base_url) + uri
|
1323
|
+
else
|
1324
|
+
u
|
1325
|
+
end
|
1326
|
+
end
|
1020
1327
|
end
|