httpclient 2.1.5 → 2.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +85 -0
- data/bin/httpclient +77 -0
- data/bin/jsonclient +85 -0
- data/lib/hexdump.rb +50 -0
- data/lib/http-access2.rb +6 -4
- data/lib/httpclient/auth.rb +575 -173
- data/lib/httpclient/cacert.pem +3952 -0
- data/lib/httpclient/cacert1024.pem +3866 -0
- data/lib/httpclient/connection.rb +6 -2
- data/lib/httpclient/cookie.rb +162 -504
- data/lib/httpclient/http.rb +334 -119
- data/lib/httpclient/include_client.rb +85 -0
- data/lib/httpclient/jruby_ssl_socket.rb +588 -0
- data/lib/httpclient/session.rb +385 -288
- data/lib/httpclient/ssl_config.rb +195 -155
- data/lib/httpclient/ssl_socket.rb +150 -0
- data/lib/httpclient/timeout.rb +14 -10
- data/lib/httpclient/util.rb +142 -6
- data/lib/httpclient/version.rb +3 -0
- data/lib/httpclient/webagent-cookie.rb +459 -0
- data/lib/httpclient.rb +509 -202
- data/lib/jsonclient.rb +63 -0
- data/lib/oauthclient.rb +111 -0
- data/sample/async.rb +8 -0
- data/sample/auth.rb +11 -0
- data/sample/cookie.rb +18 -0
- data/sample/dav.rb +103 -0
- data/sample/howto.rb +49 -0
- data/sample/jsonclient.rb +67 -0
- data/sample/oauth_buzz.rb +57 -0
- data/sample/oauth_friendfeed.rb +59 -0
- data/sample/oauth_twitter.rb +61 -0
- data/sample/ssl/0cert.pem +22 -0
- data/sample/ssl/0key.pem +30 -0
- data/sample/ssl/1000cert.pem +19 -0
- data/sample/ssl/1000key.pem +18 -0
- data/sample/ssl/htdocs/index.html +10 -0
- data/sample/ssl/ssl_client.rb +22 -0
- data/sample/ssl/webrick_httpsd.rb +29 -0
- data/sample/stream.rb +21 -0
- data/sample/thread.rb +27 -0
- data/sample/wcat.rb +21 -0
- data/test/ca-chain.pem +44 -0
- data/test/ca.cert +23 -0
- data/test/client-pass.key +18 -0
- data/test/client.cert +19 -0
- data/test/client.key +15 -0
- data/test/helper.rb +131 -0
- data/test/htdigest +1 -0
- data/test/htpasswd +2 -0
- data/test/jruby_ssl_socket/test_pemutils.rb +32 -0
- data/test/runner.rb +2 -0
- data/test/server.cert +19 -0
- data/test/server.key +15 -0
- data/test/sslsvr.rb +65 -0
- data/test/subca.cert +21 -0
- data/test/test_auth.rb +492 -0
- data/test/test_cookie.rb +309 -0
- data/test/test_hexdump.rb +14 -0
- data/test/test_http-access2.rb +508 -0
- data/test/test_httpclient.rb +2145 -0
- data/test/test_include_client.rb +52 -0
- data/test/test_jsonclient.rb +80 -0
- data/test/test_ssl.rb +559 -0
- data/test/test_webagent-cookie.rb +465 -0
- metadata +85 -44
- data/lib/httpclient/auth.rb.orig +0 -513
- data/lib/httpclient/cacert.p7s +0 -1579
- data/lib/httpclient.rb.orig +0 -1020
- data/lib/tags +0 -908
data/lib/httpclient/session.rb
CHANGED
@@ -1,22 +1,31 @@
|
|
1
1
|
# HTTPClient - HTTP client library.
|
2
|
-
# Copyright (C) 2000-
|
2
|
+
# Copyright (C) 2000-2015 NAKAMURA, Hiroshi <nahi@ruby-lang.org>.
|
3
3
|
#
|
4
4
|
# This program is copyrighted free software by NAKAMURA, Hiroshi. You can
|
5
5
|
# redistribute it and/or modify it under the same terms of Ruby's license;
|
6
6
|
# either the dual license version in 2003, or any later version.
|
7
|
-
|
8
|
-
# httpclient/session.rb is based on http-access.rb in http-access/0.0.4.
|
9
|
-
#
|
10
|
-
#
|
7
|
+
#
|
8
|
+
# httpclient/session.rb is based on http-access.rb in http-access/0.0.4. Some
|
9
|
+
# part of it is copyrighted by Maebashi-san who made and published
|
10
|
+
# http-access/0.0.4. http-access/0.0.4 did not include license notice but when
|
11
|
+
# I asked Maebashi-san he agreed that I can redistribute it under the same terms
|
12
|
+
# of Ruby. Many thanks to Maebashi-san.
|
11
13
|
|
12
14
|
|
13
15
|
require 'socket'
|
14
16
|
require 'thread'
|
17
|
+
require 'timeout'
|
15
18
|
require 'stringio'
|
19
|
+
require 'zlib'
|
16
20
|
|
17
|
-
require 'httpclient/timeout'
|
21
|
+
require 'httpclient/timeout' # TODO: remove this once we drop 1.8 support
|
18
22
|
require 'httpclient/ssl_config'
|
19
23
|
require 'httpclient/http'
|
24
|
+
if RUBY_ENGINE == 'jruby'
|
25
|
+
require 'httpclient/jruby_ssl_socket'
|
26
|
+
else
|
27
|
+
require 'httpclient/ssl_socket'
|
28
|
+
end
|
20
29
|
|
21
30
|
|
22
31
|
class HTTPClient
|
@@ -27,15 +36,16 @@ class HTTPClient
|
|
27
36
|
# Protocol scheme.
|
28
37
|
attr_accessor :scheme
|
29
38
|
# Host String.
|
30
|
-
|
39
|
+
attr_accessor :host
|
40
|
+
alias hostname host
|
31
41
|
# Port number.
|
32
|
-
|
42
|
+
attr_accessor :port
|
33
43
|
|
34
44
|
# Creates a new Site based on the given URI.
|
35
45
|
def initialize(uri = nil)
|
36
46
|
if uri
|
37
|
-
@scheme = uri.scheme
|
38
|
-
@host = uri.
|
47
|
+
@scheme = uri.scheme || 'tcp'
|
48
|
+
@host = uri.hostname || '0.0.0.0'
|
39
49
|
@port = uri.port.to_i
|
40
50
|
else
|
41
51
|
@scheme = 'tcp'
|
@@ -66,7 +76,7 @@ class HTTPClient
|
|
66
76
|
def to_s # :nodoc:
|
67
77
|
addr
|
68
78
|
end
|
69
|
-
|
79
|
+
|
70
80
|
# Returns true if scheme, host and port of the given URI matches with this.
|
71
81
|
def match(uri)
|
72
82
|
(@scheme == uri.scheme) and (@host == uri.host) and (@port == uri.port.to_i)
|
@@ -75,6 +85,8 @@ class HTTPClient
|
|
75
85
|
def inspect # :nodoc:
|
76
86
|
sprintf("#<%s:0x%x %s>", self.class.name, __id__, addr)
|
77
87
|
end
|
88
|
+
|
89
|
+
EMPTY = Site.new.freeze
|
78
90
|
end
|
79
91
|
|
80
92
|
|
@@ -93,19 +105,30 @@ class HTTPClient
|
|
93
105
|
attr_accessor :debug_dev
|
94
106
|
# Boolean value for Socket#sync
|
95
107
|
attr_accessor :socket_sync
|
108
|
+
# Boolean value to send TCP keepalive packets; no timing settings exist at present
|
109
|
+
attr_accessor :tcp_keepalive
|
96
110
|
|
97
111
|
attr_accessor :connect_timeout
|
98
112
|
# Maximum retry count. 0 for infinite.
|
99
113
|
attr_accessor :connect_retry
|
100
114
|
attr_accessor :send_timeout
|
101
115
|
attr_accessor :receive_timeout
|
116
|
+
attr_accessor :keep_alive_timeout
|
102
117
|
attr_accessor :read_block_size
|
103
118
|
attr_accessor :protocol_retry_count
|
104
119
|
|
120
|
+
# Raise BadResponseError if response size does not match with Content-Length header in response.
|
121
|
+
attr_accessor :strict_response_size_check
|
122
|
+
|
123
|
+
# Local address to bind local side of the socket to
|
124
|
+
attr_accessor :socket_local
|
125
|
+
|
105
126
|
attr_accessor :ssl_config
|
106
127
|
|
107
128
|
attr_reader :test_loopback_http_response
|
108
129
|
|
130
|
+
attr_accessor :transparent_gzip_decompression
|
131
|
+
|
109
132
|
def initialize(client)
|
110
133
|
@client = client
|
111
134
|
@proxy = client.proxy
|
@@ -116,20 +139,27 @@ class HTTPClient
|
|
116
139
|
@protocol_version = nil
|
117
140
|
@debug_dev = client.debug_dev
|
118
141
|
@socket_sync = true
|
119
|
-
@
|
142
|
+
@tcp_keepalive = false
|
143
|
+
@chunk_size = ::HTTP::Message::Body::DEFAULT_CHUNK_SIZE
|
120
144
|
|
121
145
|
@connect_timeout = 60
|
122
146
|
@connect_retry = 1
|
123
147
|
@send_timeout = 120
|
124
|
-
@receive_timeout = 60
|
148
|
+
@receive_timeout = 60 # For each read_block_size bytes
|
149
|
+
@keep_alive_timeout = 15 # '15' is from Apache 2 default
|
125
150
|
@read_block_size = 1024 * 16 # follows net/http change in 1.8.7
|
126
151
|
@protocol_retry_count = 5
|
127
152
|
|
128
153
|
@ssl_config = nil
|
129
154
|
@test_loopback_http_response = []
|
130
155
|
|
131
|
-
@
|
156
|
+
@transparent_gzip_decompression = false
|
157
|
+
@strict_response_size_check = false
|
158
|
+
@socket_local = Site.new
|
159
|
+
|
160
|
+
@sess_pool = {}
|
132
161
|
@sess_pool_mutex = Mutex.new
|
162
|
+
@sess_pool_last_checked = Time.now
|
133
163
|
end
|
134
164
|
|
135
165
|
def proxy=(proxy)
|
@@ -141,8 +171,8 @@ class HTTPClient
|
|
141
171
|
end
|
142
172
|
|
143
173
|
def query(req, via_proxy)
|
144
|
-
req.
|
145
|
-
sess =
|
174
|
+
req.http_body.chunk_size = @chunk_size if req.http_body
|
175
|
+
sess = get_session(req, via_proxy)
|
146
176
|
begin
|
147
177
|
sess.query(req)
|
148
178
|
rescue
|
@@ -161,45 +191,65 @@ class HTTPClient
|
|
161
191
|
close_all
|
162
192
|
end
|
163
193
|
|
194
|
+
# assert: sess.last_used must not be nil
|
164
195
|
def keep(sess)
|
165
196
|
add_cached_session(sess)
|
166
197
|
end
|
167
198
|
|
168
199
|
private
|
169
200
|
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
201
|
+
# TODO: create PR for webmock's httpclient adapter to use get_session
|
202
|
+
# instead of open so that we can remove duplicated Site creation for
|
203
|
+
# each session.
|
204
|
+
def get_session(req, via_proxy = false)
|
205
|
+
uri = req.header.request_uri
|
206
|
+
if uri.scheme.nil?
|
207
|
+
raise ArgumentError.new("Request URI must have schema. Possibly add 'http://' to the request URI?")
|
208
|
+
end
|
209
|
+
site = Site.new(uri)
|
210
|
+
if cached = get_cached_session(site)
|
211
|
+
cached
|
174
212
|
else
|
175
|
-
|
176
|
-
sess.proxy = via_proxy ? @proxy : nil
|
177
|
-
sess.socket_sync = @socket_sync
|
178
|
-
sess.requested_version = @protocol_version if @protocol_version
|
179
|
-
sess.connect_timeout = @connect_timeout
|
180
|
-
sess.connect_retry = @connect_retry
|
181
|
-
sess.send_timeout = @send_timeout
|
182
|
-
sess.receive_timeout = @receive_timeout
|
183
|
-
sess.read_block_size = @read_block_size
|
184
|
-
sess.protocol_retry_count = @protocol_retry_count
|
185
|
-
sess.ssl_config = @ssl_config
|
186
|
-
sess.debug_dev = @debug_dev
|
187
|
-
sess.test_loopback_http_response = @test_loopback_http_response
|
213
|
+
open(uri, via_proxy)
|
188
214
|
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def open(uri, via_proxy = false)
|
218
|
+
site = Site.new(uri)
|
219
|
+
sess = Session.new(@client, site, @agent_name, @from)
|
220
|
+
sess.proxy = via_proxy ? @proxy : nil
|
221
|
+
sess.socket_sync = @socket_sync
|
222
|
+
sess.tcp_keepalive = @tcp_keepalive
|
223
|
+
sess.requested_version = @protocol_version if @protocol_version
|
224
|
+
sess.connect_timeout = @connect_timeout
|
225
|
+
sess.connect_retry = @connect_retry
|
226
|
+
sess.send_timeout = @send_timeout
|
227
|
+
sess.receive_timeout = @receive_timeout
|
228
|
+
sess.read_block_size = @read_block_size
|
229
|
+
sess.protocol_retry_count = @protocol_retry_count
|
230
|
+
sess.ssl_config = @ssl_config
|
231
|
+
sess.debug_dev = @debug_dev
|
232
|
+
sess.strict_response_size_check = @strict_response_size_check
|
233
|
+
sess.socket_local = @socket_local
|
234
|
+
sess.test_loopback_http_response = @test_loopback_http_response
|
235
|
+
sess.transparent_gzip_decompression = @transparent_gzip_decompression
|
189
236
|
sess
|
190
237
|
end
|
191
238
|
|
192
239
|
def close_all
|
193
240
|
@sess_pool_mutex.synchronize do
|
194
|
-
@sess_pool.each do |
|
195
|
-
sess
|
241
|
+
@sess_pool.each do |site, pool|
|
242
|
+
pool.each do |sess|
|
243
|
+
sess.close
|
244
|
+
end
|
196
245
|
end
|
197
246
|
end
|
198
247
|
@sess_pool.clear
|
199
248
|
end
|
200
249
|
|
250
|
+
# This method might not work as you expected...
|
201
251
|
def close(dest)
|
202
|
-
if cached = get_cached_session(dest)
|
252
|
+
if cached = get_cached_session(Site.new(dest))
|
203
253
|
cached.close
|
204
254
|
true
|
205
255
|
else
|
@@ -207,136 +257,48 @@ class HTTPClient
|
|
207
257
|
end
|
208
258
|
end
|
209
259
|
|
210
|
-
def get_cached_session(
|
211
|
-
|
260
|
+
def get_cached_session(site)
|
261
|
+
if Thread.current[:HTTPClient_AcquireNewConnection]
|
262
|
+
return nil
|
263
|
+
end
|
212
264
|
@sess_pool_mutex.synchronize do
|
213
|
-
|
214
|
-
@
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
265
|
+
now = Time.now
|
266
|
+
if now > @sess_pool_last_checked + @keep_alive_timeout
|
267
|
+
scrub_cached_session(now)
|
268
|
+
@sess_pool_last_checked = now
|
269
|
+
end
|
270
|
+
if pool = @sess_pool[site]
|
271
|
+
pool.each_with_index do |sess, idx|
|
272
|
+
if valid_session?(sess, now)
|
273
|
+
return pool.slice!(idx)
|
274
|
+
end
|
219
275
|
end
|
220
276
|
end
|
221
|
-
@sess_pool = new_pool
|
222
277
|
end
|
223
|
-
|
224
|
-
end
|
225
|
-
|
226
|
-
def add_cached_session(sess)
|
227
|
-
@sess_pool_mutex.synchronize do
|
228
|
-
@sess_pool << sess
|
229
|
-
end
|
230
|
-
end
|
231
|
-
end
|
232
|
-
|
233
|
-
|
234
|
-
# Wraps up OpenSSL::SSL::SSLSocket and offers debugging features.
|
235
|
-
class SSLSocketWrap
|
236
|
-
def initialize(socket, context, debug_dev = nil)
|
237
|
-
unless SSLEnabled
|
238
|
-
raise ConfigurationError.new('Ruby/OpenSSL module is required')
|
239
|
-
end
|
240
|
-
@context = context
|
241
|
-
@socket = socket
|
242
|
-
@ssl_socket = create_openssl_socket(@socket)
|
243
|
-
@debug_dev = debug_dev
|
244
|
-
end
|
245
|
-
|
246
|
-
def ssl_connect
|
247
|
-
@ssl_socket.connect
|
278
|
+
nil
|
248
279
|
end
|
249
280
|
|
250
|
-
def
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
@ssl_socket.post_connection_check(hostname)
|
261
|
-
else
|
262
|
-
@context.post_connection_check(@ssl_socket.peer_cert, hostname)
|
281
|
+
def scrub_cached_session(now)
|
282
|
+
@sess_pool.each do |site, pool|
|
283
|
+
pool.replace(pool.select { |sess|
|
284
|
+
if valid_session?(sess, now)
|
285
|
+
true
|
286
|
+
else
|
287
|
+
sess.close # close & remove from the pool
|
288
|
+
false
|
289
|
+
end
|
290
|
+
})
|
263
291
|
end
|
264
292
|
end
|
265
293
|
|
266
|
-
def
|
267
|
-
@
|
268
|
-
end
|
269
|
-
|
270
|
-
def close
|
271
|
-
@ssl_socket.close
|
272
|
-
@socket.close
|
273
|
-
end
|
274
|
-
|
275
|
-
def closed?
|
276
|
-
@socket.closed?
|
277
|
-
end
|
278
|
-
|
279
|
-
def eof?
|
280
|
-
@ssl_socket.eof?
|
281
|
-
end
|
282
|
-
|
283
|
-
def gets(*args)
|
284
|
-
str = @ssl_socket.gets(*args)
|
285
|
-
debug(str)
|
286
|
-
str
|
287
|
-
end
|
288
|
-
|
289
|
-
def read(*args)
|
290
|
-
str = @ssl_socket.read(*args)
|
291
|
-
debug(str)
|
292
|
-
str
|
293
|
-
end
|
294
|
-
|
295
|
-
def readpartial(*args)
|
296
|
-
str = @ssl_socket.readpartial(*args)
|
297
|
-
debug(str)
|
298
|
-
str
|
299
|
-
end
|
300
|
-
|
301
|
-
def <<(str)
|
302
|
-
rv = @ssl_socket.write(str)
|
303
|
-
debug(str)
|
304
|
-
rv
|
305
|
-
end
|
306
|
-
|
307
|
-
def flush
|
308
|
-
@ssl_socket.flush
|
309
|
-
end
|
310
|
-
|
311
|
-
def sync
|
312
|
-
@ssl_socket.sync
|
313
|
-
end
|
314
|
-
|
315
|
-
def sync=(sync)
|
316
|
-
@ssl_socket.sync = sync
|
294
|
+
def valid_session?(sess, now)
|
295
|
+
(now <= sess.last_used + @keep_alive_timeout)
|
317
296
|
end
|
318
297
|
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
value & mask == mask
|
323
|
-
end
|
324
|
-
|
325
|
-
def create_openssl_socket(socket)
|
326
|
-
ssl_socket = nil
|
327
|
-
if OpenSSL::SSL.const_defined?("SSLContext")
|
328
|
-
ctx = OpenSSL::SSL::SSLContext.new
|
329
|
-
@context.set_context(ctx)
|
330
|
-
ssl_socket = OpenSSL::SSL::SSLSocket.new(socket, ctx)
|
331
|
-
else
|
332
|
-
ssl_socket = OpenSSL::SSL::SSLSocket.new(socket)
|
333
|
-
@context.set_context(ssl_socket)
|
298
|
+
def add_cached_session(sess)
|
299
|
+
@sess_pool_mutex.synchronize do
|
300
|
+
(@sess_pool[sess.dest] ||= []).unshift(sess)
|
334
301
|
end
|
335
|
-
ssl_socket
|
336
|
-
end
|
337
|
-
|
338
|
-
def debug(str)
|
339
|
-
@debug_dev << str if @debug_dev && str
|
340
302
|
end
|
341
303
|
end
|
342
304
|
|
@@ -360,20 +322,20 @@ class HTTPClient
|
|
360
322
|
@socket.eof?
|
361
323
|
end
|
362
324
|
|
363
|
-
def gets(
|
364
|
-
@socket.gets(
|
325
|
+
def gets(rs)
|
326
|
+
@socket.gets(rs)
|
365
327
|
end
|
366
328
|
|
367
|
-
def read(
|
368
|
-
@socket.read(
|
329
|
+
def read(size, buf = nil)
|
330
|
+
@socket.read(size, buf)
|
369
331
|
end
|
370
332
|
|
371
|
-
def readpartial(
|
333
|
+
def readpartial(size, buf = nil)
|
372
334
|
# StringIO doesn't support :readpartial
|
373
335
|
if @socket.respond_to?(:readpartial)
|
374
|
-
@socket.readpartial(
|
336
|
+
@socket.readpartial(size, buf)
|
375
337
|
else
|
376
|
-
@socket.read(
|
338
|
+
@socket.read(size, buf)
|
377
339
|
end
|
378
340
|
end
|
379
341
|
|
@@ -409,19 +371,19 @@ class HTTPClient
|
|
409
371
|
debug("! CONNECTION CLOSED\n")
|
410
372
|
end
|
411
373
|
|
412
|
-
def gets(
|
374
|
+
def gets(rs)
|
413
375
|
str = super
|
414
376
|
debug(str)
|
415
377
|
str
|
416
378
|
end
|
417
379
|
|
418
|
-
def read(
|
380
|
+
def read(size, buf = nil)
|
419
381
|
str = super
|
420
382
|
debug(str)
|
421
383
|
str
|
422
384
|
end
|
423
385
|
|
424
|
-
def readpartial(
|
386
|
+
def readpartial(size, buf = nil)
|
425
387
|
str = super
|
426
388
|
debug(str)
|
427
389
|
str
|
@@ -435,7 +397,15 @@ class HTTPClient
|
|
435
397
|
private
|
436
398
|
|
437
399
|
def debug(str)
|
438
|
-
|
400
|
+
if str && @debug_dev
|
401
|
+
if str.index("\0")
|
402
|
+
require 'hexdump'
|
403
|
+
str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
|
404
|
+
@debug_dev << HexDump.encode(str).join("\n")
|
405
|
+
else
|
406
|
+
@debug_dev << str
|
407
|
+
end
|
408
|
+
end
|
439
409
|
end
|
440
410
|
end
|
441
411
|
|
@@ -453,12 +423,17 @@ class HTTPClient
|
|
453
423
|
def <<(str)
|
454
424
|
# ignored
|
455
425
|
end
|
426
|
+
|
427
|
+
def peer_cert
|
428
|
+
nil
|
429
|
+
end
|
456
430
|
end
|
457
431
|
|
458
432
|
|
459
433
|
# Manages a HTTP session with a Site.
|
460
434
|
class Session
|
461
435
|
include HTTPClient::Timeout
|
436
|
+
include Util
|
462
437
|
|
463
438
|
# Destination site
|
464
439
|
attr_reader :dest
|
@@ -466,6 +441,8 @@ class HTTPClient
|
|
466
441
|
attr_accessor :proxy
|
467
442
|
# Boolean value for Socket#sync
|
468
443
|
attr_accessor :socket_sync
|
444
|
+
# Boolean value to send TCP keepalive packets; no timing settings exist at present
|
445
|
+
attr_accessor :tcp_keepalive
|
469
446
|
# Requested protocol version
|
470
447
|
attr_accessor :requested_version
|
471
448
|
# Device for dumping log for debugging
|
@@ -478,15 +455,22 @@ class HTTPClient
|
|
478
455
|
attr_accessor :read_block_size
|
479
456
|
attr_accessor :protocol_retry_count
|
480
457
|
|
458
|
+
attr_accessor :strict_response_size_check
|
459
|
+
attr_accessor :socket_local
|
460
|
+
|
481
461
|
attr_accessor :ssl_config
|
482
462
|
attr_reader :ssl_peer_cert
|
483
463
|
attr_accessor :test_loopback_http_response
|
484
464
|
|
465
|
+
attr_accessor :transparent_gzip_decompression
|
466
|
+
attr_reader :last_used
|
467
|
+
|
485
468
|
def initialize(client, dest, agent_name, from)
|
486
469
|
@client = client
|
487
470
|
@dest = dest
|
488
471
|
@proxy = nil
|
489
472
|
@socket_sync = true
|
473
|
+
@tcp_keepalive = false
|
490
474
|
@requested_version = nil
|
491
475
|
|
492
476
|
@debug_dev = nil
|
@@ -502,6 +486,8 @@ class HTTPClient
|
|
502
486
|
@ssl_peer_cert = nil
|
503
487
|
|
504
488
|
@test_loopback_http_response = nil
|
489
|
+
@strict_response_size_check = false
|
490
|
+
@socket_local = Site::EMPTY
|
505
491
|
|
506
492
|
@agent_name = agent_name
|
507
493
|
@from = from
|
@@ -515,28 +501,34 @@ class HTTPClient
|
|
515
501
|
|
516
502
|
@socket = nil
|
517
503
|
@readbuf = nil
|
504
|
+
|
505
|
+
@transparent_gzip_decompression = false
|
506
|
+
@last_used = nil
|
518
507
|
end
|
519
508
|
|
520
509
|
# Send a request to the server
|
521
510
|
def query(req)
|
522
511
|
connect if @state == :INIT
|
523
|
-
|
512
|
+
# Use absolute URI (not absolute path) iif via proxy AND not HTTPS.
|
513
|
+
req.header.request_absolute_uri = !@proxy.nil? && !https?(@dest)
|
524
514
|
begin
|
525
|
-
timeout(@send_timeout, SendTimeoutError) do
|
515
|
+
::Timeout.timeout(@send_timeout, SendTimeoutError) do
|
526
516
|
set_header(req)
|
527
517
|
req.dump(@socket)
|
528
518
|
# flush the IO stream as IO::sync mode is false
|
529
519
|
@socket.flush unless @socket_sync
|
530
520
|
end
|
531
|
-
rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE
|
521
|
+
rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
|
522
|
+
# JRuby can raise IOError instead of ECONNRESET for now
|
532
523
|
close
|
533
|
-
raise KeepAliveDisconnected.new
|
524
|
+
raise KeepAliveDisconnected.new(self, $!)
|
534
525
|
rescue HTTPClient::TimeoutError
|
535
526
|
close
|
536
527
|
raise
|
537
|
-
rescue
|
538
|
-
|
539
|
-
|
528
|
+
rescue => e
|
529
|
+
close
|
530
|
+
if SSLEnabled and e.is_a?(OpenSSL::SSL::SSLError)
|
531
|
+
raise KeepAliveDisconnected.new(self, e)
|
540
532
|
else
|
541
533
|
raise
|
542
534
|
end
|
@@ -545,6 +537,7 @@ class HTTPClient
|
|
545
537
|
@state = :META if @state == :WAIT
|
546
538
|
@next_connection = nil
|
547
539
|
@requests.push(req)
|
540
|
+
@last_used = Time.now
|
548
541
|
end
|
549
542
|
|
550
543
|
def close
|
@@ -585,6 +578,9 @@ class HTTPClient
|
|
585
578
|
begin
|
586
579
|
read_header if @state == :META
|
587
580
|
return nil if @state != :DATA
|
581
|
+
if @transparent_gzip_decompression
|
582
|
+
block = content_inflater_block(@content_encoding, block)
|
583
|
+
end
|
588
584
|
if @chunked
|
589
585
|
read_body_chunked(&block)
|
590
586
|
elsif @content_length
|
@@ -606,21 +602,142 @@ class HTTPClient
|
|
606
602
|
nil
|
607
603
|
end
|
608
604
|
|
605
|
+
def create_socket(host, port)
|
606
|
+
socket = nil
|
607
|
+
begin
|
608
|
+
@debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
|
609
|
+
clean_host = host.delete("[]")
|
610
|
+
if @socket_local == Site::EMPTY
|
611
|
+
socket = TCPSocket.new(clean_host, port)
|
612
|
+
else
|
613
|
+
clean_local = @socket_local.host.delete("[]")
|
614
|
+
socket = TCPSocket.new(clean_host, port, clean_local, @socket_local.port)
|
615
|
+
end
|
616
|
+
socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) if @tcp_keepalive
|
617
|
+
if @debug_dev
|
618
|
+
@debug_dev << "! CONNECTION ESTABLISHED\n"
|
619
|
+
socket.extend(DebugSocket)
|
620
|
+
socket.debug_dev = @debug_dev
|
621
|
+
end
|
622
|
+
rescue SystemCallError => e
|
623
|
+
raise e.class, e.message + " (#{host}:#{port})"
|
624
|
+
rescue SocketError => e
|
625
|
+
raise e.class, e.message + " (#{host}:#{port})"
|
626
|
+
end
|
627
|
+
socket
|
628
|
+
end
|
629
|
+
|
630
|
+
def create_loopback_socket(host, port, str)
|
631
|
+
@debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
|
632
|
+
socket = LoopBackSocket.new(host, port, str)
|
633
|
+
if @debug_dev
|
634
|
+
@debug_dev << "! CONNECTION ESTABLISHED\n"
|
635
|
+
socket.extend(DebugSocket)
|
636
|
+
socket.debug_dev = @debug_dev
|
637
|
+
end
|
638
|
+
if https?(@dest) && @proxy
|
639
|
+
connect_ssl_proxy(socket, Util.urify(@dest.to_s))
|
640
|
+
end
|
641
|
+
socket
|
642
|
+
end
|
643
|
+
|
644
|
+
def connect_ssl_proxy(socket, uri)
|
645
|
+
req = HTTP::Message.new_connect_request(uri)
|
646
|
+
@client.request_filter.each do |filter|
|
647
|
+
filter.filter_request(req)
|
648
|
+
end
|
649
|
+
set_header(req)
|
650
|
+
req.dump(socket)
|
651
|
+
socket.flush unless @socket_sync
|
652
|
+
res = HTTP::Message.new_response('')
|
653
|
+
parse_header(socket)
|
654
|
+
res.http_version, res.status, res.reason = @version, @status, @reason
|
655
|
+
@headers.each do |key, value|
|
656
|
+
res.header.set(key.to_s, value)
|
657
|
+
end
|
658
|
+
commands = @client.request_filter.collect { |filter|
|
659
|
+
filter.filter_response(req, res)
|
660
|
+
}
|
661
|
+
if commands.find { |command| command == :retry }
|
662
|
+
raise RetryableResponse.new(res)
|
663
|
+
end
|
664
|
+
unless @status == 200
|
665
|
+
raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
|
666
|
+
end
|
667
|
+
end
|
668
|
+
|
609
669
|
private
|
610
670
|
|
671
|
+
# This inflater allows deflate compression with/without zlib header
|
672
|
+
class LenientInflater
|
673
|
+
def initialize
|
674
|
+
@inflater = Zlib::Inflate.new(Zlib::MAX_WBITS)
|
675
|
+
@first = true
|
676
|
+
end
|
677
|
+
|
678
|
+
def inflate(body)
|
679
|
+
if @first
|
680
|
+
first_inflate(body)
|
681
|
+
else
|
682
|
+
@inflater.inflate(body)
|
683
|
+
end
|
684
|
+
end
|
685
|
+
|
686
|
+
private
|
687
|
+
|
688
|
+
def first_inflate(body)
|
689
|
+
@first = false
|
690
|
+
begin
|
691
|
+
@inflater.inflate(body)
|
692
|
+
rescue Zlib::DataError
|
693
|
+
# fallback to deflate without zlib header
|
694
|
+
@inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS)
|
695
|
+
@inflater.inflate(body)
|
696
|
+
end
|
697
|
+
end
|
698
|
+
end
|
699
|
+
|
700
|
+
def content_inflater_block(content_encoding, block)
|
701
|
+
case content_encoding
|
702
|
+
when 'gzip', 'x-gzip'
|
703
|
+
# zlib itself has a functionality to decompress gzip stream.
|
704
|
+
# - zlib 1.2.5 Manual
|
705
|
+
# http://www.zlib.net/manual.html#Advanced
|
706
|
+
# > windowBits can also be greater than 15 for optional gzip decoding. Add 32 to
|
707
|
+
# > windowBits to enable zlib and gzip decoding with automatic header detection,
|
708
|
+
# > or add 16 to decode only the gzip format
|
709
|
+
inflate_stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
|
710
|
+
when 'deflate'
|
711
|
+
inflate_stream = LenientInflater.new
|
712
|
+
else
|
713
|
+
return block
|
714
|
+
end
|
715
|
+
Proc.new { |buf|
|
716
|
+
block.call(inflate_stream.inflate(buf))
|
717
|
+
}
|
718
|
+
end
|
719
|
+
|
611
720
|
def set_header(req)
|
612
721
|
if @requested_version
|
613
722
|
if /^(?:HTTP\/|)(\d+.\d+)$/ =~ @requested_version
|
614
|
-
req.
|
723
|
+
req.http_version = $1
|
615
724
|
end
|
616
725
|
end
|
617
|
-
if @agent_name
|
726
|
+
if @agent_name && req.header.get('User-Agent').empty?
|
618
727
|
req.header.set('User-Agent', "#{@agent_name} #{LIB_NAME}")
|
619
728
|
end
|
620
|
-
if @from
|
729
|
+
if @from && req.header.get('From').empty?
|
621
730
|
req.header.set('From', @from)
|
622
731
|
end
|
623
|
-
req.header.
|
732
|
+
if req.header.get('Accept').empty?
|
733
|
+
req.header.set('Accept', '*/*')
|
734
|
+
end
|
735
|
+
if @transparent_gzip_decompression
|
736
|
+
req.header.set('Accept-Encoding', 'gzip,deflate')
|
737
|
+
end
|
738
|
+
if req.header.get('Date').empty?
|
739
|
+
req.header.set_date_header
|
740
|
+
end
|
624
741
|
end
|
625
742
|
|
626
743
|
# Connect to the server
|
@@ -628,22 +745,15 @@ class HTTPClient
|
|
628
745
|
site = @proxy || @dest
|
629
746
|
retry_number = 0
|
630
747
|
begin
|
631
|
-
timeout(@connect_timeout, ConnectTimeoutError) do
|
632
|
-
|
633
|
-
|
634
|
-
|
635
|
-
|
636
|
-
|
637
|
-
|
638
|
-
|
639
|
-
@socket.ssl_connect
|
640
|
-
@socket.post_connection_check(@dest)
|
641
|
-
@ssl_peer_cert = @socket.peer_cert
|
642
|
-
end
|
748
|
+
::Timeout.timeout(@connect_timeout, ConnectTimeoutError) do
|
749
|
+
if str = @test_loopback_http_response.shift
|
750
|
+
@socket = create_loopback_socket(site.host, site.port, str)
|
751
|
+
elsif https?(@dest)
|
752
|
+
@socket = SSLSocket.create_socket(self)
|
753
|
+
@ssl_peer_cert = @socket.peer_cert
|
754
|
+
else
|
755
|
+
@socket = create_socket(site.host, site.port)
|
643
756
|
end
|
644
|
-
# Use Ruby internal buffering instead of passing data immediately
|
645
|
-
# to the underlying layer
|
646
|
-
# => we need to to call explicitly flush on the socket
|
647
757
|
@socket.sync = @socket_sync
|
648
758
|
end
|
649
759
|
rescue RetryableResponse
|
@@ -665,72 +775,17 @@ class HTTPClient
|
|
665
775
|
@state = :WAIT
|
666
776
|
end
|
667
777
|
|
668
|
-
def create_socket(site)
|
669
|
-
socket = nil
|
670
|
-
begin
|
671
|
-
@debug_dev << "! CONNECT TO #{site.host}:#{site.port}\n" if @debug_dev
|
672
|
-
if str = @test_loopback_http_response.shift
|
673
|
-
socket = LoopBackSocket.new(site.host, site.port, str)
|
674
|
-
else
|
675
|
-
socket = TCPSocket.new(site.host, site.port)
|
676
|
-
end
|
677
|
-
if @debug_dev
|
678
|
-
@debug_dev << "! CONNECTION ESTABLISHED\n"
|
679
|
-
socket.extend(DebugSocket)
|
680
|
-
socket.debug_dev = @debug_dev
|
681
|
-
end
|
682
|
-
rescue SystemCallError => e
|
683
|
-
e.message << " (#{site})"
|
684
|
-
raise
|
685
|
-
rescue SocketError => e
|
686
|
-
e.message << " (#{site})"
|
687
|
-
raise
|
688
|
-
end
|
689
|
-
socket
|
690
|
-
end
|
691
|
-
|
692
|
-
# wrap socket with OpenSSL.
|
693
|
-
def create_ssl_socket(raw_socket)
|
694
|
-
SSLSocketWrap.new(raw_socket, @ssl_config, @debug_dev)
|
695
|
-
end
|
696
|
-
|
697
|
-
def connect_ssl_proxy(socket, uri)
|
698
|
-
req = HTTP::Message.new_connect_request(uri)
|
699
|
-
@client.request_filter.each do |filter|
|
700
|
-
filter.filter_request(req)
|
701
|
-
end
|
702
|
-
set_header(req)
|
703
|
-
req.dump(@socket)
|
704
|
-
@socket.flush unless @socket_sync
|
705
|
-
res = HTTP::Message.new_response('')
|
706
|
-
parse_header
|
707
|
-
res.version, res.status, res.reason = @version, @status, @reason
|
708
|
-
@headers.each do |key, value|
|
709
|
-
res.header.set(key, value)
|
710
|
-
end
|
711
|
-
commands = @client.request_filter.collect { |filter|
|
712
|
-
filter.filter_response(req, res)
|
713
|
-
}
|
714
|
-
if commands.find { |command| command == :retry }
|
715
|
-
raise RetryableResponse.new
|
716
|
-
end
|
717
|
-
unless @status == 200
|
718
|
-
raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
|
719
|
-
end
|
720
|
-
end
|
721
|
-
|
722
778
|
# Read status block.
|
723
779
|
def read_header
|
724
780
|
@content_length = nil
|
725
781
|
@chunked = false
|
782
|
+
@content_encoding = nil
|
726
783
|
@chunk_length = 0
|
727
|
-
parse_header
|
728
|
-
|
729
|
-
# Head of the request has been parsed.
|
784
|
+
parse_header(@socket)
|
785
|
+
# Header of the request has been parsed.
|
730
786
|
@state = :DATA
|
731
787
|
req = @requests.shift
|
732
|
-
|
733
|
-
if req.header.request_method == 'HEAD'
|
788
|
+
if req.header.request_method == 'HEAD' or no_message_body?(@status)
|
734
789
|
@content_length = 0
|
735
790
|
if @next_connection
|
736
791
|
@state = :WAIT
|
@@ -738,16 +793,24 @@ class HTTPClient
|
|
738
793
|
close
|
739
794
|
end
|
740
795
|
end
|
741
|
-
@next_connection = false
|
796
|
+
@next_connection = false if !@content_length and !@chunked
|
742
797
|
end
|
743
798
|
|
744
799
|
StatusParseRegexp = %r(\AHTTP/(\d+\.\d+)\s+(\d\d\d)\s*([^\r\n]+)?\r?\n\z)
|
745
|
-
def parse_header
|
746
|
-
timeout(@receive_timeout, ReceiveTimeoutError) do
|
800
|
+
def parse_header(socket)
|
801
|
+
::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
|
802
|
+
initial_line = nil
|
747
803
|
begin
|
748
|
-
|
749
|
-
|
750
|
-
|
804
|
+
begin
|
805
|
+
initial_line = socket.gets("\n")
|
806
|
+
if initial_line.nil?
|
807
|
+
close
|
808
|
+
raise KeepAliveDisconnected.new(self)
|
809
|
+
end
|
810
|
+
rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
|
811
|
+
# JRuby can raise IOError instead of ECONNRESET for now
|
812
|
+
close
|
813
|
+
raise KeepAliveDisconnected.new(self, $!)
|
751
814
|
end
|
752
815
|
if StatusParseRegexp !~ initial_line
|
753
816
|
@version = '0.9'
|
@@ -759,32 +822,48 @@ class HTTPClient
|
|
759
822
|
break
|
760
823
|
end
|
761
824
|
@version, @status, @reason = $1, $2.to_i, $3
|
762
|
-
@next_connection = HTTP::Message.keep_alive_enabled?(@version
|
825
|
+
@next_connection = HTTP::Message.keep_alive_enabled?(@version)
|
763
826
|
@headers = []
|
764
827
|
while true
|
765
|
-
line =
|
828
|
+
line = socket.gets("\n")
|
766
829
|
unless line
|
767
830
|
raise BadResponseError.new('unexpected EOF')
|
768
831
|
end
|
769
832
|
line.chomp!
|
770
833
|
break if line.empty?
|
771
|
-
|
772
|
-
|
773
|
-
|
834
|
+
if line[0] == ?\ or line[0] == ?\t
|
835
|
+
last = @headers.last[1]
|
836
|
+
last << ' ' unless last.empty?
|
837
|
+
last << line.strip
|
838
|
+
else
|
839
|
+
key, value = line.strip.split(/\s*:\s*/, 2)
|
840
|
+
parse_content_header(key, value)
|
841
|
+
@headers << [key, value]
|
842
|
+
end
|
774
843
|
end
|
775
844
|
end while (@version == '1.1' && @status == 100)
|
776
845
|
end
|
777
846
|
end
|
778
847
|
|
779
|
-
def
|
848
|
+
def no_message_body?(status)
|
849
|
+
!status.nil? && # HTTP/0.9
|
850
|
+
((status >= 100 && status < 200) || status == 204 || status == 304)
|
851
|
+
end
|
852
|
+
|
853
|
+
def parse_content_header(key, value)
|
780
854
|
key = key.downcase
|
781
|
-
|
855
|
+
case key
|
856
|
+
when 'content-length'
|
782
857
|
@content_length = value.to_i
|
783
|
-
|
784
|
-
@
|
785
|
-
|
786
|
-
|
787
|
-
|
858
|
+
when 'content-encoding'
|
859
|
+
@content_encoding = value.downcase
|
860
|
+
when 'transfer-encoding'
|
861
|
+
if value.downcase == 'chunked'
|
862
|
+
@chunked = true
|
863
|
+
@chunk_length = 0
|
864
|
+
@content_length = nil
|
865
|
+
end
|
866
|
+
when 'connection', 'proxy-connection'
|
788
867
|
if value.downcase == 'keep-alive'
|
789
868
|
@next_connection = true
|
790
869
|
else
|
@@ -795,19 +874,23 @@ class HTTPClient
|
|
795
874
|
|
796
875
|
def read_body_length(&block)
|
797
876
|
return nil if @content_length == 0
|
798
|
-
buf = ''
|
799
877
|
while true
|
878
|
+
buf = empty_bin_str
|
800
879
|
maxbytes = @read_block_size
|
801
|
-
maxbytes = @content_length if maxbytes > @content_length
|
802
|
-
timeout(@receive_timeout, ReceiveTimeoutError) do
|
880
|
+
maxbytes = @content_length if maxbytes > @content_length && @content_length > 0
|
881
|
+
::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
|
803
882
|
begin
|
804
883
|
@socket.readpartial(maxbytes, buf)
|
805
884
|
rescue EOFError
|
885
|
+
close
|
806
886
|
buf = nil
|
887
|
+
if @strict_response_size_check
|
888
|
+
raise BadResponseError.new("EOF while reading rest #{@content_length} bytes")
|
889
|
+
end
|
807
890
|
end
|
808
891
|
end
|
809
|
-
if buf && buf.
|
810
|
-
@content_length -= buf.
|
892
|
+
if buf && buf.bytesize > 0
|
893
|
+
@content_length -= buf.bytesize
|
811
894
|
yield buf
|
812
895
|
else
|
813
896
|
@content_length = 0
|
@@ -818,45 +901,59 @@ class HTTPClient
|
|
818
901
|
|
819
902
|
RS = "\r\n"
|
820
903
|
def read_body_chunked(&block)
|
821
|
-
buf =
|
904
|
+
buf = empty_bin_str
|
822
905
|
while true
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
|
827
|
-
|
828
|
-
|
829
|
-
|
830
|
-
|
831
|
-
|
906
|
+
::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
|
907
|
+
len = @socket.gets(RS)
|
908
|
+
if len.nil? # EOF
|
909
|
+
close
|
910
|
+
return
|
911
|
+
end
|
912
|
+
@chunk_length = len.hex
|
913
|
+
if @chunk_length == 0
|
914
|
+
@content_length = 0
|
915
|
+
@socket.gets(RS)
|
916
|
+
return
|
917
|
+
end
|
918
|
+
@socket.read(@chunk_length, buf)
|
919
|
+
@socket.read(2)
|
832
920
|
end
|
833
921
|
unless buf.empty?
|
834
|
-
yield buf
|
922
|
+
yield buf
|
835
923
|
end
|
836
924
|
end
|
837
925
|
end
|
838
926
|
|
839
927
|
def read_body_rest
|
840
|
-
if @readbuf and @readbuf.
|
928
|
+
if @readbuf and @readbuf.bytesize > 0
|
841
929
|
yield @readbuf
|
842
930
|
@readbuf = nil
|
843
931
|
end
|
844
|
-
buf = ''
|
845
932
|
while true
|
846
|
-
|
933
|
+
buf = empty_bin_str
|
934
|
+
::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
|
847
935
|
begin
|
848
936
|
@socket.readpartial(@read_block_size, buf)
|
849
937
|
rescue EOFError
|
850
938
|
buf = nil
|
939
|
+
if @strict_response_size_check
|
940
|
+
raise BadResponseError.new("EOF while reading chunked response")
|
941
|
+
end
|
851
942
|
end
|
852
943
|
end
|
853
|
-
if buf && buf.
|
944
|
+
if buf && buf.bytesize > 0
|
854
945
|
yield buf
|
855
946
|
else
|
856
947
|
return
|
857
948
|
end
|
858
949
|
end
|
859
950
|
end
|
951
|
+
|
952
|
+
def empty_bin_str
|
953
|
+
str = ''
|
954
|
+
str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
|
955
|
+
str
|
956
|
+
end
|
860
957
|
end
|
861
958
|
|
862
959
|
|