httpclient 2.3.0.1 → 2.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +85 -0
- data/bin/httpclient +18 -6
- data/bin/jsonclient +85 -0
- data/lib/http-access2.rb +1 -1
- data/lib/httpclient.rb +262 -88
- data/lib/httpclient/auth.rb +269 -244
- data/lib/httpclient/cacert.pem +3952 -0
- data/lib/httpclient/cacert1024.pem +3866 -0
- data/lib/httpclient/connection.rb +1 -1
- data/lib/httpclient/cookie.rb +161 -514
- data/lib/httpclient/http.rb +57 -21
- data/lib/httpclient/include_client.rb +2 -0
- data/lib/httpclient/jruby_ssl_socket.rb +588 -0
- data/lib/httpclient/session.rb +259 -317
- data/lib/httpclient/ssl_config.rb +141 -188
- data/lib/httpclient/ssl_socket.rb +150 -0
- data/lib/httpclient/timeout.rb +1 -1
- data/lib/httpclient/util.rb +62 -1
- data/lib/httpclient/version.rb +1 -1
- data/lib/httpclient/webagent-cookie.rb +459 -0
- data/lib/jsonclient.rb +63 -0
- data/lib/oauthclient.rb +2 -1
- data/sample/jsonclient.rb +67 -0
- data/sample/oauth_twitter.rb +4 -4
- data/test/{ca-chain.cert → ca-chain.pem} +0 -0
- data/test/client-pass.key +18 -0
- data/test/helper.rb +10 -8
- data/test/jruby_ssl_socket/test_pemutils.rb +32 -0
- data/test/test_auth.rb +175 -4
- data/test/test_cookie.rb +147 -243
- data/test/test_http-access2.rb +17 -16
- data/test/test_httpclient.rb +458 -77
- data/test/test_jsonclient.rb +80 -0
- data/test/test_ssl.rb +341 -17
- data/test/test_webagent-cookie.rb +465 -0
- metadata +57 -55
- data/README.txt +0 -721
- data/lib/httpclient/cacert.p7s +0 -1858
- data/lib/httpclient/cacert_sha1.p7s +0 -1858
- data/sample/oauth_salesforce_10.rb +0 -63
data/lib/httpclient/session.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# HTTPClient - HTTP client library.
|
2
|
-
# Copyright (C) 2000-
|
2
|
+
# Copyright (C) 2000-2015 NAKAMURA, Hiroshi <nahi@ruby-lang.org>.
|
3
3
|
#
|
4
4
|
# This program is copyrighted free software by NAKAMURA, Hiroshi. You can
|
5
5
|
# redistribute it and/or modify it under the same terms of Ruby's license;
|
@@ -14,12 +14,18 @@
|
|
14
14
|
|
15
15
|
require 'socket'
|
16
16
|
require 'thread'
|
17
|
+
require 'timeout'
|
17
18
|
require 'stringio'
|
18
19
|
require 'zlib'
|
19
20
|
|
20
|
-
require 'httpclient/timeout'
|
21
|
+
require 'httpclient/timeout' # TODO: remove this once we drop 1.8 support
|
21
22
|
require 'httpclient/ssl_config'
|
22
23
|
require 'httpclient/http'
|
24
|
+
if RUBY_ENGINE == 'jruby'
|
25
|
+
require 'httpclient/jruby_ssl_socket'
|
26
|
+
else
|
27
|
+
require 'httpclient/ssl_socket'
|
28
|
+
end
|
23
29
|
|
24
30
|
|
25
31
|
class HTTPClient
|
@@ -31,14 +37,15 @@ class HTTPClient
|
|
31
37
|
attr_accessor :scheme
|
32
38
|
# Host String.
|
33
39
|
attr_accessor :host
|
40
|
+
alias hostname host
|
34
41
|
# Port number.
|
35
42
|
attr_accessor :port
|
36
43
|
|
37
44
|
# Creates a new Site based on the given URI.
|
38
45
|
def initialize(uri = nil)
|
39
46
|
if uri
|
40
|
-
@scheme = uri.scheme
|
41
|
-
@host = uri.
|
47
|
+
@scheme = uri.scheme || 'tcp'
|
48
|
+
@host = uri.hostname || '0.0.0.0'
|
42
49
|
@port = uri.port.to_i
|
43
50
|
else
|
44
51
|
@scheme = 'tcp'
|
@@ -69,7 +76,7 @@ class HTTPClient
|
|
69
76
|
def to_s # :nodoc:
|
70
77
|
addr
|
71
78
|
end
|
72
|
-
|
79
|
+
|
73
80
|
# Returns true if scheme, host and port of the given URI matches with this.
|
74
81
|
def match(uri)
|
75
82
|
(@scheme == uri.scheme) and (@host == uri.host) and (@port == uri.port.to_i)
|
@@ -98,6 +105,8 @@ class HTTPClient
|
|
98
105
|
attr_accessor :debug_dev
|
99
106
|
# Boolean value for Socket#sync
|
100
107
|
attr_accessor :socket_sync
|
108
|
+
# Boolean value to send TCP keepalive packets; no timing settings exist at present
|
109
|
+
attr_accessor :tcp_keepalive
|
101
110
|
|
102
111
|
attr_accessor :connect_timeout
|
103
112
|
# Maximum retry count. 0 for infinite.
|
@@ -108,6 +117,9 @@ class HTTPClient
|
|
108
117
|
attr_accessor :read_block_size
|
109
118
|
attr_accessor :protocol_retry_count
|
110
119
|
|
120
|
+
# Raise BadResponseError if response size does not match with Content-Length header in response.
|
121
|
+
attr_accessor :strict_response_size_check
|
122
|
+
|
111
123
|
# Local address to bind local side of the socket to
|
112
124
|
attr_accessor :socket_local
|
113
125
|
|
@@ -127,6 +139,7 @@ class HTTPClient
|
|
127
139
|
@protocol_version = nil
|
128
140
|
@debug_dev = client.debug_dev
|
129
141
|
@socket_sync = true
|
142
|
+
@tcp_keepalive = false
|
130
143
|
@chunk_size = ::HTTP::Message::Body::DEFAULT_CHUNK_SIZE
|
131
144
|
|
132
145
|
@connect_timeout = 60
|
@@ -141,6 +154,7 @@ class HTTPClient
|
|
141
154
|
@test_loopback_http_response = []
|
142
155
|
|
143
156
|
@transparent_gzip_decompression = false
|
157
|
+
@strict_response_size_check = false
|
144
158
|
@socket_local = Site.new
|
145
159
|
|
146
160
|
@sess_pool = {}
|
@@ -157,8 +171,8 @@ class HTTPClient
|
|
157
171
|
end
|
158
172
|
|
159
173
|
def query(req, via_proxy)
|
160
|
-
req.http_body.chunk_size = @chunk_size
|
161
|
-
sess =
|
174
|
+
req.http_body.chunk_size = @chunk_size if req.http_body
|
175
|
+
sess = get_session(req, via_proxy)
|
162
176
|
begin
|
163
177
|
sess.query(req)
|
164
178
|
rescue
|
@@ -182,40 +196,43 @@ class HTTPClient
|
|
182
196
|
add_cached_session(sess)
|
183
197
|
end
|
184
198
|
|
185
|
-
def invalidate(site)
|
186
|
-
@sess_pool_mutex.synchronize do
|
187
|
-
if pool = @sess_pool[site]
|
188
|
-
pool.each do |sess|
|
189
|
-
sess.invalidate
|
190
|
-
end
|
191
|
-
end
|
192
|
-
end
|
193
|
-
end
|
194
|
-
|
195
199
|
private
|
196
200
|
|
197
|
-
|
201
|
+
# TODO: create PR for webmock's httpclient adapter to use get_session
|
202
|
+
# instead of open so that we can remove duplicated Site creation for
|
203
|
+
# each session.
|
204
|
+
def get_session(req, via_proxy = false)
|
205
|
+
uri = req.header.request_uri
|
206
|
+
if uri.scheme.nil?
|
207
|
+
raise ArgumentError.new("Request URI must have schema. Possibly add 'http://' to the request URI?")
|
208
|
+
end
|
198
209
|
site = Site.new(uri)
|
199
|
-
sess = nil
|
200
210
|
if cached = get_cached_session(site)
|
201
|
-
|
211
|
+
cached
|
202
212
|
else
|
203
|
-
|
204
|
-
sess.proxy = via_proxy ? @proxy : nil
|
205
|
-
sess.socket_sync = @socket_sync
|
206
|
-
sess.requested_version = @protocol_version if @protocol_version
|
207
|
-
sess.connect_timeout = @connect_timeout
|
208
|
-
sess.connect_retry = @connect_retry
|
209
|
-
sess.send_timeout = @send_timeout
|
210
|
-
sess.receive_timeout = @receive_timeout
|
211
|
-
sess.read_block_size = @read_block_size
|
212
|
-
sess.protocol_retry_count = @protocol_retry_count
|
213
|
-
sess.ssl_config = @ssl_config
|
214
|
-
sess.debug_dev = @debug_dev
|
215
|
-
sess.socket_local = @socket_local
|
216
|
-
sess.test_loopback_http_response = @test_loopback_http_response
|
217
|
-
sess.transparent_gzip_decompression = @transparent_gzip_decompression
|
213
|
+
open(uri, via_proxy)
|
218
214
|
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def open(uri, via_proxy = false)
|
218
|
+
site = Site.new(uri)
|
219
|
+
sess = Session.new(@client, site, @agent_name, @from)
|
220
|
+
sess.proxy = via_proxy ? @proxy : nil
|
221
|
+
sess.socket_sync = @socket_sync
|
222
|
+
sess.tcp_keepalive = @tcp_keepalive
|
223
|
+
sess.requested_version = @protocol_version if @protocol_version
|
224
|
+
sess.connect_timeout = @connect_timeout
|
225
|
+
sess.connect_retry = @connect_retry
|
226
|
+
sess.send_timeout = @send_timeout
|
227
|
+
sess.receive_timeout = @receive_timeout
|
228
|
+
sess.read_block_size = @read_block_size
|
229
|
+
sess.protocol_retry_count = @protocol_retry_count
|
230
|
+
sess.ssl_config = @ssl_config
|
231
|
+
sess.debug_dev = @debug_dev
|
232
|
+
sess.strict_response_size_check = @strict_response_size_check
|
233
|
+
sess.socket_local = @socket_local
|
234
|
+
sess.test_loopback_http_response = @test_loopback_http_response
|
235
|
+
sess.transparent_gzip_decompression = @transparent_gzip_decompression
|
219
236
|
sess
|
220
237
|
end
|
221
238
|
|
@@ -241,6 +258,9 @@ class HTTPClient
|
|
241
258
|
end
|
242
259
|
|
243
260
|
def get_cached_session(site)
|
261
|
+
if Thread.current[:HTTPClient_AcquireNewConnection]
|
262
|
+
return nil
|
263
|
+
end
|
244
264
|
@sess_pool_mutex.synchronize do
|
245
265
|
now = Time.now
|
246
266
|
if now > @sess_pool_last_checked + @keep_alive_timeout
|
@@ -272,7 +292,7 @@ class HTTPClient
|
|
272
292
|
end
|
273
293
|
|
274
294
|
def valid_session?(sess, now)
|
275
|
-
|
295
|
+
(now <= sess.last_used + @keep_alive_timeout)
|
276
296
|
end
|
277
297
|
|
278
298
|
def add_cached_session(sess)
|
@@ -283,131 +303,6 @@ class HTTPClient
|
|
283
303
|
end
|
284
304
|
|
285
305
|
|
286
|
-
# Wraps up OpenSSL::SSL::SSLSocket and offers debugging features.
|
287
|
-
class SSLSocketWrap
|
288
|
-
def initialize(socket, context, debug_dev = nil)
|
289
|
-
unless SSLEnabled
|
290
|
-
raise ConfigurationError.new('Ruby/OpenSSL module is required')
|
291
|
-
end
|
292
|
-
@context = context
|
293
|
-
@socket = socket
|
294
|
-
@ssl_socket = create_openssl_socket(@socket)
|
295
|
-
@debug_dev = debug_dev
|
296
|
-
end
|
297
|
-
|
298
|
-
def ssl_connect(hostname = nil)
|
299
|
-
if hostname && @ssl_socket.respond_to?(:hostname=)
|
300
|
-
@ssl_socket.hostname = hostname
|
301
|
-
end
|
302
|
-
@ssl_socket.connect
|
303
|
-
end
|
304
|
-
|
305
|
-
def post_connection_check(host)
|
306
|
-
verify_mode = @context.verify_mode || OpenSSL::SSL::VERIFY_NONE
|
307
|
-
if verify_mode == OpenSSL::SSL::VERIFY_NONE
|
308
|
-
return
|
309
|
-
elsif @ssl_socket.peer_cert.nil? and
|
310
|
-
check_mask(verify_mode, OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT)
|
311
|
-
raise OpenSSL::SSL::SSLError.new('no peer cert')
|
312
|
-
end
|
313
|
-
hostname = host.host
|
314
|
-
if @ssl_socket.respond_to?(:post_connection_check) and RUBY_VERSION > "1.8.4"
|
315
|
-
@ssl_socket.post_connection_check(hostname)
|
316
|
-
else
|
317
|
-
@context.post_connection_check(@ssl_socket.peer_cert, hostname)
|
318
|
-
end
|
319
|
-
end
|
320
|
-
|
321
|
-
def ssl_version
|
322
|
-
@ssl_socket.ssl_version if @ssl_socket.respond_to?(:ssl_version)
|
323
|
-
end
|
324
|
-
|
325
|
-
def ssl_cipher
|
326
|
-
@ssl_socket.cipher
|
327
|
-
end
|
328
|
-
|
329
|
-
def ssl_state
|
330
|
-
@ssl_socket.state
|
331
|
-
end
|
332
|
-
|
333
|
-
def peer_cert
|
334
|
-
@ssl_socket.peer_cert
|
335
|
-
end
|
336
|
-
|
337
|
-
def close
|
338
|
-
@ssl_socket.close
|
339
|
-
@socket.close
|
340
|
-
end
|
341
|
-
|
342
|
-
def closed?
|
343
|
-
@socket.closed?
|
344
|
-
end
|
345
|
-
|
346
|
-
def eof?
|
347
|
-
@ssl_socket.eof?
|
348
|
-
end
|
349
|
-
|
350
|
-
def gets(*args)
|
351
|
-
str = @ssl_socket.gets(*args)
|
352
|
-
debug(str)
|
353
|
-
str
|
354
|
-
end
|
355
|
-
|
356
|
-
def read(*args)
|
357
|
-
str = @ssl_socket.read(*args)
|
358
|
-
debug(str)
|
359
|
-
str
|
360
|
-
end
|
361
|
-
|
362
|
-
def readpartial(*args)
|
363
|
-
str = @ssl_socket.readpartial(*args)
|
364
|
-
debug(str)
|
365
|
-
str
|
366
|
-
end
|
367
|
-
|
368
|
-
def <<(str)
|
369
|
-
rv = @ssl_socket.write(str)
|
370
|
-
debug(str)
|
371
|
-
rv
|
372
|
-
end
|
373
|
-
|
374
|
-
def flush
|
375
|
-
@ssl_socket.flush
|
376
|
-
end
|
377
|
-
|
378
|
-
def sync
|
379
|
-
@ssl_socket.sync
|
380
|
-
end
|
381
|
-
|
382
|
-
def sync=(sync)
|
383
|
-
@ssl_socket.sync = sync
|
384
|
-
end
|
385
|
-
|
386
|
-
private
|
387
|
-
|
388
|
-
def check_mask(value, mask)
|
389
|
-
value & mask == mask
|
390
|
-
end
|
391
|
-
|
392
|
-
def create_openssl_socket(socket)
|
393
|
-
ssl_socket = nil
|
394
|
-
if OpenSSL::SSL.const_defined?("SSLContext")
|
395
|
-
ctx = OpenSSL::SSL::SSLContext.new
|
396
|
-
@context.set_context(ctx)
|
397
|
-
ssl_socket = OpenSSL::SSL::SSLSocket.new(socket, ctx)
|
398
|
-
else
|
399
|
-
ssl_socket = OpenSSL::SSL::SSLSocket.new(socket)
|
400
|
-
@context.set_context(ssl_socket)
|
401
|
-
end
|
402
|
-
ssl_socket
|
403
|
-
end
|
404
|
-
|
405
|
-
def debug(str)
|
406
|
-
@debug_dev << str if @debug_dev && str
|
407
|
-
end
|
408
|
-
end
|
409
|
-
|
410
|
-
|
411
306
|
# Wraps up a Socket for method interception.
|
412
307
|
module SocketWrap
|
413
308
|
def initialize(socket, *args)
|
@@ -427,20 +322,20 @@ class HTTPClient
|
|
427
322
|
@socket.eof?
|
428
323
|
end
|
429
324
|
|
430
|
-
def gets(
|
431
|
-
@socket.gets(
|
325
|
+
def gets(rs)
|
326
|
+
@socket.gets(rs)
|
432
327
|
end
|
433
328
|
|
434
|
-
def read(
|
435
|
-
@socket.read(
|
329
|
+
def read(size, buf = nil)
|
330
|
+
@socket.read(size, buf)
|
436
331
|
end
|
437
332
|
|
438
|
-
def readpartial(
|
333
|
+
def readpartial(size, buf = nil)
|
439
334
|
# StringIO doesn't support :readpartial
|
440
335
|
if @socket.respond_to?(:readpartial)
|
441
|
-
@socket.readpartial(
|
336
|
+
@socket.readpartial(size, buf)
|
442
337
|
else
|
443
|
-
@socket.read(
|
338
|
+
@socket.read(size, buf)
|
444
339
|
end
|
445
340
|
end
|
446
341
|
|
@@ -476,19 +371,19 @@ class HTTPClient
|
|
476
371
|
debug("! CONNECTION CLOSED\n")
|
477
372
|
end
|
478
373
|
|
479
|
-
def gets(
|
374
|
+
def gets(rs)
|
480
375
|
str = super
|
481
376
|
debug(str)
|
482
377
|
str
|
483
378
|
end
|
484
379
|
|
485
|
-
def read(
|
380
|
+
def read(size, buf = nil)
|
486
381
|
str = super
|
487
382
|
debug(str)
|
488
383
|
str
|
489
384
|
end
|
490
385
|
|
491
|
-
def readpartial(
|
386
|
+
def readpartial(size, buf = nil)
|
492
387
|
str = super
|
493
388
|
debug(str)
|
494
389
|
str
|
@@ -528,6 +423,10 @@ class HTTPClient
|
|
528
423
|
def <<(str)
|
529
424
|
# ignored
|
530
425
|
end
|
426
|
+
|
427
|
+
def peer_cert
|
428
|
+
nil
|
429
|
+
end
|
531
430
|
end
|
532
431
|
|
533
432
|
|
@@ -542,6 +441,8 @@ class HTTPClient
|
|
542
441
|
attr_accessor :proxy
|
543
442
|
# Boolean value for Socket#sync
|
544
443
|
attr_accessor :socket_sync
|
444
|
+
# Boolean value to send TCP keepalive packets; no timing settings exist at present
|
445
|
+
attr_accessor :tcp_keepalive
|
545
446
|
# Requested protocol version
|
546
447
|
attr_accessor :requested_version
|
547
448
|
# Device for dumping log for debugging
|
@@ -554,6 +455,7 @@ class HTTPClient
|
|
554
455
|
attr_accessor :read_block_size
|
555
456
|
attr_accessor :protocol_retry_count
|
556
457
|
|
458
|
+
attr_accessor :strict_response_size_check
|
557
459
|
attr_accessor :socket_local
|
558
460
|
|
559
461
|
attr_accessor :ssl_config
|
@@ -566,9 +468,9 @@ class HTTPClient
|
|
566
468
|
def initialize(client, dest, agent_name, from)
|
567
469
|
@client = client
|
568
470
|
@dest = dest
|
569
|
-
@invalidated = false
|
570
471
|
@proxy = nil
|
571
472
|
@socket_sync = true
|
473
|
+
@tcp_keepalive = false
|
572
474
|
@requested_version = nil
|
573
475
|
|
574
476
|
@debug_dev = nil
|
@@ -584,6 +486,7 @@ class HTTPClient
|
|
584
486
|
@ssl_peer_cert = nil
|
585
487
|
|
586
488
|
@test_loopback_http_response = nil
|
489
|
+
@strict_response_size_check = false
|
587
490
|
@socket_local = Site::EMPTY
|
588
491
|
|
589
492
|
@agent_name = agent_name
|
@@ -607,9 +510,9 @@ class HTTPClient
|
|
607
510
|
def query(req)
|
608
511
|
connect if @state == :INIT
|
609
512
|
# Use absolute URI (not absolute path) iif via proxy AND not HTTPS.
|
610
|
-
req.header.request_absolute_uri = !@proxy.nil?
|
513
|
+
req.header.request_absolute_uri = !@proxy.nil? && !https?(@dest)
|
611
514
|
begin
|
612
|
-
timeout(@send_timeout, SendTimeoutError) do
|
515
|
+
::Timeout.timeout(@send_timeout, SendTimeoutError) do
|
613
516
|
set_header(req)
|
614
517
|
req.dump(@socket)
|
615
518
|
# flush the IO stream as IO::sync mode is false
|
@@ -618,14 +521,14 @@ class HTTPClient
|
|
618
521
|
rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
|
619
522
|
# JRuby can raise IOError instead of ECONNRESET for now
|
620
523
|
close
|
621
|
-
raise KeepAliveDisconnected.new(self)
|
524
|
+
raise KeepAliveDisconnected.new(self, $!)
|
622
525
|
rescue HTTPClient::TimeoutError
|
623
526
|
close
|
624
527
|
raise
|
625
|
-
rescue
|
528
|
+
rescue => e
|
626
529
|
close
|
627
|
-
if SSLEnabled and
|
628
|
-
raise KeepAliveDisconnected.new(self)
|
530
|
+
if SSLEnabled and e.is_a?(OpenSSL::SSL::SSLError)
|
531
|
+
raise KeepAliveDisconnected.new(self, e)
|
629
532
|
else
|
630
533
|
raise
|
631
534
|
end
|
@@ -650,14 +553,6 @@ class HTTPClient
|
|
650
553
|
@state == :INIT
|
651
554
|
end
|
652
555
|
|
653
|
-
def invalidate
|
654
|
-
@invalidated = true
|
655
|
-
end
|
656
|
-
|
657
|
-
def invalidated?
|
658
|
-
@invalidated
|
659
|
-
end
|
660
|
-
|
661
556
|
def get_header
|
662
557
|
begin
|
663
558
|
if @state != :META
|
@@ -683,18 +578,8 @@ class HTTPClient
|
|
683
578
|
begin
|
684
579
|
read_header if @state == :META
|
685
580
|
return nil if @state != :DATA
|
686
|
-
if @
|
687
|
-
|
688
|
-
# - zlib 1.2.5 Manual
|
689
|
-
# http://www.zlib.net/manual.html#Advanced
|
690
|
-
# > windowBits can also be greater than 15 for optional gzip decoding. Add 32 to
|
691
|
-
# > windowBits to enable zlib and gzip decoding with automatic header detection,
|
692
|
-
# > or add 16 to decode only the gzip format
|
693
|
-
inflate_stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
|
694
|
-
original_block = block
|
695
|
-
block = Proc.new { |buf|
|
696
|
-
original_block.call(inflate_stream.inflate(buf))
|
697
|
-
}
|
581
|
+
if @transparent_gzip_decompression
|
582
|
+
block = content_inflater_block(@content_encoding, block)
|
698
583
|
end
|
699
584
|
if @chunked
|
700
585
|
read_body_chunked(&block)
|
@@ -717,18 +602,131 @@ class HTTPClient
|
|
717
602
|
nil
|
718
603
|
end
|
719
604
|
|
605
|
+
def create_socket(host, port)
|
606
|
+
socket = nil
|
607
|
+
begin
|
608
|
+
@debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
|
609
|
+
clean_host = host.delete("[]")
|
610
|
+
if @socket_local == Site::EMPTY
|
611
|
+
socket = TCPSocket.new(clean_host, port)
|
612
|
+
else
|
613
|
+
clean_local = @socket_local.host.delete("[]")
|
614
|
+
socket = TCPSocket.new(clean_host, port, clean_local, @socket_local.port)
|
615
|
+
end
|
616
|
+
socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) if @tcp_keepalive
|
617
|
+
if @debug_dev
|
618
|
+
@debug_dev << "! CONNECTION ESTABLISHED\n"
|
619
|
+
socket.extend(DebugSocket)
|
620
|
+
socket.debug_dev = @debug_dev
|
621
|
+
end
|
622
|
+
rescue SystemCallError => e
|
623
|
+
raise e.class, e.message + " (#{host}:#{port})"
|
624
|
+
rescue SocketError => e
|
625
|
+
raise e.class, e.message + " (#{host}:#{port})"
|
626
|
+
end
|
627
|
+
socket
|
628
|
+
end
|
629
|
+
|
630
|
+
def create_loopback_socket(host, port, str)
|
631
|
+
@debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
|
632
|
+
socket = LoopBackSocket.new(host, port, str)
|
633
|
+
if @debug_dev
|
634
|
+
@debug_dev << "! CONNECTION ESTABLISHED\n"
|
635
|
+
socket.extend(DebugSocket)
|
636
|
+
socket.debug_dev = @debug_dev
|
637
|
+
end
|
638
|
+
if https?(@dest) && @proxy
|
639
|
+
connect_ssl_proxy(socket, Util.urify(@dest.to_s))
|
640
|
+
end
|
641
|
+
socket
|
642
|
+
end
|
643
|
+
|
644
|
+
def connect_ssl_proxy(socket, uri)
|
645
|
+
req = HTTP::Message.new_connect_request(uri)
|
646
|
+
@client.request_filter.each do |filter|
|
647
|
+
filter.filter_request(req)
|
648
|
+
end
|
649
|
+
set_header(req)
|
650
|
+
req.dump(socket)
|
651
|
+
socket.flush unless @socket_sync
|
652
|
+
res = HTTP::Message.new_response('')
|
653
|
+
parse_header(socket)
|
654
|
+
res.http_version, res.status, res.reason = @version, @status, @reason
|
655
|
+
@headers.each do |key, value|
|
656
|
+
res.header.set(key.to_s, value)
|
657
|
+
end
|
658
|
+
commands = @client.request_filter.collect { |filter|
|
659
|
+
filter.filter_response(req, res)
|
660
|
+
}
|
661
|
+
if commands.find { |command| command == :retry }
|
662
|
+
raise RetryableResponse.new(res)
|
663
|
+
end
|
664
|
+
unless @status == 200
|
665
|
+
raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
|
666
|
+
end
|
667
|
+
end
|
668
|
+
|
720
669
|
private
|
721
670
|
|
671
|
+
# This inflater allows deflate compression with/without zlib header
|
672
|
+
class LenientInflater
|
673
|
+
def initialize
|
674
|
+
@inflater = Zlib::Inflate.new(Zlib::MAX_WBITS)
|
675
|
+
@first = true
|
676
|
+
end
|
677
|
+
|
678
|
+
def inflate(body)
|
679
|
+
if @first
|
680
|
+
first_inflate(body)
|
681
|
+
else
|
682
|
+
@inflater.inflate(body)
|
683
|
+
end
|
684
|
+
end
|
685
|
+
|
686
|
+
private
|
687
|
+
|
688
|
+
def first_inflate(body)
|
689
|
+
@first = false
|
690
|
+
begin
|
691
|
+
@inflater.inflate(body)
|
692
|
+
rescue Zlib::DataError
|
693
|
+
# fallback to deflate without zlib header
|
694
|
+
@inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS)
|
695
|
+
@inflater.inflate(body)
|
696
|
+
end
|
697
|
+
end
|
698
|
+
end
|
699
|
+
|
700
|
+
def content_inflater_block(content_encoding, block)
|
701
|
+
case content_encoding
|
702
|
+
when 'gzip', 'x-gzip'
|
703
|
+
# zlib itself has a functionality to decompress gzip stream.
|
704
|
+
# - zlib 1.2.5 Manual
|
705
|
+
# http://www.zlib.net/manual.html#Advanced
|
706
|
+
# > windowBits can also be greater than 15 for optional gzip decoding. Add 32 to
|
707
|
+
# > windowBits to enable zlib and gzip decoding with automatic header detection,
|
708
|
+
# > or add 16 to decode only the gzip format
|
709
|
+
inflate_stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
|
710
|
+
when 'deflate'
|
711
|
+
inflate_stream = LenientInflater.new
|
712
|
+
else
|
713
|
+
return block
|
714
|
+
end
|
715
|
+
Proc.new { |buf|
|
716
|
+
block.call(inflate_stream.inflate(buf))
|
717
|
+
}
|
718
|
+
end
|
719
|
+
|
722
720
|
def set_header(req)
|
723
721
|
if @requested_version
|
724
722
|
if /^(?:HTTP\/|)(\d+.\d+)$/ =~ @requested_version
|
725
723
|
req.http_version = $1
|
726
724
|
end
|
727
725
|
end
|
728
|
-
if @agent_name
|
726
|
+
if @agent_name && req.header.get('User-Agent').empty?
|
729
727
|
req.header.set('User-Agent', "#{@agent_name} #{LIB_NAME}")
|
730
728
|
end
|
731
|
-
if @from
|
729
|
+
if @from && req.header.get('From').empty?
|
732
730
|
req.header.set('From', @from)
|
733
731
|
end
|
734
732
|
if req.header.get('Accept').empty?
|
@@ -747,30 +745,15 @@ class HTTPClient
|
|
747
745
|
site = @proxy || @dest
|
748
746
|
retry_number = 0
|
749
747
|
begin
|
750
|
-
timeout(@connect_timeout, ConnectTimeoutError) do
|
751
|
-
|
752
|
-
|
753
|
-
|
754
|
-
|
755
|
-
|
756
|
-
|
757
|
-
|
758
|
-
begin
|
759
|
-
@socket.ssl_connect(@dest.host)
|
760
|
-
ensure
|
761
|
-
if $DEBUG
|
762
|
-
warn("Protocol version: #{@socket.ssl_version}")
|
763
|
-
warn("Cipher: #{@socket.ssl_cipher.inspect}")
|
764
|
-
warn("State: #{@socket.ssl_state}")
|
765
|
-
end
|
766
|
-
end
|
767
|
-
@socket.post_connection_check(@dest)
|
768
|
-
@ssl_peer_cert = @socket.peer_cert
|
769
|
-
end
|
748
|
+
::Timeout.timeout(@connect_timeout, ConnectTimeoutError) do
|
749
|
+
if str = @test_loopback_http_response.shift
|
750
|
+
@socket = create_loopback_socket(site.host, site.port, str)
|
751
|
+
elsif https?(@dest)
|
752
|
+
@socket = SSLSocket.create_socket(self)
|
753
|
+
@ssl_peer_cert = @socket.peer_cert
|
754
|
+
else
|
755
|
+
@socket = create_socket(site.host, site.port)
|
770
756
|
end
|
771
|
-
# Use Ruby internal buffering instead of passing data immediately
|
772
|
-
# to the underlying layer
|
773
|
-
# => we need to to call explicitly flush on the socket
|
774
757
|
@socket.sync = @socket_sync
|
775
758
|
end
|
776
759
|
rescue RetryableResponse
|
@@ -792,69 +775,13 @@ class HTTPClient
|
|
792
775
|
@state = :WAIT
|
793
776
|
end
|
794
777
|
|
795
|
-
def create_socket(site)
|
796
|
-
socket = nil
|
797
|
-
begin
|
798
|
-
@debug_dev << "! CONNECT TO #{site.host}:#{site.port}\n" if @debug_dev
|
799
|
-
if str = @test_loopback_http_response.shift
|
800
|
-
socket = LoopBackSocket.new(site.host, site.port, str)
|
801
|
-
elsif @socket_local == Site::EMPTY
|
802
|
-
socket = TCPSocket.new(site.host, site.port)
|
803
|
-
else
|
804
|
-
socket = TCPSocket.new(site.host, site.port, @socket_local.host, @socket_local.port)
|
805
|
-
end
|
806
|
-
if @debug_dev
|
807
|
-
@debug_dev << "! CONNECTION ESTABLISHED\n"
|
808
|
-
socket.extend(DebugSocket)
|
809
|
-
socket.debug_dev = @debug_dev
|
810
|
-
end
|
811
|
-
rescue SystemCallError => e
|
812
|
-
e.message << " (#{site})"
|
813
|
-
raise
|
814
|
-
rescue SocketError => e
|
815
|
-
e.message << " (#{site})"
|
816
|
-
raise
|
817
|
-
end
|
818
|
-
socket
|
819
|
-
end
|
820
|
-
|
821
|
-
# wrap socket with OpenSSL.
|
822
|
-
def create_ssl_socket(raw_socket)
|
823
|
-
SSLSocketWrap.new(raw_socket, @ssl_config, @debug_dev)
|
824
|
-
end
|
825
|
-
|
826
|
-
def connect_ssl_proxy(socket, uri)
|
827
|
-
req = HTTP::Message.new_connect_request(uri)
|
828
|
-
@client.request_filter.each do |filter|
|
829
|
-
filter.filter_request(req)
|
830
|
-
end
|
831
|
-
set_header(req)
|
832
|
-
req.dump(@socket)
|
833
|
-
@socket.flush unless @socket_sync
|
834
|
-
res = HTTP::Message.new_response('')
|
835
|
-
parse_header
|
836
|
-
res.http_version, res.status, res.reason = @version, @status, @reason
|
837
|
-
@headers.each do |key, value|
|
838
|
-
res.header.set(key.to_s, value)
|
839
|
-
end
|
840
|
-
commands = @client.request_filter.collect { |filter|
|
841
|
-
filter.filter_response(req, res)
|
842
|
-
}
|
843
|
-
if commands.find { |command| command == :retry }
|
844
|
-
raise RetryableResponse.new
|
845
|
-
end
|
846
|
-
unless @status == 200
|
847
|
-
raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
|
848
|
-
end
|
849
|
-
end
|
850
|
-
|
851
778
|
# Read status block.
|
852
779
|
def read_header
|
853
780
|
@content_length = nil
|
854
781
|
@chunked = false
|
855
|
-
@
|
782
|
+
@content_encoding = nil
|
856
783
|
@chunk_length = 0
|
857
|
-
parse_header
|
784
|
+
parse_header(@socket)
|
858
785
|
# Header of the request has been parsed.
|
859
786
|
@state = :DATA
|
860
787
|
req = @requests.shift
|
@@ -870,12 +797,12 @@ class HTTPClient
|
|
870
797
|
end
|
871
798
|
|
872
799
|
StatusParseRegexp = %r(\AHTTP/(\d+\.\d+)\s+(\d\d\d)\s*([^\r\n]+)?\r?\n\z)
|
873
|
-
def parse_header
|
874
|
-
timeout(@receive_timeout, ReceiveTimeoutError) do
|
800
|
+
def parse_header(socket)
|
801
|
+
::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
|
875
802
|
initial_line = nil
|
876
803
|
begin
|
877
804
|
begin
|
878
|
-
initial_line =
|
805
|
+
initial_line = socket.gets("\n")
|
879
806
|
if initial_line.nil?
|
880
807
|
close
|
881
808
|
raise KeepAliveDisconnected.new(self)
|
@@ -883,7 +810,7 @@ class HTTPClient
|
|
883
810
|
rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
|
884
811
|
# JRuby can raise IOError instead of ECONNRESET for now
|
885
812
|
close
|
886
|
-
raise KeepAliveDisconnected.new(self)
|
813
|
+
raise KeepAliveDisconnected.new(self, $!)
|
887
814
|
end
|
888
815
|
if StatusParseRegexp !~ initial_line
|
889
816
|
@version = '0.9'
|
@@ -898,7 +825,7 @@ class HTTPClient
|
|
898
825
|
@next_connection = HTTP::Message.keep_alive_enabled?(@version)
|
899
826
|
@headers = []
|
900
827
|
while true
|
901
|
-
line =
|
828
|
+
line = socket.gets("\n")
|
902
829
|
unless line
|
903
830
|
raise BadResponseError.new('unexpected EOF')
|
904
831
|
end
|
@@ -910,7 +837,7 @@ class HTTPClient
|
|
910
837
|
last << line.strip
|
911
838
|
else
|
912
839
|
key, value = line.strip.split(/\s*:\s*/, 2)
|
913
|
-
|
840
|
+
parse_content_header(key, value)
|
914
841
|
@headers << [key, value]
|
915
842
|
end
|
916
843
|
end
|
@@ -923,18 +850,20 @@ class HTTPClient
|
|
923
850
|
((status >= 100 && status < 200) || status == 204 || status == 304)
|
924
851
|
end
|
925
852
|
|
926
|
-
def
|
853
|
+
def parse_content_header(key, value)
|
927
854
|
key = key.downcase
|
928
|
-
|
855
|
+
case key
|
856
|
+
when 'content-length'
|
929
857
|
@content_length = value.to_i
|
930
|
-
|
931
|
-
|
932
|
-
|
933
|
-
|
934
|
-
|
935
|
-
|
936
|
-
|
937
|
-
|
858
|
+
when 'content-encoding'
|
859
|
+
@content_encoding = value.downcase
|
860
|
+
when 'transfer-encoding'
|
861
|
+
if value.downcase == 'chunked'
|
862
|
+
@chunked = true
|
863
|
+
@chunk_length = 0
|
864
|
+
@content_length = nil
|
865
|
+
end
|
866
|
+
when 'connection', 'proxy-connection'
|
938
867
|
if value.downcase == 'keep-alive'
|
939
868
|
@next_connection = true
|
940
869
|
else
|
@@ -946,15 +875,18 @@ class HTTPClient
|
|
946
875
|
def read_body_length(&block)
|
947
876
|
return nil if @content_length == 0
|
948
877
|
while true
|
949
|
-
buf =
|
878
|
+
buf = empty_bin_str
|
950
879
|
maxbytes = @read_block_size
|
951
|
-
maxbytes = @content_length if maxbytes > @content_length
|
952
|
-
timeout(@receive_timeout, ReceiveTimeoutError) do
|
880
|
+
maxbytes = @content_length if maxbytes > @content_length && @content_length > 0
|
881
|
+
::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
|
953
882
|
begin
|
954
883
|
@socket.readpartial(maxbytes, buf)
|
955
884
|
rescue EOFError
|
956
885
|
close
|
957
886
|
buf = nil
|
887
|
+
if @strict_response_size_check
|
888
|
+
raise BadResponseError.new("EOF while reading rest #{@content_length} bytes")
|
889
|
+
end
|
958
890
|
end
|
959
891
|
end
|
960
892
|
if buf && buf.bytesize > 0
|
@@ -969,24 +901,25 @@ class HTTPClient
|
|
969
901
|
|
970
902
|
RS = "\r\n"
|
971
903
|
def read_body_chunked(&block)
|
972
|
-
buf =
|
904
|
+
buf = empty_bin_str
|
973
905
|
while true
|
974
|
-
|
975
|
-
|
976
|
-
|
977
|
-
|
978
|
-
|
979
|
-
|
980
|
-
|
981
|
-
@
|
982
|
-
|
983
|
-
|
984
|
-
|
985
|
-
|
986
|
-
@socket.read(@chunk_length
|
906
|
+
::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
|
907
|
+
len = @socket.gets(RS)
|
908
|
+
if len.nil? # EOF
|
909
|
+
close
|
910
|
+
return
|
911
|
+
end
|
912
|
+
@chunk_length = len.hex
|
913
|
+
if @chunk_length == 0
|
914
|
+
@content_length = 0
|
915
|
+
@socket.gets(RS)
|
916
|
+
return
|
917
|
+
end
|
918
|
+
@socket.read(@chunk_length, buf)
|
919
|
+
@socket.read(2)
|
987
920
|
end
|
988
921
|
unless buf.empty?
|
989
|
-
yield buf
|
922
|
+
yield buf
|
990
923
|
end
|
991
924
|
end
|
992
925
|
end
|
@@ -997,12 +930,15 @@ class HTTPClient
|
|
997
930
|
@readbuf = nil
|
998
931
|
end
|
999
932
|
while true
|
1000
|
-
buf =
|
1001
|
-
timeout(@receive_timeout, ReceiveTimeoutError) do
|
933
|
+
buf = empty_bin_str
|
934
|
+
::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
|
1002
935
|
begin
|
1003
936
|
@socket.readpartial(@read_block_size, buf)
|
1004
937
|
rescue EOFError
|
1005
938
|
buf = nil
|
939
|
+
if @strict_response_size_check
|
940
|
+
raise BadResponseError.new("EOF while reading chunked response")
|
941
|
+
end
|
1006
942
|
end
|
1007
943
|
end
|
1008
944
|
if buf && buf.bytesize > 0
|
@@ -1012,6 +948,12 @@ class HTTPClient
|
|
1012
948
|
end
|
1013
949
|
end
|
1014
950
|
end
|
951
|
+
|
952
|
+
def empty_bin_str
|
953
|
+
str = ''
|
954
|
+
str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
|
955
|
+
str
|
956
|
+
end
|
1015
957
|
end
|
1016
958
|
|
1017
959
|
|