httpclient-fixcerts 2.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +98 -0
- data/bin/httpclient +77 -0
- data/bin/jsonclient +85 -0
- data/lib/hexdump.rb +50 -0
- data/lib/http-access2/cookie.rb +1 -0
- data/lib/http-access2/http.rb +1 -0
- data/lib/http-access2.rb +55 -0
- data/lib/httpclient/auth.rb +924 -0
- data/lib/httpclient/cacert.pem +3952 -0
- data/lib/httpclient/cacert1024.pem +3866 -0
- data/lib/httpclient/connection.rb +88 -0
- data/lib/httpclient/cookie.rb +220 -0
- data/lib/httpclient/http.rb +1082 -0
- data/lib/httpclient/include_client.rb +85 -0
- data/lib/httpclient/jruby_ssl_socket.rb +594 -0
- data/lib/httpclient/session.rb +960 -0
- data/lib/httpclient/ssl_config.rb +433 -0
- data/lib/httpclient/ssl_socket.rb +150 -0
- data/lib/httpclient/timeout.rb +140 -0
- data/lib/httpclient/util.rb +222 -0
- data/lib/httpclient/version.rb +3 -0
- data/lib/httpclient/webagent-cookie.rb +459 -0
- data/lib/httpclient.rb +1332 -0
- data/lib/jsonclient.rb +66 -0
- data/lib/oauthclient.rb +111 -0
- data/sample/async.rb +8 -0
- data/sample/auth.rb +11 -0
- data/sample/cookie.rb +18 -0
- data/sample/dav.rb +103 -0
- data/sample/howto.rb +49 -0
- data/sample/jsonclient.rb +67 -0
- data/sample/oauth_buzz.rb +57 -0
- data/sample/oauth_friendfeed.rb +59 -0
- data/sample/oauth_twitter.rb +61 -0
- data/sample/ssl/0cert.pem +22 -0
- data/sample/ssl/0key.pem +30 -0
- data/sample/ssl/1000cert.pem +19 -0
- data/sample/ssl/1000key.pem +18 -0
- data/sample/ssl/htdocs/index.html +10 -0
- data/sample/ssl/ssl_client.rb +22 -0
- data/sample/ssl/webrick_httpsd.rb +29 -0
- data/sample/stream.rb +21 -0
- data/sample/thread.rb +27 -0
- data/sample/wcat.rb +21 -0
- data/test/ca-chain.pem +44 -0
- data/test/ca.cert +23 -0
- data/test/client-pass.key +18 -0
- data/test/client.cert +19 -0
- data/test/client.key +15 -0
- data/test/helper.rb +131 -0
- data/test/htdigest +1 -0
- data/test/htpasswd +2 -0
- data/test/jruby_ssl_socket/test_pemutils.rb +32 -0
- data/test/runner.rb +2 -0
- data/test/server.cert +19 -0
- data/test/server.key +15 -0
- data/test/sslsvr.rb +65 -0
- data/test/subca.cert +21 -0
- data/test/test_auth.rb +492 -0
- data/test/test_cookie.rb +309 -0
- data/test/test_hexdump.rb +14 -0
- data/test/test_http-access2.rb +508 -0
- data/test/test_httpclient.rb +2145 -0
- data/test/test_include_client.rb +52 -0
- data/test/test_jsonclient.rb +98 -0
- data/test/test_ssl.rb +562 -0
- data/test/test_webagent-cookie.rb +465 -0
- metadata +124 -0
@@ -0,0 +1,960 @@
|
|
1
|
+
# HTTPClient - HTTP client library.
|
2
|
+
# Copyright (C) 2000-2015 NAKAMURA, Hiroshi <nahi@ruby-lang.org>.
|
3
|
+
#
|
4
|
+
# This program is copyrighted free software by NAKAMURA, Hiroshi. You can
|
5
|
+
# redistribute it and/or modify it under the same terms of Ruby's license;
|
6
|
+
# either the dual license version in 2003, or any later version.
|
7
|
+
#
|
8
|
+
# httpclient/session.rb is based on http-access.rb in http-access/0.0.4. Some
|
9
|
+
# part of it is copyrighted by Maebashi-san who made and published
|
10
|
+
# http-access/0.0.4. http-access/0.0.4 did not include license notice but when
|
11
|
+
# I asked Maebashi-san he agreed that I can redistribute it under the same terms
|
12
|
+
# of Ruby. Many thanks to Maebashi-san.
|
13
|
+
|
14
|
+
|
15
|
+
require 'socket'
|
16
|
+
require 'thread'
|
17
|
+
require 'timeout'
|
18
|
+
require 'stringio'
|
19
|
+
require 'zlib'
|
20
|
+
|
21
|
+
require 'httpclient/timeout' # TODO: remove this once we drop 1.8 support
|
22
|
+
require 'httpclient/ssl_config'
|
23
|
+
require 'httpclient/http'
|
24
|
+
if defined? JRUBY_VERSION
|
25
|
+
require 'httpclient/jruby_ssl_socket'
|
26
|
+
else
|
27
|
+
require 'httpclient/ssl_socket'
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
class HTTPClient
|
32
|
+
|
33
|
+
|
34
|
+
# Represents a Site: protocol scheme, host String and port Number.
|
35
|
+
class Site
|
36
|
+
# Protocol scheme.
|
37
|
+
attr_accessor :scheme
|
38
|
+
# Host String.
|
39
|
+
attr_accessor :host
|
40
|
+
alias hostname host
|
41
|
+
# Port number.
|
42
|
+
attr_accessor :port
|
43
|
+
|
44
|
+
# Creates a new Site based on the given URI.
|
45
|
+
def initialize(uri = nil)
|
46
|
+
if uri
|
47
|
+
@scheme = uri.scheme || 'tcp'
|
48
|
+
@host = uri.hostname || '0.0.0.0'
|
49
|
+
@port = uri.port.to_i
|
50
|
+
else
|
51
|
+
@scheme = 'tcp'
|
52
|
+
@host = '0.0.0.0'
|
53
|
+
@port = 0
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# Returns address String.
|
58
|
+
def addr
|
59
|
+
"#{@scheme}://#{@host}:#{@port.to_s}"
|
60
|
+
end
|
61
|
+
|
62
|
+
# Returns true is scheme, host and port are '=='
|
63
|
+
def ==(rhs)
|
64
|
+
(@scheme == rhs.scheme) and (@host == rhs.host) and (@port == rhs.port)
|
65
|
+
end
|
66
|
+
|
67
|
+
# Same as ==.
|
68
|
+
def eql?(rhs)
|
69
|
+
self == rhs
|
70
|
+
end
|
71
|
+
|
72
|
+
def hash # :nodoc:
|
73
|
+
[@scheme, @host, @port].hash
|
74
|
+
end
|
75
|
+
|
76
|
+
def to_s # :nodoc:
|
77
|
+
addr
|
78
|
+
end
|
79
|
+
|
80
|
+
# Returns true if scheme, host and port of the given URI matches with this.
|
81
|
+
def match(uri)
|
82
|
+
(@scheme == uri.scheme) and (@host == uri.host) and (@port == uri.port.to_i)
|
83
|
+
end
|
84
|
+
|
85
|
+
def inspect # :nodoc:
|
86
|
+
sprintf("#<%s:0x%x %s>", self.class.name, __id__, addr)
|
87
|
+
end
|
88
|
+
|
89
|
+
EMPTY = Site.new.freeze
|
90
|
+
end
|
91
|
+
|
92
|
+
|
93
|
+
# Manages sessions for a HTTPClient instance.
|
94
|
+
class SessionManager
|
95
|
+
# Name of this client. Used for 'User-Agent' header in HTTP request.
|
96
|
+
attr_accessor :agent_name
|
97
|
+
# Owner of this client. Used for 'From' header in HTTP request.
|
98
|
+
attr_accessor :from
|
99
|
+
|
100
|
+
# Requested protocol version
|
101
|
+
attr_accessor :protocol_version
|
102
|
+
# Chunk size for chunked request
|
103
|
+
attr_accessor :chunk_size
|
104
|
+
# Device for dumping log for debugging
|
105
|
+
attr_accessor :debug_dev
|
106
|
+
# Boolean value for Socket#sync
|
107
|
+
attr_accessor :socket_sync
|
108
|
+
# Boolean value to send TCP keepalive packets; no timing settings exist at present
|
109
|
+
attr_accessor :tcp_keepalive
|
110
|
+
|
111
|
+
attr_accessor :connect_timeout
|
112
|
+
# Maximum retry count. 0 for infinite.
|
113
|
+
attr_accessor :connect_retry
|
114
|
+
attr_accessor :send_timeout
|
115
|
+
attr_accessor :receive_timeout
|
116
|
+
attr_accessor :keep_alive_timeout
|
117
|
+
attr_accessor :read_block_size
|
118
|
+
attr_accessor :protocol_retry_count
|
119
|
+
|
120
|
+
# Raise BadResponseError if response size does not match with Content-Length header in response.
|
121
|
+
attr_accessor :strict_response_size_check
|
122
|
+
|
123
|
+
# Local address to bind local side of the socket to
|
124
|
+
attr_accessor :socket_local
|
125
|
+
|
126
|
+
attr_accessor :ssl_config
|
127
|
+
|
128
|
+
attr_reader :test_loopback_http_response
|
129
|
+
|
130
|
+
attr_accessor :transparent_gzip_decompression
|
131
|
+
|
132
|
+
def initialize(client)
|
133
|
+
@client = client
|
134
|
+
@proxy = client.proxy
|
135
|
+
|
136
|
+
@agent_name = nil
|
137
|
+
@from = nil
|
138
|
+
|
139
|
+
@protocol_version = nil
|
140
|
+
@debug_dev = client.debug_dev
|
141
|
+
@socket_sync = true
|
142
|
+
@tcp_keepalive = false
|
143
|
+
@chunk_size = ::HTTP::Message::Body::DEFAULT_CHUNK_SIZE
|
144
|
+
|
145
|
+
@connect_timeout = 60
|
146
|
+
@connect_retry = 1
|
147
|
+
@send_timeout = 120
|
148
|
+
@receive_timeout = 60 # For each read_block_size bytes
|
149
|
+
@keep_alive_timeout = 15 # '15' is from Apache 2 default
|
150
|
+
@read_block_size = 1024 * 16 # follows net/http change in 1.8.7
|
151
|
+
@protocol_retry_count = 5
|
152
|
+
|
153
|
+
@ssl_config = nil
|
154
|
+
@test_loopback_http_response = []
|
155
|
+
|
156
|
+
@transparent_gzip_decompression = false
|
157
|
+
@strict_response_size_check = false
|
158
|
+
@socket_local = Site.new
|
159
|
+
|
160
|
+
@sess_pool = {}
|
161
|
+
@sess_pool_mutex = Mutex.new
|
162
|
+
@sess_pool_last_checked = Time.now
|
163
|
+
end
|
164
|
+
|
165
|
+
def proxy=(proxy)
|
166
|
+
if proxy.nil?
|
167
|
+
@proxy = nil
|
168
|
+
else
|
169
|
+
@proxy = Site.new(proxy)
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
def query(req, via_proxy)
|
174
|
+
req.http_body.chunk_size = @chunk_size if req.http_body
|
175
|
+
sess = get_session(req, via_proxy)
|
176
|
+
begin
|
177
|
+
sess.query(req)
|
178
|
+
rescue
|
179
|
+
sess.close
|
180
|
+
raise
|
181
|
+
end
|
182
|
+
sess
|
183
|
+
end
|
184
|
+
|
185
|
+
def reset(uri)
|
186
|
+
site = Site.new(uri)
|
187
|
+
close(site)
|
188
|
+
end
|
189
|
+
|
190
|
+
def reset_all
|
191
|
+
close_all
|
192
|
+
end
|
193
|
+
|
194
|
+
# assert: sess.last_used must not be nil
|
195
|
+
def keep(sess)
|
196
|
+
add_cached_session(sess)
|
197
|
+
end
|
198
|
+
|
199
|
+
private
|
200
|
+
|
201
|
+
# TODO: create PR for webmock's httpclient adapter to use get_session
|
202
|
+
# instead of open so that we can remove duplicated Site creation for
|
203
|
+
# each session.
|
204
|
+
def get_session(req, via_proxy = false)
|
205
|
+
uri = req.header.request_uri
|
206
|
+
if uri.scheme.nil?
|
207
|
+
raise ArgumentError.new("Request URI must have schema. Possibly add 'http://' to the request URI?")
|
208
|
+
end
|
209
|
+
site = Site.new(uri)
|
210
|
+
if cached = get_cached_session(site)
|
211
|
+
cached
|
212
|
+
else
|
213
|
+
open(uri, via_proxy)
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def open(uri, via_proxy = false)
|
218
|
+
site = Site.new(uri)
|
219
|
+
sess = Session.new(@client, site, @agent_name, @from)
|
220
|
+
sess.proxy = via_proxy ? @proxy : nil
|
221
|
+
sess.socket_sync = @socket_sync
|
222
|
+
sess.tcp_keepalive = @tcp_keepalive
|
223
|
+
sess.requested_version = @protocol_version if @protocol_version
|
224
|
+
sess.connect_timeout = @connect_timeout
|
225
|
+
sess.connect_retry = @connect_retry
|
226
|
+
sess.send_timeout = @send_timeout
|
227
|
+
sess.receive_timeout = @receive_timeout
|
228
|
+
sess.read_block_size = @read_block_size
|
229
|
+
sess.protocol_retry_count = @protocol_retry_count
|
230
|
+
sess.ssl_config = @ssl_config
|
231
|
+
sess.debug_dev = @debug_dev
|
232
|
+
sess.strict_response_size_check = @strict_response_size_check
|
233
|
+
sess.socket_local = @socket_local
|
234
|
+
sess.test_loopback_http_response = @test_loopback_http_response
|
235
|
+
sess.transparent_gzip_decompression = @transparent_gzip_decompression
|
236
|
+
sess
|
237
|
+
end
|
238
|
+
|
239
|
+
def close_all
|
240
|
+
@sess_pool_mutex.synchronize do
|
241
|
+
@sess_pool.each do |site, pool|
|
242
|
+
pool.each do |sess|
|
243
|
+
sess.close
|
244
|
+
end
|
245
|
+
end
|
246
|
+
end
|
247
|
+
@sess_pool.clear
|
248
|
+
end
|
249
|
+
|
250
|
+
# This method might not work as you expected...
|
251
|
+
def close(dest)
|
252
|
+
if cached = get_cached_session(Site.new(dest))
|
253
|
+
cached.close
|
254
|
+
true
|
255
|
+
else
|
256
|
+
false
|
257
|
+
end
|
258
|
+
end
|
259
|
+
|
260
|
+
def get_cached_session(site)
|
261
|
+
if Thread.current[:HTTPClient_AcquireNewConnection]
|
262
|
+
return nil
|
263
|
+
end
|
264
|
+
@sess_pool_mutex.synchronize do
|
265
|
+
now = Time.now
|
266
|
+
if now > @sess_pool_last_checked + @keep_alive_timeout
|
267
|
+
scrub_cached_session(now)
|
268
|
+
@sess_pool_last_checked = now
|
269
|
+
end
|
270
|
+
if pool = @sess_pool[site]
|
271
|
+
pool.each_with_index do |sess, idx|
|
272
|
+
if valid_session?(sess, now)
|
273
|
+
return pool.slice!(idx)
|
274
|
+
end
|
275
|
+
end
|
276
|
+
end
|
277
|
+
end
|
278
|
+
nil
|
279
|
+
end
|
280
|
+
|
281
|
+
def scrub_cached_session(now)
|
282
|
+
@sess_pool.each do |site, pool|
|
283
|
+
pool.replace(pool.select { |sess|
|
284
|
+
if valid_session?(sess, now)
|
285
|
+
true
|
286
|
+
else
|
287
|
+
sess.close # close & remove from the pool
|
288
|
+
false
|
289
|
+
end
|
290
|
+
})
|
291
|
+
end
|
292
|
+
end
|
293
|
+
|
294
|
+
def valid_session?(sess, now)
|
295
|
+
(now <= sess.last_used + @keep_alive_timeout)
|
296
|
+
end
|
297
|
+
|
298
|
+
def add_cached_session(sess)
|
299
|
+
@sess_pool_mutex.synchronize do
|
300
|
+
(@sess_pool[sess.dest] ||= []).unshift(sess)
|
301
|
+
end
|
302
|
+
end
|
303
|
+
end
|
304
|
+
|
305
|
+
|
306
|
+
# Wraps up a Socket for method interception.
|
307
|
+
module SocketWrap
|
308
|
+
def initialize(socket, *args)
|
309
|
+
super(*args)
|
310
|
+
@socket = socket
|
311
|
+
end
|
312
|
+
|
313
|
+
def close
|
314
|
+
@socket.close
|
315
|
+
end
|
316
|
+
|
317
|
+
def closed?
|
318
|
+
@socket.closed?
|
319
|
+
end
|
320
|
+
|
321
|
+
def eof?
|
322
|
+
@socket.eof?
|
323
|
+
end
|
324
|
+
|
325
|
+
def gets(rs)
|
326
|
+
@socket.gets(rs)
|
327
|
+
end
|
328
|
+
|
329
|
+
def read(size, buf = nil)
|
330
|
+
@socket.read(size, buf)
|
331
|
+
end
|
332
|
+
|
333
|
+
def readpartial(size, buf = nil)
|
334
|
+
# StringIO doesn't support :readpartial
|
335
|
+
if @socket.respond_to?(:readpartial)
|
336
|
+
@socket.readpartial(size, buf)
|
337
|
+
else
|
338
|
+
@socket.read(size, buf)
|
339
|
+
end
|
340
|
+
end
|
341
|
+
|
342
|
+
def <<(str)
|
343
|
+
@socket << str
|
344
|
+
end
|
345
|
+
|
346
|
+
def flush
|
347
|
+
@socket.flush
|
348
|
+
end
|
349
|
+
|
350
|
+
def sync
|
351
|
+
@socket.sync
|
352
|
+
end
|
353
|
+
|
354
|
+
def sync=(sync)
|
355
|
+
@socket.sync = sync
|
356
|
+
end
|
357
|
+
end
|
358
|
+
|
359
|
+
|
360
|
+
# Module for intercepting Socket methods and dumps in/out to given debugging
|
361
|
+
# device. debug_dev must respond to <<.
|
362
|
+
module DebugSocket
|
363
|
+
extend SocketWrap
|
364
|
+
|
365
|
+
def debug_dev=(debug_dev)
|
366
|
+
@debug_dev = debug_dev
|
367
|
+
end
|
368
|
+
|
369
|
+
def close
|
370
|
+
super
|
371
|
+
debug("! CONNECTION CLOSED\n")
|
372
|
+
end
|
373
|
+
|
374
|
+
def gets(rs)
|
375
|
+
str = super
|
376
|
+
debug(str)
|
377
|
+
str
|
378
|
+
end
|
379
|
+
|
380
|
+
def read(size, buf = nil)
|
381
|
+
str = super
|
382
|
+
debug(str)
|
383
|
+
str
|
384
|
+
end
|
385
|
+
|
386
|
+
def readpartial(size, buf = nil)
|
387
|
+
str = super
|
388
|
+
debug(str)
|
389
|
+
str
|
390
|
+
end
|
391
|
+
|
392
|
+
def <<(str)
|
393
|
+
super
|
394
|
+
debug(str)
|
395
|
+
end
|
396
|
+
|
397
|
+
private
|
398
|
+
|
399
|
+
def debug(str)
|
400
|
+
if str && @debug_dev
|
401
|
+
if str.index("\0")
|
402
|
+
require 'hexdump'
|
403
|
+
str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
|
404
|
+
@debug_dev << HexDump.encode(str).join("\n")
|
405
|
+
else
|
406
|
+
@debug_dev << str
|
407
|
+
end
|
408
|
+
end
|
409
|
+
end
|
410
|
+
end
|
411
|
+
|
412
|
+
|
413
|
+
# Dummy Socket for emulating loopback test.
|
414
|
+
class LoopBackSocket
|
415
|
+
include SocketWrap
|
416
|
+
|
417
|
+
def initialize(host, port, response)
|
418
|
+
super(response.is_a?(StringIO) ? response : StringIO.new(response))
|
419
|
+
@host = host
|
420
|
+
@port = port
|
421
|
+
end
|
422
|
+
|
423
|
+
def <<(str)
|
424
|
+
# ignored
|
425
|
+
end
|
426
|
+
|
427
|
+
def peer_cert
|
428
|
+
nil
|
429
|
+
end
|
430
|
+
end
|
431
|
+
|
432
|
+
|
433
|
+
# Manages a HTTP session with a Site.
|
434
|
+
class Session
|
435
|
+
include HTTPClient::Timeout
|
436
|
+
include Util
|
437
|
+
|
438
|
+
# Destination site
|
439
|
+
attr_reader :dest
|
440
|
+
# Proxy site
|
441
|
+
attr_accessor :proxy
|
442
|
+
# Boolean value for Socket#sync
|
443
|
+
attr_accessor :socket_sync
|
444
|
+
# Boolean value to send TCP keepalive packets; no timing settings exist at present
|
445
|
+
attr_accessor :tcp_keepalive
|
446
|
+
# Requested protocol version
|
447
|
+
attr_accessor :requested_version
|
448
|
+
# Device for dumping log for debugging
|
449
|
+
attr_accessor :debug_dev
|
450
|
+
|
451
|
+
attr_accessor :connect_timeout
|
452
|
+
attr_accessor :connect_retry
|
453
|
+
attr_accessor :send_timeout
|
454
|
+
attr_accessor :receive_timeout
|
455
|
+
attr_accessor :read_block_size
|
456
|
+
attr_accessor :protocol_retry_count
|
457
|
+
|
458
|
+
attr_accessor :strict_response_size_check
|
459
|
+
attr_accessor :socket_local
|
460
|
+
|
461
|
+
attr_accessor :ssl_config
|
462
|
+
attr_reader :ssl_peer_cert
|
463
|
+
attr_accessor :test_loopback_http_response
|
464
|
+
|
465
|
+
attr_accessor :transparent_gzip_decompression
|
466
|
+
attr_reader :last_used
|
467
|
+
|
468
|
+
def initialize(client, dest, agent_name, from)
|
469
|
+
@client = client
|
470
|
+
@dest = dest
|
471
|
+
@proxy = nil
|
472
|
+
@socket_sync = true
|
473
|
+
@tcp_keepalive = false
|
474
|
+
@requested_version = nil
|
475
|
+
|
476
|
+
@debug_dev = nil
|
477
|
+
|
478
|
+
@connect_timeout = nil
|
479
|
+
@connect_retry = 1
|
480
|
+
@send_timeout = nil
|
481
|
+
@receive_timeout = nil
|
482
|
+
@read_block_size = nil
|
483
|
+
@protocol_retry_count = 5
|
484
|
+
|
485
|
+
@ssl_config = nil
|
486
|
+
@ssl_peer_cert = nil
|
487
|
+
|
488
|
+
@test_loopback_http_response = nil
|
489
|
+
@strict_response_size_check = false
|
490
|
+
@socket_local = Site::EMPTY
|
491
|
+
|
492
|
+
@agent_name = agent_name
|
493
|
+
@from = from
|
494
|
+
@state = :INIT
|
495
|
+
|
496
|
+
@requests = []
|
497
|
+
|
498
|
+
@status = nil
|
499
|
+
@reason = nil
|
500
|
+
@headers = []
|
501
|
+
|
502
|
+
@socket = nil
|
503
|
+
@readbuf = nil
|
504
|
+
|
505
|
+
@transparent_gzip_decompression = false
|
506
|
+
@last_used = nil
|
507
|
+
end
|
508
|
+
|
509
|
+
# Send a request to the server
|
510
|
+
def query(req)
|
511
|
+
connect if @state == :INIT
|
512
|
+
# Use absolute URI (not absolute path) iif via proxy AND not HTTPS.
|
513
|
+
req.header.request_absolute_uri = !@proxy.nil? && !https?(@dest)
|
514
|
+
begin
|
515
|
+
::Timeout.timeout(@send_timeout, SendTimeoutError) do
|
516
|
+
set_header(req)
|
517
|
+
req.dump(@socket)
|
518
|
+
# flush the IO stream as IO::sync mode is false
|
519
|
+
@socket.flush unless @socket_sync
|
520
|
+
end
|
521
|
+
rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
|
522
|
+
# JRuby can raise IOError instead of ECONNRESET for now
|
523
|
+
close
|
524
|
+
raise KeepAliveDisconnected.new(self, $!)
|
525
|
+
rescue HTTPClient::TimeoutError
|
526
|
+
close
|
527
|
+
raise
|
528
|
+
rescue => e
|
529
|
+
close
|
530
|
+
if SSLEnabled and e.is_a?(OpenSSL::SSL::SSLError)
|
531
|
+
raise KeepAliveDisconnected.new(self, e)
|
532
|
+
else
|
533
|
+
raise
|
534
|
+
end
|
535
|
+
end
|
536
|
+
|
537
|
+
@state = :META if @state == :WAIT
|
538
|
+
@next_connection = nil
|
539
|
+
@requests.push(req)
|
540
|
+
@last_used = Time.now
|
541
|
+
end
|
542
|
+
|
543
|
+
def close
|
544
|
+
if !@socket.nil? and !@socket.closed?
|
545
|
+
# @socket.flush may block when it the socket is already closed by
|
546
|
+
# foreign host and the client runs under MT-condition.
|
547
|
+
@socket.close
|
548
|
+
end
|
549
|
+
@state = :INIT
|
550
|
+
end
|
551
|
+
|
552
|
+
def closed?
|
553
|
+
@state == :INIT
|
554
|
+
end
|
555
|
+
|
556
|
+
def get_header
|
557
|
+
begin
|
558
|
+
if @state != :META
|
559
|
+
raise RuntimeError.new("get_status must be called at the beginning of a session")
|
560
|
+
end
|
561
|
+
read_header
|
562
|
+
rescue
|
563
|
+
close
|
564
|
+
raise
|
565
|
+
end
|
566
|
+
[@version, @status, @reason, @headers]
|
567
|
+
end
|
568
|
+
|
569
|
+
def eof?
|
570
|
+
if !@content_length.nil?
|
571
|
+
@content_length == 0
|
572
|
+
else
|
573
|
+
@socket.closed? or @socket.eof?
|
574
|
+
end
|
575
|
+
end
|
576
|
+
|
577
|
+
def get_body(&block)
|
578
|
+
begin
|
579
|
+
read_header if @state == :META
|
580
|
+
return nil if @state != :DATA
|
581
|
+
if @transparent_gzip_decompression
|
582
|
+
block = content_inflater_block(@content_encoding, block)
|
583
|
+
end
|
584
|
+
if @chunked
|
585
|
+
read_body_chunked(&block)
|
586
|
+
elsif @content_length
|
587
|
+
read_body_length(&block)
|
588
|
+
else
|
589
|
+
read_body_rest(&block)
|
590
|
+
end
|
591
|
+
rescue
|
592
|
+
close
|
593
|
+
raise
|
594
|
+
end
|
595
|
+
if eof?
|
596
|
+
if @next_connection
|
597
|
+
@state = :WAIT
|
598
|
+
else
|
599
|
+
close
|
600
|
+
end
|
601
|
+
end
|
602
|
+
nil
|
603
|
+
end
|
604
|
+
|
605
|
+
def create_socket(host, port)
|
606
|
+
socket = nil
|
607
|
+
begin
|
608
|
+
@debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
|
609
|
+
clean_host = host.delete("[]")
|
610
|
+
if @socket_local == Site::EMPTY
|
611
|
+
socket = TCPSocket.new(clean_host, port)
|
612
|
+
else
|
613
|
+
clean_local = @socket_local.host.delete("[]")
|
614
|
+
socket = TCPSocket.new(clean_host, port, clean_local, @socket_local.port)
|
615
|
+
end
|
616
|
+
socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) if @tcp_keepalive
|
617
|
+
if @debug_dev
|
618
|
+
@debug_dev << "! CONNECTION ESTABLISHED\n"
|
619
|
+
socket.extend(DebugSocket)
|
620
|
+
socket.debug_dev = @debug_dev
|
621
|
+
end
|
622
|
+
rescue SystemCallError => e
|
623
|
+
raise e.class, e.message + " (#{host}:#{port})"
|
624
|
+
rescue SocketError => e
|
625
|
+
raise e.class, e.message + " (#{host}:#{port})"
|
626
|
+
end
|
627
|
+
socket
|
628
|
+
end
|
629
|
+
|
630
|
+
def create_loopback_socket(host, port, str)
|
631
|
+
@debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
|
632
|
+
socket = LoopBackSocket.new(host, port, str)
|
633
|
+
if @debug_dev
|
634
|
+
@debug_dev << "! CONNECTION ESTABLISHED\n"
|
635
|
+
socket.extend(DebugSocket)
|
636
|
+
socket.debug_dev = @debug_dev
|
637
|
+
end
|
638
|
+
if https?(@dest) && @proxy
|
639
|
+
connect_ssl_proxy(socket, Util.urify(@dest.to_s))
|
640
|
+
end
|
641
|
+
socket
|
642
|
+
end
|
643
|
+
|
644
|
+
def connect_ssl_proxy(socket, uri)
|
645
|
+
req = HTTP::Message.new_connect_request(uri)
|
646
|
+
@client.request_filter.each do |filter|
|
647
|
+
filter.filter_request(req)
|
648
|
+
end
|
649
|
+
set_header(req)
|
650
|
+
req.dump(socket)
|
651
|
+
socket.flush unless @socket_sync
|
652
|
+
res = HTTP::Message.new_response('')
|
653
|
+
parse_header(socket)
|
654
|
+
res.http_version, res.status, res.reason = @version, @status, @reason
|
655
|
+
@headers.each do |key, value|
|
656
|
+
res.header.set(key.to_s, value)
|
657
|
+
end
|
658
|
+
commands = @client.request_filter.collect { |filter|
|
659
|
+
filter.filter_response(req, res)
|
660
|
+
}
|
661
|
+
if commands.find { |command| command == :retry }
|
662
|
+
raise RetryableResponse.new(res)
|
663
|
+
end
|
664
|
+
unless @status == 200
|
665
|
+
raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
|
666
|
+
end
|
667
|
+
end
|
668
|
+
|
669
|
+
private
|
670
|
+
|
671
|
+
# This inflater allows deflate compression with/without zlib header
|
672
|
+
class LenientInflater
|
673
|
+
def initialize
|
674
|
+
@inflater = Zlib::Inflate.new(Zlib::MAX_WBITS)
|
675
|
+
@first = true
|
676
|
+
end
|
677
|
+
|
678
|
+
def inflate(body)
|
679
|
+
if @first
|
680
|
+
first_inflate(body)
|
681
|
+
else
|
682
|
+
@inflater.inflate(body)
|
683
|
+
end
|
684
|
+
end
|
685
|
+
|
686
|
+
private
|
687
|
+
|
688
|
+
def first_inflate(body)
|
689
|
+
@first = false
|
690
|
+
begin
|
691
|
+
@inflater.inflate(body)
|
692
|
+
rescue Zlib::DataError
|
693
|
+
# fallback to deflate without zlib header
|
694
|
+
@inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS)
|
695
|
+
@inflater.inflate(body)
|
696
|
+
end
|
697
|
+
end
|
698
|
+
end
|
699
|
+
|
700
|
+
def content_inflater_block(content_encoding, block)
|
701
|
+
case content_encoding
|
702
|
+
when 'gzip', 'x-gzip'
|
703
|
+
# zlib itself has a functionality to decompress gzip stream.
|
704
|
+
# - zlib 1.2.5 Manual
|
705
|
+
# http://www.zlib.net/manual.html#Advanced
|
706
|
+
# > windowBits can also be greater than 15 for optional gzip decoding. Add 32 to
|
707
|
+
# > windowBits to enable zlib and gzip decoding with automatic header detection,
|
708
|
+
# > or add 16 to decode only the gzip format
|
709
|
+
inflate_stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
|
710
|
+
when 'deflate'
|
711
|
+
inflate_stream = LenientInflater.new
|
712
|
+
else
|
713
|
+
return block
|
714
|
+
end
|
715
|
+
Proc.new { |buf|
|
716
|
+
block.call(inflate_stream.inflate(buf))
|
717
|
+
}
|
718
|
+
end
|
719
|
+
|
720
|
+
def set_header(req)
|
721
|
+
if @requested_version
|
722
|
+
if /^(?:HTTP\/|)(\d+.\d+)$/ =~ @requested_version
|
723
|
+
req.http_version = $1
|
724
|
+
end
|
725
|
+
end
|
726
|
+
if @agent_name && req.header.get('User-Agent').empty?
|
727
|
+
req.header.set('User-Agent', "#{@agent_name} #{LIB_NAME}")
|
728
|
+
end
|
729
|
+
if @from && req.header.get('From').empty?
|
730
|
+
req.header.set('From', @from)
|
731
|
+
end
|
732
|
+
if req.header.get('Accept').empty?
|
733
|
+
req.header.set('Accept', '*/*')
|
734
|
+
end
|
735
|
+
if @transparent_gzip_decompression
|
736
|
+
req.header.set('Accept-Encoding', 'gzip,deflate')
|
737
|
+
end
|
738
|
+
if req.header.get('Date').empty?
|
739
|
+
req.header.set_date_header
|
740
|
+
end
|
741
|
+
end
|
742
|
+
|
743
|
+
# Connect to the server
|
744
|
+
def connect
|
745
|
+
site = @proxy || @dest
|
746
|
+
retry_number = 0
|
747
|
+
begin
|
748
|
+
::Timeout.timeout(@connect_timeout, ConnectTimeoutError) do
|
749
|
+
if str = @test_loopback_http_response.shift
|
750
|
+
@socket = create_loopback_socket(site.host, site.port, str)
|
751
|
+
elsif https?(@dest)
|
752
|
+
@socket = SSLSocket.create_socket(self)
|
753
|
+
@ssl_peer_cert = @socket.peer_cert
|
754
|
+
else
|
755
|
+
@socket = create_socket(site.host, site.port)
|
756
|
+
end
|
757
|
+
@socket.sync = @socket_sync
|
758
|
+
end
|
759
|
+
rescue RetryableResponse
|
760
|
+
retry_number += 1
|
761
|
+
if retry_number < @protocol_retry_count
|
762
|
+
retry
|
763
|
+
end
|
764
|
+
raise BadResponseError.new("connect to the server failed with status #{@status} #{@reason}")
|
765
|
+
rescue TimeoutError
|
766
|
+
if @connect_retry == 0
|
767
|
+
retry
|
768
|
+
else
|
769
|
+
retry_number += 1
|
770
|
+
retry if retry_number < @connect_retry
|
771
|
+
end
|
772
|
+
close
|
773
|
+
raise
|
774
|
+
end
|
775
|
+
@state = :WAIT
|
776
|
+
end
|
777
|
+
|
778
|
+
# Read status block.
|
779
|
+
def read_header
|
780
|
+
@content_length = nil
|
781
|
+
@chunked = false
|
782
|
+
@content_encoding = nil
|
783
|
+
@chunk_length = 0
|
784
|
+
parse_header(@socket)
|
785
|
+
# Header of the request has been parsed.
|
786
|
+
@state = :DATA
|
787
|
+
req = @requests.shift
|
788
|
+
if req.header.request_method == 'HEAD' or no_message_body?(@status)
|
789
|
+
@content_length = 0
|
790
|
+
if @next_connection
|
791
|
+
@state = :WAIT
|
792
|
+
else
|
793
|
+
close
|
794
|
+
end
|
795
|
+
end
|
796
|
+
@next_connection = false if !@content_length and !@chunked
|
797
|
+
end
|
798
|
+
|
799
|
+
StatusParseRegexp = %r(\AHTTP/(\d+\.\d+)\s+(\d\d\d)\s*([^\r\n]+)?\r?\n\z)
|
800
|
+
def parse_header(socket)
|
801
|
+
::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
|
802
|
+
initial_line = nil
|
803
|
+
begin
|
804
|
+
begin
|
805
|
+
initial_line = socket.gets("\n")
|
806
|
+
if initial_line.nil?
|
807
|
+
close
|
808
|
+
raise KeepAliveDisconnected.new(self)
|
809
|
+
end
|
810
|
+
rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
|
811
|
+
# JRuby can raise IOError instead of ECONNRESET for now
|
812
|
+
close
|
813
|
+
raise KeepAliveDisconnected.new(self, $!)
|
814
|
+
end
|
815
|
+
if StatusParseRegexp !~ initial_line
|
816
|
+
@version = '0.9'
|
817
|
+
@status = nil
|
818
|
+
@reason = nil
|
819
|
+
@next_connection = false
|
820
|
+
@content_length = nil
|
821
|
+
@readbuf = initial_line
|
822
|
+
break
|
823
|
+
end
|
824
|
+
@version, @status, @reason = $1, $2.to_i, $3
|
825
|
+
@next_connection = HTTP::Message.keep_alive_enabled?(@version)
|
826
|
+
@headers = []
|
827
|
+
while true
|
828
|
+
line = socket.gets("\n")
|
829
|
+
unless line
|
830
|
+
raise BadResponseError.new('unexpected EOF')
|
831
|
+
end
|
832
|
+
line.chomp!
|
833
|
+
break if line.empty?
|
834
|
+
if line[0] == ?\ or line[0] == ?\t
|
835
|
+
last = @headers.last[1]
|
836
|
+
last << ' ' unless last.empty?
|
837
|
+
last << line.strip
|
838
|
+
else
|
839
|
+
key, value = line.strip.split(/\s*:\s*/, 2)
|
840
|
+
parse_content_header(key, value)
|
841
|
+
@headers << [key, value]
|
842
|
+
end
|
843
|
+
end
|
844
|
+
end while (@version == '1.1' && @status == 100)
|
845
|
+
end
|
846
|
+
end
|
847
|
+
|
848
|
+
def no_message_body?(status)
|
849
|
+
!status.nil? && # HTTP/0.9
|
850
|
+
((status >= 100 && status < 200) || status == 204 || status == 304)
|
851
|
+
end
|
852
|
+
|
853
|
+
def parse_content_header(key, value)
|
854
|
+
key = key.downcase
|
855
|
+
case key
|
856
|
+
when 'content-length'
|
857
|
+
@content_length = value.to_i
|
858
|
+
when 'content-encoding'
|
859
|
+
@content_encoding = value.downcase
|
860
|
+
when 'transfer-encoding'
|
861
|
+
if value.downcase == 'chunked'
|
862
|
+
@chunked = true
|
863
|
+
@chunk_length = 0
|
864
|
+
@content_length = nil
|
865
|
+
end
|
866
|
+
when 'connection', 'proxy-connection'
|
867
|
+
if value.downcase == 'keep-alive'
|
868
|
+
@next_connection = true
|
869
|
+
else
|
870
|
+
@next_connection = false
|
871
|
+
end
|
872
|
+
end
|
873
|
+
end
|
874
|
+
|
875
|
+
def read_body_length(&block)
|
876
|
+
return nil if @content_length == 0
|
877
|
+
while true
|
878
|
+
buf = empty_bin_str
|
879
|
+
maxbytes = @read_block_size
|
880
|
+
maxbytes = @content_length if maxbytes > @content_length && @content_length > 0
|
881
|
+
::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
|
882
|
+
begin
|
883
|
+
@socket.readpartial(maxbytes, buf)
|
884
|
+
rescue EOFError
|
885
|
+
close
|
886
|
+
buf = nil
|
887
|
+
if @strict_response_size_check
|
888
|
+
raise BadResponseError.new("EOF while reading rest #{@content_length} bytes")
|
889
|
+
end
|
890
|
+
end
|
891
|
+
end
|
892
|
+
if buf && buf.bytesize > 0
|
893
|
+
@content_length -= buf.bytesize
|
894
|
+
yield buf
|
895
|
+
else
|
896
|
+
@content_length = 0
|
897
|
+
end
|
898
|
+
return if @content_length == 0
|
899
|
+
end
|
900
|
+
end
|
901
|
+
|
902
|
+
RS = "\r\n"
|
903
|
+
def read_body_chunked(&block)
|
904
|
+
buf = empty_bin_str
|
905
|
+
while true
|
906
|
+
::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
|
907
|
+
len = @socket.gets(RS)
|
908
|
+
if len.nil? # EOF
|
909
|
+
close
|
910
|
+
return
|
911
|
+
end
|
912
|
+
@chunk_length = len.hex
|
913
|
+
if @chunk_length == 0
|
914
|
+
@content_length = 0
|
915
|
+
@socket.gets(RS)
|
916
|
+
return
|
917
|
+
end
|
918
|
+
@socket.read(@chunk_length, buf)
|
919
|
+
@socket.read(2)
|
920
|
+
end
|
921
|
+
unless buf.empty?
|
922
|
+
yield buf
|
923
|
+
end
|
924
|
+
end
|
925
|
+
end
|
926
|
+
|
927
|
+
def read_body_rest
|
928
|
+
if @readbuf and @readbuf.bytesize > 0
|
929
|
+
yield @readbuf
|
930
|
+
@readbuf = nil
|
931
|
+
end
|
932
|
+
while true
|
933
|
+
buf = empty_bin_str
|
934
|
+
::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
|
935
|
+
begin
|
936
|
+
@socket.readpartial(@read_block_size, buf)
|
937
|
+
rescue EOFError
|
938
|
+
buf = nil
|
939
|
+
if @strict_response_size_check
|
940
|
+
raise BadResponseError.new("EOF while reading chunked response")
|
941
|
+
end
|
942
|
+
end
|
943
|
+
end
|
944
|
+
if buf && buf.bytesize > 0
|
945
|
+
yield buf
|
946
|
+
else
|
947
|
+
return
|
948
|
+
end
|
949
|
+
end
|
950
|
+
end
|
951
|
+
|
952
|
+
def empty_bin_str
|
953
|
+
str = ''
|
954
|
+
str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
|
955
|
+
str
|
956
|
+
end
|
957
|
+
end
|
958
|
+
|
959
|
+
|
960
|
+
end
|