httpclient 2.1.5 → 2.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +85 -0
  3. data/bin/httpclient +77 -0
  4. data/bin/jsonclient +85 -0
  5. data/lib/hexdump.rb +50 -0
  6. data/lib/http-access2.rb +6 -4
  7. data/lib/httpclient/auth.rb +575 -173
  8. data/lib/httpclient/cacert.pem +3952 -0
  9. data/lib/httpclient/cacert1024.pem +3866 -0
  10. data/lib/httpclient/connection.rb +6 -2
  11. data/lib/httpclient/cookie.rb +162 -504
  12. data/lib/httpclient/http.rb +334 -119
  13. data/lib/httpclient/include_client.rb +85 -0
  14. data/lib/httpclient/jruby_ssl_socket.rb +588 -0
  15. data/lib/httpclient/session.rb +385 -288
  16. data/lib/httpclient/ssl_config.rb +195 -155
  17. data/lib/httpclient/ssl_socket.rb +150 -0
  18. data/lib/httpclient/timeout.rb +14 -10
  19. data/lib/httpclient/util.rb +142 -6
  20. data/lib/httpclient/version.rb +3 -0
  21. data/lib/httpclient/webagent-cookie.rb +459 -0
  22. data/lib/httpclient.rb +509 -202
  23. data/lib/jsonclient.rb +63 -0
  24. data/lib/oauthclient.rb +111 -0
  25. data/sample/async.rb +8 -0
  26. data/sample/auth.rb +11 -0
  27. data/sample/cookie.rb +18 -0
  28. data/sample/dav.rb +103 -0
  29. data/sample/howto.rb +49 -0
  30. data/sample/jsonclient.rb +67 -0
  31. data/sample/oauth_buzz.rb +57 -0
  32. data/sample/oauth_friendfeed.rb +59 -0
  33. data/sample/oauth_twitter.rb +61 -0
  34. data/sample/ssl/0cert.pem +22 -0
  35. data/sample/ssl/0key.pem +30 -0
  36. data/sample/ssl/1000cert.pem +19 -0
  37. data/sample/ssl/1000key.pem +18 -0
  38. data/sample/ssl/htdocs/index.html +10 -0
  39. data/sample/ssl/ssl_client.rb +22 -0
  40. data/sample/ssl/webrick_httpsd.rb +29 -0
  41. data/sample/stream.rb +21 -0
  42. data/sample/thread.rb +27 -0
  43. data/sample/wcat.rb +21 -0
  44. data/test/ca-chain.pem +44 -0
  45. data/test/ca.cert +23 -0
  46. data/test/client-pass.key +18 -0
  47. data/test/client.cert +19 -0
  48. data/test/client.key +15 -0
  49. data/test/helper.rb +131 -0
  50. data/test/htdigest +1 -0
  51. data/test/htpasswd +2 -0
  52. data/test/jruby_ssl_socket/test_pemutils.rb +32 -0
  53. data/test/runner.rb +2 -0
  54. data/test/server.cert +19 -0
  55. data/test/server.key +15 -0
  56. data/test/sslsvr.rb +65 -0
  57. data/test/subca.cert +21 -0
  58. data/test/test_auth.rb +492 -0
  59. data/test/test_cookie.rb +309 -0
  60. data/test/test_hexdump.rb +14 -0
  61. data/test/test_http-access2.rb +508 -0
  62. data/test/test_httpclient.rb +2145 -0
  63. data/test/test_include_client.rb +52 -0
  64. data/test/test_jsonclient.rb +80 -0
  65. data/test/test_ssl.rb +559 -0
  66. data/test/test_webagent-cookie.rb +465 -0
  67. metadata +85 -44
  68. data/lib/httpclient/auth.rb.orig +0 -513
  69. data/lib/httpclient/cacert.p7s +0 -1579
  70. data/lib/httpclient.rb.orig +0 -1020
  71. data/lib/tags +0 -908
@@ -1,22 +1,31 @@
1
1
  # HTTPClient - HTTP client library.
2
- # Copyright (C) 2000-2009 NAKAMURA, Hiroshi <nahi@ruby-lang.org>.
2
+ # Copyright (C) 2000-2015 NAKAMURA, Hiroshi <nahi@ruby-lang.org>.
3
3
  #
4
4
  # This program is copyrighted free software by NAKAMURA, Hiroshi. You can
5
5
  # redistribute it and/or modify it under the same terms of Ruby's license;
6
6
  # either the dual license version in 2003, or any later version.
7
-
8
- # httpclient/session.rb is based on http-access.rb in http-access/0.0.4.
9
- # Some part of code in http-access.rb was recycled in httpclient.rb.
10
- # Those part is copyrighted by Maehashi-san.
7
+ #
8
+ # httpclient/session.rb is based on http-access.rb in http-access/0.0.4. Some
9
+ # part of it is copyrighted by Maebashi-san who made and published
10
+ # http-access/0.0.4. http-access/0.0.4 did not include license notice but when
11
+ # I asked Maebashi-san he agreed that I can redistribute it under the same terms
12
+ # of Ruby. Many thanks to Maebashi-san.
11
13
 
12
14
 
13
15
  require 'socket'
14
16
  require 'thread'
17
+ require 'timeout'
15
18
  require 'stringio'
19
+ require 'zlib'
16
20
 
17
- require 'httpclient/timeout'
21
+ require 'httpclient/timeout' # TODO: remove this once we drop 1.8 support
18
22
  require 'httpclient/ssl_config'
19
23
  require 'httpclient/http'
24
+ if RUBY_ENGINE == 'jruby'
25
+ require 'httpclient/jruby_ssl_socket'
26
+ else
27
+ require 'httpclient/ssl_socket'
28
+ end
20
29
 
21
30
 
22
31
  class HTTPClient
@@ -27,15 +36,16 @@ class HTTPClient
27
36
  # Protocol scheme.
28
37
  attr_accessor :scheme
29
38
  # Host String.
30
- attr_reader :host
39
+ attr_accessor :host
40
+ alias hostname host
31
41
  # Port number.
32
- attr_reader :port
42
+ attr_accessor :port
33
43
 
34
44
  # Creates a new Site based on the given URI.
35
45
  def initialize(uri = nil)
36
46
  if uri
37
- @scheme = uri.scheme
38
- @host = uri.host
47
+ @scheme = uri.scheme || 'tcp'
48
+ @host = uri.hostname || '0.0.0.0'
39
49
  @port = uri.port.to_i
40
50
  else
41
51
  @scheme = 'tcp'
@@ -66,7 +76,7 @@ class HTTPClient
66
76
  def to_s # :nodoc:
67
77
  addr
68
78
  end
69
-
79
+
70
80
  # Returns true if scheme, host and port of the given URI matches with this.
71
81
  def match(uri)
72
82
  (@scheme == uri.scheme) and (@host == uri.host) and (@port == uri.port.to_i)
@@ -75,6 +85,8 @@ class HTTPClient
75
85
  def inspect # :nodoc:
76
86
  sprintf("#<%s:0x%x %s>", self.class.name, __id__, addr)
77
87
  end
88
+
89
+ EMPTY = Site.new.freeze
78
90
  end
79
91
 
80
92
 
@@ -93,19 +105,30 @@ class HTTPClient
93
105
  attr_accessor :debug_dev
94
106
  # Boolean value for Socket#sync
95
107
  attr_accessor :socket_sync
108
+ # Boolean value to send TCP keepalive packets; no timing settings exist at present
109
+ attr_accessor :tcp_keepalive
96
110
 
97
111
  attr_accessor :connect_timeout
98
112
  # Maximum retry count. 0 for infinite.
99
113
  attr_accessor :connect_retry
100
114
  attr_accessor :send_timeout
101
115
  attr_accessor :receive_timeout
116
+ attr_accessor :keep_alive_timeout
102
117
  attr_accessor :read_block_size
103
118
  attr_accessor :protocol_retry_count
104
119
 
120
+ # Raise BadResponseError if response size does not match with Content-Length header in response.
121
+ attr_accessor :strict_response_size_check
122
+
123
+ # Local address to bind local side of the socket to
124
+ attr_accessor :socket_local
125
+
105
126
  attr_accessor :ssl_config
106
127
 
107
128
  attr_reader :test_loopback_http_response
108
129
 
130
+ attr_accessor :transparent_gzip_decompression
131
+
109
132
  def initialize(client)
110
133
  @client = client
111
134
  @proxy = client.proxy
@@ -116,20 +139,27 @@ class HTTPClient
116
139
  @protocol_version = nil
117
140
  @debug_dev = client.debug_dev
118
141
  @socket_sync = true
119
- @chunk_size = 4096
142
+ @tcp_keepalive = false
143
+ @chunk_size = ::HTTP::Message::Body::DEFAULT_CHUNK_SIZE
120
144
 
121
145
  @connect_timeout = 60
122
146
  @connect_retry = 1
123
147
  @send_timeout = 120
124
- @receive_timeout = 60 # For each read_block_size bytes
148
+ @receive_timeout = 60 # For each read_block_size bytes
149
+ @keep_alive_timeout = 15 # '15' is from Apache 2 default
125
150
  @read_block_size = 1024 * 16 # follows net/http change in 1.8.7
126
151
  @protocol_retry_count = 5
127
152
 
128
153
  @ssl_config = nil
129
154
  @test_loopback_http_response = []
130
155
 
131
- @sess_pool = []
156
+ @transparent_gzip_decompression = false
157
+ @strict_response_size_check = false
158
+ @socket_local = Site.new
159
+
160
+ @sess_pool = {}
132
161
  @sess_pool_mutex = Mutex.new
162
+ @sess_pool_last_checked = Time.now
133
163
  end
134
164
 
135
165
  def proxy=(proxy)
@@ -141,8 +171,8 @@ class HTTPClient
141
171
  end
142
172
 
143
173
  def query(req, via_proxy)
144
- req.body.chunk_size = @chunk_size
145
- sess = open(req.header.request_uri, via_proxy)
174
+ req.http_body.chunk_size = @chunk_size if req.http_body
175
+ sess = get_session(req, via_proxy)
146
176
  begin
147
177
  sess.query(req)
148
178
  rescue
@@ -161,45 +191,65 @@ class HTTPClient
161
191
  close_all
162
192
  end
163
193
 
194
+ # assert: sess.last_used must not be nil
164
195
  def keep(sess)
165
196
  add_cached_session(sess)
166
197
  end
167
198
 
168
199
  private
169
200
 
170
- def open(uri, via_proxy = false)
171
- sess = nil
172
- if cached = get_cached_session(uri)
173
- sess = cached
201
+ # TODO: create PR for webmock's httpclient adapter to use get_session
202
+ # instead of open so that we can remove duplicated Site creation for
203
+ # each session.
204
+ def get_session(req, via_proxy = false)
205
+ uri = req.header.request_uri
206
+ if uri.scheme.nil?
207
+ raise ArgumentError.new("Request URI must have schema. Possibly add 'http://' to the request URI?")
208
+ end
209
+ site = Site.new(uri)
210
+ if cached = get_cached_session(site)
211
+ cached
174
212
  else
175
- sess = Session.new(@client, Site.new(uri), @agent_name, @from)
176
- sess.proxy = via_proxy ? @proxy : nil
177
- sess.socket_sync = @socket_sync
178
- sess.requested_version = @protocol_version if @protocol_version
179
- sess.connect_timeout = @connect_timeout
180
- sess.connect_retry = @connect_retry
181
- sess.send_timeout = @send_timeout
182
- sess.receive_timeout = @receive_timeout
183
- sess.read_block_size = @read_block_size
184
- sess.protocol_retry_count = @protocol_retry_count
185
- sess.ssl_config = @ssl_config
186
- sess.debug_dev = @debug_dev
187
- sess.test_loopback_http_response = @test_loopback_http_response
213
+ open(uri, via_proxy)
188
214
  end
215
+ end
216
+
217
+ def open(uri, via_proxy = false)
218
+ site = Site.new(uri)
219
+ sess = Session.new(@client, site, @agent_name, @from)
220
+ sess.proxy = via_proxy ? @proxy : nil
221
+ sess.socket_sync = @socket_sync
222
+ sess.tcp_keepalive = @tcp_keepalive
223
+ sess.requested_version = @protocol_version if @protocol_version
224
+ sess.connect_timeout = @connect_timeout
225
+ sess.connect_retry = @connect_retry
226
+ sess.send_timeout = @send_timeout
227
+ sess.receive_timeout = @receive_timeout
228
+ sess.read_block_size = @read_block_size
229
+ sess.protocol_retry_count = @protocol_retry_count
230
+ sess.ssl_config = @ssl_config
231
+ sess.debug_dev = @debug_dev
232
+ sess.strict_response_size_check = @strict_response_size_check
233
+ sess.socket_local = @socket_local
234
+ sess.test_loopback_http_response = @test_loopback_http_response
235
+ sess.transparent_gzip_decompression = @transparent_gzip_decompression
189
236
  sess
190
237
  end
191
238
 
192
239
  def close_all
193
240
  @sess_pool_mutex.synchronize do
194
- @sess_pool.each do |sess|
195
- sess.close
241
+ @sess_pool.each do |site, pool|
242
+ pool.each do |sess|
243
+ sess.close
244
+ end
196
245
  end
197
246
  end
198
247
  @sess_pool.clear
199
248
  end
200
249
 
250
+ # This method might not work as you expected...
201
251
  def close(dest)
202
- if cached = get_cached_session(dest)
252
+ if cached = get_cached_session(Site.new(dest))
203
253
  cached.close
204
254
  true
205
255
  else
@@ -207,136 +257,48 @@ class HTTPClient
207
257
  end
208
258
  end
209
259
 
210
- def get_cached_session(uri)
211
- cached = nil
260
+ def get_cached_session(site)
261
+ if Thread.current[:HTTPClient_AcquireNewConnection]
262
+ return nil
263
+ end
212
264
  @sess_pool_mutex.synchronize do
213
- new_pool = []
214
- @sess_pool.each do |s|
215
- if s.dest.match(uri)
216
- cached = s
217
- else
218
- new_pool << s
265
+ now = Time.now
266
+ if now > @sess_pool_last_checked + @keep_alive_timeout
267
+ scrub_cached_session(now)
268
+ @sess_pool_last_checked = now
269
+ end
270
+ if pool = @sess_pool[site]
271
+ pool.each_with_index do |sess, idx|
272
+ if valid_session?(sess, now)
273
+ return pool.slice!(idx)
274
+ end
219
275
  end
220
276
  end
221
- @sess_pool = new_pool
222
277
  end
223
- cached
224
- end
225
-
226
- def add_cached_session(sess)
227
- @sess_pool_mutex.synchronize do
228
- @sess_pool << sess
229
- end
230
- end
231
- end
232
-
233
-
234
- # Wraps up OpenSSL::SSL::SSLSocket and offers debugging features.
235
- class SSLSocketWrap
236
- def initialize(socket, context, debug_dev = nil)
237
- unless SSLEnabled
238
- raise ConfigurationError.new('Ruby/OpenSSL module is required')
239
- end
240
- @context = context
241
- @socket = socket
242
- @ssl_socket = create_openssl_socket(@socket)
243
- @debug_dev = debug_dev
244
- end
245
-
246
- def ssl_connect
247
- @ssl_socket.connect
278
+ nil
248
279
  end
249
280
 
250
- def post_connection_check(host)
251
- verify_mode = @context.verify_mode || OpenSSL::SSL::VERIFY_NONE
252
- if verify_mode == OpenSSL::SSL::VERIFY_NONE
253
- return
254
- elsif @ssl_socket.peer_cert.nil? and
255
- check_mask(verify_mode, OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT)
256
- raise OpenSSL::SSL::SSLError.new('no peer cert')
257
- end
258
- hostname = host.host
259
- if @ssl_socket.respond_to?(:post_connection_check) and RUBY_VERSION > "1.8.4"
260
- @ssl_socket.post_connection_check(hostname)
261
- else
262
- @context.post_connection_check(@ssl_socket.peer_cert, hostname)
281
+ def scrub_cached_session(now)
282
+ @sess_pool.each do |site, pool|
283
+ pool.replace(pool.select { |sess|
284
+ if valid_session?(sess, now)
285
+ true
286
+ else
287
+ sess.close # close & remove from the pool
288
+ false
289
+ end
290
+ })
263
291
  end
264
292
  end
265
293
 
266
- def peer_cert
267
- @ssl_socket.peer_cert
268
- end
269
-
270
- def close
271
- @ssl_socket.close
272
- @socket.close
273
- end
274
-
275
- def closed?
276
- @socket.closed?
277
- end
278
-
279
- def eof?
280
- @ssl_socket.eof?
281
- end
282
-
283
- def gets(*args)
284
- str = @ssl_socket.gets(*args)
285
- debug(str)
286
- str
287
- end
288
-
289
- def read(*args)
290
- str = @ssl_socket.read(*args)
291
- debug(str)
292
- str
293
- end
294
-
295
- def readpartial(*args)
296
- str = @ssl_socket.readpartial(*args)
297
- debug(str)
298
- str
299
- end
300
-
301
- def <<(str)
302
- rv = @ssl_socket.write(str)
303
- debug(str)
304
- rv
305
- end
306
-
307
- def flush
308
- @ssl_socket.flush
309
- end
310
-
311
- def sync
312
- @ssl_socket.sync
313
- end
314
-
315
- def sync=(sync)
316
- @ssl_socket.sync = sync
294
+ def valid_session?(sess, now)
295
+ (now <= sess.last_used + @keep_alive_timeout)
317
296
  end
318
297
 
319
- private
320
-
321
- def check_mask(value, mask)
322
- value & mask == mask
323
- end
324
-
325
- def create_openssl_socket(socket)
326
- ssl_socket = nil
327
- if OpenSSL::SSL.const_defined?("SSLContext")
328
- ctx = OpenSSL::SSL::SSLContext.new
329
- @context.set_context(ctx)
330
- ssl_socket = OpenSSL::SSL::SSLSocket.new(socket, ctx)
331
- else
332
- ssl_socket = OpenSSL::SSL::SSLSocket.new(socket)
333
- @context.set_context(ssl_socket)
298
+ def add_cached_session(sess)
299
+ @sess_pool_mutex.synchronize do
300
+ (@sess_pool[sess.dest] ||= []).unshift(sess)
334
301
  end
335
- ssl_socket
336
- end
337
-
338
- def debug(str)
339
- @debug_dev << str if @debug_dev && str
340
302
  end
341
303
  end
342
304
 
@@ -360,20 +322,20 @@ class HTTPClient
360
322
  @socket.eof?
361
323
  end
362
324
 
363
- def gets(*args)
364
- @socket.gets(*args)
325
+ def gets(rs)
326
+ @socket.gets(rs)
365
327
  end
366
328
 
367
- def read(*args)
368
- @socket.read(*args)
329
+ def read(size, buf = nil)
330
+ @socket.read(size, buf)
369
331
  end
370
332
 
371
- def readpartial(*args)
333
+ def readpartial(size, buf = nil)
372
334
  # StringIO doesn't support :readpartial
373
335
  if @socket.respond_to?(:readpartial)
374
- @socket.readpartial(*args)
336
+ @socket.readpartial(size, buf)
375
337
  else
376
- @socket.read(*args)
338
+ @socket.read(size, buf)
377
339
  end
378
340
  end
379
341
 
@@ -409,19 +371,19 @@ class HTTPClient
409
371
  debug("! CONNECTION CLOSED\n")
410
372
  end
411
373
 
412
- def gets(*args)
374
+ def gets(rs)
413
375
  str = super
414
376
  debug(str)
415
377
  str
416
378
  end
417
379
 
418
- def read(*args)
380
+ def read(size, buf = nil)
419
381
  str = super
420
382
  debug(str)
421
383
  str
422
384
  end
423
385
 
424
- def readpartial(*args)
386
+ def readpartial(size, buf = nil)
425
387
  str = super
426
388
  debug(str)
427
389
  str
@@ -435,7 +397,15 @@ class HTTPClient
435
397
  private
436
398
 
437
399
  def debug(str)
438
- @debug_dev << str if str && @debug_dev
400
+ if str && @debug_dev
401
+ if str.index("\0")
402
+ require 'hexdump'
403
+ str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
404
+ @debug_dev << HexDump.encode(str).join("\n")
405
+ else
406
+ @debug_dev << str
407
+ end
408
+ end
439
409
  end
440
410
  end
441
411
 
@@ -453,12 +423,17 @@ class HTTPClient
453
423
  def <<(str)
454
424
  # ignored
455
425
  end
426
+
427
+ def peer_cert
428
+ nil
429
+ end
456
430
  end
457
431
 
458
432
 
459
433
  # Manages a HTTP session with a Site.
460
434
  class Session
461
435
  include HTTPClient::Timeout
436
+ include Util
462
437
 
463
438
  # Destination site
464
439
  attr_reader :dest
@@ -466,6 +441,8 @@ class HTTPClient
466
441
  attr_accessor :proxy
467
442
  # Boolean value for Socket#sync
468
443
  attr_accessor :socket_sync
444
+ # Boolean value to send TCP keepalive packets; no timing settings exist at present
445
+ attr_accessor :tcp_keepalive
469
446
  # Requested protocol version
470
447
  attr_accessor :requested_version
471
448
  # Device for dumping log for debugging
@@ -478,15 +455,22 @@ class HTTPClient
478
455
  attr_accessor :read_block_size
479
456
  attr_accessor :protocol_retry_count
480
457
 
458
+ attr_accessor :strict_response_size_check
459
+ attr_accessor :socket_local
460
+
481
461
  attr_accessor :ssl_config
482
462
  attr_reader :ssl_peer_cert
483
463
  attr_accessor :test_loopback_http_response
484
464
 
465
+ attr_accessor :transparent_gzip_decompression
466
+ attr_reader :last_used
467
+
485
468
  def initialize(client, dest, agent_name, from)
486
469
  @client = client
487
470
  @dest = dest
488
471
  @proxy = nil
489
472
  @socket_sync = true
473
+ @tcp_keepalive = false
490
474
  @requested_version = nil
491
475
 
492
476
  @debug_dev = nil
@@ -502,6 +486,8 @@ class HTTPClient
502
486
  @ssl_peer_cert = nil
503
487
 
504
488
  @test_loopback_http_response = nil
489
+ @strict_response_size_check = false
490
+ @socket_local = Site::EMPTY
505
491
 
506
492
  @agent_name = agent_name
507
493
  @from = from
@@ -515,28 +501,34 @@ class HTTPClient
515
501
 
516
502
  @socket = nil
517
503
  @readbuf = nil
504
+
505
+ @transparent_gzip_decompression = false
506
+ @last_used = nil
518
507
  end
519
508
 
520
509
  # Send a request to the server
521
510
  def query(req)
522
511
  connect if @state == :INIT
523
- req.header.request_via_proxy = !@proxy.nil?
512
+ # Use absolute URI (not absolute path) iif via proxy AND not HTTPS.
513
+ req.header.request_absolute_uri = !@proxy.nil? && !https?(@dest)
524
514
  begin
525
- timeout(@send_timeout, SendTimeoutError) do
515
+ ::Timeout.timeout(@send_timeout, SendTimeoutError) do
526
516
  set_header(req)
527
517
  req.dump(@socket)
528
518
  # flush the IO stream as IO::sync mode is false
529
519
  @socket.flush unless @socket_sync
530
520
  end
531
- rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE
521
+ rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
522
+ # JRuby can raise IOError instead of ECONNRESET for now
532
523
  close
533
- raise KeepAliveDisconnected.new
524
+ raise KeepAliveDisconnected.new(self, $!)
534
525
  rescue HTTPClient::TimeoutError
535
526
  close
536
527
  raise
537
- rescue
538
- if SSLEnabled and $!.is_a?(OpenSSL::SSL::SSLError)
539
- raise KeepAliveDisconnected.new
528
+ rescue => e
529
+ close
530
+ if SSLEnabled and e.is_a?(OpenSSL::SSL::SSLError)
531
+ raise KeepAliveDisconnected.new(self, e)
540
532
  else
541
533
  raise
542
534
  end
@@ -545,6 +537,7 @@ class HTTPClient
545
537
  @state = :META if @state == :WAIT
546
538
  @next_connection = nil
547
539
  @requests.push(req)
540
+ @last_used = Time.now
548
541
  end
549
542
 
550
543
  def close
@@ -585,6 +578,9 @@ class HTTPClient
585
578
  begin
586
579
  read_header if @state == :META
587
580
  return nil if @state != :DATA
581
+ if @transparent_gzip_decompression
582
+ block = content_inflater_block(@content_encoding, block)
583
+ end
588
584
  if @chunked
589
585
  read_body_chunked(&block)
590
586
  elsif @content_length
@@ -606,21 +602,142 @@ class HTTPClient
606
602
  nil
607
603
  end
608
604
 
605
+ def create_socket(host, port)
606
+ socket = nil
607
+ begin
608
+ @debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
609
+ clean_host = host.delete("[]")
610
+ if @socket_local == Site::EMPTY
611
+ socket = TCPSocket.new(clean_host, port)
612
+ else
613
+ clean_local = @socket_local.host.delete("[]")
614
+ socket = TCPSocket.new(clean_host, port, clean_local, @socket_local.port)
615
+ end
616
+ socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) if @tcp_keepalive
617
+ if @debug_dev
618
+ @debug_dev << "! CONNECTION ESTABLISHED\n"
619
+ socket.extend(DebugSocket)
620
+ socket.debug_dev = @debug_dev
621
+ end
622
+ rescue SystemCallError => e
623
+ raise e.class, e.message + " (#{host}:#{port})"
624
+ rescue SocketError => e
625
+ raise e.class, e.message + " (#{host}:#{port})"
626
+ end
627
+ socket
628
+ end
629
+
630
+ def create_loopback_socket(host, port, str)
631
+ @debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
632
+ socket = LoopBackSocket.new(host, port, str)
633
+ if @debug_dev
634
+ @debug_dev << "! CONNECTION ESTABLISHED\n"
635
+ socket.extend(DebugSocket)
636
+ socket.debug_dev = @debug_dev
637
+ end
638
+ if https?(@dest) && @proxy
639
+ connect_ssl_proxy(socket, Util.urify(@dest.to_s))
640
+ end
641
+ socket
642
+ end
643
+
644
+ def connect_ssl_proxy(socket, uri)
645
+ req = HTTP::Message.new_connect_request(uri)
646
+ @client.request_filter.each do |filter|
647
+ filter.filter_request(req)
648
+ end
649
+ set_header(req)
650
+ req.dump(socket)
651
+ socket.flush unless @socket_sync
652
+ res = HTTP::Message.new_response('')
653
+ parse_header(socket)
654
+ res.http_version, res.status, res.reason = @version, @status, @reason
655
+ @headers.each do |key, value|
656
+ res.header.set(key.to_s, value)
657
+ end
658
+ commands = @client.request_filter.collect { |filter|
659
+ filter.filter_response(req, res)
660
+ }
661
+ if commands.find { |command| command == :retry }
662
+ raise RetryableResponse.new(res)
663
+ end
664
+ unless @status == 200
665
+ raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
666
+ end
667
+ end
668
+
609
669
  private
610
670
 
671
+ # This inflater allows deflate compression with/without zlib header
672
+ class LenientInflater
673
+ def initialize
674
+ @inflater = Zlib::Inflate.new(Zlib::MAX_WBITS)
675
+ @first = true
676
+ end
677
+
678
+ def inflate(body)
679
+ if @first
680
+ first_inflate(body)
681
+ else
682
+ @inflater.inflate(body)
683
+ end
684
+ end
685
+
686
+ private
687
+
688
+ def first_inflate(body)
689
+ @first = false
690
+ begin
691
+ @inflater.inflate(body)
692
+ rescue Zlib::DataError
693
+ # fallback to deflate without zlib header
694
+ @inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS)
695
+ @inflater.inflate(body)
696
+ end
697
+ end
698
+ end
699
+
700
+ def content_inflater_block(content_encoding, block)
701
+ case content_encoding
702
+ when 'gzip', 'x-gzip'
703
+ # zlib itself has a functionality to decompress gzip stream.
704
+ # - zlib 1.2.5 Manual
705
+ # http://www.zlib.net/manual.html#Advanced
706
+ # > windowBits can also be greater than 15 for optional gzip decoding. Add 32 to
707
+ # > windowBits to enable zlib and gzip decoding with automatic header detection,
708
+ # > or add 16 to decode only the gzip format
709
+ inflate_stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
710
+ when 'deflate'
711
+ inflate_stream = LenientInflater.new
712
+ else
713
+ return block
714
+ end
715
+ Proc.new { |buf|
716
+ block.call(inflate_stream.inflate(buf))
717
+ }
718
+ end
719
+
611
720
  def set_header(req)
612
721
  if @requested_version
613
722
  if /^(?:HTTP\/|)(\d+.\d+)$/ =~ @requested_version
614
- req.version = $1.to_f
723
+ req.http_version = $1
615
724
  end
616
725
  end
617
- if @agent_name
726
+ if @agent_name && req.header.get('User-Agent').empty?
618
727
  req.header.set('User-Agent', "#{@agent_name} #{LIB_NAME}")
619
728
  end
620
- if @from
729
+ if @from && req.header.get('From').empty?
621
730
  req.header.set('From', @from)
622
731
  end
623
- req.header.set('Date', Time.now.httpdate)
732
+ if req.header.get('Accept').empty?
733
+ req.header.set('Accept', '*/*')
734
+ end
735
+ if @transparent_gzip_decompression
736
+ req.header.set('Accept-Encoding', 'gzip,deflate')
737
+ end
738
+ if req.header.get('Date').empty?
739
+ req.header.set_date_header
740
+ end
624
741
  end
625
742
 
626
743
  # Connect to the server
@@ -628,22 +745,15 @@ class HTTPClient
628
745
  site = @proxy || @dest
629
746
  retry_number = 0
630
747
  begin
631
- timeout(@connect_timeout, ConnectTimeoutError) do
632
- @socket = create_socket(site)
633
- if @dest.scheme == 'https'
634
- if @socket.is_a?(LoopBackSocket)
635
- connect_ssl_proxy(@socket, URI.parse(@dest.to_s)) if @proxy
636
- else
637
- @socket = create_ssl_socket(@socket)
638
- connect_ssl_proxy(@socket, URI.parse(@dest.to_s)) if @proxy
639
- @socket.ssl_connect
640
- @socket.post_connection_check(@dest)
641
- @ssl_peer_cert = @socket.peer_cert
642
- end
748
+ ::Timeout.timeout(@connect_timeout, ConnectTimeoutError) do
749
+ if str = @test_loopback_http_response.shift
750
+ @socket = create_loopback_socket(site.host, site.port, str)
751
+ elsif https?(@dest)
752
+ @socket = SSLSocket.create_socket(self)
753
+ @ssl_peer_cert = @socket.peer_cert
754
+ else
755
+ @socket = create_socket(site.host, site.port)
643
756
  end
644
- # Use Ruby internal buffering instead of passing data immediately
645
- # to the underlying layer
646
- # => we need to to call explicitly flush on the socket
647
757
  @socket.sync = @socket_sync
648
758
  end
649
759
  rescue RetryableResponse
@@ -665,72 +775,17 @@ class HTTPClient
665
775
  @state = :WAIT
666
776
  end
667
777
 
668
- def create_socket(site)
669
- socket = nil
670
- begin
671
- @debug_dev << "! CONNECT TO #{site.host}:#{site.port}\n" if @debug_dev
672
- if str = @test_loopback_http_response.shift
673
- socket = LoopBackSocket.new(site.host, site.port, str)
674
- else
675
- socket = TCPSocket.new(site.host, site.port)
676
- end
677
- if @debug_dev
678
- @debug_dev << "! CONNECTION ESTABLISHED\n"
679
- socket.extend(DebugSocket)
680
- socket.debug_dev = @debug_dev
681
- end
682
- rescue SystemCallError => e
683
- e.message << " (#{site})"
684
- raise
685
- rescue SocketError => e
686
- e.message << " (#{site})"
687
- raise
688
- end
689
- socket
690
- end
691
-
692
- # wrap socket with OpenSSL.
693
- def create_ssl_socket(raw_socket)
694
- SSLSocketWrap.new(raw_socket, @ssl_config, @debug_dev)
695
- end
696
-
697
- def connect_ssl_proxy(socket, uri)
698
- req = HTTP::Message.new_connect_request(uri)
699
- @client.request_filter.each do |filter|
700
- filter.filter_request(req)
701
- end
702
- set_header(req)
703
- req.dump(@socket)
704
- @socket.flush unless @socket_sync
705
- res = HTTP::Message.new_response('')
706
- parse_header
707
- res.version, res.status, res.reason = @version, @status, @reason
708
- @headers.each do |key, value|
709
- res.header.set(key, value)
710
- end
711
- commands = @client.request_filter.collect { |filter|
712
- filter.filter_response(req, res)
713
- }
714
- if commands.find { |command| command == :retry }
715
- raise RetryableResponse.new
716
- end
717
- unless @status == 200
718
- raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
719
- end
720
- end
721
-
722
778
  # Read status block.
723
779
  def read_header
724
780
  @content_length = nil
725
781
  @chunked = false
782
+ @content_encoding = nil
726
783
  @chunk_length = 0
727
- parse_header
728
-
729
- # Head of the request has been parsed.
784
+ parse_header(@socket)
785
+ # Header of the request has been parsed.
730
786
  @state = :DATA
731
787
  req = @requests.shift
732
-
733
- if req.header.request_method == 'HEAD'
788
+ if req.header.request_method == 'HEAD' or no_message_body?(@status)
734
789
  @content_length = 0
735
790
  if @next_connection
736
791
  @state = :WAIT
@@ -738,16 +793,24 @@ class HTTPClient
738
793
  close
739
794
  end
740
795
  end
741
- @next_connection = false unless @content_length
796
+ @next_connection = false if !@content_length and !@chunked
742
797
  end
743
798
 
744
799
  StatusParseRegexp = %r(\AHTTP/(\d+\.\d+)\s+(\d\d\d)\s*([^\r\n]+)?\r?\n\z)
745
- def parse_header
746
- timeout(@receive_timeout, ReceiveTimeoutError) do
800
+ def parse_header(socket)
801
+ ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
802
+ initial_line = nil
747
803
  begin
748
- initial_line = @socket.gets("\n")
749
- if initial_line.nil?
750
- raise KeepAliveDisconnected.new
804
+ begin
805
+ initial_line = socket.gets("\n")
806
+ if initial_line.nil?
807
+ close
808
+ raise KeepAliveDisconnected.new(self)
809
+ end
810
+ rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
811
+ # JRuby can raise IOError instead of ECONNRESET for now
812
+ close
813
+ raise KeepAliveDisconnected.new(self, $!)
751
814
  end
752
815
  if StatusParseRegexp !~ initial_line
753
816
  @version = '0.9'
@@ -759,32 +822,48 @@ class HTTPClient
759
822
  break
760
823
  end
761
824
  @version, @status, @reason = $1, $2.to_i, $3
762
- @next_connection = HTTP::Message.keep_alive_enabled?(@version.to_f)
825
+ @next_connection = HTTP::Message.keep_alive_enabled?(@version)
763
826
  @headers = []
764
827
  while true
765
- line = @socket.gets("\n")
828
+ line = socket.gets("\n")
766
829
  unless line
767
830
  raise BadResponseError.new('unexpected EOF')
768
831
  end
769
832
  line.chomp!
770
833
  break if line.empty?
771
- key, value = line.split(/\s*:\s*/, 2)
772
- parse_keepalive_header(key, value)
773
- @headers << [key, value]
834
+ if line[0] == ?\ or line[0] == ?\t
835
+ last = @headers.last[1]
836
+ last << ' ' unless last.empty?
837
+ last << line.strip
838
+ else
839
+ key, value = line.strip.split(/\s*:\s*/, 2)
840
+ parse_content_header(key, value)
841
+ @headers << [key, value]
842
+ end
774
843
  end
775
844
  end while (@version == '1.1' && @status == 100)
776
845
  end
777
846
  end
778
847
 
779
- def parse_keepalive_header(key, value)
848
+ def no_message_body?(status)
849
+ !status.nil? && # HTTP/0.9
850
+ ((status >= 100 && status < 200) || status == 204 || status == 304)
851
+ end
852
+
853
+ def parse_content_header(key, value)
780
854
  key = key.downcase
781
- if key == 'content-length'
855
+ case key
856
+ when 'content-length'
782
857
  @content_length = value.to_i
783
- elsif key == 'transfer-encoding' and value.downcase == 'chunked'
784
- @chunked = true
785
- @chunk_length = 0
786
- @content_length = nil
787
- elsif key == 'connection' or key == 'proxy-connection'
858
+ when 'content-encoding'
859
+ @content_encoding = value.downcase
860
+ when 'transfer-encoding'
861
+ if value.downcase == 'chunked'
862
+ @chunked = true
863
+ @chunk_length = 0
864
+ @content_length = nil
865
+ end
866
+ when 'connection', 'proxy-connection'
788
867
  if value.downcase == 'keep-alive'
789
868
  @next_connection = true
790
869
  else
@@ -795,19 +874,23 @@ class HTTPClient
795
874
 
796
875
  def read_body_length(&block)
797
876
  return nil if @content_length == 0
798
- buf = ''
799
877
  while true
878
+ buf = empty_bin_str
800
879
  maxbytes = @read_block_size
801
- maxbytes = @content_length if maxbytes > @content_length
802
- timeout(@receive_timeout, ReceiveTimeoutError) do
880
+ maxbytes = @content_length if maxbytes > @content_length && @content_length > 0
881
+ ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
803
882
  begin
804
883
  @socket.readpartial(maxbytes, buf)
805
884
  rescue EOFError
885
+ close
806
886
  buf = nil
887
+ if @strict_response_size_check
888
+ raise BadResponseError.new("EOF while reading rest #{@content_length} bytes")
889
+ end
807
890
  end
808
891
  end
809
- if buf && buf.length > 0
810
- @content_length -= buf.length
892
+ if buf && buf.bytesize > 0
893
+ @content_length -= buf.bytesize
811
894
  yield buf
812
895
  else
813
896
  @content_length = 0
@@ -818,45 +901,59 @@ class HTTPClient
818
901
 
819
902
  RS = "\r\n"
820
903
  def read_body_chunked(&block)
821
- buf = ''
904
+ buf = empty_bin_str
822
905
  while true
823
- len = @socket.gets(RS)
824
- @chunk_length = len.hex
825
- if @chunk_length == 0
826
- @content_length = 0
827
- @socket.gets(RS)
828
- return
829
- end
830
- timeout(@receive_timeout, ReceiveTimeoutError) do
831
- @socket.read(@chunk_length + 2, buf)
906
+ ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
907
+ len = @socket.gets(RS)
908
+ if len.nil? # EOF
909
+ close
910
+ return
911
+ end
912
+ @chunk_length = len.hex
913
+ if @chunk_length == 0
914
+ @content_length = 0
915
+ @socket.gets(RS)
916
+ return
917
+ end
918
+ @socket.read(@chunk_length, buf)
919
+ @socket.read(2)
832
920
  end
833
921
  unless buf.empty?
834
- yield buf.slice(0, @chunk_length)
922
+ yield buf
835
923
  end
836
924
  end
837
925
  end
838
926
 
839
927
  def read_body_rest
840
- if @readbuf and @readbuf.length > 0
928
+ if @readbuf and @readbuf.bytesize > 0
841
929
  yield @readbuf
842
930
  @readbuf = nil
843
931
  end
844
- buf = ''
845
932
  while true
846
- timeout(@receive_timeout, ReceiveTimeoutError) do
933
+ buf = empty_bin_str
934
+ ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
847
935
  begin
848
936
  @socket.readpartial(@read_block_size, buf)
849
937
  rescue EOFError
850
938
  buf = nil
939
+ if @strict_response_size_check
940
+ raise BadResponseError.new("EOF while reading chunked response")
941
+ end
851
942
  end
852
943
  end
853
- if buf && buf.length > 0
944
+ if buf && buf.bytesize > 0
854
945
  yield buf
855
946
  else
856
947
  return
857
948
  end
858
949
  end
859
950
  end
951
+
952
+ def empty_bin_str
953
+ str = ''
954
+ str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
955
+ str
956
+ end
860
957
  end
861
958
 
862
959