httpclient 2.3.0.1 → 2.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +85 -0
  3. data/bin/httpclient +18 -6
  4. data/bin/jsonclient +85 -0
  5. data/lib/http-access2.rb +1 -1
  6. data/lib/httpclient.rb +262 -88
  7. data/lib/httpclient/auth.rb +269 -244
  8. data/lib/httpclient/cacert.pem +3952 -0
  9. data/lib/httpclient/cacert1024.pem +3866 -0
  10. data/lib/httpclient/connection.rb +1 -1
  11. data/lib/httpclient/cookie.rb +161 -514
  12. data/lib/httpclient/http.rb +57 -21
  13. data/lib/httpclient/include_client.rb +2 -0
  14. data/lib/httpclient/jruby_ssl_socket.rb +588 -0
  15. data/lib/httpclient/session.rb +259 -317
  16. data/lib/httpclient/ssl_config.rb +141 -188
  17. data/lib/httpclient/ssl_socket.rb +150 -0
  18. data/lib/httpclient/timeout.rb +1 -1
  19. data/lib/httpclient/util.rb +62 -1
  20. data/lib/httpclient/version.rb +1 -1
  21. data/lib/httpclient/webagent-cookie.rb +459 -0
  22. data/lib/jsonclient.rb +63 -0
  23. data/lib/oauthclient.rb +2 -1
  24. data/sample/jsonclient.rb +67 -0
  25. data/sample/oauth_twitter.rb +4 -4
  26. data/test/{ca-chain.cert → ca-chain.pem} +0 -0
  27. data/test/client-pass.key +18 -0
  28. data/test/helper.rb +10 -8
  29. data/test/jruby_ssl_socket/test_pemutils.rb +32 -0
  30. data/test/test_auth.rb +175 -4
  31. data/test/test_cookie.rb +147 -243
  32. data/test/test_http-access2.rb +17 -16
  33. data/test/test_httpclient.rb +458 -77
  34. data/test/test_jsonclient.rb +80 -0
  35. data/test/test_ssl.rb +341 -17
  36. data/test/test_webagent-cookie.rb +465 -0
  37. metadata +57 -55
  38. data/README.txt +0 -721
  39. data/lib/httpclient/cacert.p7s +0 -1858
  40. data/lib/httpclient/cacert_sha1.p7s +0 -1858
  41. data/sample/oauth_salesforce_10.rb +0 -63
@@ -1,5 +1,5 @@
1
1
  # HTTPClient - HTTP client library.
2
- # Copyright (C) 2000-2009 NAKAMURA, Hiroshi <nahi@ruby-lang.org>.
2
+ # Copyright (C) 2000-2015 NAKAMURA, Hiroshi <nahi@ruby-lang.org>.
3
3
  #
4
4
  # This program is copyrighted free software by NAKAMURA, Hiroshi. You can
5
5
  # redistribute it and/or modify it under the same terms of Ruby's license;
@@ -14,12 +14,18 @@
14
14
 
15
15
  require 'socket'
16
16
  require 'thread'
17
+ require 'timeout'
17
18
  require 'stringio'
18
19
  require 'zlib'
19
20
 
20
- require 'httpclient/timeout'
21
+ require 'httpclient/timeout' # TODO: remove this once we drop 1.8 support
21
22
  require 'httpclient/ssl_config'
22
23
  require 'httpclient/http'
24
+ if RUBY_ENGINE == 'jruby'
25
+ require 'httpclient/jruby_ssl_socket'
26
+ else
27
+ require 'httpclient/ssl_socket'
28
+ end
23
29
 
24
30
 
25
31
  class HTTPClient
@@ -31,14 +37,15 @@ class HTTPClient
31
37
  attr_accessor :scheme
32
38
  # Host String.
33
39
  attr_accessor :host
40
+ alias hostname host
34
41
  # Port number.
35
42
  attr_accessor :port
36
43
 
37
44
  # Creates a new Site based on the given URI.
38
45
  def initialize(uri = nil)
39
46
  if uri
40
- @scheme = uri.scheme
41
- @host = uri.host
47
+ @scheme = uri.scheme || 'tcp'
48
+ @host = uri.hostname || '0.0.0.0'
42
49
  @port = uri.port.to_i
43
50
  else
44
51
  @scheme = 'tcp'
@@ -69,7 +76,7 @@ class HTTPClient
69
76
  def to_s # :nodoc:
70
77
  addr
71
78
  end
72
-
79
+
73
80
  # Returns true if scheme, host and port of the given URI matches with this.
74
81
  def match(uri)
75
82
  (@scheme == uri.scheme) and (@host == uri.host) and (@port == uri.port.to_i)
@@ -98,6 +105,8 @@ class HTTPClient
98
105
  attr_accessor :debug_dev
99
106
  # Boolean value for Socket#sync
100
107
  attr_accessor :socket_sync
108
+ # Boolean value to send TCP keepalive packets; no timing settings exist at present
109
+ attr_accessor :tcp_keepalive
101
110
 
102
111
  attr_accessor :connect_timeout
103
112
  # Maximum retry count. 0 for infinite.
@@ -108,6 +117,9 @@ class HTTPClient
108
117
  attr_accessor :read_block_size
109
118
  attr_accessor :protocol_retry_count
110
119
 
120
+ # Raise BadResponseError if response size does not match with Content-Length header in response.
121
+ attr_accessor :strict_response_size_check
122
+
111
123
  # Local address to bind local side of the socket to
112
124
  attr_accessor :socket_local
113
125
 
@@ -127,6 +139,7 @@ class HTTPClient
127
139
  @protocol_version = nil
128
140
  @debug_dev = client.debug_dev
129
141
  @socket_sync = true
142
+ @tcp_keepalive = false
130
143
  @chunk_size = ::HTTP::Message::Body::DEFAULT_CHUNK_SIZE
131
144
 
132
145
  @connect_timeout = 60
@@ -141,6 +154,7 @@ class HTTPClient
141
154
  @test_loopback_http_response = []
142
155
 
143
156
  @transparent_gzip_decompression = false
157
+ @strict_response_size_check = false
144
158
  @socket_local = Site.new
145
159
 
146
160
  @sess_pool = {}
@@ -157,8 +171,8 @@ class HTTPClient
157
171
  end
158
172
 
159
173
  def query(req, via_proxy)
160
- req.http_body.chunk_size = @chunk_size
161
- sess = open(req.header.request_uri, via_proxy)
174
+ req.http_body.chunk_size = @chunk_size if req.http_body
175
+ sess = get_session(req, via_proxy)
162
176
  begin
163
177
  sess.query(req)
164
178
  rescue
@@ -182,40 +196,43 @@ class HTTPClient
182
196
  add_cached_session(sess)
183
197
  end
184
198
 
185
- def invalidate(site)
186
- @sess_pool_mutex.synchronize do
187
- if pool = @sess_pool[site]
188
- pool.each do |sess|
189
- sess.invalidate
190
- end
191
- end
192
- end
193
- end
194
-
195
199
  private
196
200
 
197
- def open(uri, via_proxy = false)
201
+ # TODO: create PR for webmock's httpclient adapter to use get_session
202
+ # instead of open so that we can remove duplicated Site creation for
203
+ # each session.
204
+ def get_session(req, via_proxy = false)
205
+ uri = req.header.request_uri
206
+ if uri.scheme.nil?
207
+ raise ArgumentError.new("Request URI must have schema. Possibly add 'http://' to the request URI?")
208
+ end
198
209
  site = Site.new(uri)
199
- sess = nil
200
210
  if cached = get_cached_session(site)
201
- sess = cached
211
+ cached
202
212
  else
203
- sess = Session.new(@client, site, @agent_name, @from)
204
- sess.proxy = via_proxy ? @proxy : nil
205
- sess.socket_sync = @socket_sync
206
- sess.requested_version = @protocol_version if @protocol_version
207
- sess.connect_timeout = @connect_timeout
208
- sess.connect_retry = @connect_retry
209
- sess.send_timeout = @send_timeout
210
- sess.receive_timeout = @receive_timeout
211
- sess.read_block_size = @read_block_size
212
- sess.protocol_retry_count = @protocol_retry_count
213
- sess.ssl_config = @ssl_config
214
- sess.debug_dev = @debug_dev
215
- sess.socket_local = @socket_local
216
- sess.test_loopback_http_response = @test_loopback_http_response
217
- sess.transparent_gzip_decompression = @transparent_gzip_decompression
213
+ open(uri, via_proxy)
218
214
  end
215
+ end
216
+
217
+ def open(uri, via_proxy = false)
218
+ site = Site.new(uri)
219
+ sess = Session.new(@client, site, @agent_name, @from)
220
+ sess.proxy = via_proxy ? @proxy : nil
221
+ sess.socket_sync = @socket_sync
222
+ sess.tcp_keepalive = @tcp_keepalive
223
+ sess.requested_version = @protocol_version if @protocol_version
224
+ sess.connect_timeout = @connect_timeout
225
+ sess.connect_retry = @connect_retry
226
+ sess.send_timeout = @send_timeout
227
+ sess.receive_timeout = @receive_timeout
228
+ sess.read_block_size = @read_block_size
229
+ sess.protocol_retry_count = @protocol_retry_count
230
+ sess.ssl_config = @ssl_config
231
+ sess.debug_dev = @debug_dev
232
+ sess.strict_response_size_check = @strict_response_size_check
233
+ sess.socket_local = @socket_local
234
+ sess.test_loopback_http_response = @test_loopback_http_response
235
+ sess.transparent_gzip_decompression = @transparent_gzip_decompression
219
236
  sess
220
237
  end
221
238
 
@@ -241,6 +258,9 @@ class HTTPClient
241
258
  end
242
259
 
243
260
  def get_cached_session(site)
261
+ if Thread.current[:HTTPClient_AcquireNewConnection]
262
+ return nil
263
+ end
244
264
  @sess_pool_mutex.synchronize do
245
265
  now = Time.now
246
266
  if now > @sess_pool_last_checked + @keep_alive_timeout
@@ -272,7 +292,7 @@ class HTTPClient
272
292
  end
273
293
 
274
294
  def valid_session?(sess, now)
275
- !sess.invalidated? and (now <= sess.last_used + @keep_alive_timeout)
295
+ (now <= sess.last_used + @keep_alive_timeout)
276
296
  end
277
297
 
278
298
  def add_cached_session(sess)
@@ -283,131 +303,6 @@ class HTTPClient
283
303
  end
284
304
 
285
305
 
286
- # Wraps up OpenSSL::SSL::SSLSocket and offers debugging features.
287
- class SSLSocketWrap
288
- def initialize(socket, context, debug_dev = nil)
289
- unless SSLEnabled
290
- raise ConfigurationError.new('Ruby/OpenSSL module is required')
291
- end
292
- @context = context
293
- @socket = socket
294
- @ssl_socket = create_openssl_socket(@socket)
295
- @debug_dev = debug_dev
296
- end
297
-
298
- def ssl_connect(hostname = nil)
299
- if hostname && @ssl_socket.respond_to?(:hostname=)
300
- @ssl_socket.hostname = hostname
301
- end
302
- @ssl_socket.connect
303
- end
304
-
305
- def post_connection_check(host)
306
- verify_mode = @context.verify_mode || OpenSSL::SSL::VERIFY_NONE
307
- if verify_mode == OpenSSL::SSL::VERIFY_NONE
308
- return
309
- elsif @ssl_socket.peer_cert.nil? and
310
- check_mask(verify_mode, OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT)
311
- raise OpenSSL::SSL::SSLError.new('no peer cert')
312
- end
313
- hostname = host.host
314
- if @ssl_socket.respond_to?(:post_connection_check) and RUBY_VERSION > "1.8.4"
315
- @ssl_socket.post_connection_check(hostname)
316
- else
317
- @context.post_connection_check(@ssl_socket.peer_cert, hostname)
318
- end
319
- end
320
-
321
- def ssl_version
322
- @ssl_socket.ssl_version if @ssl_socket.respond_to?(:ssl_version)
323
- end
324
-
325
- def ssl_cipher
326
- @ssl_socket.cipher
327
- end
328
-
329
- def ssl_state
330
- @ssl_socket.state
331
- end
332
-
333
- def peer_cert
334
- @ssl_socket.peer_cert
335
- end
336
-
337
- def close
338
- @ssl_socket.close
339
- @socket.close
340
- end
341
-
342
- def closed?
343
- @socket.closed?
344
- end
345
-
346
- def eof?
347
- @ssl_socket.eof?
348
- end
349
-
350
- def gets(*args)
351
- str = @ssl_socket.gets(*args)
352
- debug(str)
353
- str
354
- end
355
-
356
- def read(*args)
357
- str = @ssl_socket.read(*args)
358
- debug(str)
359
- str
360
- end
361
-
362
- def readpartial(*args)
363
- str = @ssl_socket.readpartial(*args)
364
- debug(str)
365
- str
366
- end
367
-
368
- def <<(str)
369
- rv = @ssl_socket.write(str)
370
- debug(str)
371
- rv
372
- end
373
-
374
- def flush
375
- @ssl_socket.flush
376
- end
377
-
378
- def sync
379
- @ssl_socket.sync
380
- end
381
-
382
- def sync=(sync)
383
- @ssl_socket.sync = sync
384
- end
385
-
386
- private
387
-
388
- def check_mask(value, mask)
389
- value & mask == mask
390
- end
391
-
392
- def create_openssl_socket(socket)
393
- ssl_socket = nil
394
- if OpenSSL::SSL.const_defined?("SSLContext")
395
- ctx = OpenSSL::SSL::SSLContext.new
396
- @context.set_context(ctx)
397
- ssl_socket = OpenSSL::SSL::SSLSocket.new(socket, ctx)
398
- else
399
- ssl_socket = OpenSSL::SSL::SSLSocket.new(socket)
400
- @context.set_context(ssl_socket)
401
- end
402
- ssl_socket
403
- end
404
-
405
- def debug(str)
406
- @debug_dev << str if @debug_dev && str
407
- end
408
- end
409
-
410
-
411
306
  # Wraps up a Socket for method interception.
412
307
  module SocketWrap
413
308
  def initialize(socket, *args)
@@ -427,20 +322,20 @@ class HTTPClient
427
322
  @socket.eof?
428
323
  end
429
324
 
430
- def gets(*args)
431
- @socket.gets(*args)
325
+ def gets(rs)
326
+ @socket.gets(rs)
432
327
  end
433
328
 
434
- def read(*args)
435
- @socket.read(*args)
329
+ def read(size, buf = nil)
330
+ @socket.read(size, buf)
436
331
  end
437
332
 
438
- def readpartial(*args)
333
+ def readpartial(size, buf = nil)
439
334
  # StringIO doesn't support :readpartial
440
335
  if @socket.respond_to?(:readpartial)
441
- @socket.readpartial(*args)
336
+ @socket.readpartial(size, buf)
442
337
  else
443
- @socket.read(*args)
338
+ @socket.read(size, buf)
444
339
  end
445
340
  end
446
341
 
@@ -476,19 +371,19 @@ class HTTPClient
476
371
  debug("! CONNECTION CLOSED\n")
477
372
  end
478
373
 
479
- def gets(*args)
374
+ def gets(rs)
480
375
  str = super
481
376
  debug(str)
482
377
  str
483
378
  end
484
379
 
485
- def read(*args)
380
+ def read(size, buf = nil)
486
381
  str = super
487
382
  debug(str)
488
383
  str
489
384
  end
490
385
 
491
- def readpartial(*args)
386
+ def readpartial(size, buf = nil)
492
387
  str = super
493
388
  debug(str)
494
389
  str
@@ -528,6 +423,10 @@ class HTTPClient
528
423
  def <<(str)
529
424
  # ignored
530
425
  end
426
+
427
+ def peer_cert
428
+ nil
429
+ end
531
430
  end
532
431
 
533
432
 
@@ -542,6 +441,8 @@ class HTTPClient
542
441
  attr_accessor :proxy
543
442
  # Boolean value for Socket#sync
544
443
  attr_accessor :socket_sync
444
+ # Boolean value to send TCP keepalive packets; no timing settings exist at present
445
+ attr_accessor :tcp_keepalive
545
446
  # Requested protocol version
546
447
  attr_accessor :requested_version
547
448
  # Device for dumping log for debugging
@@ -554,6 +455,7 @@ class HTTPClient
554
455
  attr_accessor :read_block_size
555
456
  attr_accessor :protocol_retry_count
556
457
 
458
+ attr_accessor :strict_response_size_check
557
459
  attr_accessor :socket_local
558
460
 
559
461
  attr_accessor :ssl_config
@@ -566,9 +468,9 @@ class HTTPClient
566
468
  def initialize(client, dest, agent_name, from)
567
469
  @client = client
568
470
  @dest = dest
569
- @invalidated = false
570
471
  @proxy = nil
571
472
  @socket_sync = true
473
+ @tcp_keepalive = false
572
474
  @requested_version = nil
573
475
 
574
476
  @debug_dev = nil
@@ -584,6 +486,7 @@ class HTTPClient
584
486
  @ssl_peer_cert = nil
585
487
 
586
488
  @test_loopback_http_response = nil
489
+ @strict_response_size_check = false
587
490
  @socket_local = Site::EMPTY
588
491
 
589
492
  @agent_name = agent_name
@@ -607,9 +510,9 @@ class HTTPClient
607
510
  def query(req)
608
511
  connect if @state == :INIT
609
512
  # Use absolute URI (not absolute path) iif via proxy AND not HTTPS.
610
- req.header.request_absolute_uri = !@proxy.nil? and !https?(@dest)
513
+ req.header.request_absolute_uri = !@proxy.nil? && !https?(@dest)
611
514
  begin
612
- timeout(@send_timeout, SendTimeoutError) do
515
+ ::Timeout.timeout(@send_timeout, SendTimeoutError) do
613
516
  set_header(req)
614
517
  req.dump(@socket)
615
518
  # flush the IO stream as IO::sync mode is false
@@ -618,14 +521,14 @@ class HTTPClient
618
521
  rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
619
522
  # JRuby can raise IOError instead of ECONNRESET for now
620
523
  close
621
- raise KeepAliveDisconnected.new(self)
524
+ raise KeepAliveDisconnected.new(self, $!)
622
525
  rescue HTTPClient::TimeoutError
623
526
  close
624
527
  raise
625
- rescue
528
+ rescue => e
626
529
  close
627
- if SSLEnabled and $!.is_a?(OpenSSL::SSL::SSLError)
628
- raise KeepAliveDisconnected.new(self)
530
+ if SSLEnabled and e.is_a?(OpenSSL::SSL::SSLError)
531
+ raise KeepAliveDisconnected.new(self, e)
629
532
  else
630
533
  raise
631
534
  end
@@ -650,14 +553,6 @@ class HTTPClient
650
553
  @state == :INIT
651
554
  end
652
555
 
653
- def invalidate
654
- @invalidated = true
655
- end
656
-
657
- def invalidated?
658
- @invalidated
659
- end
660
-
661
556
  def get_header
662
557
  begin
663
558
  if @state != :META
@@ -683,18 +578,8 @@ class HTTPClient
683
578
  begin
684
579
  read_header if @state == :META
685
580
  return nil if @state != :DATA
686
- if @gzipped and @transparent_gzip_decompression
687
- # zlib itself has a functionality to decompress gzip stream.
688
- # - zlib 1.2.5 Manual
689
- # http://www.zlib.net/manual.html#Advanced
690
- # > windowBits can also be greater than 15 for optional gzip decoding. Add 32 to
691
- # > windowBits to enable zlib and gzip decoding with automatic header detection,
692
- # > or add 16 to decode only the gzip format
693
- inflate_stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
694
- original_block = block
695
- block = Proc.new { |buf|
696
- original_block.call(inflate_stream.inflate(buf))
697
- }
581
+ if @transparent_gzip_decompression
582
+ block = content_inflater_block(@content_encoding, block)
698
583
  end
699
584
  if @chunked
700
585
  read_body_chunked(&block)
@@ -717,18 +602,131 @@ class HTTPClient
717
602
  nil
718
603
  end
719
604
 
605
+ def create_socket(host, port)
606
+ socket = nil
607
+ begin
608
+ @debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
609
+ clean_host = host.delete("[]")
610
+ if @socket_local == Site::EMPTY
611
+ socket = TCPSocket.new(clean_host, port)
612
+ else
613
+ clean_local = @socket_local.host.delete("[]")
614
+ socket = TCPSocket.new(clean_host, port, clean_local, @socket_local.port)
615
+ end
616
+ socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) if @tcp_keepalive
617
+ if @debug_dev
618
+ @debug_dev << "! CONNECTION ESTABLISHED\n"
619
+ socket.extend(DebugSocket)
620
+ socket.debug_dev = @debug_dev
621
+ end
622
+ rescue SystemCallError => e
623
+ raise e.class, e.message + " (#{host}:#{port})"
624
+ rescue SocketError => e
625
+ raise e.class, e.message + " (#{host}:#{port})"
626
+ end
627
+ socket
628
+ end
629
+
630
+ def create_loopback_socket(host, port, str)
631
+ @debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
632
+ socket = LoopBackSocket.new(host, port, str)
633
+ if @debug_dev
634
+ @debug_dev << "! CONNECTION ESTABLISHED\n"
635
+ socket.extend(DebugSocket)
636
+ socket.debug_dev = @debug_dev
637
+ end
638
+ if https?(@dest) && @proxy
639
+ connect_ssl_proxy(socket, Util.urify(@dest.to_s))
640
+ end
641
+ socket
642
+ end
643
+
644
+ def connect_ssl_proxy(socket, uri)
645
+ req = HTTP::Message.new_connect_request(uri)
646
+ @client.request_filter.each do |filter|
647
+ filter.filter_request(req)
648
+ end
649
+ set_header(req)
650
+ req.dump(socket)
651
+ socket.flush unless @socket_sync
652
+ res = HTTP::Message.new_response('')
653
+ parse_header(socket)
654
+ res.http_version, res.status, res.reason = @version, @status, @reason
655
+ @headers.each do |key, value|
656
+ res.header.set(key.to_s, value)
657
+ end
658
+ commands = @client.request_filter.collect { |filter|
659
+ filter.filter_response(req, res)
660
+ }
661
+ if commands.find { |command| command == :retry }
662
+ raise RetryableResponse.new(res)
663
+ end
664
+ unless @status == 200
665
+ raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
666
+ end
667
+ end
668
+
720
669
  private
721
670
 
671
+ # This inflater allows deflate compression with/without zlib header
672
+ class LenientInflater
673
+ def initialize
674
+ @inflater = Zlib::Inflate.new(Zlib::MAX_WBITS)
675
+ @first = true
676
+ end
677
+
678
+ def inflate(body)
679
+ if @first
680
+ first_inflate(body)
681
+ else
682
+ @inflater.inflate(body)
683
+ end
684
+ end
685
+
686
+ private
687
+
688
+ def first_inflate(body)
689
+ @first = false
690
+ begin
691
+ @inflater.inflate(body)
692
+ rescue Zlib::DataError
693
+ # fallback to deflate without zlib header
694
+ @inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS)
695
+ @inflater.inflate(body)
696
+ end
697
+ end
698
+ end
699
+
700
+ def content_inflater_block(content_encoding, block)
701
+ case content_encoding
702
+ when 'gzip', 'x-gzip'
703
+ # zlib itself has a functionality to decompress gzip stream.
704
+ # - zlib 1.2.5 Manual
705
+ # http://www.zlib.net/manual.html#Advanced
706
+ # > windowBits can also be greater than 15 for optional gzip decoding. Add 32 to
707
+ # > windowBits to enable zlib and gzip decoding with automatic header detection,
708
+ # > or add 16 to decode only the gzip format
709
+ inflate_stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
710
+ when 'deflate'
711
+ inflate_stream = LenientInflater.new
712
+ else
713
+ return block
714
+ end
715
+ Proc.new { |buf|
716
+ block.call(inflate_stream.inflate(buf))
717
+ }
718
+ end
719
+
722
720
  def set_header(req)
723
721
  if @requested_version
724
722
  if /^(?:HTTP\/|)(\d+.\d+)$/ =~ @requested_version
725
723
  req.http_version = $1
726
724
  end
727
725
  end
728
- if @agent_name
726
+ if @agent_name && req.header.get('User-Agent').empty?
729
727
  req.header.set('User-Agent', "#{@agent_name} #{LIB_NAME}")
730
728
  end
731
- if @from
729
+ if @from && req.header.get('From').empty?
732
730
  req.header.set('From', @from)
733
731
  end
734
732
  if req.header.get('Accept').empty?
@@ -747,30 +745,15 @@ class HTTPClient
747
745
  site = @proxy || @dest
748
746
  retry_number = 0
749
747
  begin
750
- timeout(@connect_timeout, ConnectTimeoutError) do
751
- @socket = create_socket(site)
752
- if https?(@dest)
753
- if @socket.is_a?(LoopBackSocket)
754
- connect_ssl_proxy(@socket, urify(@dest.to_s)) if @proxy
755
- else
756
- @socket = create_ssl_socket(@socket)
757
- connect_ssl_proxy(@socket, urify(@dest.to_s)) if @proxy
758
- begin
759
- @socket.ssl_connect(@dest.host)
760
- ensure
761
- if $DEBUG
762
- warn("Protocol version: #{@socket.ssl_version}")
763
- warn("Cipher: #{@socket.ssl_cipher.inspect}")
764
- warn("State: #{@socket.ssl_state}")
765
- end
766
- end
767
- @socket.post_connection_check(@dest)
768
- @ssl_peer_cert = @socket.peer_cert
769
- end
748
+ ::Timeout.timeout(@connect_timeout, ConnectTimeoutError) do
749
+ if str = @test_loopback_http_response.shift
750
+ @socket = create_loopback_socket(site.host, site.port, str)
751
+ elsif https?(@dest)
752
+ @socket = SSLSocket.create_socket(self)
753
+ @ssl_peer_cert = @socket.peer_cert
754
+ else
755
+ @socket = create_socket(site.host, site.port)
770
756
  end
771
- # Use Ruby internal buffering instead of passing data immediately
772
- # to the underlying layer
773
- # => we need to to call explicitly flush on the socket
774
757
  @socket.sync = @socket_sync
775
758
  end
776
759
  rescue RetryableResponse
@@ -792,69 +775,13 @@ class HTTPClient
792
775
  @state = :WAIT
793
776
  end
794
777
 
795
- def create_socket(site)
796
- socket = nil
797
- begin
798
- @debug_dev << "! CONNECT TO #{site.host}:#{site.port}\n" if @debug_dev
799
- if str = @test_loopback_http_response.shift
800
- socket = LoopBackSocket.new(site.host, site.port, str)
801
- elsif @socket_local == Site::EMPTY
802
- socket = TCPSocket.new(site.host, site.port)
803
- else
804
- socket = TCPSocket.new(site.host, site.port, @socket_local.host, @socket_local.port)
805
- end
806
- if @debug_dev
807
- @debug_dev << "! CONNECTION ESTABLISHED\n"
808
- socket.extend(DebugSocket)
809
- socket.debug_dev = @debug_dev
810
- end
811
- rescue SystemCallError => e
812
- e.message << " (#{site})"
813
- raise
814
- rescue SocketError => e
815
- e.message << " (#{site})"
816
- raise
817
- end
818
- socket
819
- end
820
-
821
- # wrap socket with OpenSSL.
822
- def create_ssl_socket(raw_socket)
823
- SSLSocketWrap.new(raw_socket, @ssl_config, @debug_dev)
824
- end
825
-
826
- def connect_ssl_proxy(socket, uri)
827
- req = HTTP::Message.new_connect_request(uri)
828
- @client.request_filter.each do |filter|
829
- filter.filter_request(req)
830
- end
831
- set_header(req)
832
- req.dump(@socket)
833
- @socket.flush unless @socket_sync
834
- res = HTTP::Message.new_response('')
835
- parse_header
836
- res.http_version, res.status, res.reason = @version, @status, @reason
837
- @headers.each do |key, value|
838
- res.header.set(key.to_s, value)
839
- end
840
- commands = @client.request_filter.collect { |filter|
841
- filter.filter_response(req, res)
842
- }
843
- if commands.find { |command| command == :retry }
844
- raise RetryableResponse.new
845
- end
846
- unless @status == 200
847
- raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
848
- end
849
- end
850
-
851
778
  # Read status block.
852
779
  def read_header
853
780
  @content_length = nil
854
781
  @chunked = false
855
- @gzipped = false
782
+ @content_encoding = nil
856
783
  @chunk_length = 0
857
- parse_header
784
+ parse_header(@socket)
858
785
  # Header of the request has been parsed.
859
786
  @state = :DATA
860
787
  req = @requests.shift
@@ -870,12 +797,12 @@ class HTTPClient
870
797
  end
871
798
 
872
799
  StatusParseRegexp = %r(\AHTTP/(\d+\.\d+)\s+(\d\d\d)\s*([^\r\n]+)?\r?\n\z)
873
- def parse_header
874
- timeout(@receive_timeout, ReceiveTimeoutError) do
800
+ def parse_header(socket)
801
+ ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
875
802
  initial_line = nil
876
803
  begin
877
804
  begin
878
- initial_line = @socket.gets("\n")
805
+ initial_line = socket.gets("\n")
879
806
  if initial_line.nil?
880
807
  close
881
808
  raise KeepAliveDisconnected.new(self)
@@ -883,7 +810,7 @@ class HTTPClient
883
810
  rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
884
811
  # JRuby can raise IOError instead of ECONNRESET for now
885
812
  close
886
- raise KeepAliveDisconnected.new(self)
813
+ raise KeepAliveDisconnected.new(self, $!)
887
814
  end
888
815
  if StatusParseRegexp !~ initial_line
889
816
  @version = '0.9'
@@ -898,7 +825,7 @@ class HTTPClient
898
825
  @next_connection = HTTP::Message.keep_alive_enabled?(@version)
899
826
  @headers = []
900
827
  while true
901
- line = @socket.gets("\n")
828
+ line = socket.gets("\n")
902
829
  unless line
903
830
  raise BadResponseError.new('unexpected EOF')
904
831
  end
@@ -910,7 +837,7 @@ class HTTPClient
910
837
  last << line.strip
911
838
  else
912
839
  key, value = line.strip.split(/\s*:\s*/, 2)
913
- parse_keepalive_header(key, value)
840
+ parse_content_header(key, value)
914
841
  @headers << [key, value]
915
842
  end
916
843
  end
@@ -923,18 +850,20 @@ class HTTPClient
923
850
  ((status >= 100 && status < 200) || status == 204 || status == 304)
924
851
  end
925
852
 
926
- def parse_keepalive_header(key, value)
853
+ def parse_content_header(key, value)
927
854
  key = key.downcase
928
- if key == 'content-length'
855
+ case key
856
+ when 'content-length'
929
857
  @content_length = value.to_i
930
- elsif key == 'content-encoding' and ( value.downcase == 'gzip' or
931
- value.downcase == 'x-gzip' or value.downcase == 'deflate' )
932
- @gzipped = true
933
- elsif key == 'transfer-encoding' and value.downcase == 'chunked'
934
- @chunked = true
935
- @chunk_length = 0
936
- @content_length = nil
937
- elsif key == 'connection' or key == 'proxy-connection'
858
+ when 'content-encoding'
859
+ @content_encoding = value.downcase
860
+ when 'transfer-encoding'
861
+ if value.downcase == 'chunked'
862
+ @chunked = true
863
+ @chunk_length = 0
864
+ @content_length = nil
865
+ end
866
+ when 'connection', 'proxy-connection'
938
867
  if value.downcase == 'keep-alive'
939
868
  @next_connection = true
940
869
  else
@@ -946,15 +875,18 @@ class HTTPClient
946
875
  def read_body_length(&block)
947
876
  return nil if @content_length == 0
948
877
  while true
949
- buf = ''
878
+ buf = empty_bin_str
950
879
  maxbytes = @read_block_size
951
- maxbytes = @content_length if maxbytes > @content_length
952
- timeout(@receive_timeout, ReceiveTimeoutError) do
880
+ maxbytes = @content_length if maxbytes > @content_length && @content_length > 0
881
+ ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
953
882
  begin
954
883
  @socket.readpartial(maxbytes, buf)
955
884
  rescue EOFError
956
885
  close
957
886
  buf = nil
887
+ if @strict_response_size_check
888
+ raise BadResponseError.new("EOF while reading rest #{@content_length} bytes")
889
+ end
958
890
  end
959
891
  end
960
892
  if buf && buf.bytesize > 0
@@ -969,24 +901,25 @@ class HTTPClient
969
901
 
970
902
  RS = "\r\n"
971
903
  def read_body_chunked(&block)
972
- buf = ''
904
+ buf = empty_bin_str
973
905
  while true
974
- len = @socket.gets(RS)
975
- if len.nil? # EOF
976
- close
977
- return
978
- end
979
- @chunk_length = len.hex
980
- if @chunk_length == 0
981
- @content_length = 0
982
- @socket.gets(RS)
983
- return
984
- end
985
- timeout(@receive_timeout, ReceiveTimeoutError) do
986
- @socket.read(@chunk_length + 2, buf)
906
+ ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
907
+ len = @socket.gets(RS)
908
+ if len.nil? # EOF
909
+ close
910
+ return
911
+ end
912
+ @chunk_length = len.hex
913
+ if @chunk_length == 0
914
+ @content_length = 0
915
+ @socket.gets(RS)
916
+ return
917
+ end
918
+ @socket.read(@chunk_length, buf)
919
+ @socket.read(2)
987
920
  end
988
921
  unless buf.empty?
989
- yield buf.slice(0, @chunk_length)
922
+ yield buf
990
923
  end
991
924
  end
992
925
  end
@@ -997,12 +930,15 @@ class HTTPClient
997
930
  @readbuf = nil
998
931
  end
999
932
  while true
1000
- buf = ''
1001
- timeout(@receive_timeout, ReceiveTimeoutError) do
933
+ buf = empty_bin_str
934
+ ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
1002
935
  begin
1003
936
  @socket.readpartial(@read_block_size, buf)
1004
937
  rescue EOFError
1005
938
  buf = nil
939
+ if @strict_response_size_check
940
+ raise BadResponseError.new("EOF while reading chunked response")
941
+ end
1006
942
  end
1007
943
  end
1008
944
  if buf && buf.bytesize > 0
@@ -1012,6 +948,12 @@ class HTTPClient
1012
948
  end
1013
949
  end
1014
950
  end
951
+
952
+ def empty_bin_str
953
+ str = ''
954
+ str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
955
+ str
956
+ end
1015
957
  end
1016
958
 
1017
959