net-http-persistent 2.9.4 → 4.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,12 +1,7 @@
1
1
  require 'net/http'
2
- begin
3
- require 'net/https'
4
- rescue LoadError
5
- # net/https or openssl
6
- end if RUBY_VERSION < '1.9' # but only for 1.8
7
- require 'net/http/faster'
8
2
  require 'uri'
9
3
  require 'cgi' # for escaping
4
+ require 'connection_pool'
10
5
 
11
6
  begin
12
7
  require 'net/http/pipeline'
@@ -22,15 +17,11 @@ autoload :OpenSSL, 'openssl'
22
17
  # servers you wish to talk to. For each host:port you communicate with a
23
18
  # single persistent connection is created.
24
19
  #
25
- # Multiple Net::HTTP::Persistent objects will share the same set of
26
- # connections.
20
+ # Connections will be shared across threads through a connection pool to
21
+ # increase reuse of connections.
27
22
  #
28
- # For each thread you start a new connection will be created. A
29
- # Net::HTTP::Persistent connection will not be shared across threads.
30
- #
31
- # You can shut down the HTTP connections when done by calling #shutdown. You
32
- # should name your Net::HTTP::Persistent object if you intend to call this
33
- # method.
23
+ # You can shut down any remaining HTTP connections when done by calling
24
+ # #shutdown.
34
25
  #
35
26
  # Example:
36
27
  #
@@ -38,7 +29,7 @@ autoload :OpenSSL, 'openssl'
38
29
  #
39
30
  # uri = URI 'http://example.com/awesome/web/service'
40
31
  #
41
- # http = Net::HTTP::Persistent.new 'my_app_name'
32
+ # http = Net::HTTP::Persistent.new
42
33
  #
43
34
  # # perform a GET
44
35
  # response = http.request uri
@@ -60,23 +51,27 @@ autoload :OpenSSL, 'openssl'
60
51
  # to use URI#request_uri not URI#path. The request_uri contains the query
61
52
  # params which are sent in the body for other requests.
62
53
  #
63
- # == SSL
54
+ # == TLS/SSL
64
55
  #
65
- # SSL connections are automatically created depending upon the scheme of the
66
- # URI. SSL connections are automatically verified against the default
56
+ # TLS connections are automatically created depending upon the scheme of the
57
+ # URI. TLS connections are automatically verified against the default
67
58
  # certificate store for your computer. You can override this by changing
68
59
  # verify_mode or by specifying an alternate cert_store.
69
60
  #
70
- # Here are the SSL settings, see the individual methods for documentation:
61
+ # Here are the TLS settings, see the individual methods for documentation:
71
62
  #
72
63
  # #certificate :: This client's certificate
73
- # #ca_file :: The certificate-authority
64
+ # #ca_file :: The certificate-authorities
65
+ # #ca_path :: Directory with certificate-authorities
74
66
  # #cert_store :: An SSL certificate store
67
+ # #ciphers :: List of SSl ciphers allowed
75
68
  # #private_key :: The client's SSL private key
76
69
  # #reuse_ssl_sessions :: Reuse a previously opened SSL session for a new
77
70
  # connection
71
+ # #ssl_timeout :: Session lifetime
78
72
  # #ssl_version :: Which specific SSL version to use
79
73
  # #verify_callback :: For server certificate verification
74
+ # #verify_depth :: Depth of certificate verification
80
75
  # #verify_mode :: How connections should be verified
81
76
  #
82
77
  # == Proxies
@@ -102,14 +97,15 @@ autoload :OpenSSL, 'openssl'
102
97
  #
103
98
  # === Segregation
104
99
  #
105
- # By providing an application name to ::new you can separate your connections
106
- # from the connections of other applications.
100
+ # Each Net::HTTP::Persistent instance has its own pool of connections. There
101
+ # is no sharing with other instances (as was true in earlier versions).
107
102
  #
108
103
  # === Idle Timeout
109
104
  #
110
- # If a connection hasn't been used for this number of seconds it will automatically be
111
- # reset upon the next use to avoid attempting to send to a closed connection.
112
- # The default value is 5 seconds. nil means no timeout. Set through #idle_timeout.
105
+ # If a connection hasn't been used for this number of seconds it will
106
+ # automatically be reset upon the next use to avoid attempting to send to a
107
+ # closed connection. The default value is 5 seconds. nil means no timeout.
108
+ # Set through #idle_timeout.
113
109
  #
114
110
  # Reducing this value may help avoid the "too many connection resets" error
115
111
  # when sending non-idempotent requests while increasing this value will cause
@@ -124,8 +120,9 @@ autoload :OpenSSL, 'openssl'
124
120
  #
125
121
  # The number of requests that should be made before opening a new connection.
126
122
  # Typically many keep-alive capable servers tune this to 100 or less, so the
127
- # 101st request will fail with ECONNRESET. If unset (default), this value has no
128
- # effect, if set, connections will be reset on the request after max_requests.
123
+ # 101st request will fail with ECONNRESET. If unset (default), this value has
124
+ # no effect, if set, connections will be reset on the request after
125
+ # max_requests.
129
126
  #
130
127
  # === Open Timeout
131
128
  #
@@ -137,45 +134,6 @@ autoload :OpenSSL, 'openssl'
137
134
  # Socket options may be set on newly-created connections. See #socket_options
138
135
  # for details.
139
136
  #
140
- # === Non-Idempotent Requests
141
- #
142
- # By default non-idempotent requests will not be retried per RFC 2616. By
143
- # setting retry_change_requests to true requests will automatically be retried
144
- # once.
145
- #
146
- # Only do this when you know that retrying a POST or other non-idempotent
147
- # request is safe for your application and will not create duplicate
148
- # resources.
149
- #
150
- # The recommended way to handle non-idempotent requests is the following:
151
- #
152
- # require 'net/http/persistent'
153
- #
154
- # uri = URI 'http://example.com/awesome/web/service'
155
- # post_uri = uri + 'create'
156
- #
157
- # http = Net::HTTP::Persistent.new 'my_app_name'
158
- #
159
- # post = Net::HTTP::Post.new post_uri.path
160
- # # ... fill in POST request
161
- #
162
- # begin
163
- # response = http.request post_uri, post
164
- # rescue Net::HTTP::Persistent::Error
165
- #
166
- # # POST failed, make a new request to verify the server did not process
167
- # # the request
168
- # exists_uri = uri + '...'
169
- # response = http.get exists_uri
170
- #
171
- # # Retry if it failed
172
- # retry if response.code == '404'
173
- # end
174
- #
175
- # The method of determining if the resource was created or not is unique to
176
- # the particular service you are using. Of course, you will want to add
177
- # protection from infinite looping.
178
- #
179
137
  # === Connection Termination
180
138
  #
181
139
  # If you are done using the Net::HTTP::Persistent instance you may shut down
@@ -201,24 +159,27 @@ class Net::HTTP::Persistent
201
159
  HAVE_OPENSSL = defined? OpenSSL::SSL # :nodoc:
202
160
 
203
161
  ##
204
- # The version of Net::HTTP::Persistent you are using
162
+ # The default connection pool size is 1/4 the allowed open files
163
+ # (<code>ulimit -n</code>) or 256 if your OS does not support file handle
164
+ # limits (typically windows).
165
+
166
+ if Process.const_defined? :RLIMIT_NOFILE
167
+ open_file_limits = Process.getrlimit(Process::RLIMIT_NOFILE)
205
168
 
206
- VERSION = '2.9.4'
169
+ # Under JRuby on Windows Process responds to `getrlimit` but returns something that does not match docs
170
+ if open_file_limits.respond_to?(:first)
171
+ DEFAULT_POOL_SIZE = open_file_limits.first / 4
172
+ else
173
+ DEFAULT_POOL_SIZE = 256
174
+ end
175
+ else
176
+ DEFAULT_POOL_SIZE = 256
177
+ end
207
178
 
208
179
  ##
209
- # Exceptions rescued for automatic retry on ruby 2.0.0. This overlaps with
210
- # the exception list for ruby 1.x.
180
+ # The version of Net::HTTP::Persistent you are using
211
181
 
212
- RETRIED_EXCEPTIONS = [ # :nodoc:
213
- (Net::ReadTimeout if Net.const_defined? :ReadTimeout),
214
- IOError,
215
- EOFError,
216
- Errno::ECONNRESET,
217
- Errno::ECONNABORTED,
218
- Errno::EPIPE,
219
- (OpenSSL::SSL::SSLError if HAVE_OPENSSL),
220
- Timeout::Error,
221
- ].compact
182
+ VERSION = '4.0.2'
222
183
 
223
184
  ##
224
185
  # Error class for errors raised by Net::HTTP::Persistent. Various
@@ -248,31 +209,31 @@ class Net::HTTP::Persistent
248
209
 
249
210
  http = new 'net-http-persistent detect_idle_timeout'
250
211
 
251
- connection = http.connection_for uri
212
+ http.connection_for uri do |connection|
213
+ sleep_time = 0
252
214
 
253
- sleep_time = 0
215
+ http = connection.http
254
216
 
255
- loop do
256
- response = connection.request req
217
+ loop do
218
+ response = http.request req
257
219
 
258
- $stderr.puts "HEAD #{uri} => #{response.code}" if $DEBUG
220
+ $stderr.puts "HEAD #{uri} => #{response.code}" if $DEBUG
259
221
 
260
- unless Net::HTTPOK === response then
261
- raise Error, "bad response code #{response.code} detecting idle timeout"
262
- end
222
+ unless Net::HTTPOK === response then
223
+ raise Error, "bad response code #{response.code} detecting idle timeout"
224
+ end
263
225
 
264
- break if sleep_time >= max
226
+ break if sleep_time >= max
265
227
 
266
- sleep_time += 1
228
+ sleep_time += 1
267
229
 
268
- $stderr.puts "sleeping #{sleep_time}" if $DEBUG
269
- sleep sleep_time
230
+ $stderr.puts "sleeping #{sleep_time}" if $DEBUG
231
+ sleep sleep_time
232
+ end
270
233
  end
271
234
  rescue
272
235
  # ignore StandardErrors, we've probably found the idle timeout.
273
236
  ensure
274
- http.shutdown
275
-
276
237
  return sleep_time unless $!
277
238
  end
278
239
 
@@ -281,7 +242,9 @@ class Net::HTTP::Persistent
281
242
 
282
243
  attr_reader :certificate
283
244
 
245
+ ##
284
246
  # For Net::HTTP parity
247
+
285
248
  alias cert certificate
286
249
 
287
250
  ##
@@ -290,12 +253,23 @@ class Net::HTTP::Persistent
290
253
 
291
254
  attr_reader :ca_file
292
255
 
256
+ ##
257
+ # A directory of SSL certificates to be used as certificate authorities.
258
+ # Setting this will set verify_mode to VERIFY_PEER.
259
+
260
+ attr_reader :ca_path
261
+
293
262
  ##
294
263
  # An SSL certificate store. Setting this will override the default
295
264
  # certificate store. See verify_mode for more information.
296
265
 
297
266
  attr_reader :cert_store
298
267
 
268
+ ##
269
+ # The ciphers allowed for SSL connections
270
+
271
+ attr_reader :ciphers
272
+
299
273
  ##
300
274
  # Sends debug_output to this IO via Net::HTTP#set_debug_output.
301
275
  #
@@ -309,11 +283,6 @@ class Net::HTTP::Persistent
309
283
 
310
284
  attr_reader :generation # :nodoc:
311
285
 
312
- ##
313
- # Where this instance's connections live in the thread local variables
314
-
315
- attr_reader :generation_key # :nodoc:
316
-
317
286
  ##
318
287
  # Headers that are added to every request using Net::HTTP#add_field
319
288
 
@@ -337,6 +306,13 @@ class Net::HTTP::Persistent
337
306
 
338
307
  attr_accessor :max_requests
339
308
 
309
+ ##
310
+ # Number of retries to perform if a request fails.
311
+ #
312
+ # See also #max_retries=, Net::HTTP#max_retries=.
313
+
314
+ attr_reader :max_retries
315
+
340
316
  ##
341
317
  # The value sent in the Keep-Alive header. Defaults to 30. Not needed for
342
318
  # HTTP/1.1 servers.
@@ -349,8 +325,7 @@ class Net::HTTP::Persistent
349
325
  attr_accessor :keep_alive
350
326
 
351
327
  ##
352
- # A name for this connection. Allows you to keep your connections apart
353
- # from everybody else's.
328
+ # The name for this collection of persistent connections.
354
329
 
355
330
  attr_reader :name
356
331
 
@@ -369,7 +344,9 @@ class Net::HTTP::Persistent
369
344
 
370
345
  attr_reader :private_key
371
346
 
347
+ ##
372
348
  # For Net::HTTP parity
349
+
373
350
  alias key private_key
374
351
 
375
352
  ##
@@ -382,15 +359,20 @@ class Net::HTTP::Persistent
382
359
 
383
360
  attr_reader :no_proxy
384
361
 
362
+ ##
363
+ # Test-only accessor for the connection pool
364
+
365
+ attr_reader :pool # :nodoc:
366
+
385
367
  ##
386
368
  # Seconds to wait until reading one block. See Net::HTTP#read_timeout
387
369
 
388
370
  attr_accessor :read_timeout
389
371
 
390
372
  ##
391
- # Where this instance's request counts live in the thread local variables
373
+ # Seconds to wait until writing one block. See Net::HTTP#write_timeout
392
374
 
393
- attr_reader :request_key # :nodoc:
375
+ attr_accessor :write_timeout
394
376
 
395
377
  ##
396
378
  # By default SSL sessions are reused to avoid extra SSL handshakes. Set
@@ -418,17 +400,33 @@ class Net::HTTP::Persistent
418
400
  attr_reader :ssl_generation # :nodoc:
419
401
 
420
402
  ##
421
- # Where this instance's SSL connections live in the thread local variables
403
+ # SSL session lifetime
422
404
 
423
- attr_reader :ssl_generation_key # :nodoc:
405
+ attr_reader :ssl_timeout
424
406
 
425
407
  ##
426
408
  # SSL version to use.
427
409
  #
428
410
  # By default, the version will be negotiated automatically between client
429
- # and server. Ruby 1.9 and newer only.
411
+ # and server. Ruby 1.9 and newer only. Deprecated since Ruby 2.5.
412
+
413
+ attr_reader :ssl_version
414
+
415
+ ##
416
+ # Minimum SSL version to use, e.g. :TLS1_1
417
+ #
418
+ # By default, the version will be negotiated automatically between client
419
+ # and server. Ruby 2.5 and newer only.
430
420
 
431
- attr_reader :ssl_version if RUBY_VERSION > '1.9'
421
+ attr_reader :min_version
422
+
423
+ ##
424
+ # Maximum SSL version to use, e.g. :TLS1_2
425
+ #
426
+ # By default, the version will be negotiated automatically between client
427
+ # and server. Ruby 2.5 and newer only.
428
+
429
+ attr_reader :max_version
432
430
 
433
431
  ##
434
432
  # Where this instance's last-use times live in the thread local variables
@@ -436,38 +434,31 @@ class Net::HTTP::Persistent
436
434
  attr_reader :timeout_key # :nodoc:
437
435
 
438
436
  ##
439
- # SSL verification callback. Used when ca_file is set.
437
+ # SSL verification callback. Used when ca_file or ca_path is set.
440
438
 
441
439
  attr_reader :verify_callback
442
440
 
441
+ ##
442
+ # Sets the depth of SSL certificate verification
443
+
444
+ attr_reader :verify_depth
445
+
443
446
  ##
444
447
  # HTTPS verify mode. Defaults to OpenSSL::SSL::VERIFY_PEER which verifies
445
448
  # the server certificate.
446
449
  #
447
- # If no ca_file or cert_store is set the default system certificate store is
448
- # used.
450
+ # If no ca_file, ca_path or cert_store is set the default system certificate
451
+ # store is used.
449
452
  #
450
453
  # You can use +verify_mode+ to override any default values.
451
454
 
452
455
  attr_reader :verify_mode
453
456
 
454
- ##
455
- # Enable retries of non-idempotent requests that change data (e.g. POST
456
- # requests) when the server has disconnected.
457
- #
458
- # This will in the worst case lead to multiple requests with the same data,
459
- # but it may be useful for some applications. Take care when enabling
460
- # this option to ensure it is safe to POST or perform other non-idempotent
461
- # requests to the server.
462
-
463
- attr_accessor :retry_change_requests
464
-
465
457
  ##
466
458
  # Creates a new Net::HTTP::Persistent.
467
459
  #
468
- # Set +name+ to keep your connections apart from everybody else's. Not
469
- # required currently, but highly recommended. Your library name should be
470
- # good enough. This parameter will be required in a future version.
460
+ # Set a +name+ for fun. Your library name should be good enough, but this
461
+ # otherwise has no purpose.
471
462
  #
472
463
  # +proxy+ may be set to a URI::HTTP or :ENV to pick up proxy options from
473
464
  # the environment. See proxy_from_env for details.
@@ -478,8 +469,13 @@ class Net::HTTP::Persistent
478
469
  # proxy = URI 'http://proxy.example'
479
470
  # proxy.user = 'AzureDiamond'
480
471
  # proxy.password = 'hunter2'
472
+ #
473
+ # Set +pool_size+ to limit the maximum number of connections allowed.
474
+ # Defaults to 1/4 the number of allowed file handles or 256 if your OS does
475
+ # not support a limit on allowed file handles. You can have no more than
476
+ # this many threads with active HTTP transactions.
481
477
 
482
- def initialize name = nil, proxy = nil
478
+ def initialize name: nil, proxy: nil, pool_size: DEFAULT_POOL_SIZE
483
479
  @name = name
484
480
 
485
481
  @debug_output = nil
@@ -491,40 +487,41 @@ class Net::HTTP::Persistent
491
487
  @keep_alive = 30
492
488
  @open_timeout = nil
493
489
  @read_timeout = nil
490
+ @write_timeout = nil
494
491
  @idle_timeout = 5
495
492
  @max_requests = nil
493
+ @max_retries = 1
496
494
  @socket_options = []
495
+ @ssl_generation = 0 # incremented when SSL session variables change
497
496
 
498
497
  @socket_options << [Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1] if
499
498
  Socket.const_defined? :TCP_NODELAY
500
499
 
501
- key = ['net_http_persistent', name].compact
502
- @generation_key = [key, 'generations' ].join('_').intern
503
- @ssl_generation_key = [key, 'ssl_generations'].join('_').intern
504
- @request_key = [key, 'requests' ].join('_').intern
505
- @timeout_key = [key, 'timeouts' ].join('_').intern
500
+ @pool = Net::HTTP::Persistent::Pool.new size: pool_size do |http_args|
501
+ Net::HTTP::Persistent::Connection.new Net::HTTP, http_args, @ssl_generation
502
+ end
506
503
 
507
504
  @certificate = nil
508
505
  @ca_file = nil
506
+ @ca_path = nil
507
+ @ciphers = nil
509
508
  @private_key = nil
509
+ @ssl_timeout = nil
510
510
  @ssl_version = nil
511
+ @min_version = nil
512
+ @max_version = nil
511
513
  @verify_callback = nil
514
+ @verify_depth = nil
512
515
  @verify_mode = nil
513
516
  @cert_store = nil
514
517
 
515
518
  @generation = 0 # incremented when proxy URI changes
516
- @ssl_generation = 0 # incremented when SSL session variables change
517
519
 
518
520
  if HAVE_OPENSSL then
519
521
  @verify_mode = OpenSSL::SSL::VERIFY_PEER
520
522
  @reuse_ssl_sessions = OpenSSL::SSL.const_defined? :Session
521
523
  end
522
524
 
523
- @retry_change_requests = false
524
-
525
- @ruby_1 = RUBY_VERSION < '2'
526
- @retried_on_ruby_2 = !@ruby_1
527
-
528
525
  self.proxy = proxy if proxy
529
526
  end
530
527
 
@@ -549,6 +546,15 @@ class Net::HTTP::Persistent
549
546
  reconnect_ssl
550
547
  end
551
548
 
549
+ ##
550
+ # Sets the SSL certificate authority path.
551
+
552
+ def ca_path= path
553
+ @ca_path = path
554
+
555
+ reconnect_ssl
556
+ end
557
+
552
558
  ##
553
559
  # Overrides the default SSL certificate store used for verifying
554
560
  # connections.
@@ -560,115 +566,74 @@ class Net::HTTP::Persistent
560
566
  end
561
567
 
562
568
  ##
563
- # Finishes all connections on the given +thread+ that were created before
564
- # the given +generation+ in the threads +generation_key+ list.
565
- #
566
- # See #shutdown for a bunch of scary warning about misusing this method.
567
-
568
- def cleanup(generation, thread = Thread.current,
569
- generation_key = @generation_key) # :nodoc:
570
- timeouts = thread[@timeout_key]
571
-
572
- (0...generation).each do |old_generation|
573
- next unless thread[generation_key]
569
+ # The ciphers allowed for SSL connections
574
570
 
575
- conns = thread[generation_key].delete old_generation
571
+ def ciphers= ciphers
572
+ @ciphers = ciphers
576
573
 
577
- conns.each_value do |conn|
578
- finish conn, thread
579
-
580
- timeouts.delete conn.object_id if timeouts
581
- end if conns
582
- end
574
+ reconnect_ssl
583
575
  end
584
576
 
585
577
  ##
586
578
  # Creates a new connection for +uri+
587
579
 
588
580
  def connection_for uri
589
- Thread.current[@generation_key] ||= Hash.new { |h,k| h[k] = {} }
590
- Thread.current[@ssl_generation_key] ||= Hash.new { |h,k| h[k] = {} }
591
- Thread.current[@request_key] ||= Hash.new 0
592
- Thread.current[@timeout_key] ||= Hash.new EPOCH
593
-
594
581
  use_ssl = uri.scheme.downcase == 'https'
595
582
 
596
- if use_ssl then
597
- raise Net::HTTP::Persistent::Error, 'OpenSSL is not available' unless
598
- HAVE_OPENSSL
599
-
600
- ssl_generation = @ssl_generation
601
-
602
- ssl_cleanup ssl_generation
583
+ net_http_args = [uri.hostname, uri.port]
603
584
 
604
- connections = Thread.current[@ssl_generation_key][ssl_generation]
585
+ # I'm unsure if uri.host or uri.hostname should be checked against
586
+ # the proxy bypass list.
587
+ if @proxy_uri and not proxy_bypass? uri.host, uri.port then
588
+ net_http_args.concat @proxy_args
605
589
  else
606
- generation = @generation
607
-
608
- cleanup generation
609
-
610
- connections = Thread.current[@generation_key][generation]
590
+ net_http_args.concat [nil, nil, nil, nil]
611
591
  end
612
592
 
613
- net_http_args = [uri.host, uri.port]
614
- connection_id = net_http_args.join ':'
593
+ connection = @pool.checkout net_http_args
615
594
 
616
- if @proxy_uri and not proxy_bypass? uri.host, uri.port then
617
- connection_id << @proxy_connection_id
618
- net_http_args.concat @proxy_args
619
- end
595
+ http = connection.http
620
596
 
621
- connection = connections[connection_id]
597
+ connection.ressl @ssl_generation if
598
+ connection.ssl_generation != @ssl_generation
622
599
 
623
- unless connection = connections[connection_id] then
624
- connections[connection_id] = http_class.new(*net_http_args)
625
- connection = connections[connection_id]
626
- ssl connection if use_ssl
627
- else
628
- reset connection if expired? connection
600
+ if not http.started? then
601
+ ssl http if use_ssl
602
+ start http
603
+ elsif expired? connection then
604
+ reset connection
629
605
  end
630
606
 
631
- start connection unless connection.started?
607
+ http.keep_alive_timeout = @idle_timeout if @idle_timeout
608
+ http.max_retries = @max_retries if http.respond_to?(:max_retries=)
609
+ http.read_timeout = @read_timeout if @read_timeout
610
+ http.write_timeout = @write_timeout if
611
+ @write_timeout && http.respond_to?(:write_timeout=)
632
612
 
633
- connection.read_timeout = @read_timeout if @read_timeout
634
- connection.keep_alive_timeout = @idle_timeout if @idle_timeout && connection.respond_to?(:keep_alive_timeout=)
635
-
636
- connection
613
+ return yield connection
637
614
  rescue Errno::ECONNREFUSED
638
- address = connection.proxy_address || connection.address
639
- port = connection.proxy_port || connection.port
615
+ address = http.proxy_address || http.address
616
+ port = http.proxy_port || http.port
640
617
 
641
618
  raise Error, "connection refused: #{address}:#{port}"
642
619
  rescue Errno::EHOSTDOWN
643
- address = connection.proxy_address || connection.address
644
- port = connection.proxy_port || connection.port
620
+ address = http.proxy_address || http.address
621
+ port = http.proxy_port || http.port
645
622
 
646
623
  raise Error, "host down: #{address}:#{port}"
624
+ ensure
625
+ @pool.checkin net_http_args
647
626
  end
648
627
 
649
628
  ##
650
- # Returns an error message containing the number of requests performed on
651
- # this connection
652
-
653
- def error_message connection
654
- requests = Thread.current[@request_key][connection.object_id] - 1 # fixup
655
- last_use = Thread.current[@timeout_key][connection.object_id]
656
-
657
- age = Time.now - last_use
658
-
659
- "after #{requests} requests on #{connection.object_id}, " \
660
- "last used #{age} seconds ago"
661
- end
662
-
663
- ##
664
- # URI::escape wrapper
629
+ # CGI::escape wrapper
665
630
 
666
631
  def escape str
667
632
  CGI.escape str if str
668
633
  end
669
634
 
670
635
  ##
671
- # URI::unescape wrapper
636
+ # CGI::unescape wrapper
672
637
 
673
638
  def unescape str
674
639
  CGI.unescape str if str
@@ -680,26 +645,23 @@ class Net::HTTP::Persistent
680
645
  # maximum request count, false otherwise.
681
646
 
682
647
  def expired? connection
683
- requests = Thread.current[@request_key][connection.object_id]
684
- return true if @max_requests && requests >= @max_requests
648
+ return true if @max_requests && connection.requests >= @max_requests
685
649
  return false unless @idle_timeout
686
650
  return true if @idle_timeout.zero?
687
651
 
688
- last_used = Thread.current[@timeout_key][connection.object_id]
689
-
690
- Time.now - last_used > @idle_timeout
652
+ Time.now - connection.last_use > @idle_timeout
691
653
  end
692
654
 
693
655
  ##
694
656
  # Starts the Net::HTTP +connection+
695
657
 
696
- def start connection
697
- connection.set_debug_output @debug_output if @debug_output
698
- connection.open_timeout = @open_timeout if @open_timeout
658
+ def start http
659
+ http.set_debug_output @debug_output if @debug_output
660
+ http.open_timeout = @open_timeout if @open_timeout
699
661
 
700
- connection.start
662
+ http.start
701
663
 
702
- socket = connection.instance_variable_get :@socket
664
+ socket = http.instance_variable_get :@socket
703
665
 
704
666
  if socket then # for fakeweb
705
667
  @socket_options.each do |option|
@@ -711,120 +673,60 @@ class Net::HTTP::Persistent
711
673
  ##
712
674
  # Finishes the Net::HTTP +connection+
713
675
 
714
- def finish connection, thread = Thread.current
715
- if requests = thread[@request_key] then
716
- requests.delete connection.object_id
717
- end
718
-
676
+ def finish connection
719
677
  connection.finish
720
- rescue IOError
721
- end
722
678
 
723
- def http_class # :nodoc:
724
- if RUBY_VERSION > '2.0' then
725
- Net::HTTP
726
- elsif [:Artifice, :FakeWeb, :WebMock].any? { |klass|
727
- Object.const_defined?(klass)
728
- } or not @reuse_ssl_sessions then
729
- Net::HTTP
730
- else
731
- Net::HTTP::Persistent::SSLReuse
732
- end
679
+ connection.http.instance_variable_set :@last_communicated, nil
680
+ connection.http.instance_variable_set :@ssl_session, nil unless
681
+ @reuse_ssl_sessions
733
682
  end
734
683
 
735
684
  ##
736
685
  # Returns the HTTP protocol version for +uri+
737
686
 
738
687
  def http_version uri
739
- @http_versions["#{uri.host}:#{uri.port}"]
688
+ @http_versions["#{uri.hostname}:#{uri.port}"]
740
689
  end
741
690
 
742
691
  ##
743
- # Is +req+ idempotent according to RFC 2616?
692
+ # Adds "http://" to the String +uri+ if it is missing.
744
693
 
745
- def idempotent? req
746
- case req
747
- when Net::HTTP::Delete, Net::HTTP::Get, Net::HTTP::Head,
748
- Net::HTTP::Options, Net::HTTP::Put, Net::HTTP::Trace then
749
- true
750
- end
694
+ def normalize_uri uri
695
+ (uri =~ /^https?:/) ? uri : "http://#{uri}"
751
696
  end
752
697
 
753
698
  ##
754
- # Is the request +req+ idempotent or is retry_change_requests allowed.
699
+ # Set the maximum number of retries for a request.
755
700
  #
756
- # If +retried_on_ruby_2+ is true, true will be returned if we are on ruby,
757
- # retry_change_requests is allowed and the request is not idempotent.
758
-
759
- def can_retry? req, retried_on_ruby_2 = false
760
- return @retry_change_requests && !idempotent?(req) if retried_on_ruby_2
761
-
762
- @retry_change_requests || idempotent?(req)
763
- end
764
-
765
- if RUBY_VERSION > '1.9' then
766
- ##
767
- # Workaround for missing Net::HTTPHeader#connection_close? on Ruby 1.8
768
-
769
- def connection_close? header
770
- header.connection_close?
771
- end
772
-
773
- ##
774
- # Workaround for missing Net::HTTPHeader#connection_keep_alive? on Ruby 1.8
775
-
776
- def connection_keep_alive? header
777
- header.connection_keep_alive?
778
- end
779
- else
780
- ##
781
- # Workaround for missing Net::HTTPRequest#connection_close? on Ruby 1.8
782
-
783
- def connection_close? header
784
- header['connection'] =~ /close/ or header['proxy-connection'] =~ /close/
785
- end
786
-
787
- ##
788
- # Workaround for missing Net::HTTPRequest#connection_keep_alive? on Ruby
789
- # 1.8
701
+ # Defaults to one retry.
702
+ #
703
+ # Set this to 0 to disable retries.
790
704
 
791
- def connection_keep_alive? header
792
- header['connection'] =~ /keep-alive/ or
793
- header['proxy-connection'] =~ /keep-alive/
794
- end
795
- end
705
+ def max_retries= retries
706
+ retries = retries.to_int
796
707
 
797
- ##
798
- # Deprecated in favor of #expired?
708
+ raise ArgumentError, "max_retries must be positive" if retries < 0
799
709
 
800
- def max_age # :nodoc:
801
- return Time.now + 1 unless @idle_timeout
710
+ @max_retries = retries
802
711
 
803
- Time.now - @idle_timeout
804
- end
805
-
806
- ##
807
- # Adds "http://" to the String +uri+ if it is missing.
808
-
809
- def normalize_uri uri
810
- (uri =~ /^https?:/) ? uri : "http://#{uri}"
712
+ reconnect
811
713
  end
812
714
 
813
715
  ##
814
716
  # Pipelines +requests+ to the HTTP server at +uri+ yielding responses if a
815
- # block is given. Returns all responses recieved.
717
+ # block is given. Returns all responses received.
816
718
  #
817
719
  # See
818
- # Net::HTTP::Pipeline[http://docs.seattlerb.org/net-http-pipeline/Net/HTTP/Pipeline.html]
720
+ # Net::HTTP::Pipeline[https://rdoc.info/gems/net-http-pipeline/Net/HTTP/Pipeline]
819
721
  # for further details.
820
722
  #
821
723
  # Only if <tt>net-http-pipeline</tt> was required before
822
724
  # <tt>net-http-persistent</tt> #pipeline will be present.
823
725
 
824
726
  def pipeline uri, requests, &block # :yields: responses
825
- connection = connection_for uri
826
-
827
- connection.pipeline requests, &block
727
+ connection_for uri do |connection|
728
+ connection.http.pipeline requests, &block
729
+ end
828
730
  end
829
731
 
830
732
  ##
@@ -865,7 +767,7 @@ class Net::HTTP::Persistent
865
767
 
866
768
  if @proxy_uri then
867
769
  @proxy_args = [
868
- @proxy_uri.host,
770
+ @proxy_uri.hostname,
869
771
  @proxy_uri.port,
870
772
  unescape(@proxy_uri.user),
871
773
  unescape(@proxy_uri.password),
@@ -940,14 +842,15 @@ class Net::HTTP::Persistent
940
842
  end
941
843
 
942
844
  ##
943
- # Forces reconnection of HTTP connections.
845
+ # Forces reconnection of all HTTP connections, including TLS/SSL
846
+ # connections.
944
847
 
945
848
  def reconnect
946
849
  @generation += 1
947
850
  end
948
851
 
949
852
  ##
950
- # Forces reconnection of SSL connections.
853
+ # Forces reconnection of only TLS/SSL connections.
951
854
 
952
855
  def reconnect_ssl
953
856
  @ssl_generation += 1
@@ -957,18 +860,17 @@ class Net::HTTP::Persistent
957
860
  # Finishes then restarts the Net::HTTP +connection+
958
861
 
959
862
  def reset connection
960
- Thread.current[@request_key].delete connection.object_id
961
- Thread.current[@timeout_key].delete connection.object_id
863
+ http = connection.http
962
864
 
963
865
  finish connection
964
866
 
965
- start connection
867
+ start http
966
868
  rescue Errno::ECONNREFUSED
967
- e = Error.new "connection refused: #{connection.address}:#{connection.port}"
869
+ e = Error.new "connection refused: #{http.address}:#{http.port}"
968
870
  e.set_backtrace $@
969
871
  raise e
970
872
  rescue Errno::EHOSTDOWN
971
- e = Error.new "host down: #{connection.address}:#{connection.port}"
873
+ e = Error.new "host down: #{http.address}:#{http.port}"
972
874
  e.set_backtrace $@
973
875
  raise e
974
876
  end
@@ -980,84 +882,41 @@ class Net::HTTP::Persistent
980
882
  # If a block is passed #request behaves like Net::HTTP#request (the body of
981
883
  # the response will not have been read).
982
884
  #
983
- # +req+ must be a Net::HTTPRequest subclass (see Net::HTTP for a list).
984
- #
985
- # If there is an error and the request is idempotent according to RFC 2616
986
- # it will be retried automatically.
885
+ # +req+ must be a Net::HTTPGenericRequest subclass (see Net::HTTP for a list).
987
886
 
988
887
  def request uri, req = nil, &block
989
- retried = false
990
- bad_response = false
991
-
992
- req = request_setup req || uri
993
-
994
- connection = connection_for uri
995
- connection_id = connection.object_id
996
-
997
- begin
998
- Thread.current[@request_key][connection_id] += 1
999
- response = connection.request req, &block
1000
-
1001
- if connection_close?(req) or
1002
- (response.http_version <= '1.0' and
1003
- not connection_keep_alive?(response)) or
1004
- connection_close?(response) then
1005
- connection.finish
888
+ uri = URI uri
889
+ req = request_setup req || uri
890
+ response = nil
891
+
892
+ connection_for uri do |connection|
893
+ http = connection.http
894
+
895
+ begin
896
+ connection.requests += 1
897
+
898
+ response = http.request req, &block
899
+
900
+ if req.connection_close? or
901
+ (response.http_version <= '1.0' and
902
+ not response.connection_keep_alive?) or
903
+ response.connection_close? then
904
+ finish connection
905
+ end
906
+ rescue Exception # make sure to close the connection when it was interrupted
907
+ finish connection
908
+
909
+ raise
910
+ ensure
911
+ connection.last_use = Time.now
1006
912
  end
1007
- rescue Net::HTTPBadResponse => e
1008
- message = error_message connection
1009
-
1010
- finish connection
1011
-
1012
- raise Error, "too many bad responses #{message}" if
1013
- bad_response or not can_retry? req
1014
-
1015
- bad_response = true
1016
- retry
1017
- rescue *RETRIED_EXCEPTIONS => e # retried on ruby 2
1018
- request_failed e, req, connection if
1019
- retried or not can_retry? req, @retried_on_ruby_2
1020
-
1021
- reset connection
1022
-
1023
- retried = true
1024
- retry
1025
- rescue Errno::EINVAL, Errno::ETIMEDOUT => e # not retried on ruby 2
1026
- request_failed e, req, connection if retried or not can_retry? req
1027
-
1028
- reset connection
1029
-
1030
- retried = true
1031
- retry
1032
- rescue Exception => e
1033
- finish connection
1034
-
1035
- raise
1036
- ensure
1037
- Thread.current[@timeout_key][connection_id] = Time.now
1038
913
  end
1039
914
 
1040
- @http_versions["#{uri.host}:#{uri.port}"] ||= response.http_version
915
+ @http_versions["#{uri.hostname}:#{uri.port}"] ||= response.http_version
1041
916
 
1042
917
  response
1043
918
  end
1044
919
 
1045
- ##
1046
- # Raises an Error for +exception+ which resulted from attempting the request
1047
- # +req+ on the +connection+.
1048
- #
1049
- # Finishes the +connection+.
1050
-
1051
- def request_failed exception, req, connection # :nodoc:
1052
- due_to = "(due to #{exception.message} - #{exception.class})"
1053
- message = "too many connection resets #{due_to} #{error_message connection}"
1054
-
1055
- finish connection
1056
-
1057
-
1058
- raise Error, message, exception.backtrace
1059
- end
1060
-
1061
920
  ##
1062
921
  # Creates a GET request if +req_or_uri+ is a URI and adds headers to the
1063
922
  # request.
@@ -1065,7 +924,7 @@ class Net::HTTP::Persistent
1065
924
  # Returns the request.
1066
925
 
1067
926
  def request_setup req_or_uri # :nodoc:
1068
- req = if URI === req_or_uri then
927
+ req = if req_or_uri.respond_to? 'request_uri' then
1069
928
  Net::HTTP::Get.new req_or_uri.request_uri
1070
929
  else
1071
930
  req_or_uri
@@ -1088,45 +947,15 @@ class Net::HTTP::Persistent
1088
947
  end
1089
948
 
1090
949
  ##
1091
- # Shuts down all connections for +thread+.
1092
- #
1093
- # Uses the current thread by default.
1094
- #
1095
- # If you've used Net::HTTP::Persistent across multiple threads you should
1096
- # call this in each thread when you're done making HTTP requests.
1097
- #
1098
- # *NOTE*: Calling shutdown for another thread can be dangerous!
1099
- #
1100
- # If the thread is still using the connection it may cause an error! It is
1101
- # best to call #shutdown in the thread at the appropriate time instead!
1102
-
1103
- def shutdown thread = Thread.current
1104
- generation = reconnect
1105
- cleanup generation, thread, @generation_key
1106
-
1107
- ssl_generation = reconnect_ssl
1108
- cleanup ssl_generation, thread, @ssl_generation_key
1109
-
1110
- thread[@request_key] = nil
1111
- thread[@timeout_key] = nil
1112
- end
1113
-
1114
- ##
1115
- # Shuts down all connections in all threads
1116
- #
1117
- # *NOTE*: THIS METHOD IS VERY DANGEROUS!
950
+ # Shuts down all connections
1118
951
  #
1119
- # Do not call this method if other threads are still using their
1120
- # connections! Call #shutdown at the appropriate time instead!
952
+ # *NOTE*: Calling shutdown for can be dangerous!
1121
953
  #
1122
- # Use this method only as a last resort!
954
+ # If any thread is still using a connection it may cause an error! Call
955
+ # #shutdown when you are completely done making requests!
1123
956
 
1124
- def shutdown_in_all_threads
1125
- Thread.list.each do |thread|
1126
- shutdown thread
1127
- end
1128
-
1129
- nil
957
+ def shutdown
958
+ @pool.shutdown { |http| http.finish }
1130
959
  end
1131
960
 
1132
961
  ##
@@ -1135,9 +964,14 @@ class Net::HTTP::Persistent
1135
964
  def ssl connection
1136
965
  connection.use_ssl = true
1137
966
 
967
+ connection.ciphers = @ciphers if @ciphers
968
+ connection.ssl_timeout = @ssl_timeout if @ssl_timeout
1138
969
  connection.ssl_version = @ssl_version if @ssl_version
970
+ connection.min_version = @min_version if @min_version
971
+ connection.max_version = @max_version if @max_version
1139
972
 
1140
- connection.verify_mode = @verify_mode
973
+ connection.verify_depth = @verify_depth
974
+ connection.verify_mode = @verify_mode
1141
975
 
1142
976
  if OpenSSL::SSL::VERIFY_PEER == OpenSSL::SSL::VERIFY_NONE and
1143
977
  not Object.const_defined?(:I_KNOW_THAT_OPENSSL_VERIFY_PEER_EQUALS_VERIFY_NONE_IS_WRONG) then
@@ -1166,8 +1000,10 @@ application:
1166
1000
  WARNING
1167
1001
  end
1168
1002
 
1169
- if @ca_file then
1170
- connection.ca_file = @ca_file
1003
+ connection.ca_file = @ca_file if @ca_file
1004
+ connection.ca_path = @ca_path if @ca_path
1005
+
1006
+ if @ca_file or @ca_path then
1171
1007
  connection.verify_mode = OpenSSL::SSL::VERIFY_PEER
1172
1008
  connection.verify_callback = @verify_callback if @verify_callback
1173
1009
  end
@@ -1187,11 +1023,12 @@ application:
1187
1023
  end
1188
1024
 
1189
1025
  ##
1190
- # Finishes all connections that existed before the given SSL parameter
1191
- # +generation+.
1026
+ # SSL session lifetime
1027
+
1028
+ def ssl_timeout= ssl_timeout
1029
+ @ssl_timeout = ssl_timeout
1192
1030
 
1193
- def ssl_cleanup generation # :nodoc:
1194
- cleanup generation, Thread.current, @ssl_generation_key
1031
+ reconnect_ssl
1195
1032
  end
1196
1033
 
1197
1034
  ##
@@ -1201,7 +1038,34 @@ application:
1201
1038
  @ssl_version = ssl_version
1202
1039
 
1203
1040
  reconnect_ssl
1204
- end if RUBY_VERSION > '1.9'
1041
+ end
1042
+
1043
+ ##
1044
+ # Minimum SSL version to use
1045
+
1046
+ def min_version= min_version
1047
+ @min_version = min_version
1048
+
1049
+ reconnect_ssl
1050
+ end
1051
+
1052
+ ##
1053
+ # maximum SSL version to use
1054
+
1055
+ def max_version= max_version
1056
+ @max_version = max_version
1057
+
1058
+ reconnect_ssl
1059
+ end
1060
+
1061
+ ##
1062
+ # Sets the depth of SSL certificate verification
1063
+
1064
+ def verify_depth= verify_depth
1065
+ @verify_depth = verify_depth
1066
+
1067
+ reconnect_ssl
1068
+ end
1205
1069
 
1206
1070
  ##
1207
1071
  # Sets the HTTPS verify mode. Defaults to OpenSSL::SSL::VERIFY_PEER.
@@ -1224,8 +1088,8 @@ application:
1224
1088
 
1225
1089
  reconnect_ssl
1226
1090
  end
1227
-
1228
1091
  end
1229
1092
 
1230
- require 'net/http/persistent/ssl_reuse'
1093
+ require_relative 'persistent/connection'
1094
+ require_relative 'persistent/pool'
1231
1095