net-http-persistent-pool 2.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,25 @@
1
+ # -*- ruby -*-
2
+
3
+ require 'rubygems'
4
+ require 'hoe'
5
+
6
+ Hoe.plugin :git
7
+ Hoe.plugin :minitest
8
+ Hoe.plugin :travis
9
+
10
+ Hoe.spec 'net-http-persistent-pool' do
11
+ developer 'Getty Images', 'opensource@gettyimages.com'
12
+
13
+ self.readme_file = 'README.rdoc'
14
+ self.extra_rdoc_files += Dir['*.rdoc']
15
+
16
+ license 'MIT'
17
+
18
+ rdoc_locations <<
19
+ 'docs.seattlerb.org:/data/www/docs.seattlerb.org/net-http-persistent/'
20
+
21
+ dependency 'connection_pool', '~> 2.1'
22
+ dependency 'minitest', '~> 5.2', :development
23
+ end
24
+
25
+ # vim: syntax=Ruby
@@ -0,0 +1,27 @@
1
+ require 'net/protocol'
2
+
3
+ ##
4
+ # Aaron Patterson's monkeypatch (accepted into 1.9.1) to fix Net::HTTP's speed
5
+ # problems.
6
+ #
7
+ # http://gist.github.com/251244
8
+
9
+ class Net::BufferedIO #:nodoc:
10
+ alias :old_rbuf_fill :rbuf_fill
11
+
12
+ def rbuf_fill
13
+ if @io.respond_to? :read_nonblock then
14
+ begin
15
+ @rbuf << @io.read_nonblock(65536)
16
+ rescue Errno::EWOULDBLOCK, Errno::EAGAIN => e
17
+ retry if IO.select [@io], nil, nil, @read_timeout
18
+ raise Timeout::Error, e.message
19
+ end
20
+ else # SSL sockets do not have read_nonblock
21
+ timeout @read_timeout do
22
+ @rbuf << @io.sysread(65536)
23
+ end
24
+ end
25
+ end
26
+ end if RUBY_VERSION < '1.9'
27
+
@@ -0,0 +1,1245 @@
1
+ require 'net/http'
2
+ begin
3
+ require 'net/https'
4
+ rescue LoadError
5
+ # net/https or openssl
6
+ end if RUBY_VERSION < '1.9' # but only for 1.8
7
+ require 'net/http/faster'
8
+ require 'uri'
9
+ require 'cgi' # for escaping
10
+ require 'connection_pool'
11
+
12
+ begin
13
+ require 'net/http/pipeline'
14
+ rescue LoadError
15
+ end
16
+
17
+ autoload :OpenSSL, 'openssl'
18
+
19
+ ##
20
+ # Persistent connections for Net::HTTP
21
+ #
22
+ # Net::HTTP::Persistent maintains persistent connections across all the
23
+ # servers you wish to talk to. For each host:port you communicate with a
24
+ # single persistent connection is created.
25
+ #
26
+ # Multiple Net::HTTP::Persistent objects will share the same set of
27
+ # connections.
28
+ #
29
+ # For each thread you start a new connection will be created. A
30
+ # Net::HTTP::Persistent connection will not be shared across threads.
31
+ #
32
+ # You can shut down the HTTP connections when done by calling #shutdown. You
33
+ # should name your Net::HTTP::Persistent object if you intend to call this
34
+ # method.
35
+ #
36
+ # Example:
37
+ #
38
+ # require 'net/http/persistent'
39
+ #
40
+ # uri = URI 'http://example.com/awesome/web/service'
41
+ #
42
+ # http = Net::HTTP::Persistent.new 'my_app_name'
43
+ #
44
+ # # perform a GET
45
+ # response = http.request uri
46
+ #
47
+ # # or
48
+ #
49
+ # get = Net::HTTP::Get.new uri.request_uri
50
+ # response = http.request get
51
+ #
52
+ # # create a POST
53
+ # post_uri = uri + 'create'
54
+ # post = Net::HTTP::Post.new post_uri.path
55
+ # post.set_form_data 'some' => 'cool data'
56
+ #
57
+ # # perform the POST, the URI is always required
58
+ # response http.request post_uri, post
59
+ #
60
+ # Note that for GET, HEAD and other requests that do not have a body you want
61
+ # to use URI#request_uri not URI#path. The request_uri contains the query
62
+ # params which are sent in the body for other requests.
63
+ #
64
+ # == SSL
65
+ #
66
+ # SSL connections are automatically created depending upon the scheme of the
67
+ # URI. SSL connections are automatically verified against the default
68
+ # certificate store for your computer. You can override this by changing
69
+ # verify_mode or by specifying an alternate cert_store.
70
+ #
71
+ # Here are the SSL settings, see the individual methods for documentation:
72
+ #
73
+ # #certificate :: This client's certificate
74
+ # #ca_file :: The certificate-authorities
75
+ # #ca_path :: Directory with certificate-authorities
76
+ # #cert_store :: An SSL certificate store
77
+ # #ciphers :: List of SSl ciphers allowed
78
+ # #private_key :: The client's SSL private key
79
+ # #reuse_ssl_sessions :: Reuse a previously opened SSL session for a new
80
+ # connection
81
+ # #ssl_timeout :: SSL session lifetime
82
+ # #ssl_version :: Which specific SSL version to use
83
+ # #verify_callback :: For server certificate verification
84
+ # #verify_depth :: Depth of certificate verification
85
+ # #verify_mode :: How connections should be verified
86
+ #
87
+ # == Proxies
88
+ #
89
+ # A proxy can be set through #proxy= or at initialization time by providing a
90
+ # second argument to ::new. The proxy may be the URI of the proxy server or
91
+ # <code>:ENV</code> which will consult environment variables.
92
+ #
93
+ # See #proxy= and #proxy_from_env for details.
94
+ #
95
+ # == Headers
96
+ #
97
+ # Headers may be specified for use in every request. #headers are appended to
98
+ # any headers on the request. #override_headers replace existing headers on
99
+ # the request.
100
+ #
101
+ # The difference between the two can be seen in setting the User-Agent. Using
102
+ # <code>http.headers['User-Agent'] = 'MyUserAgent'</code> will send "Ruby,
103
+ # MyUserAgent" while <code>http.override_headers['User-Agent'] =
104
+ # 'MyUserAgent'</code> will send "MyUserAgent".
105
+ #
106
+ # == Tuning
107
+ #
108
+ # === Segregation
109
+ #
110
+ # By providing an application name to ::new you can separate your connections
111
+ # from the connections of other applications.
112
+ #
113
+ # === Idle Timeout
114
+ #
115
+ # If a connection hasn't been used for this number of seconds it will automatically be
116
+ # reset upon the next use to avoid attempting to send to a closed connection.
117
+ # The default value is 5 seconds. nil means no timeout. Set through #idle_timeout.
118
+ #
119
+ # Reducing this value may help avoid the "too many connection resets" error
120
+ # when sending non-idempotent requests while increasing this value will cause
121
+ # fewer round-trips.
122
+ #
123
+ # === Read Timeout
124
+ #
125
+ # The amount of time allowed between reading two chunks from the socket. Set
126
+ # through #read_timeout
127
+ #
128
+ # === Max Requests
129
+ #
130
+ # The number of requests that should be made before opening a new connection.
131
+ # Typically many keep-alive capable servers tune this to 100 or less, so the
132
+ # 101st request will fail with ECONNRESET. If unset (default), this value has no
133
+ # effect, if set, connections will be reset on the request after max_requests.
134
+ #
135
+ # === Open Timeout
136
+ #
137
+ # The amount of time to wait for a connection to be opened. Set through
138
+ # #open_timeout.
139
+ #
140
+ # === Socket Options
141
+ #
142
+ # Socket options may be set on newly-created connections. See #socket_options
143
+ # for details.
144
+ #
145
+ # === Non-Idempotent Requests
146
+ #
147
+ # By default non-idempotent requests will not be retried per RFC 2616. By
148
+ # setting retry_change_requests to true requests will automatically be retried
149
+ # once.
150
+ #
151
+ # Only do this when you know that retrying a POST or other non-idempotent
152
+ # request is safe for your application and will not create duplicate
153
+ # resources.
154
+ #
155
+ # The recommended way to handle non-idempotent requests is the following:
156
+ #
157
+ # require 'net/http/persistent'
158
+ #
159
+ # uri = URI 'http://example.com/awesome/web/service'
160
+ # post_uri = uri + 'create'
161
+ #
162
+ # http = Net::HTTP::Persistent.new 'my_app_name'
163
+ #
164
+ # post = Net::HTTP::Post.new post_uri.path
165
+ # # ... fill in POST request
166
+ #
167
+ # begin
168
+ # response = http.request post_uri, post
169
+ # rescue Net::HTTP::Persistent::Error
170
+ #
171
+ # # POST failed, make a new request to verify the server did not process
172
+ # # the request
173
+ # exists_uri = uri + '...'
174
+ # response = http.get exists_uri
175
+ #
176
+ # # Retry if it failed
177
+ # retry if response.code == '404'
178
+ # end
179
+ #
180
+ # The method of determining if the resource was created or not is unique to
181
+ # the particular service you are using. Of course, you will want to add
182
+ # protection from infinite looping.
183
+ #
184
+ # === Connection Termination
185
+ #
186
+ # If you are done using the Net::HTTP::Persistent instance you may shut down
187
+ # all the connections in the current thread with #shutdown. This is not
188
+ # recommended for normal use, it should only be used when it will be several
189
+ # minutes before you make another HTTP request.
190
+ #
191
+ # If you are using multiple threads, call #shutdown in each thread when the
192
+ # thread is done making requests. If you don't call shutdown, that's OK.
193
+ # Ruby will automatically garbage collect and shutdown your HTTP connections
194
+ # when the thread terminates.
195
+
196
+ class Net::HTTP::Persistent
197
+
198
+ ##
199
+ # The beginning of Time
200
+
201
+ EPOCH = Time.at 0 # :nodoc:
202
+
203
+ ##
204
+ # Is OpenSSL available? This test works with autoload
205
+
206
+ HAVE_OPENSSL = defined? OpenSSL::SSL # :nodoc:
207
+
208
+ ##
209
+ # The version of Net::HTTP::Persistent you are using
210
+
211
+ VERSION = '2.10.0'
212
+
213
+ ##
214
+ # Exceptions rescued for automatic retry on ruby 2.0.0. This overlaps with
215
+ # the exception list for ruby 1.x.
216
+
217
+ RETRIED_EXCEPTIONS = [ # :nodoc:
218
+ (Net::ReadTimeout if Net.const_defined? :ReadTimeout),
219
+ IOError,
220
+ EOFError,
221
+ Errno::ECONNRESET,
222
+ Errno::ECONNABORTED,
223
+ Errno::EPIPE,
224
+ (OpenSSL::SSL::SSLError if HAVE_OPENSSL),
225
+ Timeout::Error,
226
+ ].compact
227
+
228
+ ##
229
+ # Error class for errors raised by Net::HTTP::Persistent. Various
230
+ # SystemCallErrors are re-raised with a human-readable message under this
231
+ # class.
232
+
233
+ class Error < StandardError; end
234
+
235
+ ##
236
+ # Use this method to detect the idle timeout of the host at +uri+. The
237
+ # value returned can be used to configure #idle_timeout. +max+ controls the
238
+ # maximum idle timeout to detect.
239
+ #
240
+ # After
241
+ #
242
+ # Idle timeout detection is performed by creating a connection then
243
+ # performing a HEAD request in a loop until the connection terminates
244
+ # waiting one additional second per loop.
245
+ #
246
+ # NOTE: This may not work on ruby > 1.9.
247
+
248
+ def self.detect_idle_timeout uri, max = 10
249
+ uri = URI uri unless URI::Generic === uri
250
+ uri += '/'
251
+
252
+ req = Net::HTTP::Head.new uri.request_uri
253
+
254
+ http = new 'net-http-persistent detect_idle_timeout'
255
+
256
+ http.connection_for uri do |connection|
257
+ sleep_time = 0
258
+
259
+ http = connection.http
260
+
261
+ loop do
262
+ response = http.request req
263
+
264
+ $stderr.puts "HEAD #{uri} => #{response.code}" if $DEBUG
265
+
266
+ unless Net::HTTPOK === response then
267
+ raise Error, "bad response code #{response.code} detecting idle timeout"
268
+ end
269
+
270
+ break if sleep_time >= max
271
+
272
+ sleep_time += 1
273
+
274
+ $stderr.puts "sleeping #{sleep_time}" if $DEBUG
275
+ sleep sleep_time
276
+ end
277
+ end
278
+ rescue
279
+ # ignore StandardErrors, we've probably found the idle timeout.
280
+ ensure
281
+ return sleep_time unless $!
282
+ end
283
+
284
+ ##
285
+ # This client's OpenSSL::X509::Certificate
286
+
287
+ attr_reader :certificate
288
+
289
+ # For Net::HTTP parity
290
+ alias cert certificate
291
+
292
+ ##
293
+ # An SSL certificate authority. Setting this will set verify_mode to
294
+ # VERIFY_PEER.
295
+
296
+ attr_reader :ca_file
297
+
298
+ ##
299
+ # A directory of SSL certificates to be used as certificate authorities.
300
+ # Setting this will set verify_mode to VERIFY_PEER.
301
+
302
+ attr_reader :ca_path
303
+
304
+ ##
305
+ # An SSL certificate store. Setting this will override the default
306
+ # certificate store. See verify_mode for more information.
307
+
308
+ attr_reader :cert_store
309
+
310
+ ##
311
+ # The ciphers allowed for SSL connections
312
+
313
+ attr_reader :ciphers
314
+
315
+ ##
316
+ # Sends debug_output to this IO via Net::HTTP#set_debug_output.
317
+ #
318
+ # Never use this method in production code, it causes a serious security
319
+ # hole.
320
+
321
+ attr_accessor :debug_output
322
+
323
+ ##
324
+ # Current connection generation
325
+
326
+ attr_reader :generation # :nodoc:
327
+
328
+ ##
329
+ # Where this instance's connections live in the thread local variables
330
+
331
+ attr_reader :generation_key # :nodoc:
332
+
333
+ ##
334
+ # Headers that are added to every request using Net::HTTP#add_field
335
+
336
+ attr_reader :headers
337
+
338
+ ##
339
+ # Maps host:port to an HTTP version. This allows us to enable version
340
+ # specific features.
341
+
342
+ attr_reader :http_versions
343
+
344
+ ##
345
+ # Maximum time an unused connection can remain idle before being
346
+ # automatically closed.
347
+
348
+ attr_accessor :idle_timeout
349
+
350
+ ##
351
+ # Maximum number of requests on a connection before it is considered expired
352
+ # and automatically closed.
353
+
354
+ attr_accessor :max_requests
355
+
356
+ ##
357
+ # The value sent in the Keep-Alive header. Defaults to 30. Not needed for
358
+ # HTTP/1.1 servers.
359
+ #
360
+ # This may not work correctly for HTTP/1.0 servers
361
+ #
362
+ # This method may be removed in a future version as RFC 2616 does not
363
+ # require this header.
364
+
365
+ attr_accessor :keep_alive
366
+
367
+ ##
368
+ # A name for this connection. Allows you to keep your connections apart
369
+ # from everybody else's.
370
+
371
+ attr_reader :name
372
+
373
+ ##
374
+ # Seconds to wait until a connection is opened. See Net::HTTP#open_timeout
375
+
376
+ attr_accessor :open_timeout
377
+
378
+ ##
379
+ # Headers that are added to every request using Net::HTTP#[]=
380
+
381
+ attr_reader :override_headers
382
+
383
+ ##
384
+ # This client's SSL private key
385
+
386
+ attr_reader :private_key
387
+
388
+ # For Net::HTTP parity
389
+ alias key private_key
390
+
391
+ ##
392
+ # The URL through which requests will be proxied
393
+
394
+ attr_reader :proxy_uri
395
+
396
+ ##
397
+ # List of host suffixes which will not be proxied
398
+
399
+ attr_reader :no_proxy
400
+
401
+ ##
402
+ # Test-only accessor for the connection pool
403
+
404
+ attr_reader :pool # :nodoc:
405
+
406
+ ##
407
+ # Seconds to wait until reading one block. See Net::HTTP#read_timeout
408
+
409
+ attr_accessor :read_timeout
410
+
411
+ ##
412
+ # By default SSL sessions are reused to avoid extra SSL handshakes. Set
413
+ # this to false if you have problems communicating with an HTTPS server
414
+ # like:
415
+ #
416
+ # SSL_connect [...] read finished A: unexpected message (OpenSSL::SSL::SSLError)
417
+
418
+ attr_accessor :reuse_ssl_sessions
419
+
420
+ ##
421
+ # An array of options for Socket#setsockopt.
422
+ #
423
+ # By default the TCP_NODELAY option is set on sockets.
424
+ #
425
+ # To set additional options append them to this array:
426
+ #
427
+ # http.socket_options << [Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1]
428
+
429
+ attr_reader :socket_options
430
+
431
+ ##
432
+ # Current SSL connection generation
433
+
434
+ attr_reader :ssl_generation # :nodoc:
435
+
436
+ ##
437
+ # Where this instance's SSL connections live in the thread local variables
438
+
439
+ attr_reader :ssl_generation_key # :nodoc:
440
+
441
+ ##
442
+ # SSL session lifetime
443
+
444
+ attr_reader :ssl_timeout
445
+
446
+ ##
447
+ # SSL version to use.
448
+ #
449
+ # By default, the version will be negotiated automatically between client
450
+ # and server. Ruby 1.9 and newer only.
451
+
452
+ attr_reader :ssl_version if RUBY_VERSION > '1.9'
453
+
454
+ ##
455
+ # Where this instance's last-use times live in the thread local variables
456
+
457
+ attr_reader :timeout_key # :nodoc:
458
+
459
+ ##
460
+ # SSL verification callback. Used when ca_file or ca_path is set.
461
+
462
+ attr_reader :verify_callback
463
+
464
+ ##
465
+ # Sets the depth of SSL certificate verification
466
+
467
+ attr_reader :verify_depth
468
+
469
+ ##
470
+ # HTTPS verify mode. Defaults to OpenSSL::SSL::VERIFY_PEER which verifies
471
+ # the server certificate.
472
+ #
473
+ # If no ca_file, ca_path or cert_store is set the default system certificate
474
+ # store is used.
475
+ #
476
+ # You can use +verify_mode+ to override any default values.
477
+
478
+ attr_reader :verify_mode
479
+
480
+ ##
481
+ # Enable retries of non-idempotent requests that change data (e.g. POST
482
+ # requests) when the server has disconnected.
483
+ #
484
+ # This will in the worst case lead to multiple requests with the same data,
485
+ # but it may be useful for some applications. Take care when enabling
486
+ # this option to ensure it is safe to POST or perform other non-idempotent
487
+ # requests to the server.
488
+
489
+ attr_accessor :retry_change_requests
490
+
491
+ ##
492
+ # Creates a new Net::HTTP::Persistent.
493
+ #
494
+ # Set +name+ to keep your connections apart from everybody else's. Not
495
+ # required currently, but highly recommended. Your library name should be
496
+ # good enough. This parameter will be required in a future version.
497
+ #
498
+ # +proxy+ may be set to a URI::HTTP or :ENV to pick up proxy options from
499
+ # the environment. See proxy_from_env for details.
500
+ #
501
+ # In order to use a URI for the proxy you may need to do some extra work
502
+ # beyond URI parsing if the proxy requires a password:
503
+ #
504
+ # proxy = URI 'http://proxy.example'
505
+ # proxy.user = 'AzureDiamond'
506
+ # proxy.password = 'hunter2'
507
+
508
+ def initialize name = nil, proxy = nil
509
+ @name = name
510
+
511
+ @debug_output = nil
512
+ @proxy_uri = nil
513
+ @no_proxy = []
514
+ @headers = {}
515
+ @override_headers = {}
516
+ @http_versions = {}
517
+ @keep_alive = 30
518
+ @open_timeout = nil
519
+ @read_timeout = nil
520
+ @idle_timeout = 5
521
+ @max_requests = nil
522
+ @socket_options = []
523
+ @ssl_generation = 0 # incremented when SSL session variables change
524
+
525
+ @socket_options << [Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1] if
526
+ Socket.const_defined? :TCP_NODELAY
527
+
528
+ key = ['net_http_persistent', name].compact
529
+ @generation_key = [key, 'generations' ].join('_').intern
530
+ @ssl_generation_key = [key, 'ssl_generations'].join('_').intern
531
+ @timeout_key = [key, 'timeouts' ].join('_').intern
532
+
533
+ pool_size = Process.getrlimit(Process::RLIMIT_NOFILE).first / 4
534
+ @pool = Net::HTTP::Persistent::Pool.new size: pool_size do |http_args|
535
+ Net::HTTP::Persistent::Connection.new http_class, http_args, @ssl_generation
536
+ end
537
+
538
+ @certificate = nil
539
+ @ca_file = nil
540
+ @ca_path = nil
541
+ @ciphers = nil
542
+ @private_key = nil
543
+ @ssl_timeout = nil
544
+ @ssl_version = nil
545
+ @verify_callback = nil
546
+ @verify_depth = nil
547
+ @verify_mode = nil
548
+ @cert_store = nil
549
+
550
+ @generation = 0 # incremented when proxy URI changes
551
+
552
+ if HAVE_OPENSSL then
553
+ @verify_mode = OpenSSL::SSL::VERIFY_PEER
554
+ @reuse_ssl_sessions = OpenSSL::SSL.const_defined? :Session
555
+ end
556
+
557
+ @retry_change_requests = false
558
+
559
+ @ruby_1 = RUBY_VERSION < '2'
560
+ @retried_on_ruby_2 = !@ruby_1
561
+
562
+ self.proxy = proxy if proxy
563
+ end
564
+
565
+ ##
566
+ # Sets this client's OpenSSL::X509::Certificate
567
+
568
+ def certificate= certificate
569
+ @certificate = certificate
570
+
571
+ reconnect_ssl
572
+ end
573
+
574
+ # For Net::HTTP parity
575
+ alias cert= certificate=
576
+
577
+ ##
578
+ # Sets the SSL certificate authority file.
579
+
580
+ def ca_file= file
581
+ @ca_file = file
582
+
583
+ reconnect_ssl
584
+ end
585
+
586
+ ##
587
+ # Sets the SSL certificate authority path.
588
+
589
+ def ca_path= path
590
+ @ca_path = path
591
+
592
+ reconnect_ssl
593
+ end
594
+
595
+ ##
596
+ # Overrides the default SSL certificate store used for verifying
597
+ # connections.
598
+
599
+ def cert_store= store
600
+ @cert_store = store
601
+
602
+ reconnect_ssl
603
+ end
604
+
605
+ ##
606
+ # The ciphers allowed for SSL connections
607
+
608
+ def ciphers= ciphers
609
+ @ciphers = ciphers
610
+
611
+ reconnect_ssl
612
+ end
613
+
614
+ ##
615
+ # Finishes all connections on the given +thread+ that were created before
616
+ # the given +generation+ in the threads +generation_key+ list.
617
+ #
618
+ # See #shutdown for a bunch of scary warning about misusing this method.
619
+
620
+ def cleanup(generation, thread = Thread.current,
621
+ generation_key = @generation_key) # :nodoc:
622
+ timeouts = thread[@timeout_key]
623
+
624
+ (0...generation).each do |old_generation|
625
+ next unless thread[generation_key]
626
+
627
+ conns = thread[generation_key].delete old_generation
628
+
629
+ conns.each_value do |conn|
630
+ finish conn, thread
631
+
632
+ timeouts.delete conn.object_id if timeouts
633
+ end if conns
634
+ end
635
+ end
636
+
637
+ ##
638
+ # Creates a new connection for +uri+
639
+
640
+ def connection_for uri
641
+ Thread.current[@timeout_key] ||= Hash.new EPOCH
642
+
643
+ use_ssl = uri.scheme.downcase == 'https'
644
+
645
+ net_http_args = [uri.host, uri.port]
646
+
647
+ net_http_args.concat @proxy_args if
648
+ @proxy_uri and not proxy_bypass? uri.host, uri.port
649
+
650
+ connection = @pool.checkout net_http_args
651
+
652
+ http = connection.http
653
+
654
+ connection.ressl @ssl_generation if
655
+ connection.ssl_generation != @ssl_generation
656
+
657
+ if not http.started? then
658
+ ssl http if use_ssl
659
+ start http
660
+ elsif expired? connection then
661
+ reset connection
662
+ end
663
+
664
+ http.read_timeout = @read_timeout if @read_timeout
665
+ http.keep_alive_timeout = @idle_timeout if @idle_timeout && http.respond_to?(:keep_alive_timeout=)
666
+
667
+ return yield connection
668
+ rescue Errno::ECONNREFUSED
669
+ address = http.proxy_address || http.address
670
+ port = http.proxy_port || http.port
671
+
672
+ raise Error, "connection refused: #{address}:#{port}"
673
+ rescue Errno::EHOSTDOWN
674
+ address = http.proxy_address || http.address
675
+ port = http.proxy_port || http.port
676
+
677
+ raise Error, "host down: #{address}:#{port}"
678
+ ensure
679
+ @pool.checkin net_http_args
680
+ end
681
+
682
+ ##
683
+ # Returns an error message containing the number of requests performed on
684
+ # this connection
685
+
686
+ def error_message connection
687
+ connection.requests -= 1 # fixup
688
+
689
+ age = Time.now - connection.last_use
690
+
691
+ "after #{connection.requests} requests on #{connection.http.object_id}, " \
692
+ "last used #{age} seconds ago"
693
+ end
694
+
695
+ ##
696
+ # URI::escape wrapper
697
+
698
+ def escape str
699
+ CGI.escape str if str
700
+ end
701
+
702
+ ##
703
+ # URI::unescape wrapper
704
+
705
+ def unescape str
706
+ CGI.unescape str if str
707
+ end
708
+
709
+
710
+ ##
711
+ # Returns true if the connection should be reset due to an idle timeout, or
712
+ # maximum request count, false otherwise.
713
+
714
+ def expired? connection
715
+ return true if @max_requests && connection.requests >= @max_requests
716
+ return false unless @idle_timeout
717
+ return true if @idle_timeout.zero?
718
+
719
+ Time.now - connection.last_use > @idle_timeout
720
+ end
721
+
722
+ ##
723
+ # Starts the Net::HTTP +connection+
724
+
725
+ def start http
726
+ http.set_debug_output @debug_output if @debug_output
727
+ http.open_timeout = @open_timeout if @open_timeout
728
+
729
+ http.start
730
+
731
+ socket = http.instance_variable_get :@socket
732
+
733
+ if socket then # for fakeweb
734
+ @socket_options.each do |option|
735
+ socket.io.setsockopt(*option)
736
+ end
737
+ end
738
+ end
739
+
740
+ ##
741
+ # Finishes the Net::HTTP +connection+
742
+
743
+ def finish connection
744
+ connection.finish
745
+ end
746
+
747
+ def http_class # :nodoc:
748
+ if RUBY_VERSION > '2.0' then
749
+ Net::HTTP
750
+ elsif [:Artifice, :FakeWeb, :WebMock].any? { |klass|
751
+ Object.const_defined?(klass)
752
+ } or not @reuse_ssl_sessions then
753
+ Net::HTTP
754
+ else
755
+ Net::HTTP::Persistent::SSLReuse
756
+ end
757
+ end
758
+
759
+ ##
760
+ # Returns the HTTP protocol version for +uri+
761
+
762
+ def http_version uri
763
+ @http_versions["#{uri.host}:#{uri.port}"]
764
+ end
765
+
766
+ ##
767
+ # Is +req+ idempotent according to RFC 2616?
768
+
769
+ def idempotent? req
770
+ case req
771
+ when Net::HTTP::Delete, Net::HTTP::Get, Net::HTTP::Head,
772
+ Net::HTTP::Options, Net::HTTP::Put, Net::HTTP::Trace then
773
+ true
774
+ end
775
+ end
776
+
777
+ ##
778
+ # Is the request +req+ idempotent or is retry_change_requests allowed.
779
+ #
780
+ # If +retried_on_ruby_2+ is true, true will be returned if we are on ruby,
781
+ # retry_change_requests is allowed and the request is not idempotent.
782
+
783
+ def can_retry? req, retried_on_ruby_2 = false
784
+ return @retry_change_requests && !idempotent?(req) if retried_on_ruby_2
785
+
786
+ @retry_change_requests || idempotent?(req)
787
+ end
788
+
789
+ if RUBY_VERSION > '1.9' then
790
+ ##
791
+ # Workaround for missing Net::HTTPHeader#connection_close? on Ruby 1.8
792
+
793
+ def connection_close? header
794
+ header.connection_close?
795
+ end
796
+
797
+ ##
798
+ # Workaround for missing Net::HTTPHeader#connection_keep_alive? on Ruby 1.8
799
+
800
+ def connection_keep_alive? header
801
+ header.connection_keep_alive?
802
+ end
803
+ else
804
+ ##
805
+ # Workaround for missing Net::HTTPRequest#connection_close? on Ruby 1.8
806
+
807
+ def connection_close? header
808
+ header['connection'] =~ /close/ or header['proxy-connection'] =~ /close/
809
+ end
810
+
811
+ ##
812
+ # Workaround for missing Net::HTTPRequest#connection_keep_alive? on Ruby
813
+ # 1.8
814
+
815
+ def connection_keep_alive? header
816
+ header['connection'] =~ /keep-alive/ or
817
+ header['proxy-connection'] =~ /keep-alive/
818
+ end
819
+ end
820
+
821
+ ##
822
+ # Deprecated in favor of #expired?
823
+
824
+ def max_age # :nodoc:
825
+ return Time.now + 1 unless @idle_timeout
826
+
827
+ Time.now - @idle_timeout
828
+ end
829
+
830
+ ##
831
+ # Adds "http://" to the String +uri+ if it is missing.
832
+
833
+ def normalize_uri uri
834
+ (uri =~ /^https?:/) ? uri : "http://#{uri}"
835
+ end
836
+
837
+ ##
838
+ # Pipelines +requests+ to the HTTP server at +uri+ yielding responses if a
839
+ # block is given. Returns all responses recieved.
840
+ #
841
+ # See
842
+ # Net::HTTP::Pipeline[http://docs.seattlerb.org/net-http-pipeline/Net/HTTP/Pipeline.html]
843
+ # for further details.
844
+ #
845
+ # Only if <tt>net-http-pipeline</tt> was required before
846
+ # <tt>net-http-persistent</tt> #pipeline will be present.
847
+
848
+ def pipeline uri, requests, &block # :yields: responses
849
+ connection_for uri do |connection|
850
+ connection.http.pipeline requests, &block
851
+ end
852
+ end
853
+
854
+ ##
855
+ # Sets this client's SSL private key
856
+
857
+ def private_key= key
858
+ @private_key = key
859
+
860
+ reconnect_ssl
861
+ end
862
+
863
+ # For Net::HTTP parity
864
+ alias key= private_key=
865
+
866
+ ##
867
+ # Sets the proxy server. The +proxy+ may be the URI of the proxy server,
868
+ # the symbol +:ENV+ which will read the proxy from the environment or nil to
869
+ # disable use of a proxy. See #proxy_from_env for details on setting the
870
+ # proxy from the environment.
871
+ #
872
+ # If the proxy URI is set after requests have been made, the next request
873
+ # will shut-down and re-open all connections.
874
+ #
875
+ # The +no_proxy+ query parameter can be used to specify hosts which shouldn't
876
+ # be reached via proxy; if set it should be a comma separated list of
877
+ # hostname suffixes, optionally with +:port+ appended, for example
878
+ # <tt>example.com,some.host:8080</tt>.
879
+
880
+ def proxy= proxy
881
+ @proxy_uri = case proxy
882
+ when :ENV then proxy_from_env
883
+ when URI::HTTP then proxy
884
+ when nil then # ignore
885
+ else raise ArgumentError, 'proxy must be :ENV or a URI::HTTP'
886
+ end
887
+
888
+ @no_proxy.clear
889
+
890
+ if @proxy_uri then
891
+ @proxy_args = [
892
+ @proxy_uri.host,
893
+ @proxy_uri.port,
894
+ unescape(@proxy_uri.user),
895
+ unescape(@proxy_uri.password),
896
+ ]
897
+
898
+ @proxy_connection_id = [nil, *@proxy_args].join ':'
899
+
900
+ if @proxy_uri.query then
901
+ @no_proxy = CGI.parse(@proxy_uri.query)['no_proxy'].join(',').downcase.split(',').map { |x| x.strip }.reject { |x| x.empty? }
902
+ end
903
+ end
904
+
905
+ reconnect
906
+ reconnect_ssl
907
+ end
908
+
909
+ ##
910
+ # Creates a URI for an HTTP proxy server from ENV variables.
911
+ #
912
+ # If +HTTP_PROXY+ is set a proxy will be returned.
913
+ #
914
+ # If +HTTP_PROXY_USER+ or +HTTP_PROXY_PASS+ are set the URI is given the
915
+ # indicated user and password unless HTTP_PROXY contains either of these in
916
+ # the URI.
917
+ #
918
+ # The +NO_PROXY+ ENV variable can be used to specify hosts which shouldn't
919
+ # be reached via proxy; if set it should be a comma separated list of
920
+ # hostname suffixes, optionally with +:port+ appended, for example
921
+ # <tt>example.com,some.host:8080</tt>. When set to <tt>*</tt> no proxy will
922
+ # be returned.
923
+ #
924
+ # For Windows users, lowercase ENV variables are preferred over uppercase ENV
925
+ # variables.
926
+
927
+ def proxy_from_env
928
+ env_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY']
929
+
930
+ return nil if env_proxy.nil? or env_proxy.empty?
931
+
932
+ uri = URI normalize_uri env_proxy
933
+
934
+ env_no_proxy = ENV['no_proxy'] || ENV['NO_PROXY']
935
+
936
+ # '*' is special case for always bypass
937
+ return nil if env_no_proxy == '*'
938
+
939
+ if env_no_proxy then
940
+ uri.query = "no_proxy=#{escape(env_no_proxy)}"
941
+ end
942
+
943
+ unless uri.user or uri.password then
944
+ uri.user = escape ENV['http_proxy_user'] || ENV['HTTP_PROXY_USER']
945
+ uri.password = escape ENV['http_proxy_pass'] || ENV['HTTP_PROXY_PASS']
946
+ end
947
+
948
+ uri
949
+ end
950
+
951
+ ##
952
+ # Returns true when proxy should by bypassed for host.
953
+
954
+ def proxy_bypass? host, port
955
+ host = host.downcase
956
+ host_port = [host, port].join ':'
957
+
958
+ @no_proxy.each do |name|
959
+ return true if host[-name.length, name.length] == name or
960
+ host_port[-name.length, name.length] == name
961
+ end
962
+
963
+ false
964
+ end
965
+
966
+ ##
967
+ # Forces reconnection of HTTP connections.
968
+
969
+ def reconnect
970
+ @generation += 1
971
+ end
972
+
973
+ ##
974
+ # Forces reconnection of SSL connections.
975
+
976
+ def reconnect_ssl
977
+ @ssl_generation += 1
978
+ end
979
+
980
+ ##
981
+ # Finishes then restarts the Net::HTTP +connection+
982
+
983
+ def reset connection
984
+ http = connection.http
985
+
986
+ finish connection
987
+
988
+ start http
989
+ rescue Errno::ECONNREFUSED
990
+ e = Error.new "connection refused: #{http.address}:#{http.port}"
991
+ e.set_backtrace $@
992
+ raise e
993
+ rescue Errno::EHOSTDOWN
994
+ e = Error.new "host down: #{http.address}:#{http.port}"
995
+ e.set_backtrace $@
996
+ raise e
997
+ end
998
+
999
+ ##
1000
+ # Makes a request on +uri+. If +req+ is nil a Net::HTTP::Get is performed
1001
+ # against +uri+.
1002
+ #
1003
+ # If a block is passed #request behaves like Net::HTTP#request (the body of
1004
+ # the response will not have been read).
1005
+ #
1006
+ # +req+ must be a Net::HTTPRequest subclass (see Net::HTTP for a list).
1007
+ #
1008
+ # If there is an error and the request is idempotent according to RFC 2616
1009
+ # it will be retried automatically.
1010
+
1011
+ def request uri, req = nil, &block
1012
+ retried = false
1013
+ bad_response = false
1014
+
1015
+ req = request_setup req || uri
1016
+ response = nil
1017
+
1018
+ connection_for uri do |connection|
1019
+ http = connection.http
1020
+
1021
+ begin
1022
+ connection.requests += 1
1023
+
1024
+ response = http.request req, &block
1025
+
1026
+ if connection_close?(req) or
1027
+ (response.http_version <= '1.0' and
1028
+ not connection_keep_alive?(response)) or
1029
+ connection_close?(response) then
1030
+ finish connection
1031
+ end
1032
+ rescue Net::HTTPBadResponse => e
1033
+ message = error_message connection
1034
+
1035
+ finish connection
1036
+
1037
+ raise Error, "too many bad responses #{message}" if
1038
+ bad_response or not can_retry? req
1039
+
1040
+ bad_response = true
1041
+ retry
1042
+ rescue *RETRIED_EXCEPTIONS => e # retried on ruby 2
1043
+ request_failed e, req, connection if
1044
+ retried or not can_retry? req, @retried_on_ruby_2
1045
+
1046
+ reset connection
1047
+
1048
+ retried = true
1049
+ retry
1050
+ rescue Errno::EINVAL, Errno::ETIMEDOUT => e # not retried on ruby 2
1051
+ request_failed e, req, connection if retried or not can_retry? req
1052
+
1053
+ reset connection
1054
+
1055
+ retried = true
1056
+ retry
1057
+ rescue Exception => e
1058
+ finish connection
1059
+
1060
+ raise
1061
+ ensure
1062
+ connection.last_use = Time.now
1063
+ end
1064
+ end
1065
+
1066
+ @http_versions["#{uri.host}:#{uri.port}"] ||= response.http_version
1067
+
1068
+ response
1069
+ end
1070
+
1071
+ ##
1072
+ # Raises an Error for +exception+ which resulted from attempting the request
1073
+ # +req+ on the +connection+.
1074
+ #
1075
+ # Finishes the +connection+.
1076
+
1077
+ def request_failed exception, req, connection # :nodoc:
1078
+ due_to = "(due to #{exception.message} - #{exception.class})"
1079
+ message = "too many connection resets #{due_to} #{error_message connection}"
1080
+
1081
+ finish connection
1082
+
1083
+ raise Error, message, exception.backtrace
1084
+ end
1085
+
1086
+ ##
1087
+ # Creates a GET request if +req_or_uri+ is a URI and adds headers to the
1088
+ # request.
1089
+ #
1090
+ # Returns the request.
1091
+
1092
+ def request_setup req_or_uri # :nodoc:
1093
+ req = if URI === req_or_uri then
1094
+ Net::HTTP::Get.new req_or_uri.request_uri
1095
+ else
1096
+ req_or_uri
1097
+ end
1098
+
1099
+ @headers.each do |pair|
1100
+ req.add_field(*pair)
1101
+ end
1102
+
1103
+ @override_headers.each do |name, value|
1104
+ req[name] = value
1105
+ end
1106
+
1107
+ unless req['Connection'] then
1108
+ req.add_field 'Connection', 'keep-alive'
1109
+ req.add_field 'Keep-Alive', @keep_alive
1110
+ end
1111
+
1112
+ req
1113
+ end
1114
+
1115
+ ##
1116
+ # Shuts down all connections
1117
+ #
1118
+ # *NOTE*: Calling shutdown for can be dangerous!
1119
+ #
1120
+ # If any thread is still using a connection it may cause an error! Call
1121
+ # #shutdown when you are completely done making requests!
1122
+
1123
+ def shutdown
1124
+ @pool.available.shutdown do |http|
1125
+ http.finish
1126
+ end
1127
+ end
1128
+
1129
+ ##
1130
+ # Enables SSL on +connection+
1131
+
1132
+ def ssl connection
1133
+ connection.use_ssl = true
1134
+
1135
+ connection.ciphers = @ciphers if @ciphers
1136
+ connection.ssl_timeout = @ssl_timeout if @ssl_timeout
1137
+ connection.ssl_version = @ssl_version if @ssl_version
1138
+
1139
+ connection.verify_depth = @verify_depth
1140
+ connection.verify_mode = @verify_mode
1141
+
1142
+ if OpenSSL::SSL::VERIFY_PEER == OpenSSL::SSL::VERIFY_NONE and
1143
+ not Object.const_defined?(:I_KNOW_THAT_OPENSSL_VERIFY_PEER_EQUALS_VERIFY_NONE_IS_WRONG) then
1144
+ warn <<-WARNING
1145
+ !!!SECURITY WARNING!!!
1146
+
1147
+ The SSL HTTP connection to:
1148
+
1149
+ #{connection.address}:#{connection.port}
1150
+
1151
+ !!!MAY NOT BE VERIFIED!!!
1152
+
1153
+ On your platform your OpenSSL implementation is broken.
1154
+
1155
+ There is no difference between the values of VERIFY_NONE and VERIFY_PEER.
1156
+
1157
+ This means that attempting to verify the security of SSL connections may not
1158
+ work. This exposes you to man-in-the-middle exploits, snooping on the
1159
+ contents of your connection and other dangers to the security of your data.
1160
+
1161
+ To disable this warning define the following constant at top-level in your
1162
+ application:
1163
+
1164
+ I_KNOW_THAT_OPENSSL_VERIFY_PEER_EQUALS_VERIFY_NONE_IS_WRONG = nil
1165
+
1166
+ WARNING
1167
+ end
1168
+
1169
+ connection.ca_file = @ca_file if @ca_file
1170
+ connection.ca_path = @ca_path if @ca_path
1171
+
1172
+ if @ca_file or @ca_path then
1173
+ connection.verify_mode = OpenSSL::SSL::VERIFY_PEER
1174
+ connection.verify_callback = @verify_callback if @verify_callback
1175
+ end
1176
+
1177
+ if @certificate and @private_key then
1178
+ connection.cert = @certificate
1179
+ connection.key = @private_key
1180
+ end
1181
+
1182
+ connection.cert_store = if @cert_store then
1183
+ @cert_store
1184
+ else
1185
+ store = OpenSSL::X509::Store.new
1186
+ store.set_default_paths
1187
+ store
1188
+ end
1189
+ end
1190
+
1191
+ ##
1192
+ # SSL session lifetime
1193
+
1194
+ def ssl_timeout= ssl_timeout
1195
+ @ssl_timeout = ssl_timeout
1196
+
1197
+ reconnect_ssl
1198
+ end
1199
+
1200
+ ##
1201
+ # SSL version to use
1202
+
1203
+ def ssl_version= ssl_version
1204
+ @ssl_version = ssl_version
1205
+
1206
+ reconnect_ssl
1207
+ end if RUBY_VERSION > '1.9'
1208
+
1209
+ ##
1210
+ # Sets the depth of SSL certificate verification
1211
+
1212
+ def verify_depth= verify_depth
1213
+ @verify_depth = verify_depth
1214
+
1215
+ reconnect_ssl
1216
+ end
1217
+
1218
+ ##
1219
+ # Sets the HTTPS verify mode. Defaults to OpenSSL::SSL::VERIFY_PEER.
1220
+ #
1221
+ # Setting this to VERIFY_NONE is a VERY BAD IDEA and should NEVER be used.
1222
+ # Securely transfer the correct certificate and update the default
1223
+ # certificate store or set the ca file instead.
1224
+
1225
+ def verify_mode= verify_mode
1226
+ @verify_mode = verify_mode
1227
+
1228
+ reconnect_ssl
1229
+ end
1230
+
1231
+ ##
1232
+ # SSL verification callback.
1233
+
1234
+ def verify_callback= callback
1235
+ @verify_callback = callback
1236
+
1237
+ reconnect_ssl
1238
+ end
1239
+
1240
+ end
1241
+
1242
+ require 'net/http/persistent/connection'
1243
+ require 'net/http/persistent/pool'
1244
+ require 'net/http/persistent/ssl_reuse'
1245
+