net-http-persistent 3.0.1 → 4.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,15 +17,11 @@ autoload :OpenSSL, 'openssl'
17
17
  # servers you wish to talk to. For each host:port you communicate with a
18
18
  # single persistent connection is created.
19
19
  #
20
- # Multiple Net::HTTP::Persistent objects will share the same set of
21
- # connections.
20
+ # Connections will be shared across threads through a connection pool to
21
+ # increase reuse of connections.
22
22
  #
23
- # For each thread you start a new connection will be created. A
24
- # Net::HTTP::Persistent connection will not be shared across threads.
25
- #
26
- # You can shut down the HTTP connections when done by calling #shutdown. You
27
- # should name your Net::HTTP::Persistent object if you intend to call this
28
- # method.
23
+ # You can shut down any remaining HTTP connections when done by calling
24
+ # #shutdown.
29
25
  #
30
26
  # Example:
31
27
  #
@@ -33,7 +29,7 @@ autoload :OpenSSL, 'openssl'
33
29
  #
34
30
  # uri = URI 'http://example.com/awesome/web/service'
35
31
  #
36
- # http = Net::HTTP::Persistent.new name: 'my_app_name'
32
+ # http = Net::HTTP::Persistent.new
37
33
  #
38
34
  # # perform a GET
39
35
  # response = http.request uri
@@ -55,14 +51,14 @@ autoload :OpenSSL, 'openssl'
55
51
  # to use URI#request_uri not URI#path. The request_uri contains the query
56
52
  # params which are sent in the body for other requests.
57
53
  #
58
- # == SSL
54
+ # == TLS/SSL
59
55
  #
60
- # SSL connections are automatically created depending upon the scheme of the
61
- # URI. SSL connections are automatically verified against the default
56
+ # TLS connections are automatically created depending upon the scheme of the
57
+ # URI. TLS connections are automatically verified against the default
62
58
  # certificate store for your computer. You can override this by changing
63
59
  # verify_mode or by specifying an alternate cert_store.
64
60
  #
65
- # Here are the SSL settings, see the individual methods for documentation:
61
+ # Here are the TLS settings, see the individual methods for documentation:
66
62
  #
67
63
  # #certificate :: This client's certificate
68
64
  # #ca_file :: The certificate-authorities
@@ -72,7 +68,7 @@ autoload :OpenSSL, 'openssl'
72
68
  # #private_key :: The client's SSL private key
73
69
  # #reuse_ssl_sessions :: Reuse a previously opened SSL session for a new
74
70
  # connection
75
- # #ssl_timeout :: SSL session lifetime
71
+ # #ssl_timeout :: Session lifetime
76
72
  # #ssl_version :: Which specific SSL version to use
77
73
  # #verify_callback :: For server certificate verification
78
74
  # #verify_depth :: Depth of certificate verification
@@ -101,14 +97,15 @@ autoload :OpenSSL, 'openssl'
101
97
  #
102
98
  # === Segregation
103
99
  #
104
- # By providing an application name to ::new you can separate your connections
105
- # from the connections of other applications.
100
+ # Each Net::HTTP::Persistent instance has its own pool of connections. There
101
+ # is no sharing with other instances (as was true in earlier versions).
106
102
  #
107
103
  # === Idle Timeout
108
104
  #
109
- # If a connection hasn't been used for this number of seconds it will automatically be
110
- # reset upon the next use to avoid attempting to send to a closed connection.
111
- # The default value is 5 seconds. nil means no timeout. Set through #idle_timeout.
105
+ # If a connection hasn't been used for this number of seconds it will
106
+ # automatically be reset upon the next use to avoid attempting to send to a
107
+ # closed connection. The default value is 5 seconds. nil means no timeout.
108
+ # Set through #idle_timeout.
112
109
  #
113
110
  # Reducing this value may help avoid the "too many connection resets" error
114
111
  # when sending non-idempotent requests while increasing this value will cause
@@ -123,8 +120,9 @@ autoload :OpenSSL, 'openssl'
123
120
  #
124
121
  # The number of requests that should be made before opening a new connection.
125
122
  # Typically many keep-alive capable servers tune this to 100 or less, so the
126
- # 101st request will fail with ECONNRESET. If unset (default), this value has no
127
- # effect, if set, connections will be reset on the request after max_requests.
123
+ # 101st request will fail with ECONNRESET. If unset (default), this value has
124
+ # no effect, if set, connections will be reset on the request after
125
+ # max_requests.
128
126
  #
129
127
  # === Open Timeout
130
128
  #
@@ -136,45 +134,6 @@ autoload :OpenSSL, 'openssl'
136
134
  # Socket options may be set on newly-created connections. See #socket_options
137
135
  # for details.
138
136
  #
139
- # === Non-Idempotent Requests
140
- #
141
- # By default non-idempotent requests will not be retried per RFC 2616. By
142
- # setting retry_change_requests to true requests will automatically be retried
143
- # once.
144
- #
145
- # Only do this when you know that retrying a POST or other non-idempotent
146
- # request is safe for your application and will not create duplicate
147
- # resources.
148
- #
149
- # The recommended way to handle non-idempotent requests is the following:
150
- #
151
- # require 'net/http/persistent'
152
- #
153
- # uri = URI 'http://example.com/awesome/web/service'
154
- # post_uri = uri + 'create'
155
- #
156
- # http = Net::HTTP::Persistent.new name: 'my_app_name'
157
- #
158
- # post = Net::HTTP::Post.new post_uri.path
159
- # # ... fill in POST request
160
- #
161
- # begin
162
- # response = http.request post_uri, post
163
- # rescue Net::HTTP::Persistent::Error
164
- #
165
- # # POST failed, make a new request to verify the server did not process
166
- # # the request
167
- # exists_uri = uri + '...'
168
- # response = http.get exists_uri
169
- #
170
- # # Retry if it failed
171
- # retry if response.code == '404'
172
- # end
173
- #
174
- # The method of determining if the resource was created or not is unique to
175
- # the particular service you are using. Of course, you will want to add
176
- # protection from infinite looping.
177
- #
178
137
  # === Connection Termination
179
138
  #
180
139
  # If you are done using the Net::HTTP::Persistent instance you may shut down
@@ -200,29 +159,27 @@ class Net::HTTP::Persistent
200
159
  HAVE_OPENSSL = defined? OpenSSL::SSL # :nodoc:
201
160
 
202
161
  ##
203
- # The default connection pool size is 1/4 the allowed open files.
162
+ # The default connection pool size is 1/4 the allowed open files
163
+ # (<code>ulimit -n</code>) or 256 if your OS does not support file handle
164
+ # limits (typically windows).
204
165
 
205
- DEFAULT_POOL_SIZE = Process.getrlimit(Process::RLIMIT_NOFILE).first / 4
206
-
207
- ##
208
- # The version of Net::HTTP::Persistent you are using
166
+ if Process.const_defined? :RLIMIT_NOFILE
167
+ open_file_limits = Process.getrlimit(Process::RLIMIT_NOFILE)
209
168
 
210
- VERSION = '3.0.1'
169
+ # Under JRuby on Windows Process responds to `getrlimit` but returns something that does not match docs
170
+ if open_file_limits.respond_to?(:first)
171
+ DEFAULT_POOL_SIZE = open_file_limits.first / 4
172
+ else
173
+ DEFAULT_POOL_SIZE = 256
174
+ end
175
+ else
176
+ DEFAULT_POOL_SIZE = 256
177
+ end
211
178
 
212
179
  ##
213
- # Exceptions rescued for automatic retry on ruby 2.0.0. This overlaps with
214
- # the exception list for ruby 1.x.
180
+ # The version of Net::HTTP::Persistent you are using
215
181
 
216
- RETRIED_EXCEPTIONS = [ # :nodoc:
217
- (Net::ReadTimeout if Net.const_defined? :ReadTimeout),
218
- IOError,
219
- EOFError,
220
- Errno::ECONNRESET,
221
- Errno::ECONNABORTED,
222
- Errno::EPIPE,
223
- (OpenSSL::SSL::SSLError if HAVE_OPENSSL),
224
- Timeout::Error,
225
- ].compact
182
+ VERSION = '4.0.2'
226
183
 
227
184
  ##
228
185
  # Error class for errors raised by Net::HTTP::Persistent. Various
@@ -349,6 +306,13 @@ class Net::HTTP::Persistent
349
306
 
350
307
  attr_accessor :max_requests
351
308
 
309
+ ##
310
+ # Number of retries to perform if a request fails.
311
+ #
312
+ # See also #max_retries=, Net::HTTP#max_retries=.
313
+
314
+ attr_reader :max_retries
315
+
352
316
  ##
353
317
  # The value sent in the Keep-Alive header. Defaults to 30. Not needed for
354
318
  # HTTP/1.1 servers.
@@ -361,8 +325,7 @@ class Net::HTTP::Persistent
361
325
  attr_accessor :keep_alive
362
326
 
363
327
  ##
364
- # A name for this connection. Allows you to keep your connections apart
365
- # from everybody else's.
328
+ # The name for this collection of persistent connections.
366
329
 
367
330
  attr_reader :name
368
331
 
@@ -406,6 +369,11 @@ class Net::HTTP::Persistent
406
369
 
407
370
  attr_accessor :read_timeout
408
371
 
372
+ ##
373
+ # Seconds to wait until writing one block. See Net::HTTP#write_timeout
374
+
375
+ attr_accessor :write_timeout
376
+
409
377
  ##
410
378
  # By default SSL sessions are reused to avoid extra SSL handshakes. Set
411
379
  # this to false if you have problems communicating with an HTTPS server
@@ -486,23 +454,11 @@ class Net::HTTP::Persistent
486
454
 
487
455
  attr_reader :verify_mode
488
456
 
489
- ##
490
- # Enable retries of non-idempotent requests that change data (e.g. POST
491
- # requests) when the server has disconnected.
492
- #
493
- # This will in the worst case lead to multiple requests with the same data,
494
- # but it may be useful for some applications. Take care when enabling
495
- # this option to ensure it is safe to POST or perform other non-idempotent
496
- # requests to the server.
497
-
498
- attr_accessor :retry_change_requests
499
-
500
457
  ##
501
458
  # Creates a new Net::HTTP::Persistent.
502
459
  #
503
- # Set +name+ to keep your connections apart from everybody else's. Not
504
- # required currently, but highly recommended. Your library name should be
505
- # good enough. This parameter will be required in a future version.
460
+ # Set a +name+ for fun. Your library name should be good enough, but this
461
+ # otherwise has no purpose.
506
462
  #
507
463
  # +proxy+ may be set to a URI::HTTP or :ENV to pick up proxy options from
508
464
  # the environment. See proxy_from_env for details.
@@ -515,8 +471,9 @@ class Net::HTTP::Persistent
515
471
  # proxy.password = 'hunter2'
516
472
  #
517
473
  # Set +pool_size+ to limit the maximum number of connections allowed.
518
- # Defaults to 1/4 the number of allowed file handles. You can have no more
519
- # than this many threads with active HTTP transactions.
474
+ # Defaults to 1/4 the number of allowed file handles or 256 if your OS does
475
+ # not support a limit on allowed file handles. You can have no more than
476
+ # this many threads with active HTTP transactions.
520
477
 
521
478
  def initialize name: nil, proxy: nil, pool_size: DEFAULT_POOL_SIZE
522
479
  @name = name
@@ -530,8 +487,10 @@ class Net::HTTP::Persistent
530
487
  @keep_alive = 30
531
488
  @open_timeout = nil
532
489
  @read_timeout = nil
490
+ @write_timeout = nil
533
491
  @idle_timeout = 5
534
492
  @max_requests = nil
493
+ @max_retries = 1
535
494
  @socket_options = []
536
495
  @ssl_generation = 0 # incremented when SSL session variables change
537
496
 
@@ -563,8 +522,6 @@ class Net::HTTP::Persistent
563
522
  @reuse_ssl_sessions = OpenSSL::SSL.const_defined? :Session
564
523
  end
565
524
 
566
- @retry_change_requests = false
567
-
568
525
  self.proxy = proxy if proxy
569
526
  end
570
527
 
@@ -623,10 +580,15 @@ class Net::HTTP::Persistent
623
580
  def connection_for uri
624
581
  use_ssl = uri.scheme.downcase == 'https'
625
582
 
626
- net_http_args = [uri.host, uri.port]
583
+ net_http_args = [uri.hostname, uri.port]
627
584
 
628
- net_http_args.concat @proxy_args if
629
- @proxy_uri and not proxy_bypass? uri.host, uri.port
585
+ # I'm unsure if uri.host or uri.hostname should be checked against
586
+ # the proxy bypass list.
587
+ if @proxy_uri and not proxy_bypass? uri.host, uri.port then
588
+ net_http_args.concat @proxy_args
589
+ else
590
+ net_http_args.concat [nil, nil, nil, nil]
591
+ end
630
592
 
631
593
  connection = @pool.checkout net_http_args
632
594
 
@@ -642,8 +604,11 @@ class Net::HTTP::Persistent
642
604
  reset connection
643
605
  end
644
606
 
645
- http.read_timeout = @read_timeout if @read_timeout
646
- http.keep_alive_timeout = @idle_timeout if @idle_timeout
607
+ http.keep_alive_timeout = @idle_timeout if @idle_timeout
608
+ http.max_retries = @max_retries if http.respond_to?(:max_retries=)
609
+ http.read_timeout = @read_timeout if @read_timeout
610
+ http.write_timeout = @write_timeout if
611
+ @write_timeout && http.respond_to?(:write_timeout=)
647
612
 
648
613
  return yield connection
649
614
  rescue Errno::ECONNREFUSED
@@ -661,27 +626,14 @@ class Net::HTTP::Persistent
661
626
  end
662
627
 
663
628
  ##
664
- # Returns an error message containing the number of requests performed on
665
- # this connection
666
-
667
- def error_message connection
668
- connection.requests -= 1 # fixup
669
-
670
- age = Time.now - connection.last_use
671
-
672
- "after #{connection.requests} requests on #{connection.http.object_id}, " \
673
- "last used #{age} seconds ago"
674
- end
675
-
676
- ##
677
- # URI::escape wrapper
629
+ # CGI::escape wrapper
678
630
 
679
631
  def escape str
680
632
  CGI.escape str if str
681
633
  end
682
634
 
683
635
  ##
684
- # URI::unescape wrapper
636
+ # CGI::unescape wrapper
685
637
 
686
638
  def unescape str
687
639
  CGI.unescape str if str
@@ -724,6 +676,7 @@ class Net::HTTP::Persistent
724
676
  def finish connection
725
677
  connection.finish
726
678
 
679
+ connection.http.instance_variable_set :@last_communicated, nil
727
680
  connection.http.instance_variable_set :@ssl_session, nil unless
728
681
  @reuse_ssl_sessions
729
682
  end
@@ -732,32 +685,31 @@ class Net::HTTP::Persistent
732
685
  # Returns the HTTP protocol version for +uri+
733
686
 
734
687
  def http_version uri
735
- @http_versions["#{uri.host}:#{uri.port}"]
688
+ @http_versions["#{uri.hostname}:#{uri.port}"]
736
689
  end
737
690
 
738
691
  ##
739
- # Is +req+ idempotent according to RFC 2616?
692
+ # Adds "http://" to the String +uri+ if it is missing.
740
693
 
741
- def idempotent? req
742
- case req
743
- when Net::HTTP::Delete, Net::HTTP::Get, Net::HTTP::Head,
744
- Net::HTTP::Options, Net::HTTP::Put, Net::HTTP::Trace then
745
- true
746
- end
694
+ def normalize_uri uri
695
+ (uri =~ /^https?:/) ? uri : "http://#{uri}"
747
696
  end
748
697
 
749
698
  ##
750
- # Is the request +req+ idempotent or is retry_change_requests allowed.
699
+ # Set the maximum number of retries for a request.
700
+ #
701
+ # Defaults to one retry.
702
+ #
703
+ # Set this to 0 to disable retries.
751
704
 
752
- def can_retry? req
753
- @retry_change_requests && !idempotent?(req)
754
- end
705
+ def max_retries= retries
706
+ retries = retries.to_int
755
707
 
756
- ##
757
- # Adds "http://" to the String +uri+ if it is missing.
708
+ raise ArgumentError, "max_retries must be positive" if retries < 0
758
709
 
759
- def normalize_uri uri
760
- (uri =~ /^https?:/) ? uri : "http://#{uri}"
710
+ @max_retries = retries
711
+
712
+ reconnect
761
713
  end
762
714
 
763
715
  ##
@@ -765,7 +717,7 @@ class Net::HTTP::Persistent
765
717
  # block is given. Returns all responses received.
766
718
  #
767
719
  # See
768
- # Net::HTTP::Pipeline[http://docs.seattlerb.org/net-http-pipeline/Net/HTTP/Pipeline.html]
720
+ # Net::HTTP::Pipeline[https://rdoc.info/gems/net-http-pipeline/Net/HTTP/Pipeline]
769
721
  # for further details.
770
722
  #
771
723
  # Only if <tt>net-http-pipeline</tt> was required before
@@ -815,7 +767,7 @@ class Net::HTTP::Persistent
815
767
 
816
768
  if @proxy_uri then
817
769
  @proxy_args = [
818
- @proxy_uri.host,
770
+ @proxy_uri.hostname,
819
771
  @proxy_uri.port,
820
772
  unescape(@proxy_uri.user),
821
773
  unescape(@proxy_uri.password),
@@ -890,14 +842,15 @@ class Net::HTTP::Persistent
890
842
  end
891
843
 
892
844
  ##
893
- # Forces reconnection of HTTP connections.
845
+ # Forces reconnection of all HTTP connections, including TLS/SSL
846
+ # connections.
894
847
 
895
848
  def reconnect
896
849
  @generation += 1
897
850
  end
898
851
 
899
852
  ##
900
- # Forces reconnection of SSL connections.
853
+ # Forces reconnection of only TLS/SSL connections.
901
854
 
902
855
  def reconnect_ssl
903
856
  @ssl_generation += 1
@@ -929,15 +882,9 @@ class Net::HTTP::Persistent
929
882
  # If a block is passed #request behaves like Net::HTTP#request (the body of
930
883
  # the response will not have been read).
931
884
  #
932
- # +req+ must be a Net::HTTPRequest subclass (see Net::HTTP for a list).
933
- #
934
- # If there is an error and the request is idempotent according to RFC 2616
935
- # it will be retried automatically.
885
+ # +req+ must be a Net::HTTPGenericRequest subclass (see Net::HTTP for a list).
936
886
 
937
887
  def request uri, req = nil, &block
938
- retried = false
939
- bad_response = false
940
-
941
888
  uri = URI uri
942
889
  req = request_setup req || uri
943
890
  response = nil
@@ -951,37 +898,12 @@ class Net::HTTP::Persistent
951
898
  response = http.request req, &block
952
899
 
953
900
  if req.connection_close? or
954
- (response.http_version <= '1.0' and
901
+ (response.http_version <= '1.0' and
955
902
  not response.connection_keep_alive?) or
956
- response.connection_close? then
903
+ response.connection_close? then
957
904
  finish connection
958
905
  end
959
- rescue Net::HTTPBadResponse => e
960
- message = error_message connection
961
-
962
- finish connection
963
-
964
- raise Error, "too many bad responses #{message}" if
965
- bad_response or not can_retry? req
966
-
967
- bad_response = true
968
- retry
969
- rescue *RETRIED_EXCEPTIONS => e
970
- request_failed e, req, connection if
971
- retried or not can_retry? req
972
-
973
- reset connection
974
-
975
- retried = true
976
- retry
977
- rescue Errno::EINVAL, Errno::ETIMEDOUT => e # not retried on ruby 2
978
- request_failed e, req, connection if retried or not can_retry? req
979
-
980
- reset connection
981
-
982
- retried = true
983
- retry
984
- rescue Exception => e
906
+ rescue Exception # make sure to close the connection when it was interrupted
985
907
  finish connection
986
908
 
987
909
  raise
@@ -990,26 +912,11 @@ class Net::HTTP::Persistent
990
912
  end
991
913
  end
992
914
 
993
- @http_versions["#{uri.host}:#{uri.port}"] ||= response.http_version
915
+ @http_versions["#{uri.hostname}:#{uri.port}"] ||= response.http_version
994
916
 
995
917
  response
996
918
  end
997
919
 
998
- ##
999
- # Raises an Error for +exception+ which resulted from attempting the request
1000
- # +req+ on the +connection+.
1001
- #
1002
- # Finishes the +connection+.
1003
-
1004
- def request_failed exception, req, connection # :nodoc:
1005
- due_to = "(due to #{exception.message} - #{exception.class})"
1006
- message = "too many connection resets #{due_to} #{error_message connection}"
1007
-
1008
- finish connection
1009
-
1010
- raise Error, message, exception.backtrace
1011
- end
1012
-
1013
920
  ##
1014
921
  # Creates a GET request if +req_or_uri+ is a URI and adds headers to the
1015
922
  # request.
@@ -1017,7 +924,7 @@ class Net::HTTP::Persistent
1017
924
  # Returns the request.
1018
925
 
1019
926
  def request_setup req_or_uri # :nodoc:
1020
- req = if URI === req_or_uri then
927
+ req = if req_or_uri.respond_to? 'request_uri' then
1021
928
  Net::HTTP::Get.new req_or_uri.request_uri
1022
929
  else
1023
930
  req_or_uri
@@ -1181,9 +1088,8 @@ application:
1181
1088
 
1182
1089
  reconnect_ssl
1183
1090
  end
1184
-
1185
1091
  end
1186
1092
 
1187
- require 'net/http/persistent/connection'
1188
- require 'net/http/persistent/pool'
1093
+ require_relative 'persistent/connection'
1094
+ require_relative 'persistent/pool'
1189
1095