persistent_http 1.0.6 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/persistent_http.rb +12 -335
- data/lib/persistent_http/connection.rb +384 -0
- data/lib/persistent_http/version.rb +3 -0
- metadata +12 -11
- data/lib/persistent_http/faster.rb +0 -27
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 58881dc6c2590c4313047e0c9ec21be8be5bb36f
|
4
|
+
data.tar.gz: ea99f7c8acf7fb797dcfc6ad26a24a7a8a722628
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 28e48614043b1e6254ee6c861bcf8bf224462cac3a1be5174b4293b1cd6a24d1638bc12af82552187b573f6514875948a5ef8fff61a4f23261752a6bd2936392
|
7
|
+
data.tar.gz: 21eaea07a9f7a1e26d309374aaa70c7ecb6a3d01ffc052e3e8637306cb97a41dbae3846b3086e6288e94618af0d61d0fc1e8b017adf50845876d5153c65a50e5
|
data/lib/persistent_http.rb
CHANGED
@@ -1,7 +1,5 @@
|
|
1
|
-
require '
|
2
|
-
require '
|
3
|
-
require 'persistent_http/faster'
|
4
|
-
require 'uri'
|
1
|
+
require 'persistent_http/connection'
|
2
|
+
require 'persistent_http/version'
|
5
3
|
require 'gene_pool'
|
6
4
|
|
7
5
|
##
|
@@ -22,12 +20,12 @@ require 'gene_pool'
|
|
22
20
|
# :force_retry => true,
|
23
21
|
# :url => 'https://www.example.com/echo/foo' # equivalent to :use_ssl => true, :host => 'www.example.com', :default_path => '/echo/foo'
|
24
22
|
# )
|
25
|
-
#
|
23
|
+
#
|
26
24
|
# def send_get_message
|
27
25
|
# response = @@persistent_http.request
|
28
26
|
# ... Handle response as you would a normal Net::HTTPResponse ...
|
29
27
|
# end
|
30
|
-
#
|
28
|
+
#
|
31
29
|
# def send_post_message
|
32
30
|
# request = Net::HTTP::Post.new('/perform_service)
|
33
31
|
# ... Modify request as needed ...
|
@@ -37,66 +35,16 @@ require 'gene_pool'
|
|
37
35
|
|
38
36
|
class PersistentHTTP
|
39
37
|
|
40
|
-
##
|
41
|
-
# The version of PersistentHTTP use are using
|
42
|
-
VERSION = '1.0.0'
|
43
|
-
|
44
38
|
##
|
45
39
|
# Error class for errors raised by PersistentHTTP. Various
|
46
40
|
# SystemCallErrors are re-raised with a human-readable message under this
|
47
41
|
# class.
|
48
42
|
class Error < StandardError; end
|
49
43
|
|
50
|
-
##
|
51
|
-
# An SSL certificate authority. Setting this will set verify_mode to
|
52
|
-
# VERIFY_PEER.
|
53
|
-
attr_accessor :ca_file
|
54
|
-
|
55
|
-
##
|
56
|
-
# This client's OpenSSL::X509::Certificate
|
57
|
-
attr_accessor :certificate
|
58
|
-
|
59
|
-
##
|
60
|
-
# Sends debug_output to this IO via Net::HTTP#set_debug_output.
|
61
|
-
#
|
62
|
-
# Never use this method in production code, it causes a serious security
|
63
|
-
# hole.
|
64
|
-
attr_accessor :debug_output
|
65
|
-
|
66
|
-
##
|
67
|
-
# Default path for the request
|
68
|
-
attr_accessor :default_path
|
69
|
-
|
70
|
-
##
|
71
|
-
# Retry even for non-idempotent (POST) requests.
|
72
|
-
attr_accessor :force_retry
|
73
|
-
|
74
|
-
##
|
75
|
-
# Headers that are added to every request
|
76
|
-
attr_accessor :headers
|
77
|
-
|
78
|
-
##
|
79
|
-
# Host for the Net:HTTP connection
|
80
|
-
attr_reader :host
|
81
|
-
|
82
|
-
##
|
83
|
-
# HTTP version to enable version specific features.
|
84
|
-
attr_reader :http_version
|
85
|
-
|
86
44
|
##
|
87
45
|
# Connection will be renewed if it hasn't been used in this amount of time. Defaults to 10 seconds.
|
88
46
|
attr_reader :idle_timeout
|
89
47
|
|
90
|
-
##
|
91
|
-
# The value sent in the Keep-Alive header. Defaults to 30. Not needed for
|
92
|
-
# HTTP/1.1 servers.
|
93
|
-
#
|
94
|
-
# This may not work correctly for HTTP/1.0 servers
|
95
|
-
#
|
96
|
-
# This method may be removed in a future version as RFC 2616 does not
|
97
|
-
# require this header.
|
98
|
-
attr_accessor :keep_alive
|
99
|
-
|
100
48
|
##
|
101
49
|
# Logger for message logging.
|
102
50
|
attr_accessor :logger
|
@@ -109,47 +57,13 @@ class PersistentHTTP
|
|
109
57
|
##
|
110
58
|
# Seconds to wait for an available connection before a Timeout::Error is raised
|
111
59
|
attr_accessor :pool_timeout
|
112
|
-
##
|
113
|
-
# Seconds to wait until a connection is opened. See Net::HTTP#open_timeout
|
114
|
-
attr_accessor :open_timeout
|
115
60
|
|
116
61
|
##
|
117
62
|
# The maximum size of the connection pool
|
118
63
|
attr_reader :pool_size
|
119
64
|
|
120
65
|
##
|
121
|
-
#
|
122
|
-
attr_reader :port
|
123
|
-
|
124
|
-
##
|
125
|
-
# This client's SSL private key
|
126
|
-
attr_accessor :private_key
|
127
|
-
|
128
|
-
##
|
129
|
-
# The URL through which requests will be proxied
|
130
|
-
attr_reader :proxy_uri
|
131
|
-
|
132
|
-
##
|
133
|
-
# Seconds to wait until reading one block. See Net::HTTP#read_timeout
|
134
|
-
attr_accessor :read_timeout
|
135
|
-
|
136
|
-
##
|
137
|
-
# Use ssl if set
|
138
|
-
attr_reader :use_ssl
|
139
|
-
|
140
|
-
##
|
141
|
-
# SSL verification callback. Used when ca_file is set.
|
142
|
-
attr_accessor :verify_callback
|
143
|
-
|
144
|
-
##
|
145
|
-
# HTTPS verify mode. Defaults to OpenSSL::SSL::VERIFY_NONE which ignores
|
146
|
-
# certificate problems.
|
147
|
-
#
|
148
|
-
# You can use +verify_mode+ to override any default values.
|
149
|
-
attr_accessor :verify_mode
|
150
|
-
|
151
|
-
##
|
152
|
-
# The threshold in seconds for checking out a connection at which a warning
|
66
|
+
# The threshold in seconds for checking out a connection at which a warning
|
153
67
|
# will be logged via the logger
|
154
68
|
attr_reader :warn_timeout
|
155
69
|
|
@@ -172,92 +86,21 @@ class PersistentHTTP
|
|
172
86
|
|
173
87
|
def initialize(options={})
|
174
88
|
@name = options[:name] || 'PersistentHTTP'
|
175
|
-
@ca_file = options[:ca_file]
|
176
|
-
@certificate = options[:certificate]
|
177
|
-
@debug_output = options[:debug_output]
|
178
|
-
@default_path = options[:default_path]
|
179
|
-
@force_retry = options[:force_retry]
|
180
|
-
@headers = options[:header] || {}
|
181
|
-
@host = options[:host]
|
182
89
|
@idle_timeout = options[:idle_timeout] || 10
|
183
|
-
@keep_alive = options[:keep_alive] || 30
|
184
90
|
@logger = options[:logger]
|
185
91
|
@pool_timeout = options[:pool_timeout]
|
186
|
-
@open_timeout = options[:open_timeout]
|
187
92
|
@pool_size = options[:pool_size] || 1
|
188
|
-
@port = options[:port]
|
189
|
-
@private_key = options[:private_key]
|
190
|
-
@read_timeout = options[:read_timeout]
|
191
|
-
@use_ssl = options[:use_ssl]
|
192
|
-
@verify_callback = options[:verify_callback]
|
193
|
-
@verify_mode = options[:verify_mode]
|
194
93
|
@warn_timeout = options[:warn_timeout] || 0.5
|
195
|
-
|
196
|
-
url = options[:url]
|
197
|
-
if url
|
198
|
-
url = URI.parse(url) if url.kind_of? String
|
199
|
-
@default_path ||= url.request_uri
|
200
|
-
@host ||= url.host
|
201
|
-
@port ||= url.port
|
202
|
-
@use_ssl ||= url.scheme == 'https'
|
203
|
-
end
|
204
|
-
|
205
|
-
@port ||= (@use_ssl ? 443 : 80)
|
206
|
-
|
207
|
-
# Hash containing the request counts based on the connection
|
208
|
-
@count_hash = Hash.new(0)
|
209
|
-
|
210
|
-
raise 'host not set' unless @host
|
211
|
-
net_http_args = [@host, @port]
|
212
|
-
connection_id = net_http_args.join ':'
|
213
|
-
|
214
|
-
proxy = options[:proxy]
|
215
|
-
|
216
|
-
@proxy_uri = case proxy
|
217
|
-
when :ENV then proxy_from_env
|
218
|
-
when URI::HTTP then proxy
|
219
|
-
when nil then # ignore
|
220
|
-
else raise ArgumentError, 'proxy must be :ENV or a URI::HTTP'
|
221
|
-
end
|
222
94
|
|
223
|
-
|
224
|
-
@proxy_args = [
|
225
|
-
@proxy_uri.host,
|
226
|
-
@proxy_uri.port,
|
227
|
-
@proxy_uri.user,
|
228
|
-
@proxy_uri.password,
|
229
|
-
]
|
230
|
-
|
231
|
-
@proxy_connection_id = [nil, *@proxy_args].join ':'
|
232
|
-
|
233
|
-
connection_id << @proxy_connection_id
|
234
|
-
net_http_args.concat @proxy_args
|
235
|
-
end
|
236
|
-
|
237
|
-
@pool = GenePool.new(:name => name + '-' + connection_id,
|
95
|
+
@pool = GenePool.new(:name => name,
|
238
96
|
:pool_size => @pool_size,
|
239
97
|
:timeout => @pool_timeout,
|
240
98
|
:warn_timeout => @warn_timeout,
|
241
99
|
:idle_timeout => @idle_timeout,
|
242
|
-
:close_proc =>
|
100
|
+
:close_proc => :finish,
|
243
101
|
:logger => @logger) do
|
244
|
-
|
245
|
-
|
246
|
-
connection = Net::HTTP.new(*net_http_args)
|
247
|
-
connection.set_debug_output @debug_output if @debug_output
|
248
|
-
connection.open_timeout = @open_timeout if @open_timeout
|
249
|
-
connection.read_timeout = @read_timeout if @read_timeout
|
250
|
-
|
251
|
-
ssl connection if @use_ssl
|
252
|
-
|
253
|
-
connection.start
|
254
|
-
@logger.debug { "#{name} #{connection}: Connection created" } if @logger
|
255
|
-
connection
|
256
|
-
rescue Errno::ECONNREFUSED
|
257
|
-
raise Error, "connection refused: #{connection.address}:#{connection.port}"
|
258
|
-
rescue Errno::EHOSTDOWN
|
259
|
-
raise Error, "host down: #{connection.address}:#{connection.port}"
|
260
|
-
end
|
102
|
+
@logger.debug { "#{name}: Creating connection" } if @logger
|
103
|
+
Connection.new(options)
|
261
104
|
end
|
262
105
|
end
|
263
106
|
|
@@ -284,61 +127,12 @@ class PersistentHTTP
|
|
284
127
|
# it will be retried automatically.
|
285
128
|
|
286
129
|
def request(req = nil, options = {}, &block)
|
287
|
-
retried = false
|
288
|
-
bad_response = false
|
289
|
-
|
290
|
-
req = Net::HTTP::Get.new @default_path unless req
|
291
|
-
|
292
|
-
headers.each do |pair|
|
293
|
-
req.add_field(*pair)
|
294
|
-
end
|
295
|
-
|
296
|
-
req.add_field 'Connection', 'keep-alive'
|
297
|
-
req.add_field 'Keep-Alive', @keep_alive
|
298
|
-
|
299
130
|
@pool.with_connection do |connection|
|
300
131
|
begin
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
response = connection.request req, &block
|
305
|
-
@http_version ||= response.http_version
|
306
|
-
@count_hash[connection.object_id] += 1
|
307
|
-
return response
|
308
|
-
|
309
|
-
rescue Timeout::Error => e
|
310
|
-
due_to = "(due to #{e.message} - #{e.class})"
|
311
|
-
message = error_message connection
|
312
|
-
@logger.info "#{name}: Removing connection #{due_to} #{message}" if @logger
|
313
|
-
remove connection
|
132
|
+
connection.request req, options, &block
|
133
|
+
rescue Exception => e
|
134
|
+
@pool.remove(connection)
|
314
135
|
raise
|
315
|
-
|
316
|
-
rescue Net::HTTPBadResponse => e
|
317
|
-
message = error_message connection
|
318
|
-
if bad_response or not (idempotent? req or @force_retry)
|
319
|
-
@logger.info "#{name}: Removing connection because of too many bad responses #{message}" if @logger
|
320
|
-
remove connection
|
321
|
-
raise Error, "too many bad responses #{message}"
|
322
|
-
else
|
323
|
-
bad_response = true
|
324
|
-
@logger.info "#{name}: Renewing connection because of bad response #{message}" if @logger
|
325
|
-
connection = renew connection
|
326
|
-
retry
|
327
|
-
end
|
328
|
-
|
329
|
-
rescue IOError, EOFError, Errno::ECONNABORTED, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EPIPE => e
|
330
|
-
due_to = "(due to #{e.message} - #{e.class})"
|
331
|
-
message = error_message connection
|
332
|
-
if retried or not (idempotent? req or @force_retry)
|
333
|
-
@logger.info "#{name}: Removing connection #{due_to} #{message}" if @logger
|
334
|
-
remove connection
|
335
|
-
raise Error, "too many connection resets #{due_to} #{message}"
|
336
|
-
else
|
337
|
-
retried = true
|
338
|
-
@logger.info "#{name}: Renewing connection #{due_to} #{message}" if @logger
|
339
|
-
connection = renew connection
|
340
|
-
retry
|
341
|
-
end
|
342
136
|
end
|
343
137
|
end
|
344
138
|
end
|
@@ -348,121 +142,4 @@ class PersistentHTTP
|
|
348
142
|
def shutdown(timeout=10)
|
349
143
|
@pool.close(timeout)
|
350
144
|
end
|
351
|
-
|
352
|
-
#######
|
353
|
-
private
|
354
|
-
#######
|
355
|
-
|
356
|
-
##
|
357
|
-
# Returns an error message containing the number of requests performed on
|
358
|
-
# this connection
|
359
|
-
|
360
|
-
def error_message connection
|
361
|
-
requests = @count_hash[connection.object_id] || 0
|
362
|
-
"after #{requests} requests on #{connection.object_id}"
|
363
|
-
end
|
364
|
-
|
365
|
-
##
|
366
|
-
# URI::escape wrapper
|
367
|
-
|
368
|
-
def escape str
|
369
|
-
URI.escape str if str
|
370
|
-
end
|
371
|
-
|
372
|
-
##
|
373
|
-
# Finishes the Net::HTTP +connection+
|
374
|
-
|
375
|
-
def finish connection
|
376
|
-
@count_hash.delete(connection.object_id)
|
377
|
-
connection.finish
|
378
|
-
rescue IOError
|
379
|
-
end
|
380
|
-
|
381
|
-
##
|
382
|
-
# Is +req+ idempotent according to RFC 2616?
|
383
|
-
|
384
|
-
def idempotent? req
|
385
|
-
case req
|
386
|
-
when Net::HTTP::Delete, Net::HTTP::Get, Net::HTTP::Head,
|
387
|
-
Net::HTTP::Options, Net::HTTP::Put, Net::HTTP::Trace then
|
388
|
-
true
|
389
|
-
end
|
390
|
-
end
|
391
|
-
|
392
|
-
##
|
393
|
-
# Adds "http://" to the String +uri+ if it is missing.
|
394
|
-
|
395
|
-
def normalize_uri uri
|
396
|
-
(uri =~ /^https?:/) ? uri : "http://#{uri}"
|
397
|
-
end
|
398
|
-
|
399
|
-
##
|
400
|
-
# Creates a URI for an HTTP proxy server from ENV variables.
|
401
|
-
#
|
402
|
-
# If +HTTP_PROXY+ is set a proxy will be returned.
|
403
|
-
#
|
404
|
-
# If +HTTP_PROXY_USER+ or +HTTP_PROXY_PASS+ are set the URI is given the
|
405
|
-
# indicated user and password unless HTTP_PROXY contains either of these in
|
406
|
-
# the URI.
|
407
|
-
#
|
408
|
-
# For Windows users lowercase ENV variables are preferred over uppercase ENV
|
409
|
-
# variables.
|
410
|
-
|
411
|
-
def proxy_from_env
|
412
|
-
env_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY']
|
413
|
-
|
414
|
-
return nil if env_proxy.nil? or env_proxy.empty?
|
415
|
-
|
416
|
-
uri = URI.parse(normalize_uri(env_proxy))
|
417
|
-
|
418
|
-
unless uri.user or uri.password then
|
419
|
-
uri.user = escape ENV['http_proxy_user'] || ENV['HTTP_PROXY_USER']
|
420
|
-
uri.password = escape ENV['http_proxy_pass'] || ENV['HTTP_PROXY_PASS']
|
421
|
-
end
|
422
|
-
|
423
|
-
uri
|
424
|
-
end
|
425
|
-
|
426
|
-
##
|
427
|
-
# Finishes then removes the Net::HTTP +connection+
|
428
|
-
|
429
|
-
def remove connection
|
430
|
-
finish connection
|
431
|
-
@pool.remove(connection)
|
432
|
-
end
|
433
|
-
|
434
|
-
##
|
435
|
-
# Finishes then renews the Net::HTTP +connection+. It may be unnecessary
|
436
|
-
# to completely recreate the connection but connections that get timed out
|
437
|
-
# in JRuby leave the ssl context in a frozen object state.
|
438
|
-
|
439
|
-
def renew connection
|
440
|
-
finish connection
|
441
|
-
connection = @pool.renew(connection)
|
442
|
-
end
|
443
|
-
|
444
|
-
##
|
445
|
-
# Enables SSL on +connection+
|
446
|
-
|
447
|
-
def ssl connection
|
448
|
-
connection.use_ssl = true
|
449
|
-
|
450
|
-
# suppress warning but allow override
|
451
|
-
connection.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @verify_mode
|
452
|
-
|
453
|
-
if @ca_file then
|
454
|
-
connection.ca_file = @ca_file
|
455
|
-
connection.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
456
|
-
connection.verify_callback = @verify_callback if @verify_callback
|
457
|
-
end
|
458
|
-
|
459
|
-
if @certificate and @private_key then
|
460
|
-
connection.cert = @certificate
|
461
|
-
connection.key = @private_key
|
462
|
-
end
|
463
|
-
|
464
|
-
connection.verify_mode = @verify_mode if @verify_mode
|
465
|
-
end
|
466
|
-
|
467
145
|
end
|
468
|
-
|
@@ -0,0 +1,384 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'net/https'
|
3
|
+
require 'uri'
|
4
|
+
|
5
|
+
##
|
6
|
+
# Simplified frontend for Net::HTTP
|
7
|
+
#
|
8
|
+
# Example:
|
9
|
+
#
|
10
|
+
# @http = PersistentHTTP::Connection.new(
|
11
|
+
# :logger => Rails.logger,
|
12
|
+
# :force_retry => true,
|
13
|
+
# :url => 'https://www.example.com/echo/foo' # equivalent to :use_ssl => true, :host => 'www.example.com', :default_path => '/echo/foo'
|
14
|
+
# )
|
15
|
+
#
|
16
|
+
# def send_get_message
|
17
|
+
# response = @http.request
|
18
|
+
# ... Handle response as you would a normal Net::HTTPResponse ...
|
19
|
+
# end
|
20
|
+
#
|
21
|
+
# def send_post_message
|
22
|
+
# request = Net::HTTP::Post.new('/perform_service)
|
23
|
+
# ... Modify request as needed ...
|
24
|
+
# response = @http.request(request)
|
25
|
+
# ... Handle response as you would a normal Net::HTTPResponse ...
|
26
|
+
# end
|
27
|
+
|
28
|
+
class PersistentHTTP
|
29
|
+
class Connection
|
30
|
+
|
31
|
+
##
|
32
|
+
# An SSL certificate authority. Setting this will set verify_mode to
|
33
|
+
# VERIFY_PEER.
|
34
|
+
attr_accessor :ca_file
|
35
|
+
|
36
|
+
##
|
37
|
+
# This client's OpenSSL::X509::Certificate
|
38
|
+
attr_accessor :certificate
|
39
|
+
|
40
|
+
##
|
41
|
+
# Sends debug_output to this IO via Net::HTTP#set_debug_output.
|
42
|
+
#
|
43
|
+
# Never use this method in production code, it causes a serious security
|
44
|
+
# hole.
|
45
|
+
attr_accessor :debug_output
|
46
|
+
|
47
|
+
##
|
48
|
+
# Default path for the request
|
49
|
+
attr_accessor :default_path
|
50
|
+
|
51
|
+
##
|
52
|
+
# Retry even for non-idempotent (POST) requests.
|
53
|
+
attr_accessor :force_retry
|
54
|
+
|
55
|
+
##
|
56
|
+
# Headers that are added to every request
|
57
|
+
attr_accessor :headers
|
58
|
+
|
59
|
+
##
|
60
|
+
# Host for the Net:HTTP connection
|
61
|
+
attr_reader :host
|
62
|
+
|
63
|
+
##
|
64
|
+
# HTTP version to enable version specific features.
|
65
|
+
attr_reader :http_version
|
66
|
+
|
67
|
+
##
|
68
|
+
# The value sent in the Keep-Alive header. Defaults to 30. Not needed for
|
69
|
+
# HTTP/1.1 servers.
|
70
|
+
#
|
71
|
+
# This may not work correctly for HTTP/1.0 servers
|
72
|
+
#
|
73
|
+
# This method may be removed in a future version as RFC 2616 does not
|
74
|
+
# require this header.
|
75
|
+
attr_accessor :keep_alive
|
76
|
+
|
77
|
+
##
|
78
|
+
# Logger for message logging.
|
79
|
+
attr_accessor :logger
|
80
|
+
|
81
|
+
##
|
82
|
+
# A name for this connection. Allows you to keep your connections apart
|
83
|
+
# from everybody else's.
|
84
|
+
attr_reader :name
|
85
|
+
|
86
|
+
##
|
87
|
+
# Seconds to wait until a connection is opened. See Net::HTTP#open_timeout
|
88
|
+
attr_accessor :open_timeout
|
89
|
+
|
90
|
+
##
|
91
|
+
# Port for the Net:HTTP connection
|
92
|
+
attr_reader :port
|
93
|
+
|
94
|
+
##
|
95
|
+
# This client's SSL private key
|
96
|
+
attr_accessor :private_key
|
97
|
+
|
98
|
+
##
|
99
|
+
# The URL through which requests will be proxied
|
100
|
+
attr_reader :proxy_uri
|
101
|
+
|
102
|
+
##
|
103
|
+
# Seconds to wait until reading one block. See Net::HTTP#read_timeout
|
104
|
+
attr_accessor :read_timeout
|
105
|
+
|
106
|
+
##
|
107
|
+
# Use ssl if set
|
108
|
+
attr_reader :use_ssl
|
109
|
+
|
110
|
+
##
|
111
|
+
# SSL verification callback. Used when ca_file is set.
|
112
|
+
attr_accessor :verify_callback
|
113
|
+
|
114
|
+
##
|
115
|
+
# HTTPS verify mode. Defaults to OpenSSL::SSL::VERIFY_NONE which ignores
|
116
|
+
# certificate problems.
|
117
|
+
#
|
118
|
+
# You can use +verify_mode+ to override any default values.
|
119
|
+
attr_accessor :verify_mode
|
120
|
+
|
121
|
+
##
|
122
|
+
# Creates a new HTTP Connection.
|
123
|
+
#
|
124
|
+
# Set +name+ to keep your connections apart from everybody else's. Not
|
125
|
+
# required currently, but highly recommended. Your library name should be
|
126
|
+
# good enough. This parameter will be required in a future version.
|
127
|
+
#
|
128
|
+
# +proxy+ may be set to a URI::HTTP or :ENV to pick up proxy options from
|
129
|
+
# the environment. See proxy_from_env for details.
|
130
|
+
#
|
131
|
+
# In order to use a URI for the proxy you'll need to do some extra work
|
132
|
+
# beyond URI.parse:
|
133
|
+
#
|
134
|
+
# proxy = URI.parse 'http://proxy.example'
|
135
|
+
# proxy.user = 'AzureDiamond'
|
136
|
+
# proxy.password = 'hunter2'
|
137
|
+
|
138
|
+
def initialize(options={})
|
139
|
+
@name = options[:name] || 'PeristentHTTP::Connection'
|
140
|
+
@ca_file = options[:ca_file]
|
141
|
+
@certificate = options[:certificate]
|
142
|
+
@debug_output = options[:debug_output]
|
143
|
+
@default_path = options[:default_path]
|
144
|
+
@force_retry = options[:force_retry]
|
145
|
+
@headers = options[:header] || {}
|
146
|
+
@host = options[:host]
|
147
|
+
@keep_alive = options[:keep_alive] || 30
|
148
|
+
@logger = options[:logger]
|
149
|
+
@port = options[:port]
|
150
|
+
@private_key = options[:private_key]
|
151
|
+
@open_timeout = options[:open_timeout]
|
152
|
+
@read_timeout = options[:read_timeout]
|
153
|
+
@use_ssl = options[:use_ssl]
|
154
|
+
@verify_callback = options[:verify_callback]
|
155
|
+
@verify_mode = options[:verify_mode]
|
156
|
+
# Because maybe we want a non-persistent connection and are just using this for the proxy stuff
|
157
|
+
@non_persistent = options[:non_persistent]
|
158
|
+
|
159
|
+
url = options[:url]
|
160
|
+
if url
|
161
|
+
url = URI.parse(url) if url.kind_of? String
|
162
|
+
@default_path ||= url.request_uri
|
163
|
+
@host ||= url.host
|
164
|
+
@port ||= url.port
|
165
|
+
@use_ssl ||= url.scheme == 'https'
|
166
|
+
end
|
167
|
+
|
168
|
+
@port ||= (@use_ssl ? 443 : 80)
|
169
|
+
|
170
|
+
raise 'host not set' unless @host
|
171
|
+
@net_http_args = [@host, @port]
|
172
|
+
|
173
|
+
proxy = options[:proxy]
|
174
|
+
@proxy_uri = case proxy
|
175
|
+
when :ENV then proxy_from_env
|
176
|
+
when URI::HTTP then proxy
|
177
|
+
when nil then # ignore
|
178
|
+
else raise ArgumentError, 'proxy must be :ENV or a URI::HTTP'
|
179
|
+
end
|
180
|
+
|
181
|
+
if @proxy_uri then
|
182
|
+
@proxy_args = [
|
183
|
+
@proxy_uri.host,
|
184
|
+
@proxy_uri.port,
|
185
|
+
@proxy_uri.user,
|
186
|
+
@proxy_uri.password,
|
187
|
+
]
|
188
|
+
|
189
|
+
@net_http_args.concat @proxy_args
|
190
|
+
end
|
191
|
+
|
192
|
+
@name += ':' + @net_http_args.join(':')
|
193
|
+
@logger.debug { "#{@name}: Creating connection" } if @logger
|
194
|
+
renew
|
195
|
+
end
|
196
|
+
|
197
|
+
def renew
|
198
|
+
finish if @connection
|
199
|
+
@message_count = 0
|
200
|
+
@connection = Net::HTTP.new(*@net_http_args)
|
201
|
+
@connection.set_debug_output @debug_output if @debug_output
|
202
|
+
@connection.open_timeout = @open_timeout if @open_timeout
|
203
|
+
@connection.read_timeout = @read_timeout if @read_timeout
|
204
|
+
|
205
|
+
ssl if @use_ssl
|
206
|
+
|
207
|
+
@connection.start
|
208
|
+
@logger.debug { "#{@name} #{@connection}: Connection created" } if @logger
|
209
|
+
rescue Errno::ECONNREFUSED
|
210
|
+
raise Error, "connection refused: #{@connection.address}:#{@connection.port}"
|
211
|
+
rescue Errno::EHOSTDOWN
|
212
|
+
raise Error, "host down: #{@connection.address}:#{@connection.port}"
|
213
|
+
end
|
214
|
+
|
215
|
+
##
|
216
|
+
# Makes a request per +req+. If +req+ is nil a Net::HTTP::Get is performed
|
217
|
+
# against +default_path+.
|
218
|
+
#
|
219
|
+
# If a block is passed #request behaves like Net::HTTP#request (the body of
|
220
|
+
# the response will not have been read).
|
221
|
+
#
|
222
|
+
# +req+ must be a Net::HTTPRequest subclass (see Net::HTTP for a list).
|
223
|
+
#
|
224
|
+
# If there is an error and the request is idempontent according to RFC 2616
|
225
|
+
# it will be retried automatically.
|
226
|
+
|
227
|
+
def request(req = nil, options = {}, &block)
|
228
|
+
retried = false
|
229
|
+
bad_response = false
|
230
|
+
|
231
|
+
req = Net::HTTP::Get.new @default_path unless req
|
232
|
+
|
233
|
+
headers.each do |pair|
|
234
|
+
req.add_field(*pair)
|
235
|
+
end
|
236
|
+
|
237
|
+
unless @non_persistent
|
238
|
+
req.add_field 'Connection', 'keep-alive'
|
239
|
+
req.add_field 'Keep-Alive', @keep_alive
|
240
|
+
end
|
241
|
+
|
242
|
+
begin
|
243
|
+
options.each do |key, value|
|
244
|
+
@connection.send("#{key}=", value)
|
245
|
+
end
|
246
|
+
response = @connection.request req, &block
|
247
|
+
@http_version ||= response.http_version
|
248
|
+
@message_count += 1
|
249
|
+
return response
|
250
|
+
|
251
|
+
rescue Timeout::Error => e
|
252
|
+
due_to = "(due to #{e.message} - #{e.class})"
|
253
|
+
@logger.info "#{@name}: Removing connection #{due_to} #{error_message}" if @logger
|
254
|
+
finish
|
255
|
+
raise
|
256
|
+
|
257
|
+
rescue Net::HTTPBadResponse => e
|
258
|
+
if bad_response or not (idempotent? req or @force_retry)
|
259
|
+
@logger.info "#{@name}: Removing connection because of too many bad responses #{error_message}" if @logger
|
260
|
+
finish
|
261
|
+
raise Error, "too many bad responses #{error_message}"
|
262
|
+
else
|
263
|
+
bad_response = true
|
264
|
+
@logger.info "#{@name}: Renewing connection because of bad response #{error_message}" if @logger
|
265
|
+
renew
|
266
|
+
retry
|
267
|
+
end
|
268
|
+
|
269
|
+
rescue IOError, EOFError, Errno::ECONNABORTED, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EPIPE => e
|
270
|
+
due_to = "(due to #{e.message} - #{e.class})"
|
271
|
+
if retried or not (idempotent? req or @force_retry)
|
272
|
+
@logger.info "#{@name}: Removing connection #{due_to} #{error_message}" if @logger
|
273
|
+
finish
|
274
|
+
raise Error, "too many connection resets #{due_to} #{error_message}"
|
275
|
+
else
|
276
|
+
retried = true
|
277
|
+
@logger.info "#{@name}: Renewing connection #{due_to} #{error_message}" if @logger
|
278
|
+
renew
|
279
|
+
retry
|
280
|
+
end
|
281
|
+
end
|
282
|
+
end
|
283
|
+
|
284
|
+
##
|
285
|
+
# Finishes the Net::HTTP +connection+
|
286
|
+
|
287
|
+
def finish
|
288
|
+
@connection.finish
|
289
|
+
rescue IOError
|
290
|
+
end
|
291
|
+
|
292
|
+
def to_s
|
293
|
+
@name
|
294
|
+
end
|
295
|
+
|
296
|
+
#######
|
297
|
+
private
|
298
|
+
#######
|
299
|
+
|
300
|
+
##
|
301
|
+
# Returns an error message containing the number of requests performed on
|
302
|
+
# this connection
|
303
|
+
|
304
|
+
def error_message
|
305
|
+
"after #{@message_count} requests on #{@connection.object_id}"
|
306
|
+
end
|
307
|
+
|
308
|
+
##
|
309
|
+
# URI::escape wrapper
|
310
|
+
|
311
|
+
def escape str
|
312
|
+
URI.escape str if str
|
313
|
+
end
|
314
|
+
|
315
|
+
|
316
|
+
##
|
317
|
+
# Is +req+ idempotent according to RFC 2616?
|
318
|
+
|
319
|
+
def idempotent? req
|
320
|
+
case req
|
321
|
+
when Net::HTTP::Delete, Net::HTTP::Get, Net::HTTP::Head,
|
322
|
+
Net::HTTP::Options, Net::HTTP::Put, Net::HTTP::Trace then
|
323
|
+
true
|
324
|
+
end
|
325
|
+
end
|
326
|
+
|
327
|
+
##
|
328
|
+
# Adds "http://" to the String +uri+ if it is missing.
|
329
|
+
|
330
|
+
def normalize_uri uri
|
331
|
+
(uri =~ /^https?:/) ? uri : "http://#{uri}"
|
332
|
+
end
|
333
|
+
|
334
|
+
##
|
335
|
+
# Creates a URI for an HTTP proxy server from ENV variables.
|
336
|
+
#
|
337
|
+
# If +HTTP_PROXY+ is set a proxy will be returned.
|
338
|
+
#
|
339
|
+
# If +HTTP_PROXY_USER+ or +HTTP_PROXY_PASS+ are set the URI is given the
|
340
|
+
# indicated user and password unless HTTP_PROXY contains either of these in
|
341
|
+
# the URI.
|
342
|
+
#
|
343
|
+
# For Windows users lowercase ENV variables are preferred over uppercase ENV
|
344
|
+
# variables.
|
345
|
+
|
346
|
+
def proxy_from_env
|
347
|
+
env_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY']
|
348
|
+
|
349
|
+
return nil if env_proxy.nil? or env_proxy.empty?
|
350
|
+
|
351
|
+
uri = URI.parse(normalize_uri(env_proxy))
|
352
|
+
|
353
|
+
unless uri.user or uri.password then
|
354
|
+
uri.user = escape ENV['http_proxy_user'] || ENV['HTTP_PROXY_USER']
|
355
|
+
uri.password = escape ENV['http_proxy_pass'] || ENV['HTTP_PROXY_PASS']
|
356
|
+
end
|
357
|
+
|
358
|
+
uri
|
359
|
+
end
|
360
|
+
|
361
|
+
##
|
362
|
+
# Enables SSL on +connection+
|
363
|
+
|
364
|
+
def ssl
|
365
|
+
@connection.use_ssl = true
|
366
|
+
|
367
|
+
# suppress warning but allow override
|
368
|
+
@connection.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @verify_mode
|
369
|
+
|
370
|
+
if @ca_file then
|
371
|
+
@connection.ca_file = @ca_file
|
372
|
+
@connection.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
373
|
+
@connection.verify_callback = @verify_callback if @verify_callback
|
374
|
+
end
|
375
|
+
|
376
|
+
if @certificate and @private_key then
|
377
|
+
@connection.cert = @certificate
|
378
|
+
@connection.key = @private_key
|
379
|
+
end
|
380
|
+
|
381
|
+
@connection.verify_mode = @verify_mode if @verify_mode
|
382
|
+
end
|
383
|
+
end
|
384
|
+
end
|
metadata
CHANGED
@@ -1,27 +1,27 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: persistent_http
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brad Pardee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2014-07-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gene_pool
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '1.3'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.3'
|
27
27
|
description: Persistent HTTP connections using a connection pool
|
@@ -31,12 +31,13 @@ executables: []
|
|
31
31
|
extensions: []
|
32
32
|
extra_rdoc_files: []
|
33
33
|
files:
|
34
|
-
- lib/persistent_http/faster.rb
|
35
|
-
- lib/persistent_http.rb
|
36
|
-
- LICENSE
|
37
|
-
- Rakefile
|
38
34
|
- History.md
|
35
|
+
- LICENSE
|
39
36
|
- README.rdoc
|
37
|
+
- Rakefile
|
38
|
+
- lib/persistent_http.rb
|
39
|
+
- lib/persistent_http/connection.rb
|
40
|
+
- lib/persistent_http/version.rb
|
40
41
|
homepage: http://github.com/bpardee/persistent_http
|
41
42
|
licenses: []
|
42
43
|
metadata: {}
|
@@ -46,17 +47,17 @@ require_paths:
|
|
46
47
|
- lib
|
47
48
|
required_ruby_version: !ruby/object:Gem::Requirement
|
48
49
|
requirements:
|
49
|
-
- -
|
50
|
+
- - ">="
|
50
51
|
- !ruby/object:Gem::Version
|
51
52
|
version: '0'
|
52
53
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
53
54
|
requirements:
|
54
|
-
- -
|
55
|
+
- - ">="
|
55
56
|
- !ruby/object:Gem::Version
|
56
57
|
version: '0'
|
57
58
|
requirements: []
|
58
59
|
rubyforge_project:
|
59
|
-
rubygems_version: 2.
|
60
|
+
rubygems_version: 2.2.2
|
60
61
|
signing_key:
|
61
62
|
specification_version: 4
|
62
63
|
summary: Persistent HTTP connections using a connection pool
|
@@ -1,27 +0,0 @@
|
|
1
|
-
require 'net/protocol'
|
2
|
-
|
3
|
-
##
|
4
|
-
# Aaron Patterson's monkeypatch (accepted into 1.9.1) to fix Net::HTTP's speed
|
5
|
-
# problems.
|
6
|
-
#
|
7
|
-
# http://gist.github.com/251244
|
8
|
-
|
9
|
-
class Net::BufferedIO #:nodoc:
|
10
|
-
alias :old_rbuf_fill :rbuf_fill
|
11
|
-
|
12
|
-
def rbuf_fill
|
13
|
-
if @io.respond_to? :read_nonblock then
|
14
|
-
begin
|
15
|
-
@rbuf << @io.read_nonblock(65536)
|
16
|
-
rescue Errno::EWOULDBLOCK => e
|
17
|
-
retry if IO.select [@io], nil, nil, @read_timeout
|
18
|
-
raise Timeout::Error, e.message
|
19
|
-
end
|
20
|
-
else # SSL sockets do not have read_nonblock
|
21
|
-
timeout @read_timeout do
|
22
|
-
@rbuf << @io.sysread(65536)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end if RUBY_VERSION < '1.9'
|
27
|
-
|