http_connection 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.txt +54 -0
- data/lib/net_fix.rb +160 -0
- data/lib/right_http_connection.rb +436 -0
- metadata +58 -0
data/README.txt
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
RightScale::HttpConnection
|
2
|
+
by RightScale, Inc.
|
3
|
+
www.RightScale.com
|
4
|
+
|
5
|
+
== DESCRIPTION:
|
6
|
+
|
7
|
+
Rightscale::HttpConnection is a robust HTTP/S library. It implements a retry
|
8
|
+
algorithm for low-level network errors.
|
9
|
+
|
10
|
+
== FEATURES:
|
11
|
+
|
12
|
+
- provides put/get streaming
|
13
|
+
- does configurable retries on connect and read timeouts, DNS failures, etc.
|
14
|
+
- HTTPS certificate checking
|
15
|
+
|
16
|
+
== SYNOPSIS:
|
17
|
+
|
18
|
+
|
19
|
+
== REQUIREMENTS:
|
20
|
+
|
21
|
+
- 2/11/08: If you use RightScale::HttpConnection in conjunction with attachment_fu, the
|
22
|
+
HttpConnection gem must be included (using the require statement) AFTER
|
23
|
+
attachment_fu.
|
24
|
+
This is due to a conflict between the HttpConnection gem and another
|
25
|
+
gem required by attachment_fu.
|
26
|
+
|
27
|
+
|
28
|
+
|
29
|
+
== INSTALL:
|
30
|
+
|
31
|
+
sudo gem install right_http_connection
|
32
|
+
|
33
|
+
== LICENSE:
|
34
|
+
|
35
|
+
Copyright (c) 2007-2008 RightScale, Inc.
|
36
|
+
|
37
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
38
|
+
a copy of this software and associated documentation files (the
|
39
|
+
'Software'), to deal in the Software without restriction, including
|
40
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
41
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
42
|
+
permit persons to whom the Software is furnished to do so, subject to
|
43
|
+
the following conditions:
|
44
|
+
|
45
|
+
The above copyright notice and this permission notice shall be
|
46
|
+
included in all copies or substantial portions of the Software.
|
47
|
+
|
48
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
49
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
50
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
51
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
52
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
53
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
54
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/lib/net_fix.rb
ADDED
@@ -0,0 +1,160 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2008 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
#
|
23
|
+
#
|
24
|
+
|
25
|
+
# Net::HTTP and Net::HTTPGenericRequest fixes to support 100-continue on
|
26
|
+
# POST and PUT. The request must have 'expect' field set to '100-continue'.
|
27
|
+
|
28
|
+
|
29
|
+
module Net
|
30
|
+
|
31
|
+
class BufferedIO #:nodoc:
|
32
|
+
# Monkey-patch Net::BufferedIO to read > 1024 bytes from the socket at a time
|
33
|
+
|
34
|
+
# Default size (in bytes) of the max read from a socket into the user space read buffers for socket IO
|
35
|
+
DEFAULT_SOCKET_READ_SIZE = 16*1024
|
36
|
+
|
37
|
+
@@socket_read_size = DEFAULT_SOCKET_READ_SIZE
|
38
|
+
|
39
|
+
def self.socket_read_size=(readsize)
|
40
|
+
if(readsize <= 0)
|
41
|
+
return
|
42
|
+
end
|
43
|
+
@@socket_read_size = readsize
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.socket_read_size?()
|
47
|
+
@@socket_read_size
|
48
|
+
end
|
49
|
+
|
50
|
+
def rbuf_fill
|
51
|
+
timeout(@read_timeout) {
|
52
|
+
@rbuf << @io.sysread(@@socket_read_size)
|
53
|
+
}
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
|
58
|
+
#-- Net::HTTPGenericRequest --
|
59
|
+
|
60
|
+
class HTTPGenericRequest
|
61
|
+
# Monkey-patch Net::HTTPGenericRequest to read > 1024 bytes from the local data
|
62
|
+
# source at a time (used in streaming PUTs)
|
63
|
+
|
64
|
+
# Default size (in bytes) of the max read from a local source (File, String,
|
65
|
+
# etc.) to the user space write buffers for socket IO.
|
66
|
+
DEFAULT_LOCAL_READ_SIZE = 16*1024
|
67
|
+
|
68
|
+
@@local_read_size = DEFAULT_LOCAL_READ_SIZE
|
69
|
+
|
70
|
+
def self.local_read_size=(readsize)
|
71
|
+
if(readsize <= 0)
|
72
|
+
return
|
73
|
+
end
|
74
|
+
@@local_read_size = readsize
|
75
|
+
end
|
76
|
+
|
77
|
+
def self.local_read_size?()
|
78
|
+
@@local_read_size
|
79
|
+
end
|
80
|
+
|
81
|
+
def exec(sock, ver, path, send_only=nil) #:nodoc: internal use only
|
82
|
+
if @body
|
83
|
+
send_request_with_body sock, ver, path, @body, send_only
|
84
|
+
elsif @body_stream
|
85
|
+
send_request_with_body_stream sock, ver, path, @body_stream, send_only
|
86
|
+
else
|
87
|
+
write_header(sock, ver, path)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
private
|
92
|
+
|
93
|
+
def send_request_with_body(sock, ver, path, body, send_only=nil)
|
94
|
+
self.content_length = body.length
|
95
|
+
delete 'Transfer-Encoding'
|
96
|
+
supply_default_content_type
|
97
|
+
write_header(sock, ver, path) unless send_only == :body
|
98
|
+
sock.write(body) unless send_only == :header
|
99
|
+
end
|
100
|
+
|
101
|
+
def send_request_with_body_stream(sock, ver, path, f, send_only=nil)
|
102
|
+
unless content_length() or chunked?
|
103
|
+
raise ArgumentError,
|
104
|
+
"Content-Length not given and Transfer-Encoding is not `chunked'"
|
105
|
+
end
|
106
|
+
supply_default_content_type
|
107
|
+
write_header(sock, ver, path) unless send_only == :body
|
108
|
+
unless send_only == :header
|
109
|
+
if chunked?
|
110
|
+
while s = f.read(@@local_read_size)
|
111
|
+
sock.write(sprintf("%x\r\n", s.length) << s << "\r\n")
|
112
|
+
end
|
113
|
+
sock.write "0\r\n\r\n"
|
114
|
+
else
|
115
|
+
while s = f.read(@@local_read_size)
|
116
|
+
sock.write s
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
|
124
|
+
#-- Net::HTTP --
|
125
|
+
|
126
|
+
class HTTP
|
127
|
+
def request(req, body = nil, &block) # :yield: +response+
|
128
|
+
unless started?
|
129
|
+
start {
|
130
|
+
req['connection'] ||= 'close'
|
131
|
+
return request(req, body, &block)
|
132
|
+
}
|
133
|
+
end
|
134
|
+
if proxy_user()
|
135
|
+
unless use_ssl?
|
136
|
+
req.proxy_basic_auth proxy_user(), proxy_pass()
|
137
|
+
end
|
138
|
+
end
|
139
|
+
# set body
|
140
|
+
req.set_body_internal body
|
141
|
+
begin_transport req
|
142
|
+
# if we expect 100-continue then send a header first
|
143
|
+
send_only = ((req.is_a?(Post)||req.is_a?(Put)) && (req['expect']=='100-continue')) ? :header : nil
|
144
|
+
req.exec @socket, @curr_http_version, edit_path(req.path), send_only
|
145
|
+
begin
|
146
|
+
res = HTTPResponse.read_new(@socket)
|
147
|
+
# if we expected 100-continue then send a body
|
148
|
+
if res.is_a?(HTTPContinue) && send_only && req['content-length'].to_i > 0
|
149
|
+
req.exec @socket, @curr_http_version, edit_path(req.path), :body
|
150
|
+
end
|
151
|
+
end while res.kind_of?(HTTPContinue)
|
152
|
+
res.reading_body(@socket, req.response_body_permitted?) {
|
153
|
+
yield res if block_given?
|
154
|
+
}
|
155
|
+
end_transport req, res
|
156
|
+
res
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
end
|
@@ -0,0 +1,436 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2007-2008 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
#
|
23
|
+
|
24
|
+
require "net/https"
|
25
|
+
require "uri"
|
26
|
+
require "time"
|
27
|
+
require "logger"
|
28
|
+
|
29
|
+
$:.unshift(File.dirname(__FILE__))
|
30
|
+
require "net_fix"
|
31
|
+
|
32
|
+
|
33
|
+
module RightHttpConnection #:nodoc:
|
34
|
+
module VERSION #:nodoc:
|
35
|
+
MAJOR = 1
|
36
|
+
MINOR = 2
|
37
|
+
TINY = 4
|
38
|
+
|
39
|
+
STRING = [MAJOR, MINOR, TINY].join('.')
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
|
44
|
+
module Rightscale
|
45
|
+
|
46
|
+
=begin rdoc
|
47
|
+
HttpConnection maintains a persistent HTTP connection to a remote
|
48
|
+
server. Each instance maintains its own unique connection to the
|
49
|
+
HTTP server. HttpConnection makes a best effort to receive a proper
|
50
|
+
HTTP response from the server, although it does not guarantee that
|
51
|
+
this response contains a HTTP Success code.
|
52
|
+
|
53
|
+
On low-level errors (TCP/IP errors) HttpConnection invokes a reconnect
|
54
|
+
and retry algorithm. Note that although each HttpConnection object
|
55
|
+
has its own connection to the HTTP server, error handling is shared
|
56
|
+
across all connections to a server. For example, if there are three
|
57
|
+
connections to www.somehttpserver.com, a timeout error on one of those
|
58
|
+
connections will cause all three connections to break and reconnect.
|
59
|
+
A connection will not break and reconnect, however, unless a request
|
60
|
+
becomes active on it within a certain amount of time after the error
|
61
|
+
(as specified by HTTP_CONNECTION_RETRY_DELAY). An idle connection will not
|
62
|
+
break even if other connections to the same server experience errors.
|
63
|
+
|
64
|
+
A HttpConnection will retry a request a certain number of times (as
|
65
|
+
defined by HTTP_CONNNECTION_RETRY_COUNT). If all the retries fail,
|
66
|
+
an exception is thrown and all HttpConnections associated with a
|
67
|
+
server enter a probationary period defined by HTTP_CONNECTION_RETRY_DELAY.
|
68
|
+
If the user makes a new request subsequent to entering probation,
|
69
|
+
the request will fail immediately with the same exception thrown
|
70
|
+
on probation entry. This is so that if the HTTP server has gone
|
71
|
+
down, not every subsequent request must wait for a connect timeout
|
72
|
+
before failing. After the probation period expires, the internal
|
73
|
+
state of the HttpConnection is reset and subsequent requests have
|
74
|
+
the full number of potential reconnects and retries available to
|
75
|
+
them.
|
76
|
+
=end
|
77
|
+
|
78
|
+
class HttpConnection
|
79
|
+
|
80
|
+
# Number of times to retry the request after encountering the first error
|
81
|
+
HTTP_CONNECTION_RETRY_COUNT = 3
|
82
|
+
# Throw a Timeout::Error if a connection isn't established within this number of seconds
|
83
|
+
HTTP_CONNECTION_OPEN_TIMEOUT = 5
|
84
|
+
# Throw a Timeout::Error if no data have been read on this connnection within this number of seconds
|
85
|
+
HTTP_CONNECTION_READ_TIMEOUT = 120
|
86
|
+
# Length of the post-error probationary period during which all requests will fail
|
87
|
+
HTTP_CONNECTION_RETRY_DELAY = 15
|
88
|
+
|
89
|
+
#--------------------
|
90
|
+
# class methods
|
91
|
+
#--------------------
|
92
|
+
#
|
93
|
+
@@params = {}
|
94
|
+
@@params[:http_connection_retry_count] = HTTP_CONNECTION_RETRY_COUNT
|
95
|
+
@@params[:http_connection_open_timeout] = HTTP_CONNECTION_OPEN_TIMEOUT
|
96
|
+
@@params[:http_connection_read_timeout] = HTTP_CONNECTION_READ_TIMEOUT
|
97
|
+
@@params[:http_connection_retry_delay] = HTTP_CONNECTION_RETRY_DELAY
|
98
|
+
|
99
|
+
# Query the global (class-level) parameters:
|
100
|
+
#
|
101
|
+
# :user_agent => 'www.HostName.com' # String to report as HTTP User agent
|
102
|
+
# :ca_file => 'path_to_file' # Path to a CA certification file in PEM format. The file can contain several CA certificates. If this parameter isn't set, HTTPS certs won't be verified.
|
103
|
+
# :logger => Logger object # If omitted, HttpConnection logs to STDOUT
|
104
|
+
# :exception => Exception to raise # The type of exception to raise
|
105
|
+
# # if a request repeatedly fails. RuntimeError is raised if this parameter is omitted.
|
106
|
+
# :http_connection_retry_count # by default == Rightscale::HttpConnection::HTTP_CONNECTION_RETRY_COUNT
|
107
|
+
# :http_connection_open_timeout # by default == Rightscale::HttpConnection::HTTP_CONNECTION_OPEN_TIMEOUT
|
108
|
+
# :http_connection_read_timeout # by default == Rightscale::HttpConnection::HTTP_CONNECTION_READ_TIMEOUT
|
109
|
+
# :http_connection_retry_delay # by default == Rightscale::HttpConnection::HTTP_CONNECTION_RETRY_DELAY
|
110
|
+
def self.params
|
111
|
+
@@params
|
112
|
+
end
|
113
|
+
|
114
|
+
# Set the global (class-level) parameters
|
115
|
+
def self.params=(params)
|
116
|
+
@@params = params
|
117
|
+
end
|
118
|
+
|
119
|
+
#------------------
|
120
|
+
# instance methods
|
121
|
+
#------------------
|
122
|
+
attr_accessor :http
|
123
|
+
attr_accessor :server
|
124
|
+
attr_accessor :params # see @@params
|
125
|
+
attr_accessor :logger
|
126
|
+
|
127
|
+
# Params hash:
|
128
|
+
# :user_agent => 'www.HostName.com' # String to report as HTTP User agent
|
129
|
+
# :ca_file => 'path_to_file' # A path of a CA certification file in PEM format. The file can contain several CA certificates.
|
130
|
+
# :logger => Logger object # If omitted, HttpConnection logs to STDOUT
|
131
|
+
# :exception => Exception to raise # The type of exception to raise if a request repeatedly fails. RuntimeError is raised if this parameter is omitted.
|
132
|
+
# :http_connection_retry_count # by default == Rightscale::HttpConnection.params[:http_connection_retry_count]
|
133
|
+
# :http_connection_open_timeout # by default == Rightscale::HttpConnection.params[:http_connection_open_timeout]
|
134
|
+
# :http_connection_read_timeout # by default == Rightscale::HttpConnection.params[:http_connection_read_timeout]
|
135
|
+
# :http_connection_retry_delay # by default == Rightscale::HttpConnection.params[:http_connection_retry_delay]
|
136
|
+
#
|
137
|
+
def initialize(params={})
|
138
|
+
@params = params
|
139
|
+
@params[:http_connection_retry_count] ||= @@params[:http_connection_retry_count]
|
140
|
+
@params[:http_connection_open_timeout] ||= @@params[:http_connection_open_timeout]
|
141
|
+
@params[:http_connection_read_timeout] ||= @@params[:http_connection_read_timeout]
|
142
|
+
@params[:http_connection_retry_delay] ||= @@params[:http_connection_retry_delay]
|
143
|
+
@http = nil
|
144
|
+
@server = nil
|
145
|
+
@logger = get_param(:logger) ||
|
146
|
+
(RAILS_DEFAULT_LOGGER if defined?(RAILS_DEFAULT_LOGGER)) ||
|
147
|
+
Logger.new(STDOUT)
|
148
|
+
end
|
149
|
+
|
150
|
+
def get_param(name)
|
151
|
+
@params[name] || @@params[name]
|
152
|
+
end
|
153
|
+
|
154
|
+
# Query for the maximum size (in bytes) of a single read from the underlying
|
155
|
+
# socket. For bulk transfer, especially over fast links, this is value is
|
156
|
+
# critical to performance.
|
157
|
+
def socket_read_size?
|
158
|
+
Net::BufferedIO.socket_read_size?
|
159
|
+
end
|
160
|
+
|
161
|
+
# Set the maximum size (in bytes) of a single read from the underlying
|
162
|
+
# socket. For bulk transfer, especially over fast links, this is value is
|
163
|
+
# critical to performance.
|
164
|
+
def socket_read_size=(newsize)
|
165
|
+
Net::BufferedIO.socket_read_size=(newsize)
|
166
|
+
end
|
167
|
+
|
168
|
+
# Query for the maximum size (in bytes) of a single read from local data
|
169
|
+
# sources like files. This is important, for example, in a streaming PUT of a
|
170
|
+
# large buffer.
|
171
|
+
def local_read_size?
|
172
|
+
Net::HTTPGenericRequest.local_read_size?
|
173
|
+
end
|
174
|
+
|
175
|
+
# Set the maximum size (in bytes) of a single read from local data
|
176
|
+
# sources like files. This can be used to tune the performance of, for example, a streaming PUT of a
|
177
|
+
# large buffer.
|
178
|
+
def local_read_size=(newsize)
|
179
|
+
Net::HTTPGenericRequest.local_read_size=(newsize)
|
180
|
+
end
|
181
|
+
|
182
|
+
private
|
183
|
+
#--------------
|
184
|
+
# Retry state - Keep track of errors on a per-server basis
|
185
|
+
#--------------
|
186
|
+
@@state = {} # retry state indexed by server: consecutive error count, error time, and error
|
187
|
+
@@eof = {}
|
188
|
+
|
189
|
+
# number of consecutive errors seen for server, 0 all is ok
|
190
|
+
def error_count
|
191
|
+
@@state[@server] ? @@state[@server][:count] : 0
|
192
|
+
end
|
193
|
+
|
194
|
+
# time of last error for server, nil if all is ok
|
195
|
+
def error_time
|
196
|
+
@@state[@server] && @@state[@server][:time]
|
197
|
+
end
|
198
|
+
|
199
|
+
# message for last error for server, "" if all is ok
|
200
|
+
def error_message
|
201
|
+
@@state[@server] ? @@state[@server][:message] : ""
|
202
|
+
end
|
203
|
+
|
204
|
+
# add an error for a server
|
205
|
+
def error_add(message)
|
206
|
+
@@state[@server] = { :count => error_count+1, :time => Time.now, :message => message }
|
207
|
+
end
|
208
|
+
|
209
|
+
# reset the error state for a server (i.e. a request succeeded)
|
210
|
+
def error_reset
|
211
|
+
@@state.delete(@server)
|
212
|
+
end
|
213
|
+
|
214
|
+
# Error message stuff...
|
215
|
+
def banana_message
|
216
|
+
return "#{@server} temporarily unavailable: (#{error_message})"
|
217
|
+
end
|
218
|
+
|
219
|
+
def err_header
|
220
|
+
return "#{self.class.name} :"
|
221
|
+
end
|
222
|
+
|
223
|
+
# Adds new EOF timestamp.
|
224
|
+
# Returns the number of seconds to wait before new conection retry:
|
225
|
+
# 0.5, 1, 2, 4, 8
|
226
|
+
def add_eof
|
227
|
+
(@@eof[@server] ||= []).unshift Time.now
|
228
|
+
0.25 * 2 ** @@eof[@server].size
|
229
|
+
end
|
230
|
+
|
231
|
+
# Returns first EOF timestamp or nul if have no EOFs being tracked.
|
232
|
+
def eof_time
|
233
|
+
@@eof[@server] && @@eof[@server].last
|
234
|
+
end
|
235
|
+
|
236
|
+
# Returns true if we are receiving EOFs during last @params[:http_connection_retry_delay] seconds
|
237
|
+
# and there were no successful response from server
|
238
|
+
def raise_on_eof_exception?
|
239
|
+
@@eof[@server].blank? ? false : ( (Time.now.to_i-@params[:http_connection_retry_delay]) > @@eof[@server].last.to_i )
|
240
|
+
end
|
241
|
+
|
242
|
+
# Reset a list of EOFs for this server.
|
243
|
+
# This is being called when we have got an successful response from server.
|
244
|
+
def eof_reset
|
245
|
+
@@eof.delete(@server)
|
246
|
+
end
|
247
|
+
|
248
|
+
# Detects if an object is 'streamable' - can we read from it, and can we know the size?
|
249
|
+
def setup_streaming(request)
|
250
|
+
if(request.body && request.body.respond_to?(:read))
|
251
|
+
body = request.body
|
252
|
+
request.content_length = body.respond_to?(:lstat) ? body.lstat.size : body.size
|
253
|
+
request.body_stream = request.body
|
254
|
+
true
|
255
|
+
end
|
256
|
+
end
|
257
|
+
|
258
|
+
def get_fileptr_offset(request_params)
|
259
|
+
request_params[:request].body.pos
|
260
|
+
rescue Exception => e
|
261
|
+
# Probably caught this because the body doesn't support the pos() method, like if it is a socket.
|
262
|
+
# Just return 0 and get on with life.
|
263
|
+
0
|
264
|
+
end
|
265
|
+
|
266
|
+
def reset_fileptr_offset(request, offset = 0)
|
267
|
+
if(request.body_stream && request.body_stream.respond_to?(:pos))
|
268
|
+
begin
|
269
|
+
request.body_stream.pos = offset
|
270
|
+
rescue Exception => e
|
271
|
+
@logger.warn("Failed file pointer reset; aborting HTTP retries." +
|
272
|
+
" -- #{err_header} #{e.inspect}")
|
273
|
+
raise e
|
274
|
+
end
|
275
|
+
end
|
276
|
+
end
|
277
|
+
|
278
|
+
# Start a fresh connection. The object closes any existing connection and
|
279
|
+
# opens a new one.
|
280
|
+
def start(request_params)
|
281
|
+
# close the previous if exists
|
282
|
+
finish
|
283
|
+
# create new connection
|
284
|
+
@server = request_params[:server]
|
285
|
+
@port = request_params[:port]
|
286
|
+
@protocol = request_params[:protocol]
|
287
|
+
|
288
|
+
@logger.info("Opening new #{@protocol.upcase} connection to #@server:#@port")
|
289
|
+
@http = Net::HTTP.new(@server, @port)
|
290
|
+
@http.open_timeout = @params[:http_connection_open_timeout]
|
291
|
+
@http.read_timeout = @params[:http_connection_read_timeout]
|
292
|
+
|
293
|
+
if @protocol == 'https'
|
294
|
+
verifyCallbackProc = Proc.new{ |ok, x509_store_ctx|
|
295
|
+
code = x509_store_ctx.error
|
296
|
+
msg = x509_store_ctx.error_string
|
297
|
+
#debugger
|
298
|
+
@logger.warn("##### #{@server} certificate verify failed: #{msg}") unless code == 0
|
299
|
+
true
|
300
|
+
}
|
301
|
+
@http.use_ssl = true
|
302
|
+
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE # Looks like Ruby 1.9 defaults to VERIFY_PEER which doesn't work well
|
303
|
+
ca_file = get_param(:ca_file)
|
304
|
+
if ca_file
|
305
|
+
@http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
306
|
+
@http.verify_callback = verifyCallbackProc
|
307
|
+
@http.ca_file = ca_file
|
308
|
+
end
|
309
|
+
end
|
310
|
+
# open connection
|
311
|
+
@http.start
|
312
|
+
end
|
313
|
+
|
314
|
+
public
|
315
|
+
|
316
|
+
=begin rdoc
|
317
|
+
Send HTTP request to server
|
318
|
+
|
319
|
+
request_params hash:
|
320
|
+
:server => 'www.HostName.com' # Hostname or IP address of HTTP server
|
321
|
+
:port => '80' # Port of HTTP server
|
322
|
+
:protocol => 'https' # http and https are supported on any port
|
323
|
+
:request => 'requeststring' # Fully-formed HTTP request to make
|
324
|
+
|
325
|
+
Raises RuntimeError, Interrupt, and params[:exception] (if specified in new).
|
326
|
+
|
327
|
+
=end
|
328
|
+
def request(request_params, &block)
|
329
|
+
# We save the offset here so that if we need to retry, we can return the file pointer to its initial position
|
330
|
+
mypos = get_fileptr_offset(request_params)
|
331
|
+
loop do
|
332
|
+
# if we are inside a delay between retries: no requests this time!
|
333
|
+
if error_count > @params[:http_connection_retry_count] &&
|
334
|
+
error_time + @params[:http_connection_retry_delay] > Time.now
|
335
|
+
# store the message (otherwise it will be lost after error_reset and
|
336
|
+
# we will raise an exception with an empty text)
|
337
|
+
banana_message_text = banana_message
|
338
|
+
@logger.warn("#{err_header} re-raising same error: #{banana_message_text} " +
|
339
|
+
"-- error count: #{error_count}, error age: #{Time.now.to_i - error_time.to_i}")
|
340
|
+
exception = get_param(:exception) || RuntimeError
|
341
|
+
raise exception.new(banana_message_text)
|
342
|
+
end
|
343
|
+
|
344
|
+
# try to connect server(if connection does not exist) and get response data
|
345
|
+
begin
|
346
|
+
request_params[:protocol] ||= (request_params[:port] == 443 ? 'https' : 'http')
|
347
|
+
|
348
|
+
request = request_params[:request]
|
349
|
+
request['User-Agent'] = get_param(:user_agent) || ''
|
350
|
+
|
351
|
+
# (re)open connection to server if none exists or params has changed
|
352
|
+
unless @http &&
|
353
|
+
@http.started? &&
|
354
|
+
@server == request_params[:server] &&
|
355
|
+
@port == request_params[:port] &&
|
356
|
+
@protocol == request_params[:protocol]
|
357
|
+
start(request_params)
|
358
|
+
end
|
359
|
+
|
360
|
+
# Detect if the body is a streamable object like a file or socket. If so, stream that
|
361
|
+
# bad boy.
|
362
|
+
setup_streaming(request)
|
363
|
+
response = @http.request(request, &block)
|
364
|
+
|
365
|
+
error_reset
|
366
|
+
eof_reset
|
367
|
+
return response
|
368
|
+
|
369
|
+
# We treat EOF errors and the timeout/network errors differently. Both
|
370
|
+
# are tracked in different statistics blocks. Note below that EOF
|
371
|
+
# errors will sleep for a certain (exponentially increasing) period.
|
372
|
+
# Other errors don't sleep because there is already an inherent delay
|
373
|
+
# in them; connect and read timeouts (for example) have already
|
374
|
+
# 'slept'. It is still not clear which way we should treat errors
|
375
|
+
# like RST and resolution failures. For now, there is no additional
|
376
|
+
# delay for these errors although this may change in the future.
|
377
|
+
|
378
|
+
# EOFError means the server closed the connection on us.
|
379
|
+
rescue EOFError => e
|
380
|
+
@logger.debug("#{err_header} server #{@server} closed connection")
|
381
|
+
@http = nil
|
382
|
+
|
383
|
+
# if we have waited long enough - raise an exception...
|
384
|
+
if raise_on_eof_exception?
|
385
|
+
exception = get_param(:exception) || RuntimeError
|
386
|
+
@logger.warn("#{err_header} raising #{exception} due to permanent EOF being received from #{@server}, error age: #{Time.now.to_i - eof_time.to_i}")
|
387
|
+
raise exception.new("Permanent EOF is being received from #{@server}.")
|
388
|
+
else
|
389
|
+
# ... else just sleep a bit before new retry
|
390
|
+
sleep(add_eof)
|
391
|
+
# We will be retrying the request, so reset the file pointer
|
392
|
+
reset_fileptr_offset(request, mypos)
|
393
|
+
end
|
394
|
+
rescue Exception => e # See comment at bottom for the list of errors seen...
|
395
|
+
@http = nil
|
396
|
+
# if ctrl+c is pressed - we have to reraise exception to terminate proggy
|
397
|
+
if e.is_a?(Interrupt) && !( e.is_a?(Errno::ETIMEDOUT) || e.is_a?(Timeout::Error))
|
398
|
+
@logger.debug( "#{err_header} request to server #{@server} interrupted by ctrl-c")
|
399
|
+
raise
|
400
|
+
elsif e.is_a?(ArgumentError) && e.message.include?('wrong number of arguments (5 for 4)')
|
401
|
+
# seems our net_fix patch was overriden...
|
402
|
+
exception = get_param(:exception) || RuntimeError
|
403
|
+
raise exception.new('incompatible Net::HTTP monkey-patch')
|
404
|
+
end
|
405
|
+
# oops - we got a banana: log it
|
406
|
+
error_add(e.message)
|
407
|
+
@logger.warn("#{err_header} request failure count: #{error_count}, exception: #{e.inspect}")
|
408
|
+
|
409
|
+
# We will be retrying the request, so reset the file pointer
|
410
|
+
reset_fileptr_offset(request, mypos)
|
411
|
+
|
412
|
+
end
|
413
|
+
end
|
414
|
+
end
|
415
|
+
|
416
|
+
def finish(reason = '')
|
417
|
+
if @http && @http.started?
|
418
|
+
reason = ", reason: '#{reason}'" unless reason.blank?
|
419
|
+
@logger.info("Closing #{@http.use_ssl? ? 'HTTPS' : 'HTTP'} connection to #{@http.address}:#{@http.port}#{reason}")
|
420
|
+
@http.finish
|
421
|
+
end
|
422
|
+
end
|
423
|
+
|
424
|
+
# Errors received during testing:
|
425
|
+
#
|
426
|
+
# #<Timeout::Error: execution expired>
|
427
|
+
# #<Errno::ETIMEDOUT: Connection timed out - connect(2)>
|
428
|
+
# #<SocketError: getaddrinfo: Name or service not known>
|
429
|
+
# #<SocketError: getaddrinfo: Temporary failure in name resolution>
|
430
|
+
# #<EOFError: end of file reached>
|
431
|
+
# #<Errno::ECONNRESET: Connection reset by peer>
|
432
|
+
# #<OpenSSL::SSL::SSLError: SSL_write:: bad write retry>
|
433
|
+
end
|
434
|
+
|
435
|
+
end
|
436
|
+
|
metadata
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: http_connection
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.3.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Travis Reeder
|
8
|
+
- RightScale
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2009-07-23 00:00:00 -07:00
|
14
|
+
default_executable:
|
15
|
+
dependencies: []
|
16
|
+
|
17
|
+
description: HTTP helper library
|
18
|
+
email: travis@appoxy.com
|
19
|
+
executables: []
|
20
|
+
|
21
|
+
extensions: []
|
22
|
+
|
23
|
+
extra_rdoc_files:
|
24
|
+
- README.txt
|
25
|
+
files:
|
26
|
+
- lib/net_fix.rb
|
27
|
+
- lib/right_http_connection.rb
|
28
|
+
- README.txt
|
29
|
+
has_rdoc: true
|
30
|
+
homepage: http://github.com/appoxy/http_connection/
|
31
|
+
licenses: []
|
32
|
+
|
33
|
+
post_install_message:
|
34
|
+
rdoc_options:
|
35
|
+
- --charset=UTF-8
|
36
|
+
require_paths:
|
37
|
+
- lib
|
38
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
39
|
+
requirements:
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: "0"
|
43
|
+
version:
|
44
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: "0"
|
49
|
+
version:
|
50
|
+
requirements: []
|
51
|
+
|
52
|
+
rubyforge_project:
|
53
|
+
rubygems_version: 1.3.5
|
54
|
+
signing_key:
|
55
|
+
specification_version: 2
|
56
|
+
summary: HTTP helper library
|
57
|
+
test_files: []
|
58
|
+
|