ahoward-helene 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +274 -0
- data/helene.gemspec +26 -0
- data/lib/helene.rb +113 -0
- data/lib/helene/attempt.rb +46 -0
- data/lib/helene/aws.rb +50 -0
- data/lib/helene/config.rb +147 -0
- data/lib/helene/content_type.rb +15 -0
- data/lib/helene/content_type.yml +661 -0
- data/lib/helene/error.rb +12 -0
- data/lib/helene/logging.rb +55 -0
- data/lib/helene/objectpool.rb +220 -0
- data/lib/helene/rails.rb +21 -0
- data/lib/helene/rightscale/acf/right_acf_interface.rb +379 -0
- data/lib/helene/rightscale/awsbase/benchmark_fix.rb +39 -0
- data/lib/helene/rightscale/awsbase/right_awsbase.rb +803 -0
- data/lib/helene/rightscale/awsbase/support.rb +111 -0
- data/lib/helene/rightscale/ec2/right_ec2.rb +1737 -0
- data/lib/helene/rightscale/net_fix.rb +160 -0
- data/lib/helene/rightscale/right_aws.rb +71 -0
- data/lib/helene/rightscale/right_http_connection.rb +507 -0
- data/lib/helene/rightscale/s3/right_s3.rb +1094 -0
- data/lib/helene/rightscale/s3/right_s3_interface.rb +1180 -0
- data/lib/helene/rightscale/sdb/active_sdb.rb +930 -0
- data/lib/helene/rightscale/sdb/right_sdb_interface.rb +696 -0
- data/lib/helene/rightscale/sqs/right_sqs.rb +388 -0
- data/lib/helene/rightscale/sqs/right_sqs_gen2.rb +286 -0
- data/lib/helene/rightscale/sqs/right_sqs_gen2_interface.rb +444 -0
- data/lib/helene/rightscale/sqs/right_sqs_interface.rb +596 -0
- data/lib/helene/s3.rb +34 -0
- data/lib/helene/s3/bucket.rb +379 -0
- data/lib/helene/s3/grantee.rb +134 -0
- data/lib/helene/s3/key.rb +162 -0
- data/lib/helene/s3/owner.rb +16 -0
- data/lib/helene/sdb.rb +9 -0
- data/lib/helene/sdb/base.rb +1204 -0
- data/lib/helene/sdb/base/associations.rb +481 -0
- data/lib/helene/sdb/base/attributes.rb +90 -0
- data/lib/helene/sdb/base/connection.rb +20 -0
- data/lib/helene/sdb/base/error.rb +20 -0
- data/lib/helene/sdb/base/hooks.rb +82 -0
- data/lib/helene/sdb/base/literal.rb +52 -0
- data/lib/helene/sdb/base/logging.rb +23 -0
- data/lib/helene/sdb/base/transactions.rb +53 -0
- data/lib/helene/sdb/base/type.rb +137 -0
- data/lib/helene/sdb/base/types.rb +123 -0
- data/lib/helene/sdb/base/validations.rb +256 -0
- data/lib/helene/sdb/cast.rb +114 -0
- data/lib/helene/sdb/connection.rb +36 -0
- data/lib/helene/sdb/error.rb +5 -0
- data/lib/helene/sdb/interface.rb +412 -0
- data/lib/helene/sdb/sentinel.rb +15 -0
- data/lib/helene/sleepcycle.rb +29 -0
- data/lib/helene/superhash.rb +297 -0
- data/lib/helene/util.rb +132 -0
- data/test/auth.rb +31 -0
- data/test/helper.rb +98 -0
- data/test/integration/begin.rb +0 -0
- data/test/integration/ensure.rb +8 -0
- data/test/integration/s3/bucket.rb +106 -0
- data/test/integration/sdb/associations.rb +45 -0
- data/test/integration/sdb/creating.rb +13 -0
- data/test/integration/sdb/emptiness.rb +56 -0
- data/test/integration/sdb/hooks.rb +19 -0
- data/test/integration/sdb/limits.rb +27 -0
- data/test/integration/sdb/saving.rb +21 -0
- data/test/integration/sdb/selecting.rb +39 -0
- data/test/integration/sdb/types.rb +31 -0
- data/test/integration/sdb/validations.rb +60 -0
- data/test/integration/setup.rb +27 -0
- data/test/integration/teardown.rb +21 -0
- data/test/loader.rb +39 -0
- metadata +139 -0
@@ -0,0 +1,160 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2008 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
#
|
23
|
+
#
|
24
|
+
|
25
|
+
# Net::HTTP and Net::HTTPGenericRequest fixes to support 100-continue on
|
26
|
+
# POST and PUT. The request must have 'expect' field set to '100-continue'.
|
27
|
+
|
28
|
+
|
29
|
+
module Net
|
30
|
+
|
31
|
+
class BufferedIO #:nodoc:
|
32
|
+
# Monkey-patch Net::BufferedIO to read > 1024 bytes from the socket at a time
|
33
|
+
|
34
|
+
# Default size (in bytes) of the max read from a socket into the user space read buffers for socket IO
|
35
|
+
DEFAULT_SOCKET_READ_SIZE = 16*1024
|
36
|
+
|
37
|
+
@@socket_read_size = DEFAULT_SOCKET_READ_SIZE
|
38
|
+
|
39
|
+
def self.socket_read_size=(readsize)
|
40
|
+
if(readsize <= 0)
|
41
|
+
return
|
42
|
+
end
|
43
|
+
@@socket_read_size = readsize
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.socket_read_size?()
|
47
|
+
@@socket_read_size
|
48
|
+
end
|
49
|
+
|
50
|
+
def rbuf_fill
|
51
|
+
timeout(@read_timeout) {
|
52
|
+
@rbuf << @io.sysread(@@socket_read_size)
|
53
|
+
}
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
|
58
|
+
#-- Net::HTTPGenericRequest --
|
59
|
+
|
60
|
+
class HTTPGenericRequest
|
61
|
+
# Monkey-patch Net::HTTPGenericRequest to read > 1024 bytes from the local data
|
62
|
+
# source at a time (used in streaming PUTs)
|
63
|
+
|
64
|
+
# Default size (in bytes) of the max read from a local source (File, String,
|
65
|
+
# etc.) to the user space write buffers for socket IO.
|
66
|
+
DEFAULT_LOCAL_READ_SIZE = 16*1024
|
67
|
+
|
68
|
+
@@local_read_size = DEFAULT_LOCAL_READ_SIZE
|
69
|
+
|
70
|
+
def self.local_read_size=(readsize)
|
71
|
+
if(readsize <= 0)
|
72
|
+
return
|
73
|
+
end
|
74
|
+
@@local_read_size = readsize
|
75
|
+
end
|
76
|
+
|
77
|
+
def self.local_read_size?()
|
78
|
+
@@local_read_size
|
79
|
+
end
|
80
|
+
|
81
|
+
def exec(sock, ver, path, send_only=nil) #:nodoc: internal use only
|
82
|
+
if @body
|
83
|
+
send_request_with_body sock, ver, path, @body, send_only
|
84
|
+
elsif @body_stream
|
85
|
+
send_request_with_body_stream sock, ver, path, @body_stream, send_only
|
86
|
+
else
|
87
|
+
write_header(sock, ver, path)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
private
|
92
|
+
|
93
|
+
def send_request_with_body(sock, ver, path, body, send_only=nil)
|
94
|
+
self.content_length = body.length
|
95
|
+
delete 'Transfer-Encoding'
|
96
|
+
supply_default_content_type
|
97
|
+
write_header(sock, ver, path) unless send_only == :body
|
98
|
+
sock.write(body) unless send_only == :header
|
99
|
+
end
|
100
|
+
|
101
|
+
def send_request_with_body_stream(sock, ver, path, f, send_only=nil)
|
102
|
+
unless content_length() or chunked?
|
103
|
+
raise ArgumentError,
|
104
|
+
"Content-Length not given and Transfer-Encoding is not `chunked'"
|
105
|
+
end
|
106
|
+
supply_default_content_type
|
107
|
+
write_header(sock, ver, path) unless send_only == :body
|
108
|
+
unless send_only == :header
|
109
|
+
if chunked?
|
110
|
+
while s = f.read(@@local_read_size)
|
111
|
+
sock.write(sprintf("%x\r\n", s.length) << s << "\r\n")
|
112
|
+
end
|
113
|
+
sock.write "0\r\n\r\n"
|
114
|
+
else
|
115
|
+
while s = f.read(@@local_read_size)
|
116
|
+
sock.write s
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
|
124
|
+
#-- Net::HTTP --
|
125
|
+
|
126
|
+
class HTTP
|
127
|
+
def request(req, body = nil, &block) # :yield: +response+
|
128
|
+
unless started?
|
129
|
+
start {
|
130
|
+
req['connection'] ||= 'close'
|
131
|
+
return request(req, body, &block)
|
132
|
+
}
|
133
|
+
end
|
134
|
+
if proxy_user()
|
135
|
+
unless use_ssl?
|
136
|
+
req.proxy_basic_auth proxy_user(), proxy_pass()
|
137
|
+
end
|
138
|
+
end
|
139
|
+
# set body
|
140
|
+
req.set_body_internal body
|
141
|
+
begin_transport req
|
142
|
+
# if we expect 100-continue then send a header first
|
143
|
+
send_only = ((req.is_a?(Post)||req.is_a?(Put)) && (req['expect']=='100-continue')) ? :header : nil
|
144
|
+
req.exec @socket, @curr_http_version, edit_path(req.path), send_only
|
145
|
+
begin
|
146
|
+
res = HTTPResponse.read_new(@socket)
|
147
|
+
# if we expected 100-continue then send a body
|
148
|
+
if res.is_a?(HTTPContinue) && send_only && req['content-length'].to_i > 0
|
149
|
+
req.exec @socket, @curr_http_version, edit_path(req.path), :body
|
150
|
+
end
|
151
|
+
end while res.kind_of?(HTTPContinue)
|
152
|
+
res.reading_body(@socket, req.response_body_permitted?) {
|
153
|
+
yield res if block_given?
|
154
|
+
}
|
155
|
+
end_transport req, res
|
156
|
+
res
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2007-2008 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
#
|
23
|
+
|
24
|
+
require 'benchmark'
|
25
|
+
require 'net/https'
|
26
|
+
require 'uri'
|
27
|
+
require 'time'
|
28
|
+
require "cgi"
|
29
|
+
require "base64"
|
30
|
+
require "rexml/document"
|
31
|
+
require "openssl"
|
32
|
+
require "digest/sha1"
|
33
|
+
|
34
|
+
require 'rubygems'
|
35
|
+
require 'right_http_connection'
|
36
|
+
|
37
|
+
$:.unshift(File.dirname(__FILE__))
|
38
|
+
require 'awsbase/benchmark_fix'
|
39
|
+
require 'awsbase/support'
|
40
|
+
require 'awsbase/right_awsbase'
|
41
|
+
require 'ec2/right_ec2'
|
42
|
+
require 's3/right_s3_interface'
|
43
|
+
require 's3/right_s3'
|
44
|
+
require 'sqs/right_sqs_interface'
|
45
|
+
require 'sqs/right_sqs'
|
46
|
+
require 'sqs/right_sqs_gen2_interface'
|
47
|
+
require 'sqs/right_sqs_gen2'
|
48
|
+
require 'sdb/right_sdb_interface'
|
49
|
+
require 'acf/right_acf_interface'
|
50
|
+
|
51
|
+
|
52
|
+
|
53
|
+
module RightAws #:nodoc:
|
54
|
+
module VERSION #:nodoc:
|
55
|
+
RightAws::VERSION::MAJOR = 1 unless defined?(RightAws::VERSION::MAJOR)
|
56
|
+
RightAws::VERSION::MINOR = 10 unless defined?(RightAws::VERSION::MINOR)
|
57
|
+
RightAws::VERSION::TINY = 0 unless defined?(RightAws::VERSION::TINY)
|
58
|
+
STRING = [MAJOR, MINOR, TINY].join('.') unless defined?(RightAws::VERSION::STRING)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
__END__
|
63
|
+
|
64
|
+
#-
|
65
|
+
|
66
|
+
# We also want everything available in the Rightscale namespace for backward
|
67
|
+
# compatibility reasons.
|
68
|
+
module Rightscale #:nodoc:
|
69
|
+
include RightAws
|
70
|
+
extend RightAws
|
71
|
+
end
|
@@ -0,0 +1,507 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2007-2008 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
#
|
23
|
+
|
24
|
+
require "net/https"
|
25
|
+
require "uri"
|
26
|
+
require "time"
|
27
|
+
require "logger"
|
28
|
+
|
29
|
+
$:.unshift(File.dirname(__FILE__))
|
30
|
+
require "net_fix"
|
31
|
+
|
32
|
+
|
33
|
+
module RightHttpConnection #:nodoc:
|
34
|
+
unless defined?(RightHttpConnection::VERSION)
|
35
|
+
module VERSION #:nodoc:
|
36
|
+
MAJOR = 1
|
37
|
+
MINOR = 2
|
38
|
+
TINY = 4
|
39
|
+
|
40
|
+
STRING = [MAJOR, MINOR, TINY].join('.')
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
|
46
|
+
module Rightscale
|
47
|
+
|
48
|
+
=begin rdoc
|
49
|
+
HttpConnection maintains a persistent HTTP connection to a remote
|
50
|
+
server. Each instance maintains its own unique connection to the
|
51
|
+
HTTP server. HttpConnection makes a best effort to receive a proper
|
52
|
+
HTTP response from the server, although it does not guarantee that
|
53
|
+
this response contains a HTTP Success code.
|
54
|
+
|
55
|
+
On low-level errors (TCP/IP errors) HttpConnection invokes a reconnect
|
56
|
+
and retry algorithm. Note that although each HttpConnection object
|
57
|
+
has its own connection to the HTTP server, error handling is shared
|
58
|
+
across all connections to a server. For example, if there are three
|
59
|
+
connections to www.somehttpserver.com, a timeout error on one of those
|
60
|
+
connections will cause all three connections to break and reconnect.
|
61
|
+
A connection will not break and reconnect, however, unless a request
|
62
|
+
becomes active on it within a certain amount of time after the error
|
63
|
+
(as specified by HTTP_CONNECTION_RETRY_DELAY). An idle connection will not
|
64
|
+
break even if other connections to the same server experience errors.
|
65
|
+
|
66
|
+
A HttpConnection will retry a request a certain number of times (as
|
67
|
+
defined by HTTP_CONNNECTION_RETRY_COUNT). If all the retries fail,
|
68
|
+
an exception is thrown and all HttpConnections associated with a
|
69
|
+
server enter a probationary period defined by HTTP_CONNECTION_RETRY_DELAY.
|
70
|
+
If the user makes a new request subsequent to entering probation,
|
71
|
+
the request will fail immediately with the same exception thrown
|
72
|
+
on probation entry. This is so that if the HTTP server has gone
|
73
|
+
down, not every subsequent request must wait for a connect timeout
|
74
|
+
before failing. After the probation period expires, the internal
|
75
|
+
state of the HttpConnection is reset and subsequent requests have
|
76
|
+
the full number of potential reconnects and retries available to
|
77
|
+
them.
|
78
|
+
=end
|
79
|
+
|
80
|
+
class HttpConnection
|
81
|
+
|
82
|
+
# Number of times to retry the request after encountering the first error
|
83
|
+
unless defined?(HTTP_CONNECTION_RETRY_COUNT)
|
84
|
+
HTTP_CONNECTION_RETRY_COUNT = 42 # up the 3
|
85
|
+
end
|
86
|
+
# Throw a Timeout::Error if a connection isn't established within this number of seconds
|
87
|
+
unless defined?(HTTP_CONNECTION_OPEN_TIMEOUT)
|
88
|
+
HTTP_CONNECTION_OPEN_TIMEOUT = 5
|
89
|
+
end
|
90
|
+
# Throw a Timeout::Error if no data have been read on this connnection within this number of seconds
|
91
|
+
unless defined?(HTTP_CONNECTION_READ_TIMEOUT)
|
92
|
+
HTTP_CONNECTION_READ_TIMEOUT = 120
|
93
|
+
end
|
94
|
+
# Length of the post-error probationary period during which all requests will fail
|
95
|
+
unless defined?(HTTP_CONNECTION_RETRY_DELAY)
|
96
|
+
HTTP_CONNECTION_RETRY_DELAY = 15
|
97
|
+
end
|
98
|
+
|
99
|
+
#--------------------
|
100
|
+
# class methods
|
101
|
+
#--------------------
|
102
|
+
#
|
103
|
+
@@params = {}
|
104
|
+
@@params[:http_connection_retry_count] = HTTP_CONNECTION_RETRY_COUNT
|
105
|
+
@@params[:http_connection_open_timeout] = HTTP_CONNECTION_OPEN_TIMEOUT
|
106
|
+
@@params[:http_connection_read_timeout] = HTTP_CONNECTION_READ_TIMEOUT
|
107
|
+
@@params[:http_connection_retry_delay] = HTTP_CONNECTION_RETRY_DELAY
|
108
|
+
|
109
|
+
# Query the global (class-level) parameters:
|
110
|
+
#
|
111
|
+
# :user_agent => 'www.HostName.com' # String to report as HTTP User agent
|
112
|
+
# :ca_file => 'path_to_file' # Path to a CA certification file in PEM format. The file can contain several CA certificates. If this parameter isn't set, HTTPS certs won't be verified.
|
113
|
+
# :logger => Logger object # If omitted, HttpConnection logs to STDOUT
|
114
|
+
# :exception => Exception to raise # The type of exception to raise
|
115
|
+
# # if a request repeatedly fails. RuntimeError is raised if this parameter is omitted.
|
116
|
+
# :http_connection_retry_count # by default == Rightscale::HttpConnection::HTTP_CONNECTION_RETRY_COUNT
|
117
|
+
# :http_connection_open_timeout # by default == Rightscale::HttpConnection::HTTP_CONNECTION_OPEN_TIMEOUT
|
118
|
+
# :http_connection_read_timeout # by default == Rightscale::HttpConnection::HTTP_CONNECTION_READ_TIMEOUT
|
119
|
+
# :http_connection_retry_delay # by default == Rightscale::HttpConnection::HTTP_CONNECTION_RETRY_DELAY
|
120
|
+
def self.params
|
121
|
+
@@params
|
122
|
+
end
|
123
|
+
|
124
|
+
# Set the global (class-level) parameters
|
125
|
+
def self.params=(params)
|
126
|
+
@@params = params
|
127
|
+
end
|
128
|
+
|
129
|
+
module NullLogger
|
130
|
+
def respond_to?(*a, &b) true end
|
131
|
+
def method_missing(m, *a, &b) end
|
132
|
+
extend self
|
133
|
+
end
|
134
|
+
|
135
|
+
#------------------
|
136
|
+
# instance methods
|
137
|
+
#------------------
|
138
|
+
attr_accessor :http
|
139
|
+
attr_accessor :server
|
140
|
+
attr_accessor :params # see @@params
|
141
|
+
attr_accessor :logger
|
142
|
+
attr_accessor :thread
|
143
|
+
attr_accessor :state
|
144
|
+
attr_accessor :eof
|
145
|
+
|
146
|
+
# Params hash:
|
147
|
+
# :user_agent => 'www.HostName.com' # String to report as HTTP User agent
|
148
|
+
# :ca_file => 'path_to_file' # A path of a CA certification file in PEM format. The file can contain several CA certificates.
|
149
|
+
# :logger => Logger object # If omitted, HttpConnection logs to STDOUT
|
150
|
+
# :exception => Exception to raise # The type of exception to raise if a request repeatedly fails. RuntimeError is raised if this parameter is omitted.
|
151
|
+
# :http_connection_retry_count # by default == Rightscale::HttpConnection.params[:http_connection_retry_count]
|
152
|
+
# :http_connection_open_timeout # by default == Rightscale::HttpConnection.params[:http_connection_open_timeout]
|
153
|
+
# :http_connection_read_timeout # by default == Rightscale::HttpConnection.params[:http_connection_read_timeout]
|
154
|
+
# :http_connection_retry_delay # by default == Rightscale::HttpConnection.params[:http_connection_retry_delay]
|
155
|
+
#
|
156
|
+
def initialize(params={})
|
157
|
+
@thread = Thread.current
|
158
|
+
@caller = caller
|
159
|
+
@params = params
|
160
|
+
@params[:http_connection_retry_count] ||= @@params[:http_connection_retry_count]
|
161
|
+
@params[:http_connection_open_timeout] ||= @@params[:http_connection_open_timeout]
|
162
|
+
@params[:http_connection_read_timeout] ||= @@params[:http_connection_read_timeout]
|
163
|
+
@params[:http_connection_retry_delay] ||= @@params[:http_connection_retry_delay]
|
164
|
+
@http = nil
|
165
|
+
@server = nil
|
166
|
+
@logger = get_param(:logger)
|
167
|
+
@logger ||= RAILS_DEFAULT_LOGGER if defined?(RAILS_DEFAULT_LOGGER)
|
168
|
+
@logger ||= NullLogger
|
169
|
+
@ca_file = get_param(:ca_file)
|
170
|
+
@state = {}
|
171
|
+
@eof = {}
|
172
|
+
end
|
173
|
+
|
174
|
+
def log(*args, &block)
|
175
|
+
@logger.send(*args, &block) if @logger
|
176
|
+
end
|
177
|
+
|
178
|
+
def prevent_mt_use!
|
179
|
+
unless Thread.current==@thread
|
180
|
+
msg = <<-__
|
181
|
+
@thread : #{ @thread.inspect }
|
182
|
+
|
183
|
+
@caller : #{ @caller.inspect }
|
184
|
+
|
185
|
+
thread : #{ Thread.current.inspect }
|
186
|
+
|
187
|
+
caller : #{ caller.inspect }
|
188
|
+
__
|
189
|
+
|
190
|
+
raise ThreadError.new, msg
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
def get_param(name)
|
195
|
+
@params[name] || @@params[name]
|
196
|
+
end
|
197
|
+
|
198
|
+
# Query for the maximum size (in bytes) of a single read from the underlying
|
199
|
+
# socket. For bulk transfer, especially over fast links, this is value is
|
200
|
+
# critical to performance.
|
201
|
+
def socket_read_size?
|
202
|
+
Net::BufferedIO.socket_read_size?
|
203
|
+
end
|
204
|
+
|
205
|
+
# Set the maximum size (in bytes) of a single read from the underlying
|
206
|
+
# socket. For bulk transfer, especially over fast links, this is value is
|
207
|
+
# critical to performance.
|
208
|
+
def socket_read_size=(newsize)
|
209
|
+
Net::BufferedIO.socket_read_size=(newsize)
|
210
|
+
end
|
211
|
+
|
212
|
+
# Query for the maximum size (in bytes) of a single read from local data
|
213
|
+
# sources like files. This is important, for example, in a streaming PUT of a
|
214
|
+
# large buffer.
|
215
|
+
def local_read_size?
|
216
|
+
Net::HTTPGenericRequest.local_read_size?
|
217
|
+
end
|
218
|
+
|
219
|
+
# Set the maximum size (in bytes) of a single read from local data
|
220
|
+
# sources like files. This can be used to tune the performance of, for example, a streaming PUT of a
|
221
|
+
# large buffer.
|
222
|
+
def local_read_size=(newsize)
|
223
|
+
Net::HTTPGenericRequest.local_read_size=(newsize)
|
224
|
+
end
|
225
|
+
|
226
|
+
private
|
227
|
+
# number of consecutive errors seen for server, 0 all is ok
|
228
|
+
def error_count
|
229
|
+
state[@server] ? state[@server][:count] : 0
|
230
|
+
end
|
231
|
+
|
232
|
+
# time of last error for server, nil if all is ok
|
233
|
+
def error_time
|
234
|
+
state[@server] && state[@server][:time]
|
235
|
+
end
|
236
|
+
|
237
|
+
# message for last error for server, "" if all is ok
|
238
|
+
def error_message
|
239
|
+
state[@server] ? state[@server][:message] : ""
|
240
|
+
end
|
241
|
+
|
242
|
+
# add an error for a server
|
243
|
+
def error_add(message)
|
244
|
+
state[@server] = { :count => error_count+1, :time => Time.now, :message => message }
|
245
|
+
end
|
246
|
+
|
247
|
+
# reset the error state for a server (i.e. a request succeeded)
|
248
|
+
def error_reset
|
249
|
+
state.delete(@server)
|
250
|
+
end
|
251
|
+
|
252
|
+
# Error message stuff...
|
253
|
+
def banana_message
|
254
|
+
return "#{@server} temporarily unavailable: (#{error_message})"
|
255
|
+
end
|
256
|
+
|
257
|
+
def err_header
|
258
|
+
return "#{self.class.name} :"
|
259
|
+
end
|
260
|
+
|
261
|
+
# Adds new EOF timestamp.
|
262
|
+
# Returns the number of seconds to wait before new conection retry:
|
263
|
+
# 0.5, 1, 2, 4, 8
|
264
|
+
def add_eof
|
265
|
+
(eof[@server] ||= []).unshift Time.now
|
266
|
+
0.25 * 2 ** eof[@server].size
|
267
|
+
end
|
268
|
+
|
269
|
+
|
270
|
+
# Returns true if we are receiving EOFs during last @params[:http_connection_retry_delay] seconds
|
271
|
+
# and there were no successful response from server
|
272
|
+
def raise_on_eof_exception?
|
273
|
+
eof[@server].blank? ? false : ( (Time.now.to_i-@params[:http_connection_retry_delay]) > eof[@server].last.to_i )
|
274
|
+
end
|
275
|
+
|
276
|
+
# Returns first EOF timestamp or nul if have no EOFs being tracked.
|
277
|
+
def eof_time
|
278
|
+
eof[@server] && eof[@server].last
|
279
|
+
end
|
280
|
+
|
281
|
+
# Reset a list of EOFs for this server.
|
282
|
+
# This is being called when we have got an successful response from server.
|
283
|
+
def eof_reset
|
284
|
+
eof.delete(@server)
|
285
|
+
end
|
286
|
+
|
287
|
+
# Detects if an object is 'streamable' - can we read from it, and can we know the size?
|
288
|
+
def setup_streaming(request)
|
289
|
+
if(request.body && request.body.respond_to?(:read))
|
290
|
+
body = request.body
|
291
|
+
request.content_length = body.respond_to?(:lstat) ? body.lstat.size : body.size
|
292
|
+
request.body_stream = request.body
|
293
|
+
true
|
294
|
+
end
|
295
|
+
end
|
296
|
+
|
297
|
+
def get_fileptr_offset(request_params)
|
298
|
+
request_params[:request].body.pos
|
299
|
+
rescue Exception => e
|
300
|
+
# Probably caught this because the body doesn't support the pos() method, like if it is a socket.
|
301
|
+
# Just return 0 and get on with life.
|
302
|
+
0
|
303
|
+
end
|
304
|
+
|
305
|
+
def reset_fileptr_offset(request, offset = 0)
|
306
|
+
if(request.body_stream && request.body_stream.respond_to?(:pos))
|
307
|
+
begin
|
308
|
+
request.body_stream.pos = offset
|
309
|
+
rescue Exception => e
|
310
|
+
log(:warn, "Failed file pointer reset; aborting HTTP retries. -- #{err_header} #{e.inspect}")
|
311
|
+
raise e
|
312
|
+
end
|
313
|
+
end
|
314
|
+
end
|
315
|
+
|
316
|
+
# Start a fresh connection. The object closes any existing connection and
|
317
|
+
# opens a new one.
|
318
|
+
def start(request_params)
|
319
|
+
# close the previous if exists
|
320
|
+
finish
|
321
|
+
# create new connection
|
322
|
+
@server = request_params[:server]
|
323
|
+
@port = request_params[:port]
|
324
|
+
@protocol = request_params[:protocol]
|
325
|
+
|
326
|
+
log(:info, "Opening new #{@protocol.upcase} connection to #@server:#@port")
|
327
|
+
@http = Net::HTTP.new(@server, @port)
|
328
|
+
@http.open_timeout = @params[:http_connection_open_timeout]
|
329
|
+
@http.read_timeout = @params[:http_connection_read_timeout]
|
330
|
+
|
331
|
+
if @protocol == 'https'
|
332
|
+
verifyCallbackProc = Proc.new{ |ok, x509_store_ctx|
|
333
|
+
code = x509_store_ctx.error
|
334
|
+
msg = x509_store_ctx.error_string
|
335
|
+
#debugger
|
336
|
+
log(:warn, "##### #{@server} certificate verify failed: #{msg}") unless code == 0
|
337
|
+
true
|
338
|
+
}
|
339
|
+
@http.use_ssl = true
|
340
|
+
ca_file = get_param(:ca_file)
|
341
|
+
if ca_file
|
342
|
+
#@http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
343
|
+
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
344
|
+
#@http.verify_callback = verifyCallbackProc
|
345
|
+
#@http.ca_file = ca_file
|
346
|
+
end
|
347
|
+
end
|
348
|
+
# open connection
|
349
|
+
@http.start
|
350
|
+
end
|
351
|
+
|
352
|
+
public
|
353
|
+
|
354
|
+
=begin rdoc
|
355
|
+
Send HTTP request to server
|
356
|
+
|
357
|
+
request_params hash:
|
358
|
+
:server => 'www.HostName.com' # Hostname or IP address of HTTP server
|
359
|
+
:port => '80' # Port of HTTP server
|
360
|
+
:protocol => 'https' # http and https are supported on any port
|
361
|
+
:request => 'requeststring' # Fully-formed HTTP request to make
|
362
|
+
|
363
|
+
Raises RuntimeError, Interrupt, and params[:exception] (if specified in new).
|
364
|
+
|
365
|
+
=end
|
366
|
+
def request(request_params, &block)
|
367
|
+
# prevent_mt_use!
|
368
|
+
log_request(request_params) rescue nil
|
369
|
+
# We save the offset here so that if we need to retry, we can return the file pointer to its initial position
|
370
|
+
mypos = get_fileptr_offset(request_params)
|
371
|
+
loop do
|
372
|
+
# if we are inside a delay between retries: no requests this time!
|
373
|
+
if error_count > @params[:http_connection_retry_count] &&
|
374
|
+
error_time + @params[:http_connection_retry_delay] > Time.now
|
375
|
+
# store the message (otherwise it will be lost after error_reset and
|
376
|
+
# we will raise an exception with an empty text)
|
377
|
+
banana_message_text = banana_message
|
378
|
+
log(:warn, "#{err_header} re-raising same error: #{banana_message_text} " +
|
379
|
+
"-- error count: #{error_count}, error age: #{Time.now.to_i - error_time.to_i}")
|
380
|
+
exception = get_param(:exception) || RuntimeError
|
381
|
+
raise exception.new(banana_message_text)
|
382
|
+
end
|
383
|
+
|
384
|
+
# try to connect server(if connection does not exist) and get response data
|
385
|
+
begin
|
386
|
+
request_params[:protocol] ||= (request_params[:port] == 443 ? 'https' : 'http')
|
387
|
+
|
388
|
+
request = request_params[:request]
|
389
|
+
request['User-Agent'] = get_param(:user_agent) || ''
|
390
|
+
|
391
|
+
# (re)open connection to server if none exists or params has changed
|
392
|
+
unless @http &&
|
393
|
+
@http.started? &&
|
394
|
+
@server == request_params[:server] &&
|
395
|
+
@port == request_params[:port] &&
|
396
|
+
@protocol == request_params[:protocol]
|
397
|
+
start(request_params)
|
398
|
+
end
|
399
|
+
|
400
|
+
# Detect if the body is a streamable object like a file or socket. If so, stream that
|
401
|
+
# bad boy.
|
402
|
+
setup_streaming(request)
|
403
|
+
response = @http.request(request, &block)
|
404
|
+
log_response(response) rescue nil
|
405
|
+
|
406
|
+
error_reset
|
407
|
+
eof_reset
|
408
|
+
return response
|
409
|
+
|
410
|
+
# We treat EOF errors and the timeout/network errors differently. Both
|
411
|
+
# are tracked in different statistics blocks. Note below that EOF
|
412
|
+
# errors will sleep for a certain (exponentially increasing) period.
|
413
|
+
# Other errors don't sleep because there is already an inherent delay
|
414
|
+
# in them; connect and read timeouts (for example) have already
|
415
|
+
# 'slept'. It is still not clear which way we should treat errors
|
416
|
+
# like RST and resolution failures. For now, there is no additional
|
417
|
+
# delay for these errors although this may change in the future.
|
418
|
+
|
419
|
+
# EOFError means the server closed the connection on us.
|
420
|
+
rescue EOFError => e
|
421
|
+
log(:debug, "#{err_header} server #{@server} closed connection")
|
422
|
+
@http = nil
|
423
|
+
|
424
|
+
# if we have waited long enough - raise an exception...
|
425
|
+
if raise_on_eof_exception?
|
426
|
+
exception = get_param(:exception) || RuntimeError
|
427
|
+
log(:warn, "#{err_header} raising #{exception} due to permanent EOF being received from #{@server}, error age: #{Time.now.to_i - eof_time.to_i}")
|
428
|
+
raise exception.new("Permanent EOF is being received from #{@server}.")
|
429
|
+
else
|
430
|
+
# ... else just sleep a bit before new retry
|
431
|
+
sleep(add_eof)
|
432
|
+
# We will be retrying the request, so reset the file pointer
|
433
|
+
reset_fileptr_offset(request, mypos)
|
434
|
+
end
|
435
|
+
rescue Exception => e # See comment at bottom for the list of errors seen...
|
436
|
+
@http = nil
|
437
|
+
# if ctrl+c is pressed - we have to reraise exception to terminate proggy
|
438
|
+
if e.is_a?(Interrupt) && !( e.is_a?(Errno::ETIMEDOUT) || e.is_a?(Timeout::Error))
|
439
|
+
log(:debug, "#{err_header} request to server #{@server} interrupted by ctrl-c")
|
440
|
+
raise
|
441
|
+
elsif e.is_a?(ArgumentError) && e.message.include?('wrong number of arguments (5 for 4)')
|
442
|
+
# seems our net_fix patch was overriden...
|
443
|
+
exception = get_param(:exception) || RuntimeError
|
444
|
+
raise exception.new('incompatible Net::HTTP monkey-patch')
|
445
|
+
end
|
446
|
+
# oops - we got a banana: log it
|
447
|
+
error_add(e.message)
|
448
|
+
log(:warn, "#{err_header} request failure count: #{error_count}, exception: #{e.inspect}")
|
449
|
+
|
450
|
+
# We will be retrying the request, so reset the file pointer
|
451
|
+
reset_fileptr_offset(request, mypos)
|
452
|
+
|
453
|
+
end
|
454
|
+
end
|
455
|
+
end
|
456
|
+
|
457
|
+
def log_request(request_params)
|
458
|
+
return if logger.nil?
|
459
|
+
return unless logger.debug?
|
460
|
+
logger.debug{ "HttpConnection - request_params=#{ request_params.inspect }" }
|
461
|
+
if((request = request_params[:request]))
|
462
|
+
method = request.method
|
463
|
+
path = request.path
|
464
|
+
#body = (request.body.size > 42 ? (request.body[0,42] + '...') : request.body) if request.body
|
465
|
+
body = request.body
|
466
|
+
logger.debug{ "HttpConnection - request.method=#{ method.inspect }" }
|
467
|
+
logger.debug{ "HttpConnection - request.path=#{ path.inspect }" }
|
468
|
+
logger.debug{ "HttpConnection - request.body=#{ body.inspect }" }
|
469
|
+
end
|
470
|
+
end
|
471
|
+
|
472
|
+
def log_response(response)
|
473
|
+
return if logger.nil?
|
474
|
+
return unless logger.debug?
|
475
|
+
if response
|
476
|
+
code = response.code
|
477
|
+
message = response.message
|
478
|
+
#body = (response.body.size > 42 ? (response.body[0,42] + '...') : response.body) if response.body
|
479
|
+
body = response.body
|
480
|
+
logger.debug{ "HttpConnection - response.code=#{ code.inspect }" }
|
481
|
+
logger.debug{ "HttpConnection - response.message=#{ message.inspect }" }
|
482
|
+
logger.debug{ "HttpConnection - response.body=#{ body.inspect }" }
|
483
|
+
end
|
484
|
+
end
|
485
|
+
|
486
|
+
def finish(reason = '')
|
487
|
+
# prevent_mt_use!
|
488
|
+
if @http && @http.started?
|
489
|
+
reason = ", reason: '#{reason}'" unless reason.blank?
|
490
|
+
log(:info, "Closing #{@http.use_ssl? ? 'HTTPS' : 'HTTP'} connection to #{@http.address}:#{@http.port}#{reason}")
|
491
|
+
@http.finish rescue nil # it's possible for a Thread to reset this before we can finish it (so it wouldn't be started)
|
492
|
+
end
|
493
|
+
end
|
494
|
+
|
495
|
+
# Errors received during testing:
|
496
|
+
#
|
497
|
+
# #<Timeout::Error: execution expired>
|
498
|
+
# #<Errno::ETIMEDOUT: Connection timed out - connect(2)>
|
499
|
+
# #<SocketError: getaddrinfo: Name or service not known>
|
500
|
+
# #<SocketError: getaddrinfo: Temporary failure in name resolution>
|
501
|
+
# #<EOFError: end of file reached>
|
502
|
+
# #<Errno::ECONNRESET: Connection reset by peer>
|
503
|
+
# #<OpenSSL::SSL::SSLError: SSL_write:: bad write retry>
|
504
|
+
end
|
505
|
+
|
506
|
+
end
|
507
|
+
|