em_aws 0.1.10 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile.lock +22 -22
- data/LICENSE.txt +1 -1
- data/README.md +49 -8
- data/lib/aws/core/http/em_connection_pool.rb +120 -0
- data/lib/aws/core/http/em_http_handler.rb +99 -52
- data/lib/em_aws.rb +3 -1
- data/lib/em_aws/version.rb +1 -1
- data/spec/em_connection_pool_spec.rb +103 -0
- data/spec/em_http_handler_spec.rb +21 -24
- data/spec/spec_helper.rb +7 -2
- metadata +5 -2
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
em_aws (0.
|
4
|
+
em_aws (0.2.0)
|
5
5
|
aws-sdk
|
6
6
|
em-http-request
|
7
7
|
em-synchrony
|
@@ -9,47 +9,47 @@ PATH
|
|
9
9
|
GEM
|
10
10
|
remote: http://rubygems.org/
|
11
11
|
specs:
|
12
|
-
addressable (2.2
|
13
|
-
aws-sdk (1.
|
12
|
+
addressable (2.3.2)
|
13
|
+
aws-sdk (1.8.0)
|
14
14
|
httparty (~> 0.7)
|
15
15
|
json (~> 1.4)
|
16
16
|
nokogiri (>= 1.4.4)
|
17
17
|
uuidtools (~> 2.1)
|
18
|
-
builder (3.
|
18
|
+
builder (3.1.4)
|
19
19
|
cookiejar (0.3.0)
|
20
20
|
diff-lcs (1.1.3)
|
21
|
-
em-http-request (1.0.
|
21
|
+
em-http-request (1.0.3)
|
22
22
|
addressable (>= 2.2.3)
|
23
23
|
cookiejar
|
24
24
|
em-socksify
|
25
25
|
eventmachine (>= 1.0.0.beta.4)
|
26
26
|
http_parser.rb (>= 0.5.3)
|
27
|
-
em-socksify (0.2.
|
27
|
+
em-socksify (0.2.1)
|
28
28
|
eventmachine (>= 1.0.0.beta.4)
|
29
29
|
em-synchrony (1.0.2)
|
30
30
|
eventmachine (>= 1.0.0.beta.1)
|
31
|
-
eventmachine (1.0.0
|
32
|
-
eventmachine (1.0.0
|
31
|
+
eventmachine (1.0.0)
|
32
|
+
eventmachine (1.0.0-java)
|
33
33
|
http_parser.rb (0.5.3)
|
34
34
|
http_parser.rb (0.5.3-java)
|
35
|
-
httparty (0.
|
35
|
+
httparty (0.9.0)
|
36
36
|
multi_json (~> 1.0)
|
37
37
|
multi_xml
|
38
|
-
json (1.7.
|
39
|
-
json (1.7.
|
40
|
-
multi_json (1.
|
38
|
+
json (1.7.6)
|
39
|
+
json (1.7.6-java)
|
40
|
+
multi_json (1.5.0)
|
41
41
|
multi_xml (0.5.1)
|
42
|
-
nokogiri (1.5.
|
43
|
-
nokogiri (1.5.
|
44
|
-
rspec (2.
|
45
|
-
rspec-core (~> 2.
|
46
|
-
rspec-expectations (~> 2.
|
47
|
-
rspec-mocks (~> 2.
|
48
|
-
rspec-core (2.
|
49
|
-
rspec-expectations (2.
|
42
|
+
nokogiri (1.5.6)
|
43
|
+
nokogiri (1.5.6-java)
|
44
|
+
rspec (2.12.0)
|
45
|
+
rspec-core (~> 2.12.0)
|
46
|
+
rspec-expectations (~> 2.12.0)
|
47
|
+
rspec-mocks (~> 2.12.0)
|
48
|
+
rspec-core (2.12.2)
|
49
|
+
rspec-expectations (2.12.1)
|
50
50
|
diff-lcs (~> 1.1.3)
|
51
|
-
rspec-mocks (2.
|
52
|
-
uuidtools (2.1.
|
51
|
+
rspec-mocks (2.12.1)
|
52
|
+
uuidtools (2.1.3)
|
53
53
|
|
54
54
|
PLATFORMS
|
55
55
|
java
|
data/LICENSE.txt
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# EmAws
|
2
|
-
An EM-Synchrony handler for Ruby [AWS-SDK](https://github.com/
|
2
|
+
An EM-Synchrony handler for Ruby [AWS-SDK-Ruby](https://github.com/aws/aws-sdk-ruby)
|
3
3
|
|
4
4
|
## Installation
|
5
5
|
|
@@ -8,7 +8,7 @@ em_aws is available through [Rubygems](https://rubygems.org/gems/em_aws) and can
|
|
8
8
|
$ gem install em_aws
|
9
9
|
|
10
10
|
## Rails 3 setup (no rails 2 sorry)
|
11
|
-
Setup [AWS-
|
11
|
+
Setup [AWS-SDK-Ruby](https://github.com/aws/aws-sdk-ruby/blob/master/README.rdoc) as you would normally.
|
12
12
|
|
13
13
|
Assuming you've already setup async-rails, add em_aws to you gemfile:
|
14
14
|
|
@@ -25,16 +25,57 @@ In your environments files add:
|
|
25
25
|
AWS.eager_autoload! # AWS lazyloading is not threadsafe
|
26
26
|
AWS.config(
|
27
27
|
:http_handler => AWS::Http::EMHttpHandler.new(
|
28
|
-
:proxy => {:host => "http://myproxy.com", :port => 80}
|
29
|
-
|
28
|
+
:proxy => {:host => "http://myproxy.com", :port => 80}))
|
29
|
+
:pool_size => 0 # by default connection pooling is off
|
30
|
+
:async => false # if set to true all requests for this client will be asynchronous
|
30
31
|
|
31
32
|
Your done.
|
32
33
|
|
33
|
-
All requests to AWS will use EM-Synchrony's implementation of em-http-request for non-block HTTP
|
34
|
+
All requests to AWS will use EM-Synchrony's implementation of em-http-request for non-block HTTP requests and fiber management.
|
35
|
+
|
36
|
+
## Connection Pooling (keep-alive)
|
37
|
+
To enable connection pooling set the :pool_size to anything greater than 0. By default :inactivity_timeout is set
|
38
|
+
to 0 which will leave the connection open for as long as the client allows. Connects
|
39
|
+
are created lazy, so pools grow until they meet the set pool size.
|
40
|
+
|
41
|
+
require 'aws-sdk'
|
42
|
+
require 'aws/core/http/em_http_handler'
|
43
|
+
AWS.config(
|
44
|
+
:http_handler => AWS::Http::EMHttpHandler.new({
|
45
|
+
:pool_size => 20,
|
46
|
+
:inactivity_timeout => 0, # number of seconds to timeout stale connections in the pool,
|
47
|
+
:never_block => true, # if we run out of connections, create a new one
|
48
|
+
:proxy => {:host => "http://myproxy.com",:port => 80})
|
49
|
+
)
|
50
|
+
|
51
|
+
## Streaming
|
52
|
+
Requires [AWS-SKD-Ruby >= 1.6.3] (http://aws.amazon.com/releasenotes/Ruby/5728376747252106)
|
53
|
+
EM.synchrony do
|
54
|
+
s3 = AWS::S3.new
|
55
|
+
file = File.open("path_to_file")
|
56
|
+
s3.buckets['bucket_name'].objects["foo.txt"].write(file)
|
57
|
+
EM.stop
|
58
|
+
end
|
59
|
+
|
60
|
+
## Asynchronous Requests
|
61
|
+
Requests can be set to perform asynchronously, returning nil initially and performing
|
62
|
+
the actions in the background. If the request option :async are set to true that only
|
63
|
+
that request request will handled asynchronously. If the client option :async all requests will
|
64
|
+
be handled asynchronously.
|
65
|
+
|
66
|
+
EM.synchrony do
|
67
|
+
s3 = AWS::S3.new
|
68
|
+
s3.buckets['bucket-name'].objects["foo"].write('test', :async => true) # => nil
|
69
|
+
EM::Synchrony.sleep(2) # Let the pending fibers run
|
70
|
+
s3.buckets['bucket-name'].objects["foo"].read # => # 'test'
|
71
|
+
s3.buckets['bucket-name'].objects["foo"].delete(:async => true) # => nil
|
72
|
+
EM::Synchrony.sleep(2) # Let the pending fibers run
|
73
|
+
EM.stop
|
74
|
+
end
|
34
75
|
|
35
76
|
## References
|
36
77
|
|
37
|
-
[
|
78
|
+
[AWS-SDK-Ruby](https://github.com/aws/aws-sdk-ruby)
|
38
79
|
|
39
80
|
[Async-Rails](https://github.com/igrigorik/async-rails)
|
40
81
|
|
@@ -52,9 +93,9 @@ All requests to AWS will use EM-Synchrony's implementation of em-http-request fo
|
|
52
93
|
|
53
94
|
## Thanks
|
54
95
|
|
55
|
-
Code based on
|
96
|
+
Code based on HTTP Handers in [AWS-SDK-Ruby](https://github.com/aws/aws-sdk-ruby/blob/master/README.rdoc)
|
56
97
|
|
57
98
|
## License
|
58
99
|
|
59
100
|
EmAws [license](https://github.com/JoshMcKin/em_aws/blob/master/LICENSE.txt)
|
60
|
-
AWS-SDK [license](https://github.com/
|
101
|
+
AWS-SDK-Ruby [license](https://github.com/aws/aws-sdk-for-ruby/blob/master/LICENSE.txt)
|
@@ -0,0 +1,120 @@
|
|
1
|
+
require 'thread'
|
2
|
+
require "em-synchrony/em-http"
|
3
|
+
require 'em-synchrony/thread'
|
4
|
+
|
5
|
+
module AWS
|
6
|
+
module Core
|
7
|
+
module Http
|
8
|
+
class EMConnectionPool
|
9
|
+
# Since AWS connections may be to any number of urls, the connection
|
10
|
+
# pool is a hash of connection arrays, instead of a simple array like
|
11
|
+
# most connection pools
|
12
|
+
# Stores data concerning pools, like current size, last fetched
|
13
|
+
#
|
14
|
+
# Options:
|
15
|
+
# * :pool_size - number of connections for each pool
|
16
|
+
# * :inactivity_timeout - number of seconds to wait before disconnecting,
|
17
|
+
# setting to 0 means the connection will not be closed
|
18
|
+
# * :pool_timeout - the amount of seconds to block waiting for an available connection,
|
19
|
+
# because this is blocking it should be an extremely short amount of
|
20
|
+
# time default to 0.5 seconds, if you need more consider enlarging your pool
|
21
|
+
# instead of raising this number
|
22
|
+
# :never_block - if set to true, a connection will always be returned
|
23
|
+
def initialize(options={})
|
24
|
+
options[:never_block] ||= true
|
25
|
+
@pools = {}
|
26
|
+
@pool_data = {}
|
27
|
+
@pool_size = (options[:pool_size] || 5)
|
28
|
+
@never_block = (options[:never_block])
|
29
|
+
@inactivity_timeout = (options[:inactivity_timeout].to_i)
|
30
|
+
@pool_timeout = (options[:pool_timeout] || 0.5)
|
31
|
+
end
|
32
|
+
|
33
|
+
# A fiber safe mutex
|
34
|
+
def _fiber_mutex
|
35
|
+
@fibered_mutex ||= EM::Synchrony::Thread::Mutex.new
|
36
|
+
end
|
37
|
+
|
38
|
+
# Returns a pool for the associated url
|
39
|
+
def available_pools(url)
|
40
|
+
_fiber_mutex.synchronize do
|
41
|
+
add_connection(url) if add_connection?(url)
|
42
|
+
@pools[url]
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def add_connection?(url)
|
47
|
+
(@pool_data[url].nil? || (@pools[url].length == 0 && (@pool_data[url][:current_size] < @pool_size)))
|
48
|
+
end
|
49
|
+
|
50
|
+
def add_connection(url)
|
51
|
+
AWS.config.logger.info "Adding AWS connection to #{url}"
|
52
|
+
add_connection_data(url)
|
53
|
+
@pools[url] ||= []
|
54
|
+
@pools[url] << new_connection(url)
|
55
|
+
@pools[url]
|
56
|
+
end
|
57
|
+
|
58
|
+
def add_connection_data(url)
|
59
|
+
@pool_data[url] ||= {:current_size => 0}
|
60
|
+
@pool_data[url][:current_size] += 1
|
61
|
+
end
|
62
|
+
|
63
|
+
def new_connection(url)
|
64
|
+
EM::HttpRequest.new(url, :inactivity_timeout => @inactivity_timeout)
|
65
|
+
end
|
66
|
+
|
67
|
+
# Run the block on the retrieved connection, then return the connection
|
68
|
+
# back to the pool.
|
69
|
+
def run(url, &block)
|
70
|
+
connection = santize_connection(fetch_connection(url))
|
71
|
+
block.call(connection)
|
72
|
+
ensure
|
73
|
+
return_connection(url,connection)
|
74
|
+
end
|
75
|
+
|
76
|
+
# Fetch an available connection or raise an error
|
77
|
+
def fetch_connection(url)
|
78
|
+
connection = nil
|
79
|
+
alarm = (Time.now + @pool_timeout)
|
80
|
+
# block until we get an available connection or Timeout::Error
|
81
|
+
while connection.nil?
|
82
|
+
if alarm <= Time.now
|
83
|
+
message = "Could not fetch a free connection in time. Consider increasing your connection pool for em_aws or setting :never_block to true."
|
84
|
+
AWS.config.logger.error message
|
85
|
+
raise Timeout::Error, message
|
86
|
+
end
|
87
|
+
connection = available_pools(url).shift
|
88
|
+
if connection.nil? && (@never_block)
|
89
|
+
AWS.config.logger.info "Adding AWS connection to #{url} for never_block, will not be returned to pool."
|
90
|
+
connection = new_connection(url)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
connection
|
94
|
+
end
|
95
|
+
|
96
|
+
# Make sure we have a good connection.
|
97
|
+
def santize_connection(connection)
|
98
|
+
if connection.conn && connection.conn.error?
|
99
|
+
AWS.config.logger.info "Reconnecting to AWS: #{EventMachine::report_connection_error_status(connection.conn.instance_variable_get(:@signature))}"
|
100
|
+
connection.conn.close_connection
|
101
|
+
connection.instance_variable_set(:@deferred, true)
|
102
|
+
end
|
103
|
+
connection
|
104
|
+
end
|
105
|
+
|
106
|
+
# Return connections to pool if allowed, otherwise closes connection
|
107
|
+
def return_connection(url,connection)
|
108
|
+
_fiber_mutex.synchronize do
|
109
|
+
if (@pools[url].nil? || (@pools[url].length == @pool_size))
|
110
|
+
connection.conn.close_connection if connection.conn
|
111
|
+
else
|
112
|
+
@pools[url] << connection
|
113
|
+
end
|
114
|
+
@pools[url]
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
@@ -3,7 +3,6 @@ require "em-synchrony"
|
|
3
3
|
require "em-synchrony/em-http"
|
4
4
|
require 'em-synchrony/thread'
|
5
5
|
module AWS
|
6
|
-
|
7
6
|
module Core
|
8
7
|
module Http
|
9
8
|
|
@@ -17,18 +16,19 @@ module AWS
|
|
17
16
|
# require 'aws-sdk'
|
18
17
|
# require 'aws/core/http/em_http_handler'
|
19
18
|
# AWS.config(
|
20
|
-
#
|
21
|
-
#
|
22
|
-
#
|
23
|
-
#
|
24
|
-
#
|
19
|
+
# :http_handler => AWS::Http::EMHttpHandler.new(
|
20
|
+
# :proxy => {:host => "http://myproxy.com",
|
21
|
+
# :port => 80,
|
22
|
+
# :pool_size => 20, # not set by default which disables connection pooling
|
23
|
+
# :async => false # if set to true all requests are handle asynchronously and initially return nil
|
24
|
+
# }))
|
25
25
|
class EMHttpHandler
|
26
|
-
|
27
|
-
# request.
|
26
|
+
|
27
|
+
# @return [Hash] The default options to send to EM-Synchrony on each request.
|
28
28
|
attr_reader :default_request_options
|
29
|
-
|
29
|
+
attr_accessor :status_0_retries
|
30
|
+
|
30
31
|
# Constructs a new HTTP handler using EM-Synchrony.
|
31
|
-
#
|
32
32
|
# @param [Hash] options Default options to send to EM-Synchrony on
|
33
33
|
# each request. These options will be sent to +get+, +post+,
|
34
34
|
# +head+, +put+, or +delete+ when a request is made. Note
|
@@ -36,12 +36,48 @@ module AWS
|
|
36
36
|
# ignored. If you need to set the CA file, you should use the
|
37
37
|
# +:ssl_ca_file+ option to {AWS.config} or
|
38
38
|
# {AWS::Configuration} instead.
|
39
|
-
# Defaults pool_size to 5
|
40
39
|
def initialize options = {}
|
41
|
-
#puts "Using EM-Synchrony for AWS requests"
|
42
40
|
@default_request_options = options
|
41
|
+
@pool = EMConnectionPool.new(options) if options[:pool_size].to_i > 0
|
42
|
+
@status_0_retries = 2 # set to 0 for no retries
|
43
|
+
end
|
44
|
+
|
45
|
+
def handle(request,response,&read_block)
|
46
|
+
if EM::reactor_running?
|
47
|
+
process_request(request,response,&read_block)
|
48
|
+
else
|
49
|
+
EM.synchrony do
|
50
|
+
process_request(request,response,&read_block)
|
51
|
+
EM.stop
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# If the request option :async are set to true that request will handled
|
57
|
+
# asynchronously returning nil initially and processing in the background
|
58
|
+
# managed by EM-Synchrony. If the client option :async all requests will
|
59
|
+
# be handled asynchronously.
|
60
|
+
# EX:
|
61
|
+
# EM.synchrony do
|
62
|
+
# s3 = AWS::S3.new
|
63
|
+
# s3.obj.write('test', :async => true) => nil
|
64
|
+
# EM::Synchrony.sleep(2)
|
65
|
+
# s3.obj.read => # 'test'
|
66
|
+
# EM.stop
|
67
|
+
# end
|
68
|
+
def handle_async(request,response,handle,&read_block)
|
69
|
+
if EM::reactor_running?
|
70
|
+
process_request(request,response,true,&read_block)
|
71
|
+
else
|
72
|
+
EM.synchrony do
|
73
|
+
process_request(request,response,true,&read_block)
|
74
|
+
EM.stop
|
75
|
+
end
|
76
|
+
end
|
43
77
|
end
|
44
78
|
|
79
|
+
private
|
80
|
+
|
45
81
|
def fetch_url(request)
|
46
82
|
url = nil
|
47
83
|
if request.use_ssl?
|
@@ -53,15 +89,10 @@ module AWS
|
|
53
89
|
end
|
54
90
|
|
55
91
|
def fetch_headers(request)
|
56
|
-
# Net::HTTP adds this header for us when the body is
|
57
|
-
# provided, but it messes up signing
|
58
92
|
headers = { 'content-type' => '' }
|
59
|
-
|
60
|
-
# headers must have string values (net http calls .strip on them)
|
61
93
|
request.headers.each_pair do |key,value|
|
62
94
|
headers[key] = value.to_s
|
63
95
|
end
|
64
|
-
|
65
96
|
{:head => headers}
|
66
97
|
end
|
67
98
|
|
@@ -82,51 +113,38 @@ module AWS
|
|
82
113
|
opts
|
83
114
|
end
|
84
115
|
|
85
|
-
def
|
86
|
-
|
87
|
-
merge(
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
def fetch_response(url,method,opts={})
|
92
|
-
return EM::HttpRequest.new("#{url}").send(method, opts)
|
93
|
-
end
|
94
|
-
|
95
|
-
def handle(request,response)
|
96
|
-
if EM::reactor_running?
|
97
|
-
handle_it(request, response)
|
98
|
-
else
|
99
|
-
EM.synchrony do
|
100
|
-
handle_it(request, response)
|
101
|
-
EM.stop
|
102
|
-
end
|
103
|
-
end
|
104
|
-
end
|
105
|
-
|
106
|
-
def handle_it(request, response)
|
107
|
-
#puts "Using EM!!!!"
|
108
|
-
# get, post, put, delete, head
|
109
|
-
method = request.http_method.downcase.to_sym
|
110
|
-
opts = default_request_options.merge(request_options(request))
|
116
|
+
def fetch_request_options(request,method)
|
117
|
+
opts = default_request_options.
|
118
|
+
merge(fetch_headers(request).
|
119
|
+
merge(fetch_proxy(request)).
|
120
|
+
merge(fetch_ssl(request)))
|
111
121
|
if (method == :get)
|
112
122
|
opts[:query] = request.body
|
113
123
|
else
|
114
124
|
opts[:body] = request.body
|
115
125
|
end
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
126
|
+
opts
|
127
|
+
end
|
128
|
+
|
129
|
+
def fetch_response(url,method,opts={},&read_block)
|
130
|
+
if @pool
|
131
|
+
@pool.run(url) do |connection|
|
132
|
+
req = connection.send(method, {:keepalive => true}.merge(opts))
|
133
|
+
req.stream &read_block if block_given?
|
134
|
+
return EM::Synchrony.sync req unless opts[:async]
|
135
|
+
end
|
136
|
+
else
|
137
|
+
req = EM::HttpRequest.new(url).send(method,opts)
|
138
|
+
req.stream &read_block if block_given?
|
139
|
+
return EM::Synchrony.sync req unless opts[:async]
|
124
140
|
end
|
141
|
+
nil
|
125
142
|
end
|
126
143
|
|
127
144
|
# AWS needs all headers downcased, and for some reason x-amz-expiration and
|
128
145
|
# x-amz-restore need to be arrays
|
129
|
-
def
|
146
|
+
def fetch_response_headers(response)
|
147
|
+
response_headers = response.response_header.raw.to_hash
|
130
148
|
aws_headers = {}
|
131
149
|
response_headers.each_pair do |k,v|
|
132
150
|
key = k.downcase
|
@@ -138,6 +156,35 @@ module AWS
|
|
138
156
|
end
|
139
157
|
response_headers.merge(aws_headers)
|
140
158
|
end
|
159
|
+
|
160
|
+
# Builds and attempts the request. Occasionally under load em-http-request
|
161
|
+
# returns a status of 0 with nil for header and body, in such situations
|
162
|
+
# we retry as many times as status_0_retries is set. If our retries exceed
|
163
|
+
# status_0_retries we assume there is a network error
|
164
|
+
def process_request(request,response,async=false,retries=0,&read_block)
|
165
|
+
method = "a#{request.http_method}".downcase.to_sym # aget, apost, aput, adelete, ahead
|
166
|
+
opts = fetch_request_options(request,method)
|
167
|
+
opts[:async] = (async || opts[:async])
|
168
|
+
url = fetch_url(request)
|
169
|
+
begin
|
170
|
+
http_response = fetch_response(url,method,opts,&read_block)
|
171
|
+
unless opts[:async]
|
172
|
+
response.status = http_response.response_header.status.to_i
|
173
|
+
if response.status == 0
|
174
|
+
if retries <= status_0_retries.to_i
|
175
|
+
process_request(request,response,(retries + 1),&read_block)
|
176
|
+
else
|
177
|
+
response.network_error = true
|
178
|
+
end
|
179
|
+
else
|
180
|
+
response.headers = fetch_response_headers(http_response)
|
181
|
+
response.body = http_response.response
|
182
|
+
end
|
183
|
+
end
|
184
|
+
rescue *AWS::Core::Http::NetHttpHandler::NETWORK_ERRORS
|
185
|
+
response.network_error = true
|
186
|
+
end
|
187
|
+
end
|
141
188
|
end
|
142
189
|
end
|
143
190
|
end
|
data/lib/em_aws.rb
CHANGED
@@ -5,6 +5,8 @@ require 'em-http'
|
|
5
5
|
require 'em-synchrony'
|
6
6
|
require 'em-synchrony/em-http'
|
7
7
|
require 'aws/core/autoloader'
|
8
|
+
require 'aws/core/http/em_connection_pool'
|
9
|
+
require 'aws/core/http/em_http_handler'
|
8
10
|
|
9
11
|
AWS.eager_autoload! # lazy load isn't thread safe
|
10
|
-
module EmAws;end
|
12
|
+
module EmAws;end
|
data/lib/em_aws/version.rb
CHANGED
@@ -0,0 +1,103 @@
|
|
1
|
+
# To change this template, choose Tools | Templates
|
2
|
+
# and open the template in the editor.
|
3
|
+
require 'thread'
|
4
|
+
require "em-synchrony"
|
5
|
+
require "em-synchrony/em-http"
|
6
|
+
require 'spec_helper'
|
7
|
+
module AWS
|
8
|
+
module Core
|
9
|
+
module Http
|
10
|
+
describe EMConnectionPool do
|
11
|
+
before(:each) do
|
12
|
+
@em_connection_pool = EMConnectionPool.new
|
13
|
+
end
|
14
|
+
|
15
|
+
context 'default configuration' do
|
16
|
+
it "should have @pool_size of 5" do
|
17
|
+
@em_connection_pool.instance_variable_get(:@pool_size).should eql(5)
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should have @inactivity_timeout of 0" do
|
21
|
+
@em_connection_pool.instance_variable_get(:@inactivity_timeout).should eql(0)
|
22
|
+
end
|
23
|
+
|
24
|
+
it "should have @pool_timeout of 0" do
|
25
|
+
@em_connection_pool.instance_variable_get(:@pool_timeout).should eql(0.5)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
describe '#add_connection?' do
|
30
|
+
it "should be true if @pool_data does not have data for the url"do
|
31
|
+
@em_connection_pool.add_connection?("http://www.testurl123.com/").should be_true
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should be true if @pool_data has data but the number of connnections has not reached the pool_size" do
|
35
|
+
@em_connection_pool.instance_variable_set(:@pools,{"http://www.testurl123.com/" => ["connection"]})
|
36
|
+
@em_connection_pool.add_connection?("http://www.testurl123.com/").should be_true
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should be false pool has reached pool_size" do
|
40
|
+
@em_connection_pool.instance_variable_set(:@pools,
|
41
|
+
{"http://www.testurl123.com/" => ["connection","connection","connection","connection","connection"]})
|
42
|
+
@em_connection_pool.add_connection?("http://www.testurl123.com/").should be_true
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
describe '#add_connection' do
|
47
|
+
it "should add connections for supplied url"do
|
48
|
+
@em_connection_pool.add_connection("http://www.testurl123.com/")
|
49
|
+
@em_connection_pool.instance_variable_get(:@pools)["http://www.testurl123.com/"].should_not be_nil
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
describe '#fetch_connection' do
|
54
|
+
it "should raise Timeout::Error if an available is not found in time"do
|
55
|
+
@em_connection_pool.stub(:available_pools).and_return([])
|
56
|
+
@em_connection_pool.instance_variable_set(:@never_block, false)
|
57
|
+
lambda { @em_connection_pool.fetch_connection('http://some_url.com')}.should raise_error(Timeout::Error)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
context 'integration test with parallel requests' do
|
62
|
+
# 10 parallel requests
|
63
|
+
|
64
|
+
it "should work" do
|
65
|
+
@requests_made = []
|
66
|
+
EM.synchrony do
|
67
|
+
@em_connection_pool.instance_variable_set(:@never_block, true)
|
68
|
+
fibers = []
|
69
|
+
10.times do
|
70
|
+
fibers << Fiber.new do
|
71
|
+
@em_connection_pool.run "http://www.testurl123.com/" do |connection|
|
72
|
+
@requests_made << connection.get(:keepalive => true).response_header.status
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
fibers.each do |f|
|
78
|
+
f.resume
|
79
|
+
end
|
80
|
+
|
81
|
+
loop do
|
82
|
+
done = true
|
83
|
+
fibers.each do |f|
|
84
|
+
done = false if f.alive?
|
85
|
+
end
|
86
|
+
if done
|
87
|
+
break
|
88
|
+
else
|
89
|
+
EM::Synchrony.sleep(0.01)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
@requests_made.length.should eql(10)
|
94
|
+
@em_connection_pool.instance_variable_get(:@pools)["http://www.testurl123.com/"].length.should eql(@em_connection_pool.instance_variable_get(:@pool_size))
|
95
|
+
|
96
|
+
EM.stop
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
@@ -46,17 +46,8 @@ module AWS::Core
|
|
46
46
|
be_an(AWS::Core::Http::EMHttpHandler)
|
47
47
|
end
|
48
48
|
|
49
|
-
describe '#initialize' do
|
50
|
-
it 'should set the default request options' do
|
51
|
-
described_class.new(:foo => "BAR").default_request_options.
|
52
|
-
should == { :foo => "BAR" }
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
49
|
describe '#handle' do
|
57
|
-
|
58
50
|
context 'timeouts' do
|
59
|
-
|
60
51
|
it 'should rescue Timeout::Error' do
|
61
52
|
handler.stub(:fetch_response).
|
62
53
|
and_raise(Timeout::Error)
|
@@ -71,13 +62,12 @@ module AWS::Core
|
|
71
62
|
should_not raise_error
|
72
63
|
end
|
73
64
|
|
74
|
-
it 'should indicate that there was a
|
65
|
+
it 'should indicate that there was a network_error' do
|
75
66
|
handler.stub(:fetch_response).
|
76
67
|
and_raise(Errno::ETIMEDOUT)
|
77
68
|
handler.handle(req, resp)
|
78
|
-
resp.
|
69
|
+
resp.network_error?.should be_true
|
79
70
|
end
|
80
|
-
|
81
71
|
end
|
82
72
|
|
83
73
|
context 'default request options' do
|
@@ -93,19 +83,27 @@ module AWS::Core
|
|
93
83
|
it 'uses the default when the request option is not set' do
|
94
84
|
#puts handler.default_request_options
|
95
85
|
handler.default_request_options[:private_key_file].should == "blarg"
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
describe '#process_request' do
|
90
|
+
context 'too many retries' do
|
91
|
+
it "should have network error" do
|
92
|
+
EM.synchrony do
|
93
|
+
handler.send(:process_request,(req),(resp),3)
|
94
|
+
resp.network_error?.should be_true
|
95
|
+
EM.stop
|
96
|
+
end
|
96
97
|
end
|
97
|
-
|
98
98
|
end
|
99
|
-
|
100
99
|
end
|
101
100
|
describe '#fetch_proxy' do
|
102
101
|
context ':proxy_uri' do
|
103
102
|
it 'passes proxy address and port from the request' do
|
104
103
|
req.proxy_uri = URI.parse('https://user:pass@proxy.com:443/path?query')
|
105
|
-
handler.fetch_proxy(req)[:proxy][:host].should == 'proxy.com'
|
106
|
-
handler.fetch_proxy(req)[:proxy][:port].should == 443
|
104
|
+
handler.send(:fetch_proxy,(req))[:proxy][:host].should == 'proxy.com'
|
105
|
+
handler.send(:fetch_proxy,(req))[:proxy][:port].should == 443
|
107
106
|
end
|
108
|
-
|
109
107
|
end
|
110
108
|
|
111
109
|
describe '#fetch_ssl' do
|
@@ -113,12 +111,11 @@ module AWS::Core
|
|
113
111
|
req.use_ssl = true
|
114
112
|
req.ssl_verify_peer = true
|
115
113
|
req.ssl_ca_file = "something"
|
116
|
-
handler.fetch_ssl(req)[:private_key_file].should == "something"
|
117
|
-
handler.fetch_ssl(req)[:cert_chain_file].should == "something"
|
114
|
+
handler.send(:fetch_ssl,(req))[:private_key_file].should == "something"
|
115
|
+
handler.send(:fetch_ssl,(req))[:cert_chain_file].should == "something"
|
118
116
|
end
|
119
117
|
|
120
118
|
context 'CA cert path' do
|
121
|
-
|
122
119
|
context 'use_ssl? is true' do
|
123
120
|
|
124
121
|
before(:each) { req.use_ssl = true }
|
@@ -131,21 +128,21 @@ module AWS::Core
|
|
131
128
|
end
|
132
129
|
|
133
130
|
it 'should use the ssl_ca_file attribute of the request' do
|
134
|
-
handler.fetch_ssl(req)[:private_key_file].should == "foobar.txt"
|
131
|
+
handler.send(:fetch_ssl,(req))[:private_key_file].should == "foobar.txt"
|
135
132
|
end
|
136
133
|
|
137
134
|
it 'should use the ssl_ca_file attribute of the request' do
|
138
|
-
handler.fetch_ssl(req)[:cert_chain_file].should == "foobar.txt"
|
135
|
+
handler.send(:fetch_ssl,(req))[:cert_chain_file].should == "foobar.txt"
|
139
136
|
end
|
140
137
|
end
|
141
138
|
|
142
139
|
it 'should not set the ssl_ca_file option without ssl_verify_peer?' do
|
143
|
-
handler.fetch_ssl(req).should_not include(:private_key_file)
|
140
|
+
handler.send(:fetch_ssl,(req)).should_not include(:private_key_file)
|
144
141
|
end
|
145
142
|
end
|
146
143
|
|
147
144
|
it 'should not set the ssl_ca_file option without use_ssl?' do
|
148
|
-
handler.fetch_ssl(req).should_not include(:private_key_file)
|
145
|
+
handler.send(:fetch_ssl,(req)).should_not include(:private_key_file)
|
149
146
|
end
|
150
147
|
end
|
151
148
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -4,14 +4,19 @@ require 'em_aws'
|
|
4
4
|
require 'aws/core/http/em_http_handler'
|
5
5
|
require 'rspec'
|
6
6
|
require 'bundler/setup'
|
7
|
+
require 'logger'
|
7
8
|
|
8
9
|
|
9
10
|
|
10
11
|
# Requires supporting files with custom matchers and macros, etc,
|
11
12
|
# in ./support/ and its subdirectories.
|
12
13
|
#Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
|
13
|
-
|
14
|
-
|
14
|
+
class StubLogger
|
15
|
+
def method_missing(method, *args)
|
16
|
+
#we don't care
|
17
|
+
end
|
18
|
+
end
|
19
|
+
AWS.config(:logger => StubLogger.new)
|
15
20
|
RSpec.configure do |config|
|
16
21
|
|
17
22
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: em_aws
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-01-
|
12
|
+
date: 2013-01-07 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: aws-sdk
|
@@ -106,10 +106,12 @@ files:
|
|
106
106
|
- README.md
|
107
107
|
- Rakefile
|
108
108
|
- em_aws.gemspec
|
109
|
+
- lib/aws/core/http/em_connection_pool.rb
|
109
110
|
- lib/aws/core/http/em_http_handler.rb
|
110
111
|
- lib/em_aws.rb
|
111
112
|
- lib/em_aws/patches.rb
|
112
113
|
- lib/em_aws/version.rb
|
114
|
+
- spec/em_connection_pool_spec.rb
|
113
115
|
- spec/em_http_handler_spec.rb
|
114
116
|
- spec/patches_spec.rb
|
115
117
|
- spec/spec_helper.rb
|
@@ -138,6 +140,7 @@ signing_key:
|
|
138
140
|
specification_version: 3
|
139
141
|
summary: Adds EM-Synchrony support to AWS-SDK gem
|
140
142
|
test_files:
|
143
|
+
- spec/em_connection_pool_spec.rb
|
141
144
|
- spec/em_http_handler_spec.rb
|
142
145
|
- spec/patches_spec.rb
|
143
146
|
- spec/spec_helper.rb
|