em_aws 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile.lock +2 -2
- data/README.md +6 -5
- data/lib/aws/core/http/em_connection_pool.rb +40 -43
- data/lib/em_aws/version.rb +1 -1
- data/spec/em_http_handler_spec.rb +2 -1
- metadata +2 -2
data/Gemfile.lock
CHANGED
@@ -32,13 +32,13 @@ GEM
|
|
32
32
|
eventmachine (1.0.0-java)
|
33
33
|
http_parser.rb (0.5.3)
|
34
34
|
http_parser.rb (0.5.3-java)
|
35
|
-
httparty (0.
|
35
|
+
httparty (0.10.0)
|
36
36
|
multi_json (~> 1.0)
|
37
37
|
multi_xml
|
38
38
|
json (1.7.6)
|
39
39
|
json (1.7.6-java)
|
40
40
|
multi_json (1.5.0)
|
41
|
-
multi_xml (0.5.
|
41
|
+
multi_xml (0.5.2)
|
42
42
|
nokogiri (1.5.6)
|
43
43
|
nokogiri (1.5.6-java)
|
44
44
|
rspec (2.12.0)
|
data/README.md
CHANGED
@@ -10,7 +10,7 @@ em_aws is available through [Rubygems](https://rubygems.org/gems/em_aws) and can
|
|
10
10
|
## Rails 3 setup (no rails 2 sorry)
|
11
11
|
Setup [AWS-SDK-Ruby](https://github.com/aws/aws-sdk-ruby/blob/master/README.rdoc) as you would normally.
|
12
12
|
|
13
|
-
Assuming you've already setup async-rails, add em_aws to
|
13
|
+
Assuming you've already setup async-rails, add em_aws to your gemfile:
|
14
14
|
|
15
15
|
gem 'em_aws'
|
16
16
|
|
@@ -49,7 +49,8 @@ are created lazy, so pools grow until they meet the set pool size.
|
|
49
49
|
)
|
50
50
|
|
51
51
|
## Streaming
|
52
|
-
Requires [AWS-SKD-Ruby >= 1.6.3]
|
52
|
+
Requires [AWS-SKD-Ruby >= 1.6.3](http://aws.amazon.com/releasenotes/Ruby/5728376747252106)
|
53
|
+
|
53
54
|
EM.synchrony do
|
54
55
|
s3 = AWS::S3.new
|
55
56
|
file = File.open("path_to_file")
|
@@ -59,9 +60,9 @@ Requires [AWS-SKD-Ruby >= 1.6.3] (http://aws.amazon.com/releasenotes/Ruby/572837
|
|
59
60
|
|
60
61
|
## Asynchronous Requests
|
61
62
|
Requests can be set to perform asynchronously, returning nil initially and performing
|
62
|
-
the actions in the background. If the request option :async are set to true
|
63
|
-
that request
|
64
|
-
be handled asynchronously.
|
63
|
+
the actions in the background. If the request option :async are set to true, only
|
64
|
+
that request will be handled asynchronously. If the client option :async is set to true,
|
65
|
+
all requests will be handled asynchronously.
|
65
66
|
|
66
67
|
EM.synchrony do
|
67
68
|
s3 = AWS::S3.new
|
@@ -8,9 +8,7 @@ module AWS
|
|
8
8
|
class EMConnectionPool
|
9
9
|
# Since AWS connections may be to any number of urls, the connection
|
10
10
|
# pool is a hash of connection arrays, instead of a simple array like
|
11
|
-
# most connection pools
|
12
|
-
# Stores data concerning pools, like current size, last fetched
|
13
|
-
#
|
11
|
+
# most connection pools #
|
14
12
|
# Options:
|
15
13
|
# * :pool_size - number of connections for each pool
|
16
14
|
# * :inactivity_timeout - number of seconds to wait before disconnecting,
|
@@ -27,20 +25,24 @@ module AWS
|
|
27
25
|
@pool_size = (options[:pool_size] || 5)
|
28
26
|
@never_block = (options[:never_block])
|
29
27
|
@inactivity_timeout = (options[:inactivity_timeout].to_i)
|
28
|
+
@connect_timeout = options[:connect_timeout]
|
30
29
|
@pool_timeout = (options[:pool_timeout] || 0.5)
|
30
|
+
@fibered_mutex = EM::Synchrony::Thread::Mutex.new # A fiber safe mutex
|
31
31
|
end
|
32
|
-
|
33
|
-
#
|
34
|
-
|
35
|
-
|
32
|
+
|
33
|
+
# Run the block on the retrieved connection. Then return the connection
|
34
|
+
# back to the pool.
|
35
|
+
def run(url, &block)
|
36
|
+
connection = santize_connection(fetch_connection(url))
|
37
|
+
block.call(connection)
|
38
|
+
ensure
|
39
|
+
return_connection(url,connection)
|
36
40
|
end
|
37
41
|
|
38
42
|
# Returns a pool for the associated url
|
39
43
|
def available_pools(url)
|
40
|
-
|
41
|
-
|
42
|
-
@pools[url]
|
43
|
-
end
|
44
|
+
add_connection(url) if add_connection?(url)
|
45
|
+
@pools[url]
|
44
46
|
end
|
45
47
|
|
46
48
|
def add_connection?(url)
|
@@ -61,36 +63,9 @@ module AWS
|
|
61
63
|
end
|
62
64
|
|
63
65
|
def new_connection(url)
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
# Run the block on the retrieved connection, then return the connection
|
68
|
-
# back to the pool.
|
69
|
-
def run(url, &block)
|
70
|
-
connection = santize_connection(fetch_connection(url))
|
71
|
-
block.call(connection)
|
72
|
-
ensure
|
73
|
-
return_connection(url,connection)
|
74
|
-
end
|
75
|
-
|
76
|
-
# Fetch an available connection or raise an error
|
77
|
-
def fetch_connection(url)
|
78
|
-
connection = nil
|
79
|
-
alarm = (Time.now + @pool_timeout)
|
80
|
-
# block until we get an available connection or Timeout::Error
|
81
|
-
while connection.nil?
|
82
|
-
if alarm <= Time.now
|
83
|
-
message = "Could not fetch a free connection in time. Consider increasing your connection pool for em_aws or setting :never_block to true."
|
84
|
-
AWS.config.logger.error message
|
85
|
-
raise Timeout::Error, message
|
86
|
-
end
|
87
|
-
connection = available_pools(url).shift
|
88
|
-
if connection.nil? && (@never_block)
|
89
|
-
AWS.config.logger.info "Adding AWS connection to #{url} for never_block, will not be returned to pool."
|
90
|
-
connection = new_connection(url)
|
91
|
-
end
|
92
|
-
end
|
93
|
-
connection
|
66
|
+
opts = {:inactivity_timeout => @inactivity_timeout}
|
67
|
+
opts[:connect_timeout] = @connect_timeout if @connect_timeout
|
68
|
+
EM::HttpRequest.new(url, opts)
|
94
69
|
end
|
95
70
|
|
96
71
|
# Make sure we have a good connection.
|
@@ -103,13 +78,35 @@ module AWS
|
|
103
78
|
connection
|
104
79
|
end
|
105
80
|
|
81
|
+
# Fetch an available connection or raise an error
|
82
|
+
def fetch_connection(url)
|
83
|
+
alarm = (Time.now + @pool_timeout)
|
84
|
+
# block until we get an available connection or Timeout::Error
|
85
|
+
@fibered_mutex.synchronize do
|
86
|
+
loop do
|
87
|
+
if alarm <= Time.now
|
88
|
+
message = "Could not fetch a free connection in time. Consider increasing your connection pool for em_aws or setting :never_block to true."
|
89
|
+
AWS.config.logger.error message
|
90
|
+
raise Timeout::Error, message
|
91
|
+
end
|
92
|
+
connection = available_pools(url).shift
|
93
|
+
if connection.nil? && (@never_block)
|
94
|
+
AWS.config.logger.info "Adding AWS connection to #{url} for never_block, will not be returned to pool."
|
95
|
+
connection = new_connection(url)
|
96
|
+
end
|
97
|
+
return connection if connection
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
106
102
|
# Return connections to pool if allowed, otherwise closes connection
|
103
|
+
# We return connections to the front of the pool to keep them active.
|
107
104
|
def return_connection(url,connection)
|
108
|
-
|
105
|
+
@fibered_mutex.synchronize do
|
109
106
|
if (@pools[url].nil? || (@pools[url].length == @pool_size))
|
110
107
|
connection.conn.close_connection if connection.conn
|
111
108
|
else
|
112
|
-
@pools[url]
|
109
|
+
@pools[url].insert(0,connection)
|
113
110
|
end
|
114
111
|
@pools[url]
|
115
112
|
end
|
data/lib/em_aws/version.rb
CHANGED
@@ -90,7 +90,8 @@ module AWS::Core
|
|
90
90
|
context 'too many retries' do
|
91
91
|
it "should have network error" do
|
92
92
|
EM.synchrony do
|
93
|
-
|
93
|
+
resp.stub(:status).and_return(0)
|
94
|
+
handler.send(:process_request,(req),(resp),false,3)
|
94
95
|
resp.network_error?.should be_true
|
95
96
|
EM.stop
|
96
97
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: em_aws
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-01-
|
12
|
+
date: 2013-01-17 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: aws-sdk
|