em_aws 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +4 -0
- data/Gemfile.lock +7 -5
- data/lib/aws/core/http/em_connection_pool.rb +27 -20
- data/lib/em_aws/version.rb +1 -1
- data/spec/em_connection_pool_spec.rb +21 -9
- metadata +2 -2
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
em_aws (0.2.
|
4
|
+
em_aws (0.2.2)
|
5
5
|
aws-sdk
|
6
6
|
em-http-request
|
7
7
|
em-synchrony
|
@@ -9,6 +9,7 @@ PATH
|
|
9
9
|
GEM
|
10
10
|
remote: http://rubygems.org/
|
11
11
|
specs:
|
12
|
+
ZenTest (4.8.2)
|
12
13
|
addressable (2.3.2)
|
13
14
|
aws-sdk (1.8.0)
|
14
15
|
httparty (~> 0.7)
|
@@ -26,15 +27,15 @@ GEM
|
|
26
27
|
http_parser.rb (>= 0.5.3)
|
27
28
|
em-socksify (0.2.1)
|
28
29
|
eventmachine (>= 1.0.0.beta.4)
|
29
|
-
em-synchrony (1.0.
|
30
|
+
em-synchrony (1.0.3)
|
30
31
|
eventmachine (>= 1.0.0.beta.1)
|
31
32
|
eventmachine (1.0.0)
|
32
33
|
eventmachine (1.0.0-java)
|
33
34
|
http_parser.rb (0.5.3)
|
34
35
|
http_parser.rb (0.5.3-java)
|
35
|
-
httparty (0.10.
|
36
|
+
httparty (0.10.2)
|
36
37
|
multi_json (~> 1.0)
|
37
|
-
multi_xml
|
38
|
+
multi_xml (>= 0.5.2)
|
38
39
|
json (1.7.6)
|
39
40
|
json (1.7.6-java)
|
40
41
|
multi_json (1.5.0)
|
@@ -48,7 +49,7 @@ GEM
|
|
48
49
|
rspec-core (2.12.2)
|
49
50
|
rspec-expectations (2.12.1)
|
50
51
|
diff-lcs (~> 1.1.3)
|
51
|
-
rspec-mocks (2.12.
|
52
|
+
rspec-mocks (2.12.2)
|
52
53
|
uuidtools (2.1.3)
|
53
54
|
|
54
55
|
PLATFORMS
|
@@ -56,6 +57,7 @@ PLATFORMS
|
|
56
57
|
ruby
|
57
58
|
|
58
59
|
DEPENDENCIES
|
60
|
+
ZenTest (= 4.8.2)
|
59
61
|
builder
|
60
62
|
em_aws!
|
61
63
|
rspec
|
@@ -33,12 +33,14 @@ module AWS
|
|
33
33
|
# Run the block on the retrieved connection. Then return the connection
|
34
34
|
# back to the pool.
|
35
35
|
def run(url, &block)
|
36
|
-
|
36
|
+
url = url.to_s.split("?")[0].to_s.gsub(/\/$/, "") # homogenize
|
37
|
+
connection = santize_connection(connection(url))
|
37
38
|
block.call(connection)
|
38
39
|
ensure
|
39
40
|
return_connection(url,connection)
|
40
41
|
end
|
41
42
|
|
43
|
+
private
|
42
44
|
# Returns a pool for the associated url
|
43
45
|
def available_pools(url)
|
44
46
|
add_connection(url) if add_connection?(url)
|
@@ -77,36 +79,41 @@ module AWS
|
|
77
79
|
end
|
78
80
|
connection
|
79
81
|
end
|
80
|
-
|
82
|
+
|
81
83
|
# Fetch an available connection or raise an error
|
82
|
-
def
|
84
|
+
def connection(url)
|
83
85
|
alarm = (Time.now + @pool_timeout)
|
84
86
|
# block until we get an available connection or Timeout::Error
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
raise Timeout::Error, message
|
91
|
-
end
|
92
|
-
connection = available_pools(url).shift
|
93
|
-
if connection.nil? && (@never_block)
|
94
|
-
AWS.config.logger.info "Adding AWS connection to #{url} for never_block, will not be returned to pool."
|
95
|
-
connection = new_connection(url)
|
96
|
-
end
|
97
|
-
return connection if connection
|
87
|
+
loop do
|
88
|
+
if alarm <= Time.now
|
89
|
+
message = "Could not fetch a free connection in time. Consider increasing your connection pool for em_aws or setting :never_block to true."
|
90
|
+
AWS.config.logger.error message
|
91
|
+
raise Timeout::Error, message
|
98
92
|
end
|
93
|
+
connection = fetch_connection(url)
|
94
|
+
if connection.nil? && (@never_block)
|
95
|
+
AWS.config.logger.info "Adding AWS connection to #{url} for never_block, will not be returned to pool."
|
96
|
+
connection = new_connection(url)
|
97
|
+
end
|
98
|
+
return connection if connection
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
# Fetch an available connection
|
103
|
+
# We pop the last connection increase our chances of getting a live connection
|
104
|
+
def fetch_connection(url)
|
105
|
+
@fibered_mutex.synchronize do
|
106
|
+
available_pools(url).pop
|
99
107
|
end
|
100
108
|
end
|
101
109
|
|
102
|
-
#
|
103
|
-
# We return connections to the front of the pool to keep them active.
|
110
|
+
# If allowed, returns connections to pool end of pool; otherwise closes connection
|
104
111
|
def return_connection(url,connection)
|
105
112
|
@fibered_mutex.synchronize do
|
106
113
|
if (@pools[url].nil? || (@pools[url].length == @pool_size))
|
107
114
|
connection.conn.close_connection if connection.conn
|
108
115
|
else
|
109
|
-
@pools[url]
|
116
|
+
@pools[url] << connection
|
110
117
|
end
|
111
118
|
@pools[url]
|
112
119
|
end
|
@@ -114,4 +121,4 @@ module AWS
|
|
114
121
|
end
|
115
122
|
end
|
116
123
|
end
|
117
|
-
end
|
124
|
+
end
|
data/lib/em_aws/version.rb
CHANGED
@@ -28,47 +28,59 @@ module AWS
|
|
28
28
|
|
29
29
|
describe '#add_connection?' do
|
30
30
|
it "should be true if @pool_data does not have data for the url"do
|
31
|
-
@em_connection_pool.add_connection
|
31
|
+
@em_connection_pool.send(:add_connection?,"http://www.testurl123.com/").should be_true
|
32
32
|
end
|
33
33
|
|
34
34
|
it "should be true if @pool_data has data but the number of connnections has not reached the pool_size" do
|
35
35
|
@em_connection_pool.instance_variable_set(:@pools,{"http://www.testurl123.com/" => ["connection"]})
|
36
|
-
@em_connection_pool.add_connection
|
36
|
+
@em_connection_pool.send(:add_connection?,"http://www.testurl123.com/").should be_true
|
37
37
|
end
|
38
38
|
|
39
39
|
it "should be false pool has reached pool_size" do
|
40
40
|
@em_connection_pool.instance_variable_set(:@pools,
|
41
41
|
{"http://www.testurl123.com/" => ["connection","connection","connection","connection","connection"]})
|
42
|
-
@em_connection_pool.add_connection
|
42
|
+
@em_connection_pool.send(:add_connection?,"http://www.testurl123.com/").should be_true
|
43
43
|
end
|
44
44
|
end
|
45
45
|
|
46
46
|
describe '#add_connection' do
|
47
47
|
it "should add connections for supplied url"do
|
48
|
-
@em_connection_pool.add_connection
|
48
|
+
@em_connection_pool.send(:add_connection,"http://www.testurl123.com/")
|
49
49
|
@em_connection_pool.instance_variable_get(:@pools)["http://www.testurl123.com/"].should_not be_nil
|
50
50
|
end
|
51
51
|
end
|
52
52
|
|
53
|
-
describe '#
|
53
|
+
describe '#connection' do
|
54
54
|
it "should raise Timeout::Error if an available is not found in time"do
|
55
55
|
@em_connection_pool.stub(:available_pools).and_return([])
|
56
56
|
@em_connection_pool.instance_variable_set(:@never_block, false)
|
57
|
-
lambda { @em_connection_pool.
|
57
|
+
lambda { @em_connection_pool.send(:connection,'http://some_url.com')}.should raise_error(Timeout::Error)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
describe '#run' do
|
62
|
+
it "should homogenize url as much as possible by remove params and trailing '/'" do
|
63
|
+
url = "http://www.testurl123.com/?foo=bar"
|
64
|
+
@em_connection_pool.run(url) do {
|
65
|
+
#stuff
|
66
|
+
}
|
67
|
+
end
|
68
|
+
@em_connection_pool.instance_variable_get(:@pools)["http://www.testurl123.com"].should_not be_nil
|
58
69
|
end
|
59
70
|
end
|
60
71
|
|
61
72
|
context 'integration test with parallel requests' do
|
62
73
|
# 10 parallel requests
|
63
74
|
|
64
|
-
it "should work" do
|
75
|
+
it "should work" do
|
76
|
+
url = "http://www.testurl123.com"
|
65
77
|
@requests_made = []
|
66
78
|
EM.synchrony do
|
67
79
|
@em_connection_pool.instance_variable_set(:@never_block, true)
|
68
80
|
fibers = []
|
69
81
|
10.times do
|
70
82
|
fibers << Fiber.new do
|
71
|
-
@em_connection_pool.run
|
83
|
+
@em_connection_pool.run url do |connection|
|
72
84
|
@requests_made << connection.get(:keepalive => true).response_header.status
|
73
85
|
end
|
74
86
|
end
|
@@ -91,7 +103,7 @@ module AWS
|
|
91
103
|
end
|
92
104
|
|
93
105
|
@requests_made.length.should eql(10)
|
94
|
-
@em_connection_pool.instance_variable_get(:@pools)[
|
106
|
+
@em_connection_pool.instance_variable_get(:@pools)[url].length.should eql(@em_connection_pool.instance_variable_get(:@pool_size))
|
95
107
|
|
96
108
|
EM.stop
|
97
109
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: em_aws
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-01-
|
12
|
+
date: 2013-01-28 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: aws-sdk
|