em_aws 0.2.5 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,23 +1,24 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- em_aws (0.2.4)
4
+ em_aws (0.2.6)
5
5
  aws-sdk
6
6
  em-http-request
7
7
  em-synchrony
8
+ hot_tub (~> 0.2.2)
8
9
 
9
10
  GEM
10
11
  remote: http://rubygems.org/
11
12
  specs:
12
13
  ZenTest (4.8.4)
13
14
  addressable (2.3.3)
14
- aws-sdk (1.8.3.1)
15
+ aws-sdk (1.8.5)
15
16
  json (~> 1.4)
16
17
  nokogiri (>= 1.4.4)
17
18
  uuidtools (~> 2.1)
18
19
  builder (3.2.0)
19
20
  cookiejar (0.3.0)
20
- diff-lcs (1.2.1)
21
+ diff-lcs (1.2.2)
21
22
  em-http-request (1.0.3)
22
23
  addressable (>= 2.2.3)
23
24
  cookiejar
@@ -28,20 +29,21 @@ GEM
28
29
  eventmachine (>= 1.0.0.beta.4)
29
30
  em-synchrony (1.0.3)
30
31
  eventmachine (>= 1.0.0.beta.1)
31
- eventmachine (1.0.0)
32
- eventmachine (1.0.0-java)
32
+ eventmachine (1.0.3)
33
+ eventmachine (1.0.3-java)
33
34
  eventmachine_httpserver (0.2.1)
35
+ hot_tub (0.2.2)
34
36
  http_parser.rb (0.5.3)
35
37
  http_parser.rb (0.5.3-java)
36
38
  json (1.7.7)
37
39
  json (1.7.7-java)
38
- nokogiri (1.5.6)
39
- nokogiri (1.5.6-java)
40
+ nokogiri (1.5.9)
41
+ nokogiri (1.5.9-java)
40
42
  rspec (2.13.0)
41
43
  rspec-core (~> 2.13.0)
42
44
  rspec-expectations (~> 2.13.0)
43
45
  rspec-mocks (~> 2.13.0)
44
- rspec-core (2.13.0)
46
+ rspec-core (2.13.1)
45
47
  rspec-expectations (2.13.0)
46
48
  diff-lcs (>= 1.1.3, < 2.0)
47
49
  rspec-mocks (2.13.0)
data/HISTORY.md CHANGED
@@ -4,6 +4,8 @@ EmAws Changelog
4
4
  HEAD
5
5
  =======
6
6
 
7
+ - Use [hot_tub](https://github.com/JoshMcKin/hot_tub) for connection pooling and sessions
8
+
7
9
  0.2.5
8
10
  =======
9
11
 
@@ -17,6 +17,7 @@ Gem::Specification.new do |s|
17
17
  s.add_runtime_dependency "aws-sdk"
18
18
  s.add_runtime_dependency "em-synchrony"
19
19
  s.add_runtime_dependency "em-http-request"
20
+ s.add_runtime_dependency "hot_tub", "~> 0.2.2"
20
21
  s.add_development_dependency "rspec"
21
22
  s.add_development_dependency "builder"
22
23
 
@@ -1,16 +1,17 @@
1
1
  # http://docs.amazonwebservices.com/AWSRubySDK/latest/
2
- require "em-synchrony"
3
- require "em-synchrony/em-http"
2
+ require 'hot_tub'
3
+ require 'em-synchrony'
4
+ require 'em-synchrony/em-http'
4
5
  require 'em-synchrony/thread'
5
6
  module AWS
6
7
  module Core
7
8
  module Http
8
-
9
+
9
10
  # An EM-Synchrony implementation for Fiber based asynchronous ruby application.
10
- # See https://github.com/igrigorik/async-rails and
11
+ # See https://github.com/igrigorik/async-rails and
11
12
  # http://www.mikeperham.com/2010/04/03/introducing-phat-an-asynchronous-rails-app/
12
13
  # for examples of Aync-Rails application
13
- #
14
+ #
14
15
  # In Rails add the following to you various environment files:
15
16
  #
16
17
  # require 'aws-sdk'
@@ -23,7 +24,7 @@ module AWS
23
24
  # :async => false # if set to true all requests are handle asynchronously and initially return nil
24
25
  # }))
25
26
  class EMHttpHandler
26
-
27
+
27
28
  EM_PASS_THROUGH_ERRORS = [
28
29
  NoMethodError, FloatDomainError, TypeError, NotImplementedError,
29
30
  SystemExit, Interrupt, SyntaxError, RangeError, NoMemoryError,
@@ -33,7 +34,7 @@ module AWS
33
34
  ]
34
35
  # @return [Hash] The default options to send to EM-Synchrony on each request.
35
36
  attr_reader :default_request_options
36
-
37
+
37
38
  # Constructs a new HTTP handler using EM-Synchrony.
38
39
  # @param [Hash] options Default options to send to EM-Synchrony on
39
40
  # each request. These options will be sent to +get+, +post+,
@@ -44,23 +45,36 @@ module AWS
44
45
  # {AWS::Configuration} instead.
45
46
  def initialize options = {}
46
47
  @default_request_options = options
47
- @pool = EMConnectionPool.new(options) if options[:pool_size].to_i > 0
48
- end
49
-
48
+ @client_options = {
49
+ :inactivity_timeout => (options[:inactivity_timeout] || 0),
50
+ :connect_timeout => (options[:connect_timeout] || 10)
51
+ }
52
+ @pool_options = {
53
+ :with_pool => true,
54
+ :size => ((options[:pool_size].to_i || 5)),
55
+ :never_block => (options[:never_block].nil? ? true : options[:never_block]),
56
+ :blocking_timeout => (options[:pool_timeout] || 10)
57
+ }
58
+ if @pool_options[:size] > 0
59
+ @pool = HotTub::Session.new(@pool_options) { |url| EM::HttpRequest.new(url,@client_options)}
60
+ end
61
+ end
62
+
50
63
  def handle(request,response,&read_block)
51
- if EM::reactor_running?
52
- process_request(request,response,&read_block)
64
+ if EM::reactor_running?
65
+ process_request(request,response,&read_block)
53
66
  else
54
67
  EM.synchrony do
55
68
  process_request(request,response,&read_block)
69
+ @pool.close_all if @pool
56
70
  EM.stop
57
71
  end
58
72
  end
59
73
  end
60
-
61
- # If the request option :async are set to true that request will handled
62
- # asynchronously returning nil initially and processing in the background
63
- # managed by EM-Synchrony. If the client option :async all requests will
74
+
75
+ # If the request option :async are set to true that request will handled
76
+ # asynchronously returning nil initially and processing in the background
77
+ # managed by EM-Synchrony. If the client option :async all requests will
64
78
  # be handled asynchronously.
65
79
  # EX:
66
80
  # EM.synchrony do
@@ -71,22 +85,23 @@ module AWS
71
85
  # EM.stop
72
86
  # end
73
87
  def handle_async(request,response,handle,&read_block)
74
- if EM::reactor_running?
75
- process_request(request,response,true,&read_block)
88
+ if EM::reactor_running?
89
+ process_request(request,response,true,&read_block)
76
90
  else
77
91
  EM.synchrony do
78
92
  process_request(request,response,true,&read_block)
93
+ @pool.close_all if @pool
79
94
  EM.stop
80
95
  end
81
96
  end
82
97
  end
83
-
98
+
84
99
  private
85
-
100
+
86
101
  def fetch_url(request)
87
102
  "#{(request.use_ssl? ? "https" : "http")}://#{request.host}:#{request.port}"
88
103
  end
89
-
104
+
90
105
  def fetch_headers(request)
91
106
  headers = { 'content-type' => '' }
92
107
  request.headers.each_pair do |key,value|
@@ -94,30 +109,30 @@ module AWS
94
109
  end
95
110
  {:head => headers}
96
111
  end
97
-
112
+
98
113
  def fetch_proxy(request)
99
114
  opts={}
100
- if request.proxy_uri
115
+ if request.proxy_uri
101
116
  opts[:proxy] = {:host => request.proxy_uri.host,:port => request.proxy_uri.port}
102
117
  end
103
118
  opts
104
119
  end
105
-
120
+
106
121
  def fetch_ssl(request)
107
122
  opts = {}
108
123
  if request.use_ssl? && request.ssl_verify_peer?
109
- opts[:private_key_file] = request.ssl_ca_file
110
- opts[:cert_chain_file]= request.ssl_ca_file
124
+ opts[:private_key_file] = request.ssl_ca_file
125
+ opts[:cert_chain_file]= request.ssl_ca_file
111
126
  end
112
127
  opts
113
128
  end
114
-
129
+
115
130
  def fetch_request_options(request)
116
131
  opts = default_request_options.
117
132
  merge(fetch_headers(request).
118
- merge(fetch_proxy(request)).
119
- merge(fetch_ssl(request)))
120
- opts[:query] = request.querystring
133
+ merge(fetch_proxy(request)).
134
+ merge(fetch_ssl(request)))
135
+ opts[:query] = request.querystring
121
136
  if request.body_stream.respond_to?(:path)
122
137
  opts[:file] = request.body_stream.path
123
138
  else
@@ -126,7 +141,7 @@ module AWS
126
141
  opts[:path] = request.path if request.path
127
142
  opts
128
143
  end
129
-
144
+
130
145
  def fetch_response(request,opts={},&read_block)
131
146
  method = "a#{request.http_method}".downcase.to_sym # aget, apost, aput, adelete, ahead
132
147
  url = fetch_url(request)
@@ -137,14 +152,14 @@ module AWS
137
152
  return EM::Synchrony.sync req unless opts[:async]
138
153
  end
139
154
  else
140
- req = EM::HttpRequest.new(url,
141
- :inactivity_timeout => request.read_timeout).send(method,opts)
155
+ opts = @client_options.merge(:inactivity_timeout => request.read_timeout)
156
+ req = EM::HttpRequest.new(url,opts).send(method,opts)
142
157
  req.stream &read_block if block_given?
143
158
  return EM::Synchrony.sync req unless opts[:async]
144
159
  end
145
160
  nil
146
161
  end
147
-
162
+
148
163
  # AWS needs all header keys downcased and values need to be arrays
149
164
  def fetch_response_headers(response)
150
165
  response_headers = response.response_header.raw.to_hash
@@ -160,16 +175,16 @@ module AWS
160
175
  end
161
176
  response_headers.merge(aws_headers)
162
177
  end
163
-
178
+
164
179
  # Builds and attempts the request. Occasionally under load em-http-request
165
180
  # em-http-request returns a status of 0 for various http timeouts, see:
166
181
  # https://github.com/igrigorik/em-http-request/issues/76
167
182
  # https://github.com/eventmachine/eventmachine/issues/175
168
- def process_request(request,response,async=false,&read_block)
183
+ def process_request(request,response,async=false,&read_block)
169
184
  opts = fetch_request_options(request)
170
185
  opts[:async] = (async || opts[:async])
171
186
  begin
172
- http_response = fetch_response(request,opts,&read_block)
187
+ http_response = fetch_response(request,opts,&read_block)
173
188
  unless opts[:async]
174
189
  response.status = http_response.response_header.status.to_i
175
190
  raise Timeout::Error if response.status == 0
@@ -195,4 +210,4 @@ module AWS
195
210
  module Http
196
211
  class EMHttpHandler < Core::Http::EMHttpHandler; end
197
212
  end
198
- end
213
+ end
@@ -5,7 +5,6 @@ require 'em-http'
5
5
  require 'em-synchrony'
6
6
  require 'em-synchrony/em-http'
7
7
  require 'aws/core/autoloader'
8
- require 'aws/core/http/em_connection_pool'
9
8
  require 'aws/core/http/em_http_handler'
10
9
 
11
10
  AWS.eager_autoload! # lazy load isn't thread safe
@@ -1,3 +1,3 @@
1
1
  module EmAws
2
- VERSION = "0.2.5"
2
+ VERSION = "0.2.6"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: em_aws
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.5
4
+ version: 0.2.6
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-03-01 00:00:00.000000000 Z
12
+ date: 2013-04-05 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: aws-sdk
@@ -59,6 +59,22 @@ dependencies:
59
59
  - - ! '>='
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
+ - !ruby/object:Gem::Dependency
63
+ name: hot_tub
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ~>
68
+ - !ruby/object:Gem::Version
69
+ version: 0.2.2
70
+ type: :runtime
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ~>
76
+ - !ruby/object:Gem::Version
77
+ version: 0.2.2
62
78
  - !ruby/object:Gem::Dependency
63
79
  name: rspec
64
80
  requirement: !ruby/object:Gem::Requirement
@@ -107,12 +123,10 @@ files:
107
123
  - README.md
108
124
  - Rakefile
109
125
  - em_aws.gemspec
110
- - lib/aws/core/http/em_connection_pool.rb
111
126
  - lib/aws/core/http/em_http_handler.rb
112
127
  - lib/em_aws.rb
113
128
  - lib/em_aws/patches.rb
114
129
  - lib/em_aws/version.rb
115
- - spec/em_connection_pool_spec.rb
116
130
  - spec/em_http_handler_spec.rb
117
131
  - spec/patches_spec.rb
118
132
  - spec/spec_helper.rb
@@ -141,7 +155,6 @@ signing_key:
141
155
  specification_version: 3
142
156
  summary: Adds EM-Synchrony support to AWS-SDK gem
143
157
  test_files:
144
- - spec/em_connection_pool_spec.rb
145
158
  - spec/em_http_handler_spec.rb
146
159
  - spec/patches_spec.rb
147
160
  - spec/spec_helper.rb
@@ -1,123 +0,0 @@
1
- require 'thread'
2
- require "em-synchrony/em-http"
3
- require 'em-synchrony/thread'
4
-
5
- module AWS
6
- module Core
7
- module Http
8
- class EMConnectionPool
9
- # Since AWS connections may be to any number of urls, the connection
10
- # pool is a hash of connection arrays, instead of a simple array like
11
- # most connection pools #
12
- # Options:
13
- # * :pool_size - number of connections for each pool
14
- # * :inactivity_timeout - number of seconds to wait before disconnecting,
15
- # setting to 0 means the connection will not be closed
16
- # * :pool_timeout - the amount of seconds to block waiting for an available connection,
17
- # because this is blocking it should be an extremely short amount of
18
- # time default to 0.5 seconds, if you need more consider enlarging your pool
19
- # instead of raising this number
20
- # :never_block - if set to true, a connection will always be returned
21
- def initialize(options={})
22
- options[:never_block] ||= true
23
- @pools = {}
24
- @pool_data = {}
25
- @pool_size = (options[:pool_size] || 5)
26
- @never_block = (options[:never_block])
27
- @inactivity_timeout = (options[:inactivity_timeout].to_i)
28
- @connect_timeout = options[:connect_timeout]
29
- @pool_timeout = (options[:pool_timeout] || 0.5)
30
- @fibered_mutex = EM::Synchrony::Thread::Mutex.new # A fiber safe mutex
31
- end
32
-
33
- # Run the block on the retrieved connection. Then return the connection
34
- # back to the pool.
35
- def run(url, &block)
36
- connection = santize_connection(connection(url))
37
- block.call(connection)
38
- ensure
39
- return_connection(url,connection)
40
- end
41
-
42
- private
43
- # Returns a pool for the associated url
44
- def available_pools(url)
45
- add_connection(url) if add_connection?(url)
46
- @pools[url]
47
- end
48
-
49
- def add_connection?(url)
50
- (@pool_data[url].nil? || (@pools[url].length == 0 && (@pool_data[url][:current_size] < @pool_size)))
51
- end
52
-
53
- def add_connection(url)
54
- AWS.config.logger.info "Adding AWS connection to #{url}"
55
- add_connection_data(url)
56
- @pools[url] ||= []
57
- @pools[url] << new_connection(url)
58
- @pools[url]
59
- end
60
-
61
- def add_connection_data(url)
62
- @pool_data[url] ||= {:current_size => 0}
63
- @pool_data[url][:current_size] += 1
64
- end
65
-
66
- def new_connection(url)
67
- opts = {:inactivity_timeout => @inactivity_timeout}
68
- opts[:connect_timeout] = @connect_timeout if @connect_timeout
69
- EM::HttpRequest.new(url, opts)
70
- end
71
-
72
- # Make sure we have a good connection.
73
- def santize_connection(connection)
74
- if connection.conn && connection.conn.error?
75
- AWS.config.logger.info "Reconnecting to AWS: #{EventMachine::report_connection_error_status(connection.conn.instance_variable_get(:@signature))}"
76
- connection.conn.close_connection
77
- connection.instance_variable_set(:@deferred, true)
78
- end
79
- connection
80
- end
81
-
82
- # Fetch an available connection or raise an error
83
- def connection(url)
84
- alarm = (Time.now + @pool_timeout)
85
- # block until we get an available connection or Timeout::Error
86
- loop do
87
- if alarm <= Time.now
88
- message = "Could not fetch a free connection in time. Consider increasing your connection pool for em_aws or setting :never_block to true."
89
- AWS.config.logger.error message
90
- raise Timeout::Error, message
91
- end
92
- connection = fetch_connection(url)
93
- if connection.nil? && (@never_block)
94
- AWS.config.logger.info "Adding AWS connection to #{url} for never_block, will not be returned to pool."
95
- connection = new_connection(url)
96
- end
97
- return connection if connection
98
- end
99
- end
100
-
101
- # Fetch an available connection
102
- # We pop the last connection increase our chances of getting a live connection
103
- def fetch_connection(url)
104
- @fibered_mutex.synchronize do
105
- available_pools(url).pop
106
- end
107
- end
108
-
109
- # If allowed, returns connections to pool end of pool; otherwise closes connection
110
- def return_connection(url,connection)
111
- @fibered_mutex.synchronize do
112
- if (@pools[url].nil? || (@pools[url].length == @pool_size))
113
- connection.conn.close_connection if connection.conn
114
- else
115
- @pools[url] << connection
116
- end
117
- @pools[url]
118
- end
119
- end
120
- end
121
- end
122
- end
123
- end
@@ -1,104 +0,0 @@
1
- # To change this template, choose Tools | Templates
2
- # and open the template in the editor.
3
- require 'thread'
4
- require "em-synchrony"
5
- require "em-synchrony/em-http"
6
- require 'spec_helper'
7
- module AWS
8
- module Core
9
- module Http
10
- describe EMConnectionPool do
11
- before(:each) do
12
- @em_connection_pool = EMConnectionPool.new
13
- end
14
-
15
- context 'default configuration' do
16
- it "should have @pool_size of 5" do
17
- @em_connection_pool.instance_variable_get(:@pool_size).should eql(5)
18
- end
19
-
20
- it "should have @inactivity_timeout of 0" do
21
- @em_connection_pool.instance_variable_get(:@inactivity_timeout).should eql(0)
22
- end
23
-
24
- it "should have @pool_timeout of 0" do
25
- @em_connection_pool.instance_variable_get(:@pool_timeout).should eql(0.5)
26
- end
27
- end
28
-
29
- describe '#add_connection?' do
30
- it "should be true if @pool_data does not have data for the url"do
31
- @em_connection_pool.send(:add_connection?,"http://www.testurl123.com/").should be_true
32
- end
33
-
34
- it "should be true if @pool_data has data but the number of connnections has not reached the pool_size" do
35
- @em_connection_pool.instance_variable_set(:@pools,{"http://www.testurl123.com/" => ["connection"]})
36
- @em_connection_pool.send(:add_connection?,"http://www.testurl123.com/").should be_true
37
- end
38
-
39
- it "should be false pool has reached pool_size" do
40
- @em_connection_pool.instance_variable_set(:@pools,
41
- {"http://www.testurl123.com/" => ["connection","connection","connection","connection","connection"]})
42
- @em_connection_pool.send(:add_connection?,"http://www.testurl123.com/").should be_true
43
- end
44
- end
45
-
46
- describe '#add_connection' do
47
- it "should add connections for supplied url"do
48
- @em_connection_pool.send(:add_connection,"http://www.testurl123.com/")
49
- @em_connection_pool.instance_variable_get(:@pools)["http://www.testurl123.com/"].should_not be_nil
50
- end
51
- end
52
-
53
- describe '#connection' do
54
- it "should raise Timeout::Error if an available is not found in time"do
55
- @em_connection_pool.stub(:available_pools).and_return([])
56
- @em_connection_pool.instance_variable_set(:@never_block, false)
57
- lambda { @em_connection_pool.send(:connection,'http://some_url.com')}.should raise_error(Timeout::Error)
58
- end
59
- end
60
-
61
- context 'integration test with parallel requests' do
62
- # 10 parallel requests
63
-
64
- it "should work" do
65
- url = "http://www.testurl123.com"
66
- @requests_made = []
67
- EM.synchrony do
68
- @em_connection_pool.instance_variable_set(:@never_block, true)
69
- fibers = []
70
- 10.times do
71
- fibers << Fiber.new do
72
- @em_connection_pool.run url do |connection|
73
- @requests_made << connection.get(:keepalive => true).response_header.status
74
- end
75
- end
76
- end
77
-
78
- fibers.each do |f|
79
- f.resume
80
- end
81
-
82
- loop do
83
- done = true
84
- fibers.each do |f|
85
- done = false if f.alive?
86
- end
87
- if done
88
- break
89
- else
90
- EM::Synchrony.sleep(0.01)
91
- end
92
- end
93
-
94
- @requests_made.length.should eql(10)
95
- @em_connection_pool.instance_variable_get(:@pools)[url].length.should eql(@em_connection_pool.instance_variable_get(:@pool_size))
96
-
97
- EM.stop
98
- end
99
- end
100
- end
101
- end
102
- end
103
- end
104
- end