em_aws 0.2.5 → 0.2.6

data/Gemfile.lock

@@ -1,23 +1,24 @@
 PATH
   remote: .
   specs:
-    em_aws (0.2.4)
+    em_aws (0.2.6)
       aws-sdk
       em-http-request
       em-synchrony
+      hot_tub (~> 0.2.2)
 
 GEM
   remote: http://rubygems.org/
   specs:
     ZenTest (4.8.4)
     addressable (2.3.3)
-    aws-sdk (1.8.3.1)
+    aws-sdk (1.8.5)
       json (~> 1.4)
       nokogiri (>= 1.4.4)
       uuidtools (~> 2.1)
     builder (3.2.0)
     cookiejar (0.3.0)
-    diff-lcs (1.2.1)
+    diff-lcs (1.2.2)
     em-http-request (1.0.3)
       addressable (>= 2.2.3)
       cookiejar
@@ -28,20 +29,21 @@ GEM
       eventmachine (>= 1.0.0.beta.4)
     em-synchrony (1.0.3)
       eventmachine (>= 1.0.0.beta.1)
-    eventmachine (1.0.0)
-    eventmachine (1.0.0-java)
+    eventmachine (1.0.3)
+    eventmachine (1.0.3-java)
     eventmachine_httpserver (0.2.1)
+    hot_tub (0.2.2)
     http_parser.rb (0.5.3)
     http_parser.rb (0.5.3-java)
     json (1.7.7)
     json (1.7.7-java)
-    nokogiri (1.5.6)
-    nokogiri (1.5.6-java)
+    nokogiri (1.5.9)
+    nokogiri (1.5.9-java)
     rspec (2.13.0)
       rspec-core (~> 2.13.0)
       rspec-expectations (~> 2.13.0)
       rspec-mocks (~> 2.13.0)
-    rspec-core (2.13.0)
+    rspec-core (2.13.1)
     rspec-expectations (2.13.0)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.13.0)

data/HISTORY.md

@@ -4,6 +4,8 @@ EmAws Changelog
 HEAD
 =======
 
+- Use [hot_tub](https://github.com/JoshMcKin/hot_tub) for connection pooling and sessions
+
 0.2.5
 =======
 

data/em_aws.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "aws-sdk"
   s.add_runtime_dependency "em-synchrony"
   s.add_runtime_dependency "em-http-request"
+  s.add_runtime_dependency "hot_tub", "~> 0.2.2"
   s.add_development_dependency "rspec"
   s.add_development_dependency "builder"
   

data/lib/aws/core/http/em_http_handler.rb

@@ -1,16 +1,17 @@
 # http://docs.amazonwebservices.com/AWSRubySDK/latest/
-require "em-synchrony"
-require "em-synchrony/em-http"
+require 'hot_tub'
+require 'em-synchrony'
+require 'em-synchrony/em-http'
 require 'em-synchrony/thread'
 module AWS
   module Core
     module Http
-      
+
       # An EM-Synchrony implementation for Fiber based asynchronous ruby application.
-      # See https://github.com/igrigorik/async-rails and 
+      # See https://github.com/igrigorik/async-rails and
       # http://www.mikeperham.com/2010/04/03/introducing-phat-an-asynchronous-rails-app/
       # for examples of Aync-Rails application
-      # 
+      #
       # In Rails add the following to you various environment files:
       #
       # require 'aws-sdk'
@@ -23,7 +24,7 @@ module AWS
       #   :async => false # if set to true all requests are handle asynchronously and initially return nil
       #   }))
       class EMHttpHandler
-        
+
         EM_PASS_THROUGH_ERRORS = [
           NoMethodError, FloatDomainError, TypeError, NotImplementedError,
           SystemExit, Interrupt, SyntaxError, RangeError, NoMemoryError,
@@ -33,7 +34,7 @@ module AWS
         ]
         # @return [Hash] The default options to send to EM-Synchrony on each request.
         attr_reader :default_request_options
-        
+
         # Constructs a new HTTP handler using EM-Synchrony.
         # @param [Hash] options Default options to send to EM-Synchrony on
         # each request. These options will be sent to +get+, +post+,
@@ -44,23 +45,36 @@ module AWS
         # {AWS::Configuration} instead.
         def initialize options = {}
           @default_request_options = options
-          @pool = EMConnectionPool.new(options) if options[:pool_size].to_i > 0
-        end 
-    
+          @client_options = {
+            :inactivity_timeout => (options[:inactivity_timeout] || 0),
+            :connect_timeout => (options[:connect_timeout] || 10)
+          }
+          @pool_options = {
+            :with_pool => true,
+            :size => ((options[:pool_size].to_i || 5)),
+            :never_block => (options[:never_block].nil? ? true : options[:never_block]),
+            :blocking_timeout => (options[:pool_timeout] || 10)
+          }
+          if @pool_options[:size] > 0
+            @pool = HotTub::Session.new(@pool_options) { |url| EM::HttpRequest.new(url,@client_options)}
+          end
+        end
+
         def handle(request,response,&read_block)
-          if EM::reactor_running? 
-            process_request(request,response,&read_block)    
+          if EM::reactor_running?
+            process_request(request,response,&read_block)
           else
             EM.synchrony do
               process_request(request,response,&read_block)
+              @pool.close_all if @pool
               EM.stop
             end
           end
         end
-        
-        # If the request option :async are set to true that request will  handled 
-        # asynchronously returning nil initially and processing in the background 
-        # managed by EM-Synchrony. If the client option :async all requests will 
+
+        # If the request option :async are set to true that request will  handled
+        # asynchronously returning nil initially and processing in the background
+        # managed by EM-Synchrony. If the client option :async all requests will
         # be handled asynchronously.
         # EX:
         #     EM.synchrony do
@@ -71,22 +85,23 @@ module AWS
         #       EM.stop
         #     end
         def handle_async(request,response,handle,&read_block)
-          if EM::reactor_running? 
-            process_request(request,response,true,&read_block)    
+          if EM::reactor_running?
+            process_request(request,response,true,&read_block)
           else
             EM.synchrony do
               process_request(request,response,true,&read_block)
+              @pool.close_all if @pool
               EM.stop
             end
           end
         end
-        
+
         private
-        
+
         def fetch_url(request)
           "#{(request.use_ssl? ? "https" : "http")}://#{request.host}:#{request.port}"
         end
-                   
+
         def fetch_headers(request)
           headers = { 'content-type' => '' }
           request.headers.each_pair do |key,value|
@@ -94,30 +109,30 @@ module AWS
           end
           {:head => headers}
         end
-        
+
         def fetch_proxy(request)
           opts={}
-          if request.proxy_uri     
+          if request.proxy_uri
             opts[:proxy] = {:host => request.proxy_uri.host,:port => request.proxy_uri.port}
           end
           opts
         end
-          
+
         def fetch_ssl(request)
           opts = {}
           if request.use_ssl? && request.ssl_verify_peer?
-            opts[:private_key_file] = request.ssl_ca_file 
-            opts[:cert_chain_file]= request.ssl_ca_file 
+            opts[:private_key_file] = request.ssl_ca_file
+            opts[:cert_chain_file]= request.ssl_ca_file
           end
           opts
         end
-        
+
         def fetch_request_options(request)
           opts = default_request_options.
             merge(fetch_headers(request).
-              merge(fetch_proxy(request)).
-              merge(fetch_ssl(request))) 
-          opts[:query] = request.querystring
+                  merge(fetch_proxy(request)).
+                  merge(fetch_ssl(request)))
+            opts[:query] = request.querystring
           if request.body_stream.respond_to?(:path)
             opts[:file] = request.body_stream.path
           else
@@ -126,7 +141,7 @@ module AWS
           opts[:path] = request.path if request.path
           opts
         end
-        
+
         def fetch_response(request,opts={},&read_block)
           method = "a#{request.http_method}".downcase.to_sym  # aget, apost, aput, adelete, ahead
           url = fetch_url(request)
@@ -137,14 +152,14 @@ module AWS
               return  EM::Synchrony.sync req unless opts[:async]
             end
           else
-            req = EM::HttpRequest.new(url, 
-              :inactivity_timeout => request.read_timeout).send(method,opts)
+            opts = @client_options.merge(:inactivity_timeout => request.read_timeout)
+            req = EM::HttpRequest.new(url,opts).send(method,opts)
             req.stream &read_block if block_given?
             return  EM::Synchrony.sync req unless opts[:async]
           end
           nil
         end
-            
+
         # AWS needs all header keys downcased and values need to be arrays
         def fetch_response_headers(response)
           response_headers = response.response_header.raw.to_hash
@@ -160,16 +175,16 @@ module AWS
           end
           response_headers.merge(aws_headers)
         end
-        
+
         # Builds and attempts the request. Occasionally under load em-http-request
         # em-http-request returns a status of 0 for various http timeouts, see:
         # https://github.com/igrigorik/em-http-request/issues/76
         # https://github.com/eventmachine/eventmachine/issues/175
-        def process_request(request,response,async=false,&read_block)      
+        def process_request(request,response,async=false,&read_block)
           opts = fetch_request_options(request)
           opts[:async] = (async || opts[:async])
           begin
-            http_response = fetch_response(request,opts,&read_block) 
+            http_response = fetch_response(request,opts,&read_block)
             unless opts[:async]
               response.status = http_response.response_header.status.to_i
               raise Timeout::Error if response.status == 0
@@ -195,4 +210,4 @@ module AWS
   module Http
     class EMHttpHandler < Core::Http::EMHttpHandler; end
   end
-end
+end

data/lib/em_aws.rb

@@ -5,7 +5,6 @@ require 'em-http'
 require 'em-synchrony'
 require 'em-synchrony/em-http'
 require 'aws/core/autoloader'
-require 'aws/core/http/em_connection_pool'
 require 'aws/core/http/em_http_handler'
 
 AWS.eager_autoload! # lazy load isn't thread safe

data/lib/em_aws/version.rb

@@ -1,3 +1,3 @@
 module EmAws
-  VERSION = "0.2.5"
+  VERSION = "0.2.6"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: em_aws
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.2.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-01 00:00:00.000000000 Z
+date: 2013-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -59,6 +59,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: hot_tub
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.2
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -107,12 +123,10 @@ files:
 - README.md
 - Rakefile
 - em_aws.gemspec
-- lib/aws/core/http/em_connection_pool.rb
 - lib/aws/core/http/em_http_handler.rb
 - lib/em_aws.rb
 - lib/em_aws/patches.rb
 - lib/em_aws/version.rb
-- spec/em_connection_pool_spec.rb
 - spec/em_http_handler_spec.rb
 - spec/patches_spec.rb
 - spec/spec_helper.rb
@@ -141,7 +155,6 @@ signing_key:
 specification_version: 3
 summary: Adds EM-Synchrony support to AWS-SDK gem
 test_files:
-- spec/em_connection_pool_spec.rb
 - spec/em_http_handler_spec.rb
 - spec/patches_spec.rb
 - spec/spec_helper.rb

data/lib/aws/core/http/em_connection_pool.rb

@@ -1,123 +0,0 @@
-require 'thread'
-require "em-synchrony/em-http"
-require 'em-synchrony/thread'
-
-module AWS 
-  module Core
-    module Http
-      class EMConnectionPool
-        # Since AWS connections may be to any number of urls, the connection
-        # pool is a hash of connection arrays, instead of a simple array like
-        # most connection pools        # 
-        # Options:
-        # * :pool_size - number of connections for each pool
-        # * :inactivity_timeout - number of seconds to wait before disconnecting, 
-        #   setting to 0 means the connection will not be closed
-        # * :pool_timeout - the amount of seconds to block waiting for an available connection, 
-        #   because this is blocking it should be an extremely short amount of 
-        #   time default to 0.5 seconds, if you need more consider enlarging your pool
-        #   instead of raising this number
-        # :never_block - if set to true, a connection will always be returned
-        def initialize(options={})
-          options[:never_block] ||= true
-          @pools = {}
-          @pool_data = {}
-          @pool_size = (options[:pool_size] || 5)
-          @never_block = (options[:never_block])
-          @inactivity_timeout = (options[:inactivity_timeout].to_i)
-          @connect_timeout = options[:connect_timeout]
-          @pool_timeout = (options[:pool_timeout] || 0.5) 
-          @fibered_mutex = EM::Synchrony::Thread::Mutex.new  # A fiber safe mutex
-        end
-       
-        # Run the block on the retrieved connection. Then return the connection
-        # back to the pool.
-        def run(url, &block)
-          connection = santize_connection(connection(url))
-          block.call(connection)
-        ensure
-          return_connection(url,connection) 
-        end
-        
-        private
-        # Returns a pool for the associated url
-        def available_pools(url)
-          add_connection(url) if add_connection?(url)
-          @pools[url]
-        end
-        
-        def add_connection?(url)
-          (@pool_data[url].nil? || (@pools[url].length == 0 && (@pool_data[url][:current_size] < @pool_size)))
-        end
-        
-        def add_connection(url) 
-          AWS.config.logger.info "Adding AWS connection to #{url}"
-          add_connection_data(url)
-          @pools[url] ||= []
-          @pools[url] << new_connection(url)         
-          @pools[url]
-        end
-        
-        def add_connection_data(url)
-          @pool_data[url] ||= {:current_size => 0} 
-          @pool_data[url][:current_size] += 1 
-        end
-        
-        def new_connection(url)
-          opts = {:inactivity_timeout => @inactivity_timeout}
-          opts[:connect_timeout] = @connect_timeout if @connect_timeout
-          EM::HttpRequest.new(url, opts)
-        end
-        
-        # Make sure we have a good connection.
-        def santize_connection(connection)
-          if connection.conn && connection.conn.error?
-            AWS.config.logger.info "Reconnecting to AWS: #{EventMachine::report_connection_error_status(connection.conn.instance_variable_get(:@signature))}"
-            connection.conn.close_connection
-            connection.instance_variable_set(:@deferred, true)
-          end
-          connection
-        end
-              
-        # Fetch an available connection or raise an error
-        def connection(url)         
-          alarm = (Time.now + @pool_timeout)       
-          # block until we get an available connection or Timeout::Error   
-          loop do
-            if alarm <= Time.now
-              message = "Could not fetch a free connection in time. Consider increasing your connection pool for em_aws or setting :never_block to true."
-              AWS.config.logger.error message
-              raise Timeout::Error, message
-            end
-            connection = fetch_connection(url)
-            if connection.nil? && (@never_block)
-              AWS.config.logger.info "Adding AWS connection to #{url} for never_block, will not be returned to pool."
-              connection = new_connection(url)
-            end
-            return connection if connection
-          end
-        end
-        
-        # Fetch an available connection
-        # We pop the last connection increase our chances of getting a live connection
-        def fetch_connection(url)         
-          @fibered_mutex.synchronize do
-            available_pools(url).pop
-          end
-        end
-        
-        # If allowed, returns connections to pool end of pool; otherwise closes connection
-        def return_connection(url,connection)
-          @fibered_mutex.synchronize do
-            if (@pools[url].nil? || (@pools[url].length == @pool_size))
-              connection.conn.close_connection if connection.conn
-            else
-              @pools[url] << connection
-            end
-            @pools[url]
-          end
-        end
-      end
-    end
-  end
-end

data/spec/em_connection_pool_spec.rb

@@ -1,104 +0,0 @@
-# To change this template, choose Tools | Templates
-# and open the template in the editor.
-require 'thread'
-require "em-synchrony"
-require "em-synchrony/em-http"
-require 'spec_helper'
-module AWS 
-  module Core
-    module Http
-      describe EMConnectionPool do
-        before(:each) do
-          @em_connection_pool = EMConnectionPool.new
-        end
-
-        context 'default configuration' do
-          it "should have @pool_size of 5" do
-            @em_connection_pool.instance_variable_get(:@pool_size).should eql(5)
-          end
-    
-          it "should have @inactivity_timeout of 0" do
-            @em_connection_pool.instance_variable_get(:@inactivity_timeout).should eql(0)
-          end
-    
-          it "should have @pool_timeout of 0" do
-            @em_connection_pool.instance_variable_get(:@pool_timeout).should eql(0.5)
-          end
-        end
-        
-        describe '#add_connection?' do
-          it "should be true if @pool_data does not have data for the url"do
-            @em_connection_pool.send(:add_connection?,"http://www.testurl123.com/").should be_true 
-          end
-      
-          it "should be true if @pool_data has data but the number of connnections has not reached the pool_size" do
-            @em_connection_pool.instance_variable_set(:@pools,{"http://www.testurl123.com/" => ["connection"]})
-            @em_connection_pool.send(:add_connection?,"http://www.testurl123.com/").should be_true 
-          end
-          
-          it "should be false pool has reached pool_size" do
-            @em_connection_pool.instance_variable_set(:@pools,
-              {"http://www.testurl123.com/" => ["connection","connection","connection","connection","connection"]})
-            @em_connection_pool.send(:add_connection?,"http://www.testurl123.com/").should be_true 
-          end
-        end
-        
-        describe '#add_connection' do
-          it "should add connections for supplied url"do
-            @em_connection_pool.send(:add_connection,"http://www.testurl123.com/") 
-            @em_connection_pool.instance_variable_get(:@pools)["http://www.testurl123.com/"].should_not be_nil
-          end
-        end   
-        
-        describe '#connection' do
-          it "should raise Timeout::Error if an available is not found in time"do
-            @em_connection_pool.stub(:available_pools).and_return([])
-            @em_connection_pool.instance_variable_set(:@never_block, false)
-            lambda { @em_connection_pool.send(:connection,'http://some_url.com')}.should raise_error(Timeout::Error)
-          end
-        end
-
-        context 'integration test with parallel requests' do
-          # 10 parallel requests
-          
-          it "should work" do  
-            url = "http://www.testurl123.com"
-            @requests_made = []
-            EM.synchrony do 
-              @em_connection_pool.instance_variable_set(:@never_block, true)
-              fibers = []  
-              10.times do 
-                fibers << Fiber.new do                           
-                @em_connection_pool.run url do |connection|                 
-                  @requests_made << connection.get(:keepalive => true).response_header.status    
-                  end  
-                end  
-              end 
-              
-              fibers.each do |f|
-               f.resume
-              end
-        
-              loop do  
-                done = true
-                fibers.each do |f|
-                  done = false if f.alive?
-                end
-                if done
-                  break
-                else
-                  EM::Synchrony.sleep(0.01)   
-                end
-              end
-    
-              @requests_made.length.should eql(10)
-              @em_connection_pool.instance_variable_get(:@pools)[url].length.should eql(@em_connection_pool.instance_variable_get(:@pool_size))              
-
-              EM.stop
-            end
-          end          
-        end
-      end
-    end
-  end
-end