em_aws 0.0.3 → 0.1.0

data/.gitignore

@@ -1,4 +1,3 @@
-Gemfile.lock
 pkg/*
 *.gem
 .bundle

data/Gemfile.lock

@@ -0,0 +1,54 @@
+PATH
+  remote: .
+  specs:
+    em_aws (0.1.0)
+      aws-sdk
+      em-http-request
+      em-synchrony
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.2.7)
+    aws-sdk (1.3.9)
+      httparty (~> 0.7)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+      uuidtools (~> 2.1)
+    cookiejar (0.3.0)
+    diff-lcs (1.1.3)
+    em-http-request (1.0.2)
+      addressable (>= 2.2.3)
+      cookiejar
+      em-socksify
+      eventmachine (>= 1.0.0.beta.4)
+      http_parser.rb (>= 0.5.3)
+    em-socksify (0.2.0)
+      eventmachine (>= 1.0.0.beta.4)
+    em-synchrony (1.0.0)
+      eventmachine (>= 1.0.0.beta.1)
+    eventmachine (1.0.0.beta.4)
+    http_parser.rb (0.5.3)
+    httparty (0.8.1)
+      multi_json
+      multi_xml
+    json (1.6.6)
+    multi_json (1.2.0)
+    multi_xml (0.4.2)
+    nokogiri (1.5.2)
+    rspec (2.9.0)
+      rspec-core (~> 2.9.0)
+      rspec-expectations (~> 2.9.0)
+      rspec-mocks (~> 2.9.0)
+    rspec-core (2.9.0)
+    rspec-expectations (2.9.1)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.9.0)
+    uuidtools (2.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  em_aws!
+  rspec

data/README.md

@@ -6,27 +6,55 @@ Until approval (or if it is declined) I created this gem.
 
 ## Installation
 
-em_aws is available through [Rubygems](http://rubygems.org/gems/simple_model) and can be installed via:
+em_aws is available through [Rubygems](https://rubygems.org/gems/em_aws) and can be installed via:
 
     $ gem install em_aws
 
 ## Rails 3 setup (no rails 2 sorry)
 Setup [AWS-SKD](https://github.com/amazonwebservices/aws-sdk-for-ruby/blob/master/README.rdoc) as you would normally.
 
-Assuming you've already setup async-rails...
+Assuming you've already setup async-rails, add em_aws to you gemfile:
+    
+    gem 'em_aws'
+
+Then run:
+    
+    bundle install
 
 In your environments files add:
 
     require 'aws-sdk'
     require 'aws/core/http/em_http_handler'
     AWS.config(
-      :http_handler => AWS::Http::EMHttpHandler.new)
+      :http_handler => AWS::Http::EMHttpHandler.new(
+      :pool_size => 5,
+      :proxy => {:host => "http://myproxy.com", :port => 80}
+       ))
 
 Your done. 
 
 All requests to AWS will use EM-Synchrony's implementation of em-http-request for non-block HTTP request and fiber management.
 
-### References
+## Connection Pools
+v0.1+ implement connections EventMachine::Synchrony::ConnectionPool. The default pool_size is 5 but it can be
+increased by in the AWS.config options. Connection pools were problematic for S3 calls that required the :path option for
+em-http-request and would return AWS::S3::SignatureDoesNotMatch errors; so for now S3 calls with paths do not make use of the connection pools.
+
+## Heroku Gotcha
+When deploying to Heroku, if you get "NameError: uninitialized constant Syck::Syck", you need to vendorize em_aws
+    
+From your apps root directory run:
+
+    gem unpack em_aws --target vendor/gems
+
+Update you gemfile:
+
+    gem 'em_aws', :path => "vendor/gems/em_aws-0.0.3"
+
+Finally, run bundler and make sure you check-in your new Gemfile.lock. 
+I'm pretty sure the Syck error is do to an outdated version of rubygems and bundler on Heroku, but as of yet have not been able to reproduce it locally.
+
+## References
 
   [aws-sdk](https://github.com/amazonwebservices/aws-sdk-for-ruby)
 
@@ -45,4 +73,4 @@ All requests to AWS will use EM-Synchrony's implementation of em-http-request fo
 * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
 
 ## Thanks
-Code based on HTTParty Hander in [AWS-SKD](https://github.com/amazonwebservices/aws-sdk-for-ruby/blob/master/README.rdoc)
+Code based on HTTParty Hander in [aws-sdk](https://github.com/amazonwebservices/aws-sdk-for-ruby/blob/master/README.rdoc)

data/em_aws.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "aws-sdk"
   s.add_runtime_dependency "em-synchrony"
   s.add_runtime_dependency "em-http-request"
-  s.add_development_dependency "rspec" , '2.6.0'
+  s.add_development_dependency "rspec"
   
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/aws/core/http/em_http_handler.rb

@@ -1,9 +1,9 @@
 # http://docs.amazonwebservices.com/AWSRubySDK/latest/
-
 require "em-synchrony"
 require "em-synchrony/em-http"
-
+require 'em-synchrony/thread'
 module AWS
+  
   module Core
     module Http
       
@@ -18,36 +18,46 @@ module AWS
       # require 'aws/core/http/em_http_handler'
       # AWS.config(
       # :http_handler => AWS::Http::EMHttpHandler.new(
+      # :pool_size => 20,
       # :proxy => {:host => "http://myproxy.com",:port => 80}
       # )
       # )
       #
       class EMHttpHandler
-  
         # @return [Hash] The default options to send to EM-Synchrony on each
         # request.
         attr_reader :default_request_options
-  
+        @@pools = {}
+      
+        def self.fetch_connection(url,pool_size)
+          @@pools[url] ||= EventMachine::Synchrony::ConnectionPool.new(size: pool_size) do
+            EM::HttpRequest.new(url)
+          end
+          @@pools[url]
+        end
+             
         # Constructs a new HTTP handler using EM-Synchrony.
         #
-        # @param [Hash] options Default options to send to HTTParty on
+        # @param [Hash] options Default options to send to EM-Synchrony on
         # each request. These options will be sent to +get+, +post+,
         # +head+, +put+, or +delete+ when a request is made. Note
         # that +:body+, +:head+, +:parser+, and +:ssl_ca_file+ are
         # ignored. If you need to set the CA file, you should use the
         # +:ssl_ca_file+ option to {AWS.config} or
         # {AWS::Configuration} instead.
+        # Defaults pool_size to 5
         def initialize options = {}
           #puts "Using EM-Synchrony for AWS requests"
+          options[:pool_size] ||= 5
           @default_request_options = options
         end
         
         def fetch_url(request)
           url = nil
           if request.use_ssl?
-            url = "https://#{request.host}:443#{request.uri}"
+            url = "https://#{request.host}:443"
           else
-            url = "http://#{request.host}#{request.uri}"
+            url = "http://#{request.host}"
           end
           url
         end
@@ -88,36 +98,43 @@ module AWS
             merge(fetch_ssl(request))
         end
         
+        # We get AWS::S3::SignatureDoesNotMatch when path is used to fetch an s3 object
+        # so for now we won't use the pool for requests where the path is more than just '/'
         def fetch_response(url,method,opts={})
-          new_response = nil
+          return EM::HttpRequest.new("#{url}#{opts[:path]}").send(method, opts) if (@default_request_options[:pool_size] == 0 || opts[:path].to_s.length > 1)
+          self.class.fetch_connection(url,@default_request_options[:pool_size]).send(method, opts)
+        end
+
+        # Add thread safety.
+        def _fibered_mutex
+          @fibered_mutex ||= EM::Synchrony::Thread::Mutex.new
+        end
+        
+        def handle(request,response)
           if EM::reactor_running?
-              #puts "Reactor is running"
-            http = EM::HttpRequest.new(url).send(method, opts)
-            http.callback { new_response =  http}
-            http.errback { puts "#{method} to AWS failed." }
+            _fibered_mutex.synchronize do
+              handle_it(request, response)
+            end       
           else
             EM.synchrony do
-              #puts "WARNING: Had to start EM"
-              http = EM::HttpRequest.new(url).send(method, opts)
-              http.callback { new_response =  http}
-              http.errback { puts "#{method} to AWS failed." }
+              _fibered_mutex.synchronize do
+                handle_it(request, response)
+              end
               EM.stop
-            end           
+            end
           end
-          new_response
         end
-  
-        def handle(request, response)
-            #puts "Using EM!!!!"
+                
+        def handle_it(request, response)
+          #puts "Using EM!!!!"
           opts = default_request_options.merge({
-              :body => request.body
+              :body => request.body,
+              :path => request.uri,
+              
             }).merge(request_options(request))
-          
           url = fetch_url(request)
-          
           # get, post, put, delete, head
           method = request.http_method.downcase
-          
           begin
             http_response = fetch_response(url,method,opts)         
           rescue Timeout::Error, Errno::ETIMEDOUT => e
@@ -127,7 +144,7 @@ module AWS
             response.status = http_response.response_header.status.to_i
             response.headers = http_response.response_header.to_hash
           end
-        end   
+        end  
       end
     end
   end
@@ -138,5 +155,4 @@ module AWS
   module Http
     class EMHttpHandler < Core::Http::EMHttpHandler; end
   end
-
 end

data/lib/em_aws/version.rb

@@ -1,3 +1,3 @@
 module EmAws
-  VERSION = "0.0.3"
+  VERSION = "0.1.0"
 end

data/spec/em_http_handler_spec.rb

@@ -16,7 +16,7 @@ require 'spec_helper'
 module AWS::Core
   module Http
     describe EMHttpHandler do
-
+    
       let(:handler) { EMHttpHandler.new(default_request_options) }
 
       let(:default_request_options) { {} }
@@ -32,7 +32,6 @@ module AWS::Core
       let(:em_http_options) do
         options = nil
         EMHttpHandler.should_receive(:fetch_response).with do |url, method,opts|
-          puts "here"
           options = opts
           double("http response",
             :response => "<foo/>",
@@ -52,7 +51,12 @@ module AWS::Core
 
         it 'should set the default request options' do
           described_class.new(:foo => "BAR").default_request_options.
-            should == { :foo => "BAR" }
+            should == { :foo => "BAR", :pool_size => 5 }
+        end
+        
+        it 'should not override supplied pool_size' do
+          described_class.new(:pool_size => 20).default_request_options.
+            should == { :pool_size => 20 }
         end
 
       end
@@ -101,61 +105,59 @@ module AWS::Core
           
         end
         
-        
-        describe '#fetch_proxy' do
-          context ':proxy_uri' do
-
-            it 'passes proxy address and port from the request' do
-              req.proxy_uri = URI.parse('https://user:pass@proxy.com:443/path?query')
-              handler.fetch_proxy(req)[:proxy][:host].should == 'proxy.com'
-              handler.fetch_proxy(req)[:proxy][:port].should == 443
-            end
-
+      end
+      describe '#fetch_proxy' do
+        context ':proxy_uri' do
+          it 'passes proxy address and port from the request' do
+            req.proxy_uri = URI.parse('https://user:pass@proxy.com:443/path?query')
+            handler.fetch_proxy(req)[:proxy][:host].should == 'proxy.com'
+            handler.fetch_proxy(req)[:proxy][:port].should == 443
           end
+
+        end
         
-          describe '#fetch_ssl' do
-            it 'prefers the request option when set' do
-              req.use_ssl = true
-              req.ssl_verify_peer = true
-              req.ssl_ca_file = "something"
-              handler.fetch_ssl(req)[:private_key_file].should == "something"
-              handler.fetch_ssl(req)[:cert_chain_file].should == "something"
-            end
+        describe '#fetch_ssl' do
+          it 'prefers the request option when set' do
+            req.use_ssl = true
+            req.ssl_verify_peer = true
+            req.ssl_ca_file = "something"
+            handler.fetch_ssl(req)[:private_key_file].should == "something"
+            handler.fetch_ssl(req)[:cert_chain_file].should == "something"
+          end
            
-            context 'CA cert path' do
-
-              context 'use_ssl? is true' do
+          context 'CA cert path' do
 
-                before(:each) { req.use_ssl = true }
+            context 'use_ssl? is true' do
 
-                context 'ssl_verify_peer? is true' do
+              before(:each) { req.use_ssl = true }
 
-                  before(:each) do
-                    req.ssl_verify_peer = true
-                    req.ssl_ca_file = "foobar.txt"
-                  end
+              context 'ssl_verify_peer? is true' do
 
-                  it 'should use the ssl_ca_file attribute of the request' do
-                    handler.fetch_ssl(req)[:private_key_file].should == "foobar.txt"
-                  end
-                  
-                  it 'should use the ssl_ca_file attribute of the request' do
-                    handler.fetch_ssl(req)[:cert_chain_file].should == "foobar.txt"
-                  end
+                before(:each) do
+                  req.ssl_verify_peer = true
+                  req.ssl_ca_file = "foobar.txt"
                 end
 
-                it 'should not set the ssl_ca_file option without ssl_verify_peer?' do
-                  handler.fetch_ssl(req).should_not include(:private_key_file)
+                it 'should use the ssl_ca_file attribute of the request' do
+                  handler.fetch_ssl(req)[:private_key_file].should == "foobar.txt"
+                end
+                  
+                it 'should use the ssl_ca_file attribute of the request' do
+                  handler.fetch_ssl(req)[:cert_chain_file].should == "foobar.txt"
                 end
               end
 
-              it 'should not set the ssl_ca_file option without use_ssl?' do
+              it 'should not set the ssl_ca_file option without ssl_verify_peer?' do
                 handler.fetch_ssl(req).should_not include(:private_key_file)
               end
             end
-          end      
-        end
-      end
+
+            it 'should not set the ssl_ca_file option without use_ssl?' do
+              handler.fetch_ssl(req).should_not include(:private_key_file)
+            end
+          end
+        end      
+      end    
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: em_aws
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-18 00:00:00.000000000Z
+date: 2012-04-11 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
-  requirement: &70365956234980 !ruby/object:Gem::Requirement
+  requirement: &70247639340500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70365956234980
+  version_requirements: *70247639340500
 - !ruby/object:Gem::Dependency
   name: em-synchrony
-  requirement: &70365956234260 !ruby/object:Gem::Requirement
+  requirement: &70247639339960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70365956234260
+  version_requirements: *70247639339960
 - !ruby/object:Gem::Dependency
   name: em-http-request
-  requirement: &70365956233540 !ruby/object:Gem::Requirement
+  requirement: &70247639339420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,18 +43,18 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70365956233540
+  version_requirements: *70247639339420
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70365956232640 !ruby/object:Gem::Requirement
+  requirement: &70247639338880 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - =
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70365956232640
+  version_requirements: *70247639338880
 description: Adds EM-Synchrony support to AWS-SDK gem
 email:
 - joshmckin@gmail.com
@@ -65,6 +65,7 @@ files:
 - .gitignore
 - .rspec
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile