em_aws 0.3.2 → 1.0.0

data/lib/em-aws.rb

@@ -0,0 +1,17 @@
+require 'em-http'
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+require 'em-hot_tub'
+require 'aws-sdk-v1'
+require_relative 'em-aws/patches'
+require_relative 'em-aws/version'
+require_relative 'em-aws/http_handler'
+
+AWS.eager_autoload! # lazy load isn't thread safe
+
+module EventMachine
+  module AWS;end
+end
+
+# Backwards compatibility
+EmAws = EventMachine::AWS

data/lib/em-aws/http_handler.rb

@@ -0,0 +1,188 @@
+# http://docs.amazonwebservices.com/AWSRubySDK/latest/
+require 'hot_tub'
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+require 'em-synchrony/thread'
+module EventMachine
+  module AWS
+
+    # An em-http-request handler for the aws-sdk for fiber based asynchronous ruby application.
+    # See https://github.com/igrigorik/async-rails and
+    # http://www.mikeperham.com/2010/04/03/introducing-phat-an-asynchronous-rails-app/
+    # for examples of Aync-Rails application
+    #
+    # In Rails add the following to your aws.rb initializer
+    #
+    # require 'aws-sdk'
+    # require 'aws/core/http/em_http_handler'
+    # AWS.config(
+    #   :http_handler => AWS::Http::EMHttpHandler.new(
+    #     :proxy => {:host => '127.0.0.1',    # proxy address
+    #        :port => 9000,                 # proxy port
+    #        :type => :socks5},
+    #   :pool_size => 10,     # Default is 10
+    #   :max_size => 40,      # Maximum size of pool, nil by default so pool can grow to meet concurrency under load
+    #   :reap_timeout => 600, # How long to wait to reap connections after load dies down
+    #   :async => false))     # If set to true all requests are handle asynchronously and initially return nil
+    #
+    # EM-AWS exposes all connections options for EM-Http-Request at initialization
+    # For more information on available options see https://github.com/igrigorik/em-http-request/wiki/Issuing-Requests#available-connection--request-parameters
+    # If Options from the request section of the above link are present, they
+    # set on every request but may be over written by the request object
+    class HttpHandler < ::AWS::Core::Http::NetHttpHandler
+
+      attr_reader :default_options, :client_options, :pool_options
+
+      # Constructs a new HTTP handler using EM-Synchrony.
+      # @param [Hash] options Default options to send to EM-Synchrony on
+      # each request. These options will be sent to +get+, +post+,
+      # +head+, +put+, or +delete+ when a request is made. Note
+      # that +:body+, +:head+, +:parser+, and +:ssl_ca_file+ are
+      # ignored. If you need to set the CA file see:
+      # https://github.com/igrigorik/em-http-request/wiki/Issuing-Requests#available-connection--request-parameters
+      def initialize options = {}
+        @default_options = options
+        @pool_options = fetch_pool_options
+        @client_options = fetch_client_options
+        @verify_content_length = options[:verify_response_body_content_length]
+        @sessions = EM::HotTub::Sessions.new(pool_options) do |url|
+          EM::HttpRequest.new(url,@client_options)
+        end
+      end
+
+      def handle(request,response,&read_block)
+        process_request(request,response,&read_block)
+      end
+
+      # If the request option :async are set to true that request will  handled
+      # asynchronously returning nil initially and processing in the background
+      # managed by EM-Synchrony. If the client option :async all requests will
+      # be handled asynchronously.
+      # EX:
+      #     EM.synchrony do
+      #       s3 = AWS::S3.new
+      #       s3.obj.write('test', :async => true) => nil
+      #       EM::Synchrony.sleep(2)
+      #       s3.obj.read => # 'test'
+      #       EM.stop
+      #     end
+      def handle_async(request,response,handle,&read_block)
+        process_request(request,response,true,&read_block)
+      end
+
+      private
+
+      REMOVE_OPTIONS = [:pool_size,:max_size, :never_block, :blocking_timeout].freeze
+
+      def fetch_client_options
+        co = @default_options.select{ |k,v| !REMOVE_OPTIONS.include?(k) }
+        co[:inactivity_timeout] ||= 0.to_i
+        co[:connect_timeout] ||= 10
+        co[:keepalive] = true unless co.key?(:keepalive)
+        co
+      end
+
+      def fetch_pool_options
+        po = {}
+        po[:wait_timeout] = (@default_options[:wait_timeout] || @default_options[:blocking_timeout] || 10).to_i
+        po[:size] = (@default_options[:pool_size] || 5).to_i
+        po[:size] = 1 if po[:size] < 1
+        po[:max_size] = @default_options[:max_size].to_i if @default_options[:max_size]
+        po[:reap_timeout] = @default_options[:reap_timeout].to_i if @default_options[:reap_timeout]
+        po
+      end
+
+      def fetch_url(request)
+        "#{(request.use_ssl? ? "https" : "http")}://#{request.host}:#{request.port}"
+      end
+
+      def fetch_headers(request)
+        headers = { 'content-type' => '' }
+        request.headers.each_pair do |key,value|
+          headers[key] = value.to_s
+        end
+        {:head => headers}
+      end
+
+      def fetch_request_options(request)
+        opts = @client_options.merge(fetch_headers(request))
+        opts[:query] = request.querystring
+        if request.body_stream.respond_to?(:path)
+          opts[:file] = request.body_stream.path
+        else
+          opts[:body] = request.body.to_s
+        end
+        opts[:path] = request.path if request.path
+        opts
+      end
+
+      def pool(url)
+        @sessions.get_or_set(url, @pool_options) { EM::HttpRequest.new(url,@client_options) }
+      end
+
+      def fetch_response(request,opts={},&read_block)
+        method = "a#{request.http_method}".downcase.to_sym  # aget, apost, aput, adelete, ahead
+        url = fetch_url(request)
+        result = nil
+        if @sessions
+          pool(url).run do |connection|
+            req = connection.send(method, opts)
+            req.stream &read_block if block_given?
+            result = EM::Synchrony.sync req unless opts[:async]
+          end
+        else
+          clnt_opts = @client_options.merge(:inactivity_timeout => request.read_timeout)
+          req = EM::HttpRequest.new(url,clnt_opts).send(method,opts)
+          req.stream &read_block if block_given? and req.response_header.status.to_i < 300
+          result = EM::Synchrony.sync req unless opts[:async]
+        end
+        result
+      end
+
+      # AWS needs all header keys downcased and values need to be arrays
+      def fetch_response_headers(response)
+        response_headers = response.response_header.raw.to_hash
+        aws_headers = {}
+        response_headers.each_pair do  |k,v|
+          key = k.downcase
+          #['x-amz-crc32', 'x-amz-expiration','x-amz-restore','x-amzn-errortype']
+          if v.is_a?(Array)
+            aws_headers[key] = v
+          else
+            aws_headers[key] = [v]
+          end
+        end
+        response_headers.merge(aws_headers)
+      end
+
+      # Builds and attempts the request. Occasionally under load em-http-request
+      # em-http-request returns a status of 0 for various http timeouts, see:
+      # https://github.com/igrigorik/em-http-request/issues/76
+      # https://github.com/eventmachine/eventmachine/issues/175
+      def process_request(request,response,async=false,&read_block)
+        opts = fetch_request_options(request)
+        opts[:async] = (async || opts[:async])
+        exp_length = determine_expected_content_length(response)
+        begin
+          http_response = fetch_response(request,opts,&read_block)
+
+          unless opts[:async]
+            response.status = http_response.response_header.status.to_i
+            raise Timeout::Error if response.status == 0
+            response.headers = fetch_response_headers(http_response)
+            response.body = http_response.response
+          end
+
+          run_check = exp_length && request.http_method != "HEAD" && @verify_content_length
+          if run_check && response.body && response.body.bytesize != exp_length
+            raise TruncatedBodyError, 'content-length does not match'
+          end
+        rescue *NETWORK_ERRORS => error
+          raise error if block_given?
+          response.network_error = error
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/em-aws/version.rb

@@ -0,0 +1,5 @@
+module EventMachine
+  module AWS
+    VERSION = "1.0.0"
+  end
+end

data/lib/em_aws.rb

@@ -1,12 +1 @@
-require 'em_aws/patches'
-require 'aws-sdk'
-require 'em_aws/version'
-require 'em-http'
-require 'em-synchrony'
-require 'em-synchrony/em-http'
-require 'aws/core/http/em_http_handler'
-require 'hot_tub'
-
-AWS.eager_autoload! # lazy load isn't thread safe
-HotTub.logger = AWS.config.logger if AWS.config.logger
-module EmAws;end
+require_relative 'em-aws'

data/spec/em-aws/http_handler_spec.rb

@@ -0,0 +1,297 @@
+# Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'spec_helper'
+require 'eventmachine'
+require 'evma_httpserver'
+module AWS::Core
+  module Http
+    class EMFooIO
+      def path
+        "/my_path/test.text"
+      end
+    end
+
+    # A server for testing response,
+    # borrowed from: http://www.igvita.com/2008/05/27/ruby-eventmachine-the-speed-demon/
+    class AwsServer < EventMachine::Connection
+      include EventMachine::HttpServer
+
+      def process_http_request
+        resp = EventMachine::DelegatedHttpResponse.new( self )
+        resp.status = 200
+        resp.content = "Hello World!"
+        resp.send_response
+      end
+    end
+
+    # A slow server for testing timeout,
+    # borrowed from: http://www.igvita.com/2008/05/27/ruby-eventmachine-the-speed-demon/
+    class SlowServer < EventMachine::Connection
+      include EventMachine::HttpServer
+
+      def process_http_request
+        resp = EventMachine::DelegatedHttpResponse.new( self )
+
+        sleep 2 # Simulate a long running request
+
+        resp.status = 200
+        resp.content = "Hello World!"
+        resp.send_response
+      end
+    end
+
+    describe EM::AWS::HttpHandler do
+
+      around(:each) do |example|
+        EM.synchrony do
+          example.run
+          EM.stop
+        end
+      end
+
+      let(:handler_opts) do
+        {:verify_response_body_content_length => true}
+      end
+
+      let(:handler) { EM::AWS::HttpHandler.new(handler_opts) }
+
+      let(:request) do
+        double('aws-request',
+               :http_method => 'POST',
+               :endpoint => 'https://host.com',
+               :uri => '/path?querystring',
+               :path => '/path',
+               :host => 'host.com',
+               :port => 443,
+               :querystring => 'querystring',
+               :body_stream => StringIO.new('body'),
+               :body => 'body',
+               :use_ssl? => true,
+               :ssl_verify_peer? => true,
+               :ssl_ca_file => '/ssl/ca',
+               :ssl_ca_path => nil,
+               :read_timeout => 60,
+               :continue_timeout => 1,
+               :headers => { 'foo' => 'bar' })
+      end
+
+      let(:response) { Response.new }
+
+      let(:read_block) { }
+
+      let(:handle!) { handler.handle(request, response, &read_block) }
+
+      let(:http) { double('http-session').as_null_object }
+
+      let(:http_response) {
+        double = double('http response',
+                        :code => '200',
+                        :response => 'resp-body',
+                        :to_hash => { 'header-name' => ['header-value'] })
+        allow(double).to receive(:stream) do |&block|
+          block ? block.call('resp-body') : 'resp-body'
+        end
+        double
+      }
+
+      before(:each) do
+        allow(http).to receive(:request).and_yield(http_response)
+      end
+
+      it 'should be accessible from AWS as well as AWS::Core' do
+        expect(AWS::Http::EMHttpHandler.new).to be_an(EM::AWS::HttpHandler)
+      end
+
+      describe '#handle' do
+        context 'exceptions' do
+          it 'should rescue Timeout::Error' do
+            allow(handler).to receive(:fetch_response).and_raise(Timeout::Error)
+
+            expect {
+              handle!
+            }.to_not raise_error
+          end
+
+          it 'should rescue Errno::ETIMEDOUT' do
+            allow(handler).to receive(:fetch_response).and_raise(Errno::ETIMEDOUT)
+
+            expect {
+              handle!
+            }.to_not raise_error
+          end
+
+          it 'should indicate that there was a network_error' do
+            allow(handler).to receive(:fetch_response).and_raise(Errno::ETIMEDOUT)
+
+            handle!
+
+            expect(response).to be_network_error
+          end
+        end
+
+        context 'default request options' do
+          before(:each) do
+            allow(handler).to receive(:default_request_options).and_return(:foo => "BAR", :private_key_file => "blarg")
+          end
+
+          it 'passes extra options through to synchrony' do
+            expect(handler.default_request_options[:foo]).to eql("BAR")
+          end
+
+          it 'uses the default when the request option is not set' do
+            #puts handler.default_request_options
+            expect(handler.default_request_options[:private_key_file]).to eql("blarg")
+          end
+        end
+      end
+
+      describe '#fetch_request_options' do
+        it "should set :query and :body to request.querystring" do
+          opts = handler.send(:fetch_request_options, request)
+          expect(opts[:query]).to eql(request.querystring)
+        end
+
+        it "should set :path to request.path" do
+          opts = handler.send(:fetch_request_options, request)
+          expect(opts[:path]).to eql(request.path)
+        end
+
+        context "request.body_stream is a StringIO" do
+          it "should set :body to request.body_stream" do
+            opts = handler.send(:fetch_request_options, request)
+            expect(opts[:body]).to eql("body")
+          end
+        end
+
+        context "request.body_stream is an object that responds to :path" do
+          let(:io_object) { EMFooIO.new }
+
+          before(:each) do
+            allow(request).to receive(:body_stream).and_return(io_object)
+          end
+
+          it "should set :file to object.path " do
+            opts = handler.send(:fetch_request_options, request)
+            expect(opts[:file]).to eql(io_object.path)
+          end
+        end
+      end
+
+      describe '#fetch_client_options' do
+        it "should remove pool related options" do
+          opts = handler.send(:fetch_client_options)
+
+          expect(opts.has_key?(:size)).to eql(false)
+          expect(opts.has_key?(:never_block)).to eql(false)
+          expect(opts.has_key?(:blocking_timeout)).to eql(false)
+        end
+
+        context "when keepalive is not set" do
+
+          it "should be true" do
+            opts = handler.send(:fetch_client_options)
+
+            expect(opts[:keepalive]).to eql(true)
+          end
+        end
+
+        context "when keepalive is false" do
+
+          it "should be false" do
+            handler.default_options[:keepalive] = false
+            opts = handler.send(:fetch_client_options)
+
+            expect(opts[:keepalive]).to eql(false)
+          end
+        end
+      end
+
+      context 'content-length checking' do
+
+        let(:http_response) {
+          double = double('http-response',
+                          :code => '200',
+                          :response => 'resp-body',
+                          :to_hash => { 'content-length' => ["10000"] })
+          allow(double).to receive(:stream) do |&block|
+            block ? block.call('resp-body') : 'resp-body'
+          end
+          double
+        }
+
+        it 'should raise if content-length does not match' do
+          server = nil
+          EventMachine::run do
+            server = EventMachine::start_server '127.0.0.1', '8081', AwsServer
+          end
+          allow(handler).to receive(:fetch_url).and_return("http://127.0.0.1:8081")
+          allow(handler).to receive(:determine_expected_content_length).and_return(1)
+          handle!
+          expect(response.network_error).to be_a_kind_of(NetHttpHandler::TruncatedBodyError)
+          EventMachine.stop_server(server)
+        end
+
+        context 'can turn off length checking' do
+          let(:handler_opts) {{:verify_response_body_content_length => false}}
+
+          let(:handler) { described_class.new(handler_opts) }
+
+          it 'should not raise if length does not match but check is off' do
+            expect(response.network_error).to be_nil
+          end
+
+        end
+      end
+
+      context 'slow requests' do
+        context 'with inactivity_timeout = 0' do
+          it "should not timeout" do
+            server = nil
+            # turn on our test server
+            EventMachine::run do
+              server =  EventMachine::start_server '127.0.0.1', '8081', SlowServer
+            end
+
+            allow(handler).to receive(:fetch_url).and_return("http://127.0.0.1:8081")
+
+            handle!
+
+            expect(response.network_error).to be_nil
+
+            EventMachine.stop_server(server)
+          end
+        end
+        context 'with inactivity_timeout > 0' do
+          it "should timeout" do
+            server = nil
+            # turn on our test server
+            EventMachine::run do
+              server = EventMachine::start_server '127.0.0.1', '8081', SlowServer
+            end
+
+            allow(handler).to receive(:fetch_url).and_return("http://127.0.0.1:8081")
+            handler.client_options[:inactivity_timeout] = 0.01
+            allow(handler).to receive(:connect_timeout).and_return(1) #just to speed up the test
+
+            handle!
+
+            expect(response.network_error).to be_a(Timeout::Error)
+
+            EventMachine.stop_server(server)
+          end
+        end
+      end
+    end
+  end
+end