aws-sdk 1.1.3 → 1.1.4

data/lib/aws/core/client_logging.rb

@@ -0,0 +1,125 @@
+# Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'benchmark'
+
+module AWS
+  module Core
+
+    # @private
+    module ClientLogging
+  
+      MAX_STRING_LENGTH = 50
+  
+      def log_client_request(name, options)
+        response = nil
+        time = Benchmark.measure do
+          response = yield
+        end
+  
+        if options[:async]
+          response.on_complete do
+            log_client_request_on_success(name, options, response, time)
+          end
+        else
+          log_client_request_on_success(name, options, response, time)
+        end
+        response
+      end
+  
+      # Summarizes long strings and adds file size information
+      # @private
+      def sanitize_options(options)
+        sanitize_hash(options)
+      end
+  
+      protected
+      def log severity, message
+        config.logger.send(severity, message + "\n") if config.logger
+      end
+  
+      protected
+      def log_client_request_on_success(method_name, options, response, time)
+        status = response.http_response.status
+        service = self.class.service_name
+  
+        pattern = "[AWS %s %s %.06f] %s(%s)"
+        parts = [service, status, time.real, method_name, sanitize_options(options)]
+        severity = :info
+  
+        if response.error
+          pattern += " %s: %s"
+          parts << response.error.class
+          parts << response.error.message
+          severity = :error
+        end
+  
+        if response.cached
+          pattern << " [CACHED]"
+        end
+  
+        log(severity, pattern % parts)
+      end
+  
+      protected
+      def sanitize_value(value)
+        case value
+        when Hash
+          '{' + sanitize_hash(value) + '}'
+        when Array
+          sanitize_array(value)
+        when File
+          sanitize_file(value)
+        when String
+          sanitize_string(value)
+        else
+          value.inspect
+        end
+      end
+  
+      protected
+      def sanitize_string str
+        summary = summarize_string(str)
+        inspected = str.inspect
+        if inspected.size > summary.size
+          summary
+        else
+          inspected
+        end
+      end
+  
+      protected
+      def summarize_string str
+        "#<String #{str[0,6].inspect} ... #{str[-6,6].inspect} (#{str.size} characters)>"
+      end
+  
+      protected
+      def sanitize_file file
+        "#<File:#{file.path} (#{File.size(file.path)} bytes)>"
+      end
+  
+      protected
+      def sanitize_array array
+        "[" + array.map { |v| sanitize_value(v) }.join(",") + "]"
+      end
+  
+      protected
+      def sanitize_hash hash
+        hash.map do |k,v|
+          "#{sanitize_value(k)}=>#{sanitize_value(v)}"
+        end.sort.join(",")
+      end
+  
+    end
+  end
+end

data/lib/aws/core/collections.rb

@@ -0,0 +1,229 @@
+# Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module AWS
+  module Core
+
+    # There are 3 collection modules:
+    #
+    # AWS::Collections::Basic 
+    #   - single request returns all items
+    #   - (sqs, ec2, ses)
+    #
+    # AWS::Collections::Paged
+    #   - responses may be truncated
+    #   - truncated responses return a "next token"
+    #   - (sns, sdb)
+    #
+    # AWS::Collections::PagedWithLimits
+    #   - requests accept a "max items"
+    #   - responses may be "truncated" 
+    #   - truncated responses return a "next token"
+    #   - (s3, iam)
+    #
+    # @private
+    module Collections
+    
+      # @private
+      module Basic
+    
+        include Model
+        include Enumerable
+  
+        def enumerator options = {}
+          Enumerator.new(self, :each, options) 
+        end
+    
+        def each options = {}, &block
+          each_batch(options) do |batch|
+            batch.each(&block)
+          end
+          nil
+        end
+  
+        def each_batch options = {}, &block
+  
+          options = options.dup
+  
+          limit = options.delete(:limit)
+          batch_size = options.delete(:batch_size)
+  
+          total = 0  # count of items yeileded across all batches
+  
+          each_response(options, limit, batch_size) do |response|
+  
+            batch = []
+            each_item(response) do |item|
+              batch << item
+              if limit and (total += 1) == limit
+                yield(batch)
+                return
+              end
+            end
+  
+            yield(batch)
+  
+            batch.size
+  
+          end
+  
+          nil
+  
+        end
+  
+        def in_groups_of size, options = {}, &block
+  
+          group = []
+  
+          each_batch(options) do |batch|
+            batch.each do |item|
+              group << item
+              if group.size == size
+                yield(group)
+                group = []
+              end
+            end
+          end
+  
+          yield(group) unless group.empty?
+  
+          nil
+  
+        end
+  
+        # @note +limit+ has no effect, simply ignored
+        # @note +batch_size+ has no effect, simply ignored
+        # @private
+        protected
+        def each_response options, limit, batch_size, &block
+          response = client.send(request_method, options)
+          yield(response)
+        end
+  
+        # @note Define this method in classes including this module.
+        # @private
+        protected
+        def request_method
+          raise NotImplementedError
+        end
+  
+        # @note Define this method in classes including this module.
+        # @private
+        protected
+        def each_item response
+          raise NotImplementedError
+        end
+    
+      end
+    
+      # @private
+      module Paged
+    
+        include Basic
+  
+        # @note +limit+ has no effect, simply ignored
+        # @note +batch_size+ has no effect, simply ignored
+        protected
+        def each_response options, limit, batch_size, &block
+  
+          next_token = nil
+  
+          begin
+    
+            page_opts = {}
+            page_opts[next_token_key] = next_token if next_token
+    
+            response = client.send(request_method, options.merge(page_opts))
+    
+            yield(response)
+    
+            next_token = next_token_for(response)
+    
+          end until next_token.nil?
+    
+        end
+    
+        # Override this methid in collections that use a different name
+        # for the param that offsets the find (e.g. :marker, :next_key, etc).
+        # @private
+        protected
+        def next_token_key
+          raise NotImplementedError
+        end
+    
+        # Override this method in collections that have an alternate method
+        # for finding the next token.  
+        # @private
+        protected
+        def next_token_for response
+          method = next_token_key
+          response.respond_to?(method) ? response.send(method) : nil
+        end
+    
+      end
+    
+      # @private
+      module PagedWithLimits
+    
+        include Paged
+  
+        # A custom first method makes getting exactly one item much more 
+        # efficient.  Without the :limit => 1, an entire page of items 
+        # is received and then only one is grabbed.
+        # @private
+        def first
+          enumerator(:limit => 1).first  
+        end
+  
+        # @private
+        protected
+        def each_response options, limit, batch_size, &block
+    
+          total = 0
+          next_token = nil
+  
+          begin
+    
+            page_opts = {}
+  
+            page_opts[next_token_key] = next_token if next_token
+  
+            if limit or batch_size
+              max_items = []
+              max_items << (limit - total) if limit
+              max_items << batch_size if batch_size
+              page_opts[limit_key] = max_items.min
+            end
+    
+            response = client.send(request_method, options.merge(page_opts))
+    
+            total += yield(response)
+    
+            next_token = next_token_for(response)
+    
+          end until next_token.nil? or (limit and limit == total)
+    
+        end
+  
+        # Override this methid in collections that use a different name
+        # for the param that offsets the find (e.g. :marker, :next_key, etc).
+        # @private
+        protected
+        def limit_key
+          raise NotImplementedError
+        end
+    
+      end
+  
+    end
+  end
+end

data/lib/aws/core/configuration.rb

@@ -0,0 +1,358 @@
+# Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'set'
+require 'uri'
+
+module AWS
+  module Core
+
+    # A configuration object for AWS interfaces and clients.
+    #
+    # == Configuring Credential
+    #
+    # In order to do anything with AWS you will need to assign credentials.
+    # The simplest method is to assing your credentials into the default
+    # configuration:
+    #
+    #   AWS.config(:access_key_id => 'KEY', :secret_access_key => 'SECRET')
+    #
+    # You can also export them into your environment and they will be picked up
+    # automatically:
+    #
+    #   export AWS_ACCESS_KEY_ID='YOUR_KEY_ID_HERE'
+    #   export AWS_SECRET_ACCESS_KEY='YOUR_SECRET_KEY_HERE'
+    #
+    # For compatability with other AWS gems, the credentials can also be
+    # exported like:
+    #
+    #   export AMAZON_ACCESS_KEY_ID='YOUR_KEY_ID_HERE'
+    #   export AMAZON_SECRET_ACCESS_KEY='YOUR_SECRET_KEY_HERE'
+    #
+    # == Modifying a Configuration
+    #
+    # Configuration objects are read-only.  If you need a different set of
+    # configuration values, call {#with}, passing in the updates
+    # and a new configuration object will be returned.
+    #
+    #   config = Configuration.new(:max_retires => 3)
+    #   new_config = config.with(:max_retries => 2)
+    #
+    #   config.max_retries #=> 3
+    #   new_config.max_retries #=> 2
+    #
+    # == Global Configuration
+    #
+    # The global default configuration can be found at {AWS.config}
+    #
+    # @attr_reader [String,nil] access_key_id AWS access key id credential.
+    #   Defaults to +nil+.
+    #
+    # @attr_reader [String] ec2_endpoint The service endpoint for Amazon EC2.
+    #   Defaults to 'ec2.amazonaws.com'.
+    #
+    # @attr_reader [Object] http_handler The http handler that sends requests
+    #   to AWS.  Defaults to an HTTP handler built on net/http.
+    #
+    # @attr_reader [String] iam_endpoint The service endpoint for AWS Idenity
+    #   Access Management (IAM).  Defaults to 'iam.amazonaws.com'.
+    #
+    # @attr_reader [Object,nil] logger A logger instance that 
+    #   should receive log messages generated by service requets.  
+    #   A logger needs to respond to #log and must accept a 
+    #   severity (e.g. :info, :error, etc) and a string message.
+    #   Defaults to +nil+.
+    #
+    # @attr_reader [Integer] max_retries The maximum number of times
+    #   service errors (500) should be retried.  There is an exponential 
+    #   backoff in between service request retries, so the more retries the
+    #   longer it can take to fail.  Defautls to 3.
+    #
+    # @attr_reader [String, URI, nil] proxy_uri The URI of the proxy 
+    #    to send service requests through.  You can pass a URI object or a 
+    #    URI string.  Defautls to +nil+.
+    #
+    #      AWS.config(:proxy_uri => 'https://user:pass@my.proxy:443/path?query')
+    #
+    # @attr_reader [String] s3_endpoint The service endpoint for Amazon S3.
+    #   Defaults to 's3.amazonaws.com'.
+    #
+    # @attr_reader [Integer] s3_multipart_max_parts The maximum number of 
+    #   parts to split a file into when uploading in parts to S3.  
+    #   Defaults to 1000.
+    #
+    # @attr_reader [Integer] s3_multipart_threshold (16777216) When uploading
+    #   data to S3, if the number of bytes to send exceedes 
+    #   +:s3_multipart_threshold+ then a multi part session is automatically
+    #   started and the data is sent up in chunks.  The size of each part
+    #   is specified by +:s3_multipart_min_part_size+. Defaults to 
+    #   16777216 (16MB).
+    #
+    # @attr_reader [Integer] s3_multipart_min_part_size The absolute minimum
+    #   size (in bytes) each S3 multipart segment should be.
+    #   Defaults to 5242880 (5MB).
+    #
+    # @attr_reader [String,nil] secret_access_key AWS secret access key 
+    #   credential.  Defaults to +nil+.
+    #
+    # @attr_reader [String,nil] session_token AWS secret token credential.
+    #   Defaults to +nil+.
+    #
+    # @attr_reader [String] simple_db_endpoint The service endpoint for Amazon
+    #   SimpleDB.  Defaults to 'sdb.amazonaws.com'.
+    #
+    # @attr_reader [Boolean] simple_db_consistent_reads Determines
+    #   if all SimpleDB read requests should be done consistently.
+    #   Consistent reads are slower, but reflect all changes to SDB.
+    #   Defaults to +false+.
+    #
+    # @attr_reader [String] simple_email_service_endpoint The service endpoint
+    #   for Amazon Simple Email Service.  Defaults to 
+    #   'email.us-east-1.amazonaws.com'.
+    #
+    # @attr_reader [Object] signer The request signer.  Defaults to
+    #   a default request signer implementation.
+    #
+    # @attr_reader [String] ssl_ca_file The path to a CA cert bundle in 
+    #   PEM format.
+    #
+    #   If +ssl_verify_peer+ is true (the default) this bundle will be
+    #   used to validate the server certificate in each HTTPS request.
+    #   The AWS SDK for Ruby ships with a CA cert bundle, which is the
+    #   default value for this option.
+    #
+    # @attr_reader [Boolean] ssl_verify_peer When +true+ 
+    #   the HTTP handler validate server certificates for HTTPS requests.
+    #   Defaults to +true+.
+    #
+    #   This option should only be disabled for diagnostic purposes;
+    #   leaving this option set to +false+ exposes your application to
+    #   man-in-the-middle attacks and can pose a serious security
+    #   risk.
+    #
+    # @attr_reader [Boolean] stub_requests When +true+ requests are not
+    #   sent to AWS, instead empty reponses are generated and returned to
+    #   each service request.
+    #
+    # @attr_reader [String] sns_endpoint The service endpoint for Amazon SNS.
+    #   Defaults to 'sns.us-east-1.amazonaws.com'.
+    #
+    # @attr_reader [String] sqs_endpoint The service endpoint for Amazon SQS.
+    #   Defaults to 'sqs.us-east-1.amazonaws.com'.
+    #
+    # @attr_reader [String] sts_endpoint The service endpoint for AWS
+    #   Security Token Service.  Defaults to 'sts.amazonaws.com'.
+    #
+    # @attr_reader [Boolean] use_ssl When +true+, all requests
+    #   to AWS are sent using HTTPS instead vanilla HTTP.
+    #   Defaults to +true+.
+    #
+    # @attr_reader [String] user_agent_prefix A string prefix to 
+    #   append to all requets against AWS services.  This should be set
+    #   for clients and applications built ontop of the aws-sdk gem.
+    #   Defaults to +nil+.
+    #
+    class Configuration
+  
+      # Creates a new Configuration object.
+      # @param options (see AWS.config)
+      # @option options (see AWS.config)
+      def initialize options = {}
+  
+        @created = options.delete(:__created__) || {}
+  
+        options.each_pair do |opt_name, value|
+          opt_name = opt_name.to_sym
+          if self.class.accepted_options.include?(opt_name)
+            supplied[opt_name] = value
+          end
+        end
+  
+      end
+  
+      # Used to create a new Configuration object with the given modifications.
+      # The current configuration object is not modified.
+      #
+      #   AWS.config(:max_retries => 2)
+      #
+      #   no_retries_config = AWS.config.with(:max_retries => 0)
+      #
+      #   AWS.config.max_retries        #=> 2
+      #   no_retries_config.max_retries #=> 0
+      #
+      # You can use these configuration objects returned by #with to create
+      # AWS objects:
+      #
+      #   AWS::S3.new(:config => no_retries_config)
+      #   AWS::SQS.new(:config => no_retries_config)
+      #
+      # @param options (see AWS.config)
+      # @option options (see AWS.config)
+      # @return [Configuration] Copies the current configuration and returns
+      #   a new one with modifications as provided in +:options+.
+      def with options = {}
+  
+        # symbolize option keys
+        options = options.inject({}) {|h,kv| h[kv.first.to_sym] = kv.last; h }
+  
+        values = supplied.merge(options)
+  
+        if supplied == values
+          self # nothing changed
+        else
+          self.class.new(values.merge(:__created__ => @created.dup))
+        end
+  
+      end
+  
+      # @return [Hash] Returns a hash of all configuration values.
+      def to_h
+        self.class.accepted_options.inject({}) do |h,k|
+          h[k] = send(k)
+          h
+        end
+      end
+  
+      # @return [Boolean] Returns true if the two configuration objects have
+      #   the same values.
+      def == other
+        other.is_a?(self.class) and self.supplied == other.supplied
+      end
+  
+      alias_method :eql, :==
+  
+      # @private
+      def inspect
+        "<#{self.class.name}>"
+      end
+  
+      protected
+      def supplied
+        @supplied ||= {}
+      end
+  
+      class << self
+  
+        # @private
+        def accepted_options
+          @options ||= Set.new
+        end
+  
+        # @private
+        def add_option name, default_value = nil, options = {}, &transform
+  
+          accepted_options << name
+  
+          define_method(name) do
+            value = supplied.has_key?(name) ? supplied[name] : default_value
+            transform ? transform.call(value) : value
+          end
+  
+          alias_method("#{name}?", name) if options[:boolean]
+  
+        end
+  
+        # Configuration options that have dependencies are re-recreated
+        # anytime one of their dependendent configuration values are
+        # changed.
+        # @private
+        def add_option_with_needs name, needs, &create_block
+          
+          accepted_options << name
+  
+          define_method(name) do
+  
+            return supplied[name] if supplied.has_key?(name)
+  
+            needed = needs.collect{|need| send(need) }
+  
+            unless @created.key?(name) and @created[name][:needed] == needed
+              @created[name] = {}
+              @created[name][:object] = create_block.call(self)
+              @created[name][:needed] = needed
+            end
+  
+            @created[name][:object]
+            
+          end
+  
+        end
+
+        def add_service name, ruby_name, default_endpoint
+
+          create_block = lambda do |config| 
+            AWS.const_get(name)::Client.new(:config => config)
+          end
+
+          needs = [
+            :signer, 
+            :http_handler, 
+            :"#{ruby_name}_endpoint",
+            :max_retries,
+            :stub_requests?,
+            :proxy_uri,
+            :use_ssl?,
+            :ssl_verify_peer?,
+            :ssl_ca_file,
+            :user_agent_prefix,
+            :logger,
+          ]
+
+          add_option :"#{ruby_name}_endpoint", default_endpoint
+
+          add_option_with_needs :"#{ruby_name}_client", needs, &create_block
+
+        end
+  
+      end
+
+      add_option :access_key_id, 
+        ENV['AWS_ACCESS_KEY_ID'] || ENV['AMAZON_ACCESS_KEY_ID']
+  
+      add_option :http_handler, Core::Http::NetHttpHandler.new
+  
+      add_option :logger
+  
+      add_option :max_retries, 3
+  
+      add_option :proxy_uri do |uri| uri ? URI.parse(uri.to_s) : nil end
+  
+      add_option :secret_access_key, 
+        ENV['AWS_SECRET_ACCESS_KEY'] || ENV['AMAZON_SECRET_ACCESS_KEY']
+  
+      add_option :session_token
+  
+      add_option_with_needs :signer, 
+        [:access_key_id, :secret_access_key, :session_token] do |config|
+  
+        DefaultSigner.new(
+          config.access_key_id, 
+          config.secret_access_key, 
+          config.session_token)
+  
+      end
+  
+      add_option :ssl_verify_peer, true, :boolean => true
+  
+      add_option :ssl_ca_file, 
+        File.expand_path(File.dirname(__FILE__) + "/../../../ca-bundle.crt")
+  
+      add_option :stub_requests, false, :boolean => true
+  
+      add_option :use_ssl, true, :boolean => true
+  
+      add_option :user_agent_prefix
+  
+    end
+  end
+end