aws-sdk 1.1.4 → 1.2.0

data/lib/aws/core/client_logging.rb

@@ -19,8 +19,6 @@ module AWS
     # @private
     module ClientLogging
   
-      MAX_STRING_LENGTH = 50
-  
       def log_client_request(name, options)
         response = nil
         time = Benchmark.measure do
@@ -89,10 +87,9 @@ module AWS
   
       protected
       def sanitize_string str
-        summary = summarize_string(str)
         inspected = str.inspect
-        if inspected.size > summary.size
-          summary
+        if inspected.size > config.logger_truncate_strings_at
+          summarize_string(str)
         else
           inspected
         end
@@ -100,7 +97,9 @@ module AWS
   
       protected
       def summarize_string str
-        "#<String #{str[0,6].inspect} ... #{str[-6,6].inspect} (#{str.size} characters)>"
+        # skip the openning "
+        string_start = str.inspect[1,config.logger_truncate_strings_at]
+        "#<String \"#{string_start}\" ... (#{str.size} characters)>"
       end
   
       protected

data/lib/aws/core/collection.rb

@@ -0,0 +1,241 @@
+# Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module AWS
+  module Core
+
+    # = Different Collection Types in AWS
+    #
+    # The Collection module acts as a namespace and base implementation for
+    # the primary collection types in AWS:
+    #
+    # * {AWS::Core::Collection::Simple}
+    # * {AWS::Core::Collection::Limitable}
+    #
+    # Each AWS service allows provides a method to enumerate resources.
+    #
+    # * Services that return all results in a single response use
+    #   {AWS::Core::Collection::Simple}.
+    #
+    # * Services that truncate large results sets *AND* allow you to provide
+    #   a perfered maximum number of results use 
+    #   {AWS::Core::Collection::Limitable}.
+    #
+    module Collection
+
+      AWS.register_autoloads(self) do
+        autoload :Simple,    'simple'
+        autoload :Limitable, 'limitable'
+      end
+
+      include Enumerable
+  
+      # Yields once for every item in this collection.
+      #
+      #   collection.each {|item| ... }
+      #
+      # @note If you want fewer than all items, it is generally better 
+      #   to call #{page} than {#each} with a +:limit+.
+      #
+      # @param [Hash] options
+      #
+      # @option options [Integer] :limit (nil) The maximum number of
+      #   items to enumerate from this collection.
+      #
+      # @option options [next_token] :next_token (nil) Next tokens
+      #   act as offsets into the collection.  Next tokens vary in
+      #   format from one service to the next, (e.g. may be a number,
+      #   an opaque string, a hash of values, etc).
+      #
+      #   {#each} and {#each_batch} return a +:next_token+ when called
+      #   with +:limit+ and there were more items matching the request.
+      #
+      #   *NOTE* It is generally better to call {#page} if you only
+      #   want a few items with the ability to request more later.
+      #
+      # @return [nil_or_next_token] Returns nil if all items were enumerated.
+      #   If some items were excluded because of a +:limit+ option then
+      #   a +next_token+ is returned.  Calling an enumerable method on 
+      #   the same collection with the +next_token+ acts like an offset.
+      #
+      def each options = {}, &block
+        each_batch(options) do |batch|
+          batch.each(&block)
+        end
+      end
+
+      # Yields items from this collection in batches.
+      #
+      #   collection.each_batch do |batch|
+      #     batch.each do |item|
+      #       # ...
+      #     end
+      #   end
+      #
+      # == Variable Batch Sizes
+      #
+      # Each AWS service has its own rules on how it returns results.
+      # Because of this batch size may very based on:
+      #
+      # * Service limits (e.g. S3 limits keys to 1000 per response)
+      #
+      # * The size of response objects (SimpleDB limits responses to 1MB)
+      #
+      # * Time to process the request
+      #
+      # Because of these variables, batch sizes may not be consistent for
+      # a single collection.  Each batch represents all of the items returned
+      # in a single resopnse.
+      #
+      # @note If you require fixed size batches, see {#in_groups_of}.
+      #
+      # @param (see #each)
+      #
+      # @option (see #each)
+      #
+      # @return (see #each)
+      #
+      def each_batch options = {}, &block
+        raise NotImplementedError
+      end
+
+      # Use this method when you want to call a method provided by 
+      # Enumerable, but you need to pass options:
+      #
+      #   # raises an error because collect does not accept arguments
+      #   collection.collect(:limit => 10) {|i| i.name }
+      #
+      #   # not an issue with the enum method
+      #   collection.enum(:limit => 10).collect(&:name)
+      #
+      # @param (see #each)
+      #
+      # @option (see #each)
+      #
+      # @return [Enumerable::Enumerator] Returns an enumerator for this
+      #   collection.
+      #
+      def enum options = {}
+        Enumerator.new(self, :each, options) 
+      end
+      alias_method :enumerator, :enum
+
+      # Returns the first item from this collection.
+      #
+      # @return [item_or_nil] Returns the first item from this collection or
+      #   nil if the collection is empty.
+      #
+      def first options = {}
+        enum(options.merge(:limit => 1)).first
+      end
+
+      # Yields items from this collection in groups of an exact
+      # size (except for perhaps the last group). 
+      #
+      #   collection.in_groups_of (10, :limit => 30) do |group|
+      #
+      #     # each group should be exactly 10 items unless
+      #     # fewer than 30 items are returned by the service
+      #     group.each do |item|
+      #       #...
+      #     end
+      #
+      #   end
+      #   
+      # @param [Integer] size Size each each group of objects
+      #   should be yielded in.
+      # @param [Hash] options
+      # @option (see #each)
+      # @return (see #each)
+      def in_groups_of size, options = {}, &block
+
+        group = []
+
+        nil_or_next_token = each_batch(options) do |batch|
+          batch.each do |item|
+            group << item
+            if group.size == size
+              yield(group)
+              group = []
+            end
+          end
+        end
+
+        yield(group) unless group.empty?
+
+        nil_or_next_token
+
+      end
+
+      # Returns a single page of results in a kind-of array ({PageResult}).
+      #
+      #   items = collection.page(:per_page => 10) # defaults to 10 items
+      #   items.is_a?(Array) # => true
+      #   items.size         # => 8
+      #   items.per_page     # => 10
+      #   items.last_page?   # => true
+      #
+      # If you need to display a "next page" link in a web view you can
+      # use the #more? method.  Just make sure the generated link
+      # contains the +next_token+.
+      #
+      #   <% if items.more? %>
+      #     <%= link_to('Next Page', params.merge(:next_token => items.next_token) %>
+      #   <% end %>
+      #
+      # Then in your controller you can find the next page of results:
+      #
+      #   items = collection.page(:next_token => params[:next_token])
+      #
+      # Given a {PageResult} you can also get more results directly:
+      #
+      #   more_items = items.next_page
+      #
+      # @note This method does not accept a +:page+ option, which means you
+      #   can only start at the begining of the collection and request
+      #   the next page of results.  You can not choose an offset
+      #   or know how many pages of results there will be.
+      #
+      # @params [Hash] options A hash of options that modifies the
+      #   items returned in the page of results.
+      #
+      # @option options [Integer] :per_page (10) The number of results
+      #   to return for each page.
+      #
+      # @option options [String] :next_token (nil) A token that indicates
+      #   an offset to use when paging items.  Next tokens are returned
+      #   by {PageResult#next_token}.  
+      #
+      #   Next tokens should only be consumed by the same collection that
+      #   created them.
+      #
+      def page options = {}
+        
+        each_opts = options.dup
+
+        per_page = each_opts.delete(:per_page)
+        per_page = [nil,''].include?(per_page) ? 10 : per_page.to_i
+
+        each_opts[:limit] = per_page
+
+        items = []
+        next_token = each(each_opts) do |item|
+          items << item
+        end
+
+        Core::PageResult.new(self, items, per_page, next_token)
+
+      end
+  
+    end
+  end
+end

data/lib/aws/core/collection/batchable.rb

@@ -0,0 +1,133 @@
+# Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module AWS
+  module Core
+    module Collection
+      
+      module Batchable
+
+        include Collection
+
+        def each_batch options = {}, &block
+
+          raise NotImplementedError
+
+          each_opts  = options.dup
+
+          limit = each_opts.delete(:limit)
+
+          next_token, skip = each_opts.delete(:next_token)
+
+          total = 0  # count of items yeileded across all batches
+
+          begin
+
+            batch = []
+
+            next_token = _each_item(next_token, each_opts.dup) do |item|
+              total += 1
+              batch << item
+
+              if limit and total == limit
+                yield(batch)
+              end
+
+            end
+
+          end until next_token.nil? or (limit and limit = total)
+
+
+
+
+
+
+
+
+
+          options = options.dup
+
+          limit = options.delete(:limit)
+          batch_size = options.delete(:batch_size)
+          options.delete(:next_token) if [nil, ''].include?(options[:next_token])
+
+          total = 0  # count of items yeileded across all batches
+
+          _each_response(options, limit, batch_size) do |response|
+
+            batch = []
+            each_item(response) do |item|
+              batch << item
+              if limit and (total += 1) == limit
+                yield(batch)
+                return
+              end
+            end
+
+            yield(batch)
+
+            batch.size
+
+          end
+
+        end
+
+        # @note +limit+ is ignored because Batchable colections do not
+        #   accept a +:limit+ option at the service level.
+        # @note +batch_size+ is ignored because Batchable collections do not
+        #   accept a +:batch_size+ option at the service level.
+        protected
+        def _each_response options, limit, batch_size, &block
+  
+          next_token = nil
+  
+          begin
+    
+            page_opts = {}
+            page_opts[next_token_key] = next_token if next_token
+    
+            response = client.send(request_method, options.merge(page_opts))
+    
+            yield(response)
+    
+            next_token = _next_token_for(response)
+    
+          end until next_token.nil?
+          
+          nil
+    
+        end
+    
+        # Override this method in collections that have an alternate method
+        # for finding the next token.  
+        protected
+        def _next_token_for response
+          method = _next_token_key
+          response.respond_to?(method) ? response.send(method) : nil
+        end
+
+        protected
+        def _next_token_key
+          :next_token
+        end
+
+        protected
+        def _each_item next_token, options = {}, &block
+          raise NotImplementedError
+        end
+
+      end
+
+    end
+  end
+end

data/lib/aws/core/collection/limitable.rb

@@ -0,0 +1,92 @@
+# Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module AWS
+  module Core
+    module Collection
+      
+      # AWS::Core::Collection::Limitable is used by collections that
+      # may truncate responses but that also accept a upper limit of
+      # results to return in a single request.
+      #
+      # See {AWS::Core::Collection} for documentation on the available
+      # methods.
+      module Limitable
+
+        include Model
+        include Collection
+        include Enumerable
+
+        def each_batch options = {}, &block
+
+          each_opts  = options.dup
+          limit      = each_opts.delete(:limit) || _limit
+          next_token = each_opts.delete(:next_token)
+          batch_size = each_opts.delete(:batch_size)
+
+          total = 0  # count of items yeileded across all batches
+
+          begin
+
+            max = nil
+            if limit or batch_size
+              max = []
+              max << (limit - total) if limit
+              max << batch_size if batch_size
+              max = max.min
+            end
+
+            batch = []
+            next_token = _each_item(next_token, max, each_opts.dup) do |item|
+
+              total += 1
+              batch << item
+
+            end
+
+            yield(batch)
+
+          end until next_token.nil? or (limit and limit == total)
+
+          next_token
+
+        end
+
+        protected
+        def _each_item next_token, limit, options = {}, &block
+          raise NotImplementedError
+        end
+
+        # This method should be overriden in collection classes
+        # when there is another method to provide a "limit" besides
+        # as an option to the enumerable methods.
+        #
+        # SimpleDB::ItemCollection (for example) allows setting the limit
+        # in a method chain, e.g.
+        #
+        #   collection.limit(10).each {|item| ... }
+        #
+        # These collection classes can simply override #_limit and return
+        # their prefered limit.  This is only called in the abscense of
+        # the +:limit+ option.
+        #
+        # @private
+        protected
+        def _limit
+          nil
+        end
+
+      end
+    end
+  end
+end