aws-sdk-core 2.0.0.rc2 → 2.0.0.rc3

data/apis/source/kinesis-2013-12-02.paginators.json

@@ -0,0 +1,18 @@
+{
+  "pagination": {
+    "GetRecords": {
+      "limit_key": "Limit",
+      "input_token": "ShardIterator",
+      "output_token": "NextShardIterator",
+      "result_key": "Records"
+    },
+    "ListStreams": {
+      "limit_key": "Limit",
+      "input_token": "ExclusiveStartStreamName",
+      "output_token": "NextShardIterator",
+      "output_token": "StreamNames[-1].Key",
+      "more_results": "HasMoreStreams",
+      "result_key": "StreamNames"
+    }
+  }
+}

data/apis/source/rds-2013-01-10.paginators.json

@@ -50,6 +50,7 @@
     },
     "DescribeEngineDefaultParameters": {
       "input_token": "Marker",
+      "output_token": "EngineDefaults.Marker",
       "limit_key": "MaxRecords"
     },
     "DescribeEventSubscriptions": {

data/apis/source/rds-2013-02-12.paginators.json

@@ -50,6 +50,7 @@
     },
     "DescribeEngineDefaultParameters": {
       "input_token": "Marker",
+      "output_token": "EngineDefaults.Marker",
       "limit_key": "MaxRecords"
     },
     "DescribeEventSubscriptions": {

data/apis/source/rds-2013-05-15.paginators.json

@@ -50,6 +50,7 @@
     },
     "DescribeEngineDefaultParameters": {
       "input_token": "Marker",
+      "output_token": "EngineDefaults.Marker",
       "limit_key": "MaxRecords"
     },
     "DescribeEventSubscriptions": {

data/apis/source/redshift-2012-12-01.paginators.json

@@ -44,6 +44,7 @@
     },
     "DescribeDefaultClusterParameters": {
       "input_token": "Marker",
+      "output_token": "Marker",
       "limit_key": "MaxRecords"
     },
     "DescribeEvents": {

data/apis/source/s3-2006-03-01.paginators.json

@@ -38,7 +38,7 @@
     },
     "ListParts": {
       "limit_key": "IsTruncated",
-      "output_tokens": "NextPartNumberMarker",
+      "output_token": "NextPartNumberMarker",
       "input_token": "PartNumberMarker",
       "result_key": "Parts"
     }

data/aws-sdk-core.gemspec

@@ -18,4 +18,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency('multi_json', '~> 1.0')
   spec.add_dependency('multi_xml', '~> 0.5')
   spec.add_dependency('builder', '~> 3.0')
+  spec.add_dependency('jamespath')
 end

data/doc-src/plugins/stability.rb

@@ -0,0 +1 @@
+YARD::Tags::Library.define_tag('STABILITY', :stability)

data/features/glacier/client.feature

@@ -20,6 +20,11 @@ Feature: Amazon Glacier
     When I upload an archive from a 5.5MB large file
     Then I should be able to delete the archive
 
+  @multipart @upload @slow
+  Scenario: Using multipart upload interface
+    When I multipart-upload a 15.5MB file in 1048576 byte chunks
+    Then I should be able to delete the archive
+
   # There was an issue where the client would fail attempting to compute
   # a checksum of an empty body, this step ensures that the checksum
   # computation does not hang.

data/features/glacier/step_definitions.rb

@@ -6,15 +6,66 @@ def upload_glacier_archive(contents)
   @archive_id = resp.data.archive_id if resp.data
 end
 
+def create_file(size_in_mb)
+  file = Tempfile.new('aws-sdk-core-glacier-sample')
+  file.write('.' * bytes(size_in_mb.to_f))
+  file.rewind
+  file
+end
+
+def bytes(megabytes)
+  megabytes.to_f * 1024 * 1024
+end
+
 When(/^I upload an archive with the contents "(.*?)"$/) do |contents|
   upload_glacier_archive(contents)
 end
 
-When(/^I upload an archive from a (\d+\.\d+)MB large file$/) do |size|
-  file = Tempfile.new('aws-sdk-core-glacier-sample')
-  file.write('.' * (1024 * 1024 * size.to_f).to_i)
-  file.rewind
-  upload_glacier_archive(file)
+When(/^I upload an archive from a ([0-9\.]+)MB large file$/) do |size_in_mb|
+  upload_glacier_archive(create_file(size_in_mb))
+end
+
+When(/^I multipart\-upload a ([0-9\.]+)MB file in ([0-9\.]+) byte chunks$/) do |size_in_mb, part_size|
+
+  file = create_file(size_in_mb)
+  part_size = part_size.to_i
+
+  # start the multipart upload
+  resp = @glacier.initiate_multipart_upload(
+    vault_name: @vault_name,
+    part_size: part_size
+  )
+  @upload_id = resp.data.upload_id
+
+  # Keep a rolling tree hash of the entire file, required to complete the
+  # multipart upload at the end
+  tree_hash = Aws::TreeHash.new
+
+  # Upload the file in chunks and extract from each response the tree hash
+  # for that chunk. This eliminates the need to compute the total tree hash
+  # of the object in a second pass.
+  offset = 0
+  until file.eof?
+    chunk = file.read(part_size)
+    resp = @glacier.upload_multipart_part(
+      vault_name: @vault_name,
+      upload_id: @upload_id,
+      body: chunk,
+      range: "bytes #{offset}-#{offset+chunk.bytesize-1}/*"
+    )
+    tree_hash.hashes.concat(resp.context[:tree_hash].hashes)
+    offset += chunk.bytesize
+  end
+
+  # complete the multipart upload
+  resp = @glacier.complete_multipart_upload(
+    vault_name: @vault_name,
+    upload_id: @upload_id,
+    archive_size: file.size,
+    checksum: tree_hash.digest
+  )
+  @archive_id = resp.archive_id
+
 end
 
 Then(/^I should be able to delete the archive$/) do

data/features/kinesis/client.feature

@@ -0,0 +1,18 @@
+# language: en
+@kinesis @client
+Feature: AWS Kinesis
+
+  Scenario: Making a basic request
+    When I call "list_streams" on "kinesis"
+    Then the response "stream_names" should be an array
+
+  Scenario: Error handling
+    Given I call "describe_stream" on "kinesis" with:
+    """
+    { stream_name: 'bogus-stream-name' }
+    """
+    Then I expect the response error code to be "ResourceNotFoundException"
+    And I expect the response error message to include:
+    """
+    Stream bogus-stream-name under account
+    """

data/features/kinesis/setup.rb

@@ -0,0 +1,6 @@
+Before("@kinesis") do
+  @kinesis = Aws.kinesis
+end
+
+After("@kinesis") do
+end

data/features/s3/buckets.feature

@@ -20,3 +20,13 @@ Feature: S3 Buckets
     Given I am using the S3 "us-east-1" region
     When I create a bucket with the location constraint "EU"
     Then the bucket should have a location constraint of "EU"
+
+  Scenario: Working with bucket names that contain '.'
+    Given I am using the S3 "https://s3.amazonaws.com" endpoint
+    When I create a bucket with a DNS compatible name that contains a dot
+    Then I should be able to delete the bucket
+
+  Scenario: Working with bucket names that contain '.'
+    Given I am using the S3 "http://s3.amazonaws.com" endpoint
+    When I create a bucket with a DNS compatible name that contains a dot
+    Then I should be able to delete the bucket

data/features/s3/objects.feature

@@ -16,3 +16,12 @@ Feature: S3 Objects
   Scenario: Putting a file to an object
     When I put the test png to the key "img"
     Then the object with the key "img" should have a content length of 976
+
+  @paging
+  Scenario: Paging responses
+    Given I put nothing to the key "photos/camping/cascades.jpg"
+    Given I put nothing to the key "photos/skiing/stevens.jpg"
+    And I put nothing to the key "photos/family.jpg"
+    And I put nothing to the key "photos/friends.jpg"
+    When I page s3 objects prefixed "photos/" delimited "/" limit 1
+    Then I should have received 4 responses

data/features/s3/step_definitions.rb

@@ -63,3 +63,32 @@ Then(/^the object with the key "(.*?)" should have a content length of (\d+)$/)
   resp = @s3.head_object(bucket: @bucket_name, key: key)
   expect(resp.data.content_length).to eq(size.to_i)
 end
+
+When(/^I page s3 objects prefixed "(.*?)" delimited "(.*?)" limit (\d+)$/) do |prefix, delimiter, max_keys|
+  @responses = []
+  @s3.list_objects(
+    bucket: @bucket_name,
+    prefix: prefix,
+    delimiter: delimiter,
+    max_keys: max_keys
+  ).each do |resp|
+    @responses << resp
+  end
+end
+
+Then(/^I should have received (\d+) responses$/) do |count|
+  expect(@responses.size).to eq(count.to_i)
+end
+
+Given(/^I am using the S3 "(.*?)" endpoint$/) do |endpoint|
+  @s3 = Aws.s3(endpoint: endpoint)
+end
+
+When(/^I create a bucket with a DNS compatible name that contains a dot$/) do
+  @bucket_name = "aws.#{Time.now.to_i}.sdk"
+  @s3.create_bucket(bucket: @bucket_name)
+end
+
+Then(/^I should be able to delete the bucket$/) do
+  @s3.delete_bucket(bucket: @bucket_name)
+end

data/lib/aws.rb

@@ -12,11 +12,13 @@ module Aws
   autoload :Errors, "#{SRC}/errors"
   autoload :ErrorHandler, "#{SRC}/error_handler"
   autoload :InstanceProfileCredentials, "#{SRC}/instance_profile_credentials"
+  autoload :PageableResponse, "#{SRC}/pageable_response"
   autoload :RequestHandler, "#{SRC}/request_handler"
   autoload :ResponseHandler, "#{SRC}/response_handler"
   autoload :Service, "#{SRC}/service"
   autoload :Structure, "#{SRC}/structure"
   autoload :TimestampFormatter, "#{SRC}/timestamp_formatter"
+  autoload :TreeHash, "#{SRC}/tree_hash"
   autoload :Util, "#{SRC}/util"
   autoload :VERSION, "#{SRC}/version"
 
@@ -28,6 +30,7 @@ module Aws
     autoload :InputShapeTranslator, "#{SRC}/api/shape_translator"
     autoload :OperationTranslator, "#{SRC}/api/operation_translator"
     autoload :OutputShapeTranslator, "#{SRC}/api/shape_translator"
+    autoload :PaginationTranslator, "#{SRC}/api/pagination_translator"
     autoload :ServiceNamer, "#{SRC}/api/service_namer"
     autoload :ShapeTranslator, "#{SRC}/api/shape_translator"
     autoload :Translator, "#{SRC}/api/translator"
@@ -66,6 +69,7 @@ module Aws
     autoload :JsonRpcHeaders, "#{SRC}/plugins/json_rpc_headers"
     autoload :QueryProtocol, "#{SRC}/plugins/query_protocol"
     autoload :RegionalEndpoint, "#{SRC}/plugins/regional_endpoint"
+    autoload :ResponsePaging, "#{SRC}/plugins/response_paging"
     autoload :RetryErrors, "#{SRC}/plugins/retry_errors"
     autoload :S3BucketDns, "#{SRC}/plugins/s3_bucket_dns"
     autoload :S3CompleteMultipartUploadFix, "#{SRC}/plugins/s3_complete_multipart_upload_fix"
@@ -112,7 +116,14 @@ module Aws
 
     # @return [Hash] Returns a hash of default configuration options shared
     #   by all constructed clients.
-    attr_accessor :config
+    attr_reader :config
+
+    def config=(config)
+      unless Hash === config
+        raise ArgumentError, 'configuration object must be a hash'
+      end
+      @config = config
+    end
 
     # Adds a plugin to every AWS client class.  This registers the plugin
     # with each versioned client for each service.
@@ -144,7 +155,7 @@ module Aws
     # (class name) for the new service class and a list of client API
     # versions.
     #
-    #     Aws.add_servcie(:S3, ['apis/S3-2006-03-01.json'])
+    #     Aws.add_service(:S3, ['apis/S3-2006-03-01.json'])
     #
     # This method is called for each service defined in the apis directory of
     # this project.

data/lib/aws/api/pagination_translator.rb

@@ -0,0 +1,71 @@
+require 'multi_json'
+
+module Aws
+  module Api
+    class PaginationTranslator
+
+      include Util
+
+      def initialize(api)
+        @api = api
+      end
+
+      def apply(pagination)
+        pagination.each_pair do |operation_name, config|
+          if operation = @api.operations[underscore(operation_name)]
+            apply_paging(operation, config)
+          end
+        end
+      end
+
+      private
+
+      def apply_paging(operation, config)
+        if config && config['input_token']
+          paging = {}
+          apply_tokens(paging, config)
+          apply_truncated(paging, config)
+          operation.metadata['paging'] = paging
+        end
+      end
+
+      def apply_tokens(paging, config)
+        paging['tokens'] = extract_tokens(config)
+      end
+
+      def apply_truncated(paging, config)
+        if truncated_if = config['more_results']
+          paging['truncated_if'] = underscore(truncated_if)
+        end
+      end
+
+      def extract_tokens(config)
+        input = Array(config['input_token'])
+        output = Array(config['output_token'])
+        (0..(input.size-1)).inject({}) do |tokens, n|
+          tokens[underscore(input[n])] = underscore_jamespath(output[n])
+          tokens
+        end
+      end
+
+      def underscore_jamespath(expression)
+        expression.gsub(/\w+/) { |w| underscore(w) }.gsub(/or/, '||')
+      end
+
+      class << self
+
+        def translate(api)
+          filename = api.metadata['endpoint_prefix']
+          filename += "-#{api.version}.paginators.json"
+          path = File.join(Aws::GEM_ROOT, 'apis', 'source',filename)
+          if File.exists?(path)
+            new(api).apply(MultiJson.load(File.read(path))['pagination'])
+          else
+            puts "Missing pagination config for #{path}"
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/aws/api/service_translators/glacier.rb

@@ -6,7 +6,6 @@ module Aws::Api::ServiceTranslators::Glacier
 
     def translate(api)
       apply_plugins(api)
-      remove_checksum(api)
       convert_types(api)
     end
 
@@ -16,12 +15,6 @@ module Aws::Api::ServiceTranslators::Glacier
       api.plugins << "Aws::Plugins::GlacierChecksums"
     end
 
-    # this checksum parameter - it must be calculated on xml that is not user
-    # supplied, so the sdk has to compute the checksum after building the xml
-    def remove_checksum(api)
-      api.operations[:complete_multipart_upload].input.members.delete(:checksum)
-    end
-
     # Shapes that are bound to headers and the request URI are all typed as
     # strings, even when they *should* be integers or dates, crawling the
     # api to correct these

data/lib/aws/api/translator.rb

@@ -13,6 +13,7 @@ module Aws
         Aws::Plugins::GlobalConfiguration
         Aws::Plugins::RegionalEndpoint
         Aws::Plugins::InstanceProfileCredentials
+        Aws::Plugins::ResponsePaging
         Aws::Plugins::Credentials
       )
 
@@ -26,6 +27,7 @@ module Aws
           apply_xml_namespaces(api)
           set_service_names(api)
           apply_service_customizations(api)
+          apply_paging_metadata(api)
           sort_metadata_keys(api)
         end
       end
@@ -77,6 +79,10 @@ module Aws
         end
       end
 
+      def apply_paging_metadata(api)
+        PaginationTranslator.translate(api)
+      end
+
       def service_namer(api)
         args = []
         args << api.metadata['endpoint_prefix']
@@ -100,6 +106,7 @@ module Aws
       metadata :xmlnamespace
 
       ignore :global_endpoint
+      metadata :regional_endpoints
 
       def set_type(type)
         plugins = @properties['plugins'] ||= []
@@ -135,15 +142,15 @@ module Aws
       end
 
       def set_endpoint_prefix(prefix)
-        data = MultiJson.load(File.read('apis/configuration/endpoints.json'))
-        region_names = data['services'][prefix]
+        data = MultiJson.load(File.read(File.join(GEM_ROOT, 'apis', 'configuration', 'endpoints.json')))
+        region_names = data['services'][prefix] || []
         regions = {}
         region_names.each do |region|
           regions[region] = data['regions'][region][prefix]['hostname']
         end
         @properties['metadata'] ||= {}
         @properties['metadata']['endpoint_prefix'] = prefix
-        @properties['metadata']['regional_endpoints'] = regions
+        @properties['metadata']['regional_endpoints'] = regions unless regions.empty?
       end
 
       def set_operations(operations)