fog-aws 0.6.0 → 0.7.2

data/lib/fog/aws/requests/kinesis/merge_shards.rb

@@ -0,0 +1,85 @@
+module Fog
+  module AWS
+    class Kinesis
+      class Real
+        # Merges two adjacent shards in a stream and combines them into a single shard to reduce the stream's capacity to ingest and transport data.
+        #
+        # ==== Options
+        # * AdjacentShardToMerge<~String>: The shard ID of the adjacent shard for the merge.
+        # * ShardToMerge<~String>: The shard ID of the shard to combine with the adjacent shard for the merge.
+        # * StreamName<~String>: The name of the stream for the merge.
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #
+        # ==== See Also
+        # https://docs.aws.amazon.com/kinesis/latest/APIReference/API_MergeShards.html
+        #
+        def merge_shards(options={})
+          body = {
+            "AdjacentShardToMerge" => options.delete("AdjacentShardToMerge"),
+            "ShardToMerge" => options.delete("ShardToMerge"),
+            "StreamName" => options.delete("StreamName")
+          }.reject{ |_,v| v.nil? }
+
+          request({
+                    'X-Amz-Target' => "Kinesis_#{@version}.MergeShards",
+                    :body          => body,
+                  }.merge(options))
+        end
+      end
+
+      class Mock
+        def merge_shards(options={})
+          stream_name = options.delete("StreamName")
+          shard_to_merge_id = options.delete("ShardToMerge")
+          adjacent_shard_to_merge_id = options.delete("AdjacentShardToMerge")
+
+          unless stream = data[:kinesis_streams].detect{ |s| s["StreamName"] == stream_name }
+            raise Fog::AWS::Kinesis::ResourceNotFound.new("Stream #{stream_name} under account #{@account_id} not found.")
+          end
+
+          unless shard_to_merge = stream["Shards"].detect{ |shard| shard["ShardId"] == shard_to_merge_id }
+            raise Fog::AWS::Kinesis::ResourceNotFound.new("Could not find shard #{shard_to_merge_id} in stream #{stream_name} under account #{@account_id}.")
+          end
+
+          unless adjacent_shard_to_merge = stream["Shards"].detect{ |shard| shard["ShardId"] == adjacent_shard_to_merge_id }
+            raise Fog::AWS::Kinesis::ResourceNotFound.new("Could not find shard #{adjacent_shard_to_merge_id} in stream #{stream_name} under account #{@account_id}.")
+          end
+
+          # Close shards (set an EndingSequenceNumber on them)
+          shard_to_merge["SequenceNumberRange"]["EndingSequenceNumber"] = next_sequence_number
+          adjacent_shard_to_merge["SequenceNumberRange"]["EndingSequenceNumber"] = next_sequence_number
+
+          new_starting_hash_key = [
+            shard_to_merge["HashKeyRange"]["StartingHashKey"].to_i,
+            adjacent_shard_to_merge["HashKeyRange"]["StartingHashKey"].to_i
+          ].min.to_s
+
+          new_ending_hash_key = [
+            shard_to_merge["HashKeyRange"]["EndingHashKey"].to_i,
+            adjacent_shard_to_merge["HashKeyRange"]["EndingHashKey"].to_i
+          ].max.to_s
+
+          # create a new shard with ParentShardId and AdjacentParentShardID
+          stream["Shards"] << {
+            "HashKeyRange"=> {
+              "EndingHashKey" => new_ending_hash_key,
+              "StartingHashKey" => new_starting_hash_key
+            },
+            "SequenceNumberRange" => {
+              "StartingSequenceNumber" => next_sequence_number
+            },
+            "ShardId" => next_shard_id,
+            "ParentShardId" => shard_to_merge_id,
+            "AdjacentParentShardId" => adjacent_shard_to_merge_id
+          }
+
+          response = Excon::Response.new
+          response.status = 200
+          response.body = ""
+          response
+        end
+      end
+    end
+  end
+end

data/lib/fog/aws/requests/kinesis/put_record.rb

@@ -0,0 +1,70 @@
+module Fog
+  module AWS
+    class Kinesis
+      class Real
+        # Writes a single data record from a producer into an Amazon Kinesis stream.
+        #
+        # ==== Options
+        # * Data<~Blob>: The data blob to put into the record, which is base64-encoded when the blob is serialized.
+        # * ExplicitHashKey<~String>: The hash value used to determine explicitly the shard that the data record is assigned to by overriding the partition key hash.
+        # * PartitionKey<~String>: Determines which shard in the stream the data record is assigned to.
+        # * SequenceNumberForOrdering<~String>: Guarantees strictly increasing sequence numbers, for puts from the same client and to the same partition key.
+        # * StreamName<~String>: The stream name associated with the request.
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #
+        # ==== See Also
+        # https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecord.html
+        #
+        def put_record(options={})
+          body = {
+            "Data" => options.delete("Data"),
+            "ExplicitHashKey" => options.delete("ExplicitHashKey"),
+            "PartitionKey" => options.delete("PartitionKey"),
+            "SequenceNumberForOrdering" => options.delete("SequenceNumberForOrdering"),
+            "StreamName" => options.delete("StreamName")
+          }.reject{ |_,v| v.nil? }
+
+          response = request({
+                               'X-Amz-Target' => "Kinesis_#{@version}.PutRecord",
+                               :body          => body,
+                             }.merge(options))
+          response.body = Fog::JSON.decode(response.body) unless response.body.nil?
+          response
+        end
+      end
+
+      class Mock
+        def put_record(options={})
+          stream_name = options.delete("StreamName")
+
+          unless stream = data[:kinesis_streams].detect{ |s| s["StreamName"] == stream_name }
+            raise Fog::AWS::Kinesis::ResourceNotFound.new("Stream #{stream_name} under account #{@account_id} not found.")
+          end
+
+          sequence_number = next_sequence_number
+          data = options.delete("Data")
+          partition_key = options.delete("PartitionKey")
+
+          sample_method = RUBY_VERSION == "1.8.7" ? :choice : :sample
+          shard_id = stream["Shards"].send(sample_method)["ShardId"]
+          shard = stream["Shards"].detect{ |shard| shard["ShardId"] == shard_id }
+          # store the records on the shard(s)
+          shard["Records"] << {
+            "SequenceNumber" => sequence_number,
+            "Data" => data,
+            "PartitionKey" => partition_key
+          }
+
+          response = Excon::Response.new
+          response.status = 200
+          response.body = {
+            "SequenceNumber" => sequence_number,
+            "ShardId" => shard_id
+          }
+          response
+        end
+      end
+    end
+  end
+end

data/lib/fog/aws/requests/kinesis/put_records.rb

@@ -0,0 +1,68 @@
+module Fog
+  module AWS
+    class Kinesis
+      class Real
+        # Writes multiple data records from a producer into an Amazon Kinesis stream in a single call (also referred to as a PutRecords request).
+        #
+        # ==== Options
+        # * Records<~Array>: The records associated with the request.
+        #   * Record<~Hash>: A record.
+        #     * Data<~Blob>: The data blob to put into the record, which is base64-encoded when the blob is serialized.
+        #     * ExplicitHashKey<~String>: The hash value used to determine explicitly the shard that the data record is assigned to by overriding the partition key hash.
+        #     * PartitionKey<~String>: Determines which shard in the stream the data record is assigned to.
+        # * StreamName<~String>: The stream name associated with the request.
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #
+        # ==== See Also
+        # https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
+        #
+        def put_records(options={})
+          body = {
+            "Records" => options.delete("Records"),
+            "StreamName" => options.delete("StreamName")
+          }.reject{ |_,v| v.nil? }
+
+          response = request({
+                               'X-Amz-Target' => "Kinesis_#{@version}.PutRecords",
+                               :body          => body,
+                             }.merge(options))
+          response.body = Fog::JSON.decode(response.body) unless response.body.nil?
+          response
+        end
+      end
+
+      class Mock
+        def put_records(options={})
+          stream_name = options.delete("StreamName")
+          unless stream = data[:kinesis_streams].detect{ |s| s["StreamName"] == stream_name }
+            raise Fog::AWS::Kinesis::ResourceNotFound.new("Stream #{stream_name} under account #{@account_id} not found.")
+          end
+
+          records = options.delete("Records")
+          record_results = records.map { |r|
+            sequence_number = next_sequence_number
+
+            sample_method = RUBY_VERSION == "1.8.7" ? :choice : :sample
+            shard_id = stream["Shards"].send(sample_method)["ShardId"]
+            shard = stream["Shards"].detect{ |shard| shard["ShardId"] == shard_id }
+            # store the records on the shard(s)
+            shard["Records"] << r.merge("SequenceNumber" => sequence_number)
+            {
+              "SequenceNumber" => sequence_number,
+              "ShardId" => shard_id
+            }
+          }
+
+          response = Excon::Response.new
+          response.status = 200
+          response.body = {
+            "FailedRecordCount" => 0,
+            "Records" => record_results
+          }
+          response
+        end
+      end
+    end
+  end
+end

data/lib/fog/aws/requests/kinesis/remove_tags_from_stream.rb

@@ -0,0 +1,49 @@
+module Fog
+  module AWS
+    class Kinesis
+      class Real
+        # Deletes tags from the specified Amazon Kinesis stream.
+        #
+        # ==== Options
+        # * StreamName<~String>: The name of the stream.
+        # * TagKeys<~Array>: A list of tag keys.
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #
+        # ==== See Also
+        # https://docs.aws.amazon.com/kinesis/latest/APIReference/API_RemoveTagsFromStream.html
+        #
+        def remove_tags_from_stream(options={})
+          body = {
+            "StreamName" => options.delete("StreamName"),
+            "TagKeys" => options.delete("TagKeys")
+          }.reject{ |_,v| v.nil? }
+
+          request({
+                    'X-Amz-Target' => "Kinesis_#{@version}.RemoveTagsFromStream",
+                    :body          => body,
+                  }.merge(options))
+        end
+      end
+
+      class Mock
+        def remove_tags_from_stream(options={})
+          stream_name = options.delete("StreamName")
+          tags = options.delete("TagKeys")
+
+          unless stream = data[:kinesis_streams].detect{ |s| s["StreamName"] == stream_name }
+            raise Fog::AWS::Kinesis::ResourceNotFound.new("Stream #{stream_name} under account #{@account_id} not found.")
+          end
+
+          stream["Tags"] = stream["Tags"].delete_if { |k,_| tags.include?(k) }
+
+          response = Excon::Response.new
+          response.status = 200
+          response.body = ""
+          response
+        end
+      end
+
+    end
+  end
+end

data/lib/fog/aws/requests/kinesis/split_shard.rb

@@ -0,0 +1,85 @@
+module Fog
+  module AWS
+    class Kinesis
+      class Real
+        # Splits a shard into two new shards in the stream, to increase the stream's capacity to ingest and transport data.
+        #
+        # ==== Options
+        # * NewStartingHashKey<~String>: A hash key value for the starting hash key of one of the child shards created by the split.
+        # * ShardToSplit<~String>: The shard ID of the shard to split.
+        # * StreamName<~String>: The name of the stream for the shard split.
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #
+        # ==== See Also
+        # https://docs.aws.amazon.com/kinesis/latest/APIReference/API_SplitShard.html
+        #
+        def split_shard(options={})
+          body = {
+            "NewStartingHashKey" => options.delete("NewStartingHashKey"),
+            "ShardToSplit" => options.delete("ShardToSplit"),
+            "StreamName" => options.delete("StreamName")
+          }.reject{ |_,v| v.nil? }
+
+          request({
+                    'X-Amz-Target' => "Kinesis_#{@version}.SplitShard",
+                    :body          => body,
+                  }.merge(options))
+        end
+      end
+
+      class Mock
+        def split_shard(options={})
+          stream_name = options.delete("StreamName")
+          shard_id = options.delete("ShardToSplit")
+          stream = data[:kinesis_streams].detect{ |s| s["StreamName"] == stream_name }
+
+          unless stream = data[:kinesis_streams].detect{ |s| s["StreamName"] == stream_name }
+            raise Fog::AWS::Kinesis::ResourceNotFound.new("Stream #{stream_name} under account #{@account_id} not found.")
+          end
+
+          unless shard = stream["Shards"].detect{ |shard| shard["ShardId"] == shard_id }
+            raise Fog::AWS::Kinesis::ResourceNotFound.new("Could not find shard #{shard_id} in stream #{stream_name} under account #{@account_id}.")
+          end
+
+          # Close original shard (set an EndingSequenceNumber on it)
+          shard["SequenceNumberRange"]["EndingSequenceNumber"] = next_sequence_number
+
+          # Calculate new shard ranges
+          parent_starting_hash_key = shard["HashKeyRange"]["StartingHashKey"]
+          parent_ending_hash_key = shard["HashKeyRange"]["EndingHashKey"]
+          new_starting_hash_key = options.delete("NewStartingHashKey")
+
+          # Create two new shards using contiguous hash space based on the original shard
+          stream["Shards"] << {
+            "HashKeyRange"=> {
+              "EndingHashKey" => (new_starting_hash_key.to_i - 1).to_s,
+              "StartingHashKey" => parent_starting_hash_key
+            },
+            "SequenceNumberRange" => {
+              "StartingSequenceNumber" => next_sequence_number
+            },
+            "ShardId" => next_shard_id,
+            "ParentShardId" => shard_id
+          }
+          stream["Shards"] << {
+            "HashKeyRange" => {
+              "EndingHashKey" => parent_ending_hash_key,
+              "StartingHashKey" => new_starting_hash_key
+            },
+            "SequenceNumberRange" =>{
+              "StartingSequenceNumber" => next_sequence_number
+            },
+            "ShardId" => next_shard_id,
+            "ParentShardId" => shard_id
+          }
+
+          response = Excon::Response.new
+          response.status = 200
+          response.body = ""
+          response
+        end
+      end
+    end
+  end
+end

data/lib/fog/aws/storage.rb

@@ -559,16 +559,12 @@ module Fog
             params[:headers]['x-amz-date'] = date.to_iso8601_basic
             if params[:body].respond_to?(:read)
               # See http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+              # We ignore the bit about setting the content-encoding to aws-chunked because
+              # this can cause s3 to serve files with a blank content encoding which causes problems with some CDNs
+              # AWS have confirmed that s3 can infer that the content-encoding is aws-chunked from the x-amz-content-sha256 header
+              #
               params[:headers]['x-amz-content-sha256'] = 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
               params[:headers]['x-amz-decoded-content-length'] = params[:headers].delete 'Content-Length'
-
-              if params[:headers]['Content-Encoding'] && params[:headers]['Content-Encoding'].to_s.length > 0
-                encoding = "aws-chunked,#{params[:headers]['Content-Encoding']}"
-              else
-                encoding = "aws-chunked"
-              end
-
-              params[:headers]['Content-Encoding']  = encoding
             else
               params[:headers]['x-amz-content-sha256'] ||= Digest::SHA256.hexdigest(params[:body] || '')
             end

data/lib/fog/aws/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module AWS
-    VERSION = "0.6.0"
+    VERSION = "0.7.2"
   end
 end

data/tests/helpers/mock_helper.rb

@@ -97,7 +97,6 @@ if Fog.mock?
     :vsphere_username                 => 'apiuser',
     :vsphere_password                 => 'apipassword',
     :vsphere_expected_pubkey_hash     => 'abcdef1234567890',
-    :libvirt_uri                      => 'qemu:///system',
     :libvirt_username                 => 'root',
     :libvirt_password                 => 'password',
     :cloudsigma_username              => 'csuname',

data/tests/requests/compute/route_tests.rb

@@ -25,13 +25,14 @@ Shindo.tests('Fog::Compute[:aws] | route table requests', ['aws']) do
       }],
       'tagSet'         => Hash,
       'routeSet'       => [{
-        'destinationCidrBlock' => String,
-        'gatewayId'            => Fog::Nullable::String,
-        'instanceId'           => Fog::Nullable::String,
-        'instanceOwnerId'      => Fog::Nullable::String,
-        'networkInterfaceId'   => Fog::Nullable::String,
-        'state'                => String,
-        'origin'               => String
+        'destinationCidrBlock'   => String,
+        'gatewayId'              => Fog::Nullable::String,
+        'instanceId'             => Fog::Nullable::String,
+        'instanceOwnerId'        => Fog::Nullable::String,
+        'networkInterfaceId'     => Fog::Nullable::String,
+        'vpcPeeringConnectionId' => Fog::Nullable::String,
+        'state'                  => String,
+        'origin'                 => String
       }],
       'routeTableId'   => String,
       'vpcId'          => String,

data/tests/requests/dns/change_resource_record_sets_tests.rb

@@ -1,10 +1,34 @@
 Shindo.tests('Fog::DNS[:aws] | change_resource_record_sets', ['aws', 'dns']) do
-  @r53_connection = Fog::DNS[:aws]
-
   tests('success') do
     test('#elb_hosted_zone_mapping from DNS name') do
       zone_id = Fog::DNS::AWS.hosted_zone_for_alias_target('arbitrary-sub-domain.eu-west-1.elb.amazonaws.com')
       zone_id == Fog::DNS::AWS.elb_hosted_zone_mapping['eu-west-1']
     end
   end
+
+  tests("#change_resource_record_sets_data formats geolocation properly") do
+    change_batch = [{
+        :action=>"CREATE",
+        :name=>"ark.m.example.net.",
+        :resource_records=>["1.1.1.1"],
+        :ttl=>"300",
+        :type=>"A",
+        :set_identifier=>"ark",
+        :geo_location=>{"CountryCode"=>"US", "SubdivisionCode"=>"AR"},
+        }]
+
+    version = '2013-04-01'
+    result = Fog::DNS::AWS.change_resource_record_sets_data('zone_id123', change_batch, version)
+    doc = Nokogiri::XML(result)
+
+    returns("https://route53.amazonaws.com/doc/#{version}/") { doc.namespaces['xmlns'] }
+    returns(%w[US AR]) {
+      [
+        doc.css("GeoLocation CountryCode").text,
+        doc.css("GeoLocation SubdivisionCode").text
+      ]
+    }
+
+    result
+  end
 end