fog-aws 0.6.0 → 0.7.2

data/tests/requests/emr/helper.rb

@@ -157,7 +157,6 @@ class AWS
             }],
             'MasterInstanceType' => String,
             'SlaveInstanceType' => String,
-            'InstanceGroups' => Array,
             'TerminationProtected' => String,
             'HadoopVersion' => String
           }

data/tests/requests/kinesis/helper.rb

@@ -0,0 +1,111 @@
+class AWS
+  module Kinesis
+
+    def wait_for(&block)
+      Fog.wait_for do
+        block.call.tap do
+          print '.'
+        end
+      end
+    end
+
+    def wait_for_status(stream_name, status)
+      wait_for do
+        Fog::AWS[:kinesis].describe_stream("StreamName" => stream_name).body["StreamDescription"]["StreamStatus"] == status
+      end
+    end
+
+    def delete_if_exists(stream_name)
+      if Fog::AWS[:kinesis].list_streams.body["StreamNames"].include?(stream_name)
+        wait_for_status(stream_name, "ACTIVE")
+        Fog::AWS[:kinesis].delete_stream("StreamName" => @stream_id)
+        wait_for do
+          begin
+            Fog::AWS[:kinesis].describe_stream("StreamName" => stream_name)
+            false
+          rescue Fog::AWS::Kinesis::ResourceNotFound
+            true
+          end
+        end
+      end
+    end
+
+    module Formats # optional keys are commented out
+
+      LIST_STREAMS_FORMAT = {
+        "HasMoreStreams" => Fog::Boolean,
+        "StreamNames" => [
+          String
+        ]
+      }
+
+      DESCRIBE_STREAM_FORMAT = {
+        "StreamDescription" => {
+          "HasMoreShards" => Fog::Boolean,
+          "Shards" => [
+            {
+              #"AdjacentParentShardId" => String,
+              "HashKeyRange" => {
+                "EndingHashKey" => String,
+                "StartingHashKey" => String
+              },
+              #"ParentShardId" => String,
+              "SequenceNumberRange" => {
+                # "EndingSequenceNumber" => String,
+                "StartingSequenceNumber" => String
+              },
+              "ShardId" => String
+            }
+          ],
+          "StreamARN" => String,
+          "StreamName" => String,
+          "StreamStatus" => String
+        }
+      }
+
+      GET_SHARD_ITERATOR_FORMAT = {
+        "ShardIterator" => String
+      }
+
+      PUT_RECORDS_FORMAT = {
+        "FailedRecordCount" => Integer,
+        "Records" => [
+          {
+            # "ErrorCode" => String,
+            # "ErrorMessage" => String,
+            "SequenceNumber" => String,
+            "ShardId" => String
+          }
+        ]
+      }
+
+      PUT_RECORD_FORMAT = {
+        "SequenceNumber" => String,
+        "ShardId" => String
+      }
+
+      GET_RECORDS_FORMAT = {
+        "MillisBehindLatest" => Integer,
+        "NextShardIterator" => String,
+        "Records" => [
+          {
+            "Data" => String,
+            "PartitionKey" => String,
+            "SequenceNumber" => String
+          }
+        ]
+      }
+
+      LIST_TAGS_FOR_STREAM_FORMAT = {
+        "HasMoreTags" => Fog::Boolean,
+        "Tags" => [
+          {
+            "Key" => String,
+            "Value" => String
+          }
+        ]
+      }
+
+    end
+  end
+end

data/tests/requests/kinesis/stream_tests.rb

@@ -0,0 +1,169 @@
+include AWS::Kinesis
+
+Shindo.tests('AWS::Kinesis | stream requests', ['aws', 'kinesis']) do
+  Fog::AWS[:kinesis].reset_data if Fog.mocking?
+
+  @stream_id = 'fog-test-stream'
+
+  delete_if_exists(@stream_id) # ensure we start from a clean slate
+
+  tests("#create_stream").returns("") do
+    Fog::AWS[:kinesis].create_stream("StreamName" => @stream_id).body.tap do
+      wait_for_status(@stream_id, "ACTIVE")
+    end
+  end
+
+  tests("#list_streams").formats(Formats::LIST_STREAMS_FORMAT, false) do
+    Fog::AWS[:kinesis].list_streams.body.tap do
+      returns(true) {
+        Fog::AWS[:kinesis].list_streams.body["StreamNames"].include?(@stream_id)
+      }
+    end
+  end
+
+  tests("#describe_stream") do
+    tests("success").formats(AWS::Kinesis::Formats::DESCRIBE_STREAM_FORMAT) do
+      Fog::AWS[:kinesis].describe_stream("StreamName" => @stream_id).body
+    end
+
+    tests("ResourceNotFound").raises(Fog::AWS::Kinesis::ResourceNotFound) do
+      Fog::AWS[:kinesis].describe_stream("StreamName" => @stream_id + "-foo").body
+    end
+  end
+
+  tests("#put_records") do
+    records = [
+      {
+        "Data" => Base64.encode64("foo").chomp!,
+        "PartitionKey" => "1"
+      },
+      {
+        "Data" => Base64.encode64("bar").chomp!,
+        "PartitionKey" => "1"
+      }
+    ]
+
+    tests("success").formats(AWS::Kinesis::Formats::PUT_RECORDS_FORMAT, false) do
+      Fog::AWS[:kinesis].put_records("StreamName" => @stream_id, "Records" => records).body
+    end
+
+    tests("ResourceNotFound").raises(Fog::AWS::Kinesis::ResourceNotFound) do
+      Fog::AWS[:kinesis].put_records("StreamName" => @stream_id + "-foo", "Records" => records).body
+    end
+
+  end
+
+  tests("#put_record").formats(AWS::Kinesis::Formats::PUT_RECORD_FORMAT) do
+    Fog::AWS[:kinesis].put_record("StreamName" => @stream_id, "Data" => Base64.encode64("baz").chomp!, "PartitionKey" => "1").body
+  end
+
+  tests("#get_shard_iterator").formats(AWS::Kinesis::Formats::GET_SHARD_ITERATOR_FORMAT) do
+    first_shard_id = Fog::AWS[:kinesis].describe_stream("StreamName" => @stream_id).body["StreamDescription"]["Shards"].first["ShardId"]
+    Fog::AWS[:kinesis].get_shard_iterator("StreamName" => @stream_id, "ShardId" => first_shard_id, "ShardIteratorType" => "TRIM_HORIZON").body
+  end
+
+  tests("#get_records").formats(AWS::Kinesis::Formats::GET_RECORDS_FORMAT) do
+    first_shard_id = Fog::AWS[:kinesis].describe_stream("StreamName" => @stream_id).body["StreamDescription"]["Shards"].first["ShardId"]
+    shard_iterator = Fog::AWS[:kinesis].get_shard_iterator("StreamName" => @stream_id, "ShardId" => first_shard_id, "ShardIteratorType" => "TRIM_HORIZON").body["ShardIterator"]
+    Fog::AWS[:kinesis].get_records("ShardIterator" => shard_iterator, "Limit" => 1).body
+  end
+
+  tests("#get_records").returns(["foo", "bar"]) do
+    first_shard_id = Fog::AWS[:kinesis].describe_stream("StreamName" => @stream_id).body["StreamDescription"]["Shards"].first["ShardId"]
+    shard_iterator = Fog::AWS[:kinesis].get_shard_iterator("StreamName" => @stream_id, "ShardId" => first_shard_id, "ShardIteratorType" => "TRIM_HORIZON").body["ShardIterator"]
+
+    data = []
+    2.times do
+      response = Fog::AWS[:kinesis].get_records("ShardIterator" => shard_iterator, "Limit" => 1).body
+      response["Records"].each do |record|
+        data << Base64.decode64(record["Data"])
+      end
+      shard_iterator = response["NextShardIterator"]
+    end
+    data
+  end
+
+  tests("#split_shard").returns("") do
+    shard = Fog::AWS[:kinesis].describe_stream("StreamName" => @stream_id).body["StreamDescription"]["Shards"].first
+    shard_id = shard["ShardId"]
+    ending_hash_key = shard["HashKeyRange"]["EndingHashKey"]
+    new_starting_hash_key = (ending_hash_key.to_i / 2).to_s
+
+    result = Fog::AWS[:kinesis].split_shard("StreamName" => @stream_id, "ShardToSplit" => shard_id, "NewStartingHashKey" => new_starting_hash_key).body
+
+    wait_for_status(@stream_id, "ACTIVE")
+    shards = Fog::AWS[:kinesis].describe_stream("StreamName" => @stream_id).body["StreamDescription"]["Shards"]
+    parent_shard = shards.detect{ |shard| shard["ShardId"] == shard_id }
+    child_shards = shards.select{ |shard| shard["ParentShardId"] == shard_id }.sort_by{ |shard| shard["ShardId"] }
+
+    returns(3) { shards.size }
+    returns(2) { child_shards.size }
+    # parent is closed
+    returns(false) { parent_shard["SequenceNumberRange"]["EndingSequenceNumber"].nil? }
+
+    # ensure new ranges are what we expect (mostly for testing the mock)
+    returns([
+              {
+                "StartingHashKey" => "0",
+                "EndingHashKey" => (new_starting_hash_key.to_i - 1).to_s
+              },
+              {
+                "StartingHashKey" => new_starting_hash_key,
+                "EndingHashKey" => ending_hash_key
+              }
+            ]) { child_shards.map{ |shard| shard["HashKeyRange"] } }
+
+    result
+  end
+
+  tests("#merge_shards").returns("") do
+    shards = Fog::AWS[:kinesis].describe_stream("StreamName" => @stream_id).body["StreamDescription"]["Shards"]
+    child_shard_ids = shards.reject{ |shard| shard["SequenceNumberRange"].has_key?("EndingSequenceNumber") }.map{ |shard| shard["ShardId"] }.sort
+    result = Fog::AWS[:kinesis].merge_shards("StreamName" => @stream_id, "ShardToMerge" => child_shard_ids[0], "AdjacentShardToMerge" => child_shard_ids[1]).body
+
+    wait_for_status(@stream_id, "ACTIVE")
+    shards = Fog::AWS[:kinesis].describe_stream("StreamName" => @stream_id).body["StreamDescription"]["Shards"]
+    parent_shards = shards.select{ |shard| child_shard_ids.include?(shard["ShardId"]) }
+    child_shard = shards.detect{ |shard|
+      shard["ParentShardId"] == child_shard_ids[0] &&
+      shard["AdjacentParentShardId"] == child_shard_ids[1]
+    }
+
+    returns(2) { parent_shards.size }
+    returns(false) { child_shard.nil? }
+    returns({
+              "EndingHashKey" => "340282366920938463463374607431768211455",
+              "StartingHashKey" => "0"
+            }) {
+      child_shard["HashKeyRange"]
+    }
+
+    result
+  end
+
+  tests("#add_tags_to_stream").returns("") do
+    Fog::AWS[:kinesis].add_tags_to_stream("StreamName" => @stream_id, "Tags" => {"a" => "1", "b" => "2"}).body
+  end
+
+  tests("#list_tags_for_stream").formats(AWS::Kinesis::Formats::LIST_TAGS_FOR_STREAM_FORMAT) do
+    Fog::AWS[:kinesis].list_tags_for_stream("StreamName" => @stream_id).body.tap do |body|
+      returns({"a" => "1", "b" => "2"}) {
+        body["Tags"].inject({}){ |m, tag| m.merge(tag["Key"] => tag["Value"]) }
+      }
+    end
+  end
+
+  tests("#remove_tags_from_stream").returns("") do
+    Fog::AWS[:kinesis].remove_tags_from_stream("StreamName" => @stream_id, "TagKeys" => %w[b]).body.tap do
+      returns({"a" => "1"}) {
+        body = Fog::AWS[:kinesis].list_tags_for_stream("StreamName" => @stream_id).body
+        body["Tags"].inject({}){ |m, tag| m.merge(tag["Key"] => tag["Value"]) }
+      }
+    end
+  end
+
+  tests("#delete_stream").returns("") do
+    Fog::AWS[:kinesis].delete_stream("StreamName" => @stream_id).body
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fog-aws
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.2
 platform: ruby
 authors:
 - Josh Lane
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-23 00:00:00.000000000 Z
+date: 2015-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -168,6 +168,7 @@ files:
 - lib/fog/aws/iam/default_policies.rb
 - lib/fog/aws/iam/default_policy_versions.json
 - lib/fog/aws/iam/paged_collection.rb
+- lib/fog/aws/kinesis.rb
 - lib/fog/aws/kms.rb
 - lib/fog/aws/lambda.rb
 - lib/fog/aws/mock.rb
@@ -1108,6 +1109,19 @@ files:
 - lib/fog/aws/requests/iam/update_user.rb
 - lib/fog/aws/requests/iam/upload_server_certificate.rb
 - lib/fog/aws/requests/iam/upload_signing_certificate.rb
+- lib/fog/aws/requests/kinesis/add_tags_to_stream.rb
+- lib/fog/aws/requests/kinesis/create_stream.rb
+- lib/fog/aws/requests/kinesis/delete_stream.rb
+- lib/fog/aws/requests/kinesis/describe_stream.rb
+- lib/fog/aws/requests/kinesis/get_records.rb
+- lib/fog/aws/requests/kinesis/get_shard_iterator.rb
+- lib/fog/aws/requests/kinesis/list_streams.rb
+- lib/fog/aws/requests/kinesis/list_tags_for_stream.rb
+- lib/fog/aws/requests/kinesis/merge_shards.rb
+- lib/fog/aws/requests/kinesis/put_record.rb
+- lib/fog/aws/requests/kinesis/put_records.rb
+- lib/fog/aws/requests/kinesis/remove_tags_from_stream.rb
+- lib/fog/aws/requests/kinesis/split_shard.rb
 - lib/fog/aws/requests/kms/create_key.rb
 - lib/fog/aws/requests/kms/describe_key.rb
 - lib/fog/aws/requests/kms/list_keys.rb
@@ -1497,6 +1511,8 @@ files:
 - tests/requests/iam/server_certificate_tests.rb
 - tests/requests/iam/user_policy_tests.rb
 - tests/requests/iam/user_tests.rb
+- tests/requests/kinesis/helper.rb
+- tests/requests/kinesis/stream_tests.rb
 - tests/requests/kms/helper.rb
 - tests/requests/kms/key_tests.rb
 - tests/requests/lambda/function_sample_1.js
@@ -1566,9 +1582,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Module for the 'fog' gem to support Amazon Web Services.
 test_files: []
-has_rdoc: