fake_dynamo 0.1.4 → 0.2.0

data/lib/fake_dynamo/db.rb

@@ -90,26 +90,32 @@ module FakeDynamo
 
     def batch_get_item(data)
       response = {}
+      consumed_capacity = {}
 
       data['RequestItems'].each do |table_name, table_data|
         table = find_table(table_name)
 
         unless response[table_name]
-          response[table_name] = { 'ConsumedCapacityUnits' => 1, 'Items' => [] }
+          response[table_name] = []
+          set_consumed_capacity(consumed_capacity, table, data)
         end
 
         table_data['Keys'].each do |key|
           if item_hash = table.get_raw_item(key, table_data['AttributesToGet'])
-            response[table_name]['Items'] << item_hash
+            response[table_name] << item_hash
           end
         end
       end
 
-      { 'Responses' => response, 'UnprocessedKeys' => {}}
+      response = { 'Responses' => response, 'UnprocessedKeys' => {} }
+      merge_consumed_capacity(consumed_capacity, response)
     end
 
     def batch_write_item(data)
       response = {}
+      consumed_capacity = {}
+      item_collection_metrics = {}
+      merge_metrics = false
       items = {}
       request_count = 0
 
@@ -118,6 +124,7 @@ module FakeDynamo
         table = find_table(table_name)
 
         items[table.name] ||= {}
+        item_collection_metrics[table.name] ||= []
 
         requests.each do |request|
           if request['PutRequest']
@@ -139,21 +146,47 @@ module FakeDynamo
       # real modification
       items.each do |table_name, requests|
         table = find_table(table_name)
+        item_collection_metrics[table.name] ||= []
+
         requests.each do |key, value|
           if value == :delete
             table.batch_delete(key)
           else
             table.batch_put(value)
           end
+
+          unless (metrics = Item.from_key(key).collection_metrics(data)).empty?
+            merge_metrics = true
+            item_collection_metrics[table.name] << metrics['ItemCollectionMetrics']
+          end
+
         end
-        response[table_name] = { 'ConsumedCapacityUnits' => 1 }
+        set_consumed_capacity(consumed_capacity, table, data)
       end
 
-      { 'Responses' => response, 'UnprocessedItems' => {} }
+      response = { 'UnprocessedItems' => {} }
+      response = merge_consumed_capacity(consumed_capacity, response)
+      if merge_metrics
+        response.merge!({'ItemCollectionMetrics' => item_collection_metrics})
+      end
+      response
     end
 
     private
 
+    def set_consumed_capacity(consumed_capacity, table, data)
+      unless (capacity = table.consumed_capacity(data)).empty?
+        consumed_capacity[table.name] = capacity['ConsumedCapacity']
+      end
+    end
+
+    def merge_consumed_capacity(consumed_capacity, response)
+      unless consumed_capacity.empty?
+        response['ConsumedCapacity'] = consumed_capacity.values
+      end
+      response
+    end
+
     def check_item_conflict(items, table_name, key)
       if items[table_name][key]
         raise ValidationException, 'Provided list of item keys contains duplicates'

data/lib/fake_dynamo/item.rb

@@ -104,5 +104,15 @@ module FakeDynamo
         attributes[name] = attribute
       end
     end
+
+    def collection_metrics(data)
+      if data['ReturnItemCollectionMetrics'] == 'SIZE'
+        {  'ItemCollectionMetrics' =>
+          { 'ItemCollectionKey' => key.primary.as_hash,
+            'SizeEstimateRangeGB' => [ 0, 1 ] } }
+      else
+        {}
+      end
+    end
   end
 end

data/lib/fake_dynamo/key.rb

@@ -9,10 +9,10 @@ module FakeDynamo
       def from_data(key_data, key_schema)
         key = Key.new
         validate_key_data(key_data, key_schema)
-        key.primary = Attribute.from_hash(key_schema.hash_key.name, key_data['HashKeyElement'])
+        key.primary = Attribute.from_hash(key_schema.hash_key.name, key_data[key_schema.hash_key.name])
 
         if key_schema.range_key
-          key.range = Attribute.from_hash(key_schema.range_key.name, key_data['RangeKeyElement'])
+          key.range = Attribute.from_hash(key_schema.range_key.name, key_data[key_schema.range_key.name])
         end
         key
       end
@@ -60,14 +60,6 @@ module FakeDynamo
       result
     end
 
-    def as_key_hash
-      result = { 'HashKeyElement' => { @primary.type => @primary.value }}
-      if @range
-        result.merge!({'RangeKeyElement' => { @range.type => @range.value }})
-      end
-      result
-    end
-
     def <=>(other)
       [primary, range] <=> [other.primary, other.range]
     end

data/lib/fake_dynamo/key_schema.rb

@@ -3,23 +3,42 @@ module FakeDynamo
 
     attr_accessor :hash_key, :range_key
 
-    def initialize(data)
-      extract_values(data)
+    def initialize(key_schema, attribute_definitions)
+      extract_values(key_schema, attribute_definitions)
     end
 
     def description
-      description = { 'HashKeyElement' => hash_key.description }
+      description = [{'AttributeName' => hash_key.name, 'KeyType' => 'HASH'}]
       if range_key
-        description['RangeKeyElement'] = range_key.description
+        description << [{'AttributeName' => range_key.name, 'KeyType' => 'RANGE'}]
       end
       description
     end
 
+    def keys
+      result = [hash_key.name]
+      if range_key
+        result << range_key.name
+      end
+      result
+    end
+
     private
-    def extract_values(data)
-      @hash_key = Attribute.from_data(data['HashKeyElement'])
-      if range_key_element = data['RangeKeyElement']
-        @range_key = Attribute.from_data(range_key_element)
+    def extract_values(key_schema, attribute_definitions)
+      hash_key_name = find(key_schema, 'KeyType', 'HASH', 'AttributeName')
+      hash_key_type = find(attribute_definitions, 'AttributeName', hash_key_name, 'AttributeType')
+      @hash_key = Attribute.new(hash_key_name, nil, hash_key_type)
+      if range_key_name = find(key_schema, 'KeyType', 'RANGE', 'AttributeName', false)
+        range_key_type = find(attribute_definitions, 'AttributeName', range_key_name, 'AttributeType')
+        @range_key = Attribute.new(range_key_name, nil, range_key_type)
+      end
+    end
+
+    def find(list, key, value, pluck, raise_on_error = true)
+      if element = list.find { |e| e[key] == value }
+        element[pluck]
+      elsif raise_on_error
+        raise ValidationException, 'Some index key attributes are not defined in AttributeDefinitions'
       end
     end
   end

data/lib/fake_dynamo/local_secondary_index.rb

@@ -0,0 +1,27 @@
+module FakeDynamo
+  class LocalSecondaryIndex
+    extend Validation
+
+    attr_accessor :name, :key_schema, :projection
+
+    class << self
+      def from_data(index_data, attribute_definitions, table_key_schema)
+        index = LocalSecondaryIndex.new
+        index.name = index_data['IndexName']
+        index.key_schema = KeySchema.new(index_data['KeySchema'], attribute_definitions)
+        index.projection = Projection.from_data(index_data['Projection'])
+        validate_range_key(index.key_schema)
+        validate_hash_key(index.key_schema, table_key_schema)
+        index
+      end
+    end
+
+    def description
+      {'IndexName' => name,
+        'IndexSizeBytes' => 0,
+        'ItemCount' => 0,
+        'KeySchema' => key_schema.description,
+        'Projection' => projection.description}
+    end
+  end
+end

data/lib/fake_dynamo/projection.rb

@@ -0,0 +1,30 @@
+module FakeDynamo
+  class Projection
+    extend Validation
+    attr_accessor :type, :non_key_attributes
+
+    def initialize(type, non_key_attributes)
+      @type, @non_key_attributes = type, non_key_attributes
+    end
+
+    class << self
+      def from_data(data)
+        projection = Projection.new(data['ProjectionType'], data['NonKeyAttributes'])
+        validate_projection(projection)
+        projection
+      end
+    end
+
+    def description
+      {'ProjectionType' => type}.merge(non_key_attributes_description)
+    end
+
+    def non_key_attributes_description
+      if non_key_attributes
+        {'NonKeyAttributes' => @non_key_attributes}
+      else
+        {}
+      end
+    end
+  end
+end

data/lib/fake_dynamo/table.rb

@@ -4,8 +4,8 @@ module FakeDynamo
     include Filter
 
     attr_accessor :creation_date_time, :read_capacity_units, :write_capacity_units,
-                  :name, :status, :key_schema, :items, :size_bytes, :last_increased_time,
-                  :last_decreased_time
+                  :name, :status, :attribute_definitions, :key_schema, :items, :size_bytes,
+                  :local_secondary_indexes, :last_increased_time, :last_decreased_time
 
     def initialize(data)
       extract_values(data)
@@ -15,21 +15,48 @@ module FakeDynamo
     def description
       {
         'TableDescription' => {
+          'AttributeDefinitions' => attribute_definitions.map(&:description),
           'CreationDateTime' => creation_date_time,
           'KeySchema' => key_schema.description,
-          'ProvisionedThroughput' => {
-            'ReadCapacityUnits' => read_capacity_units,
-            'WriteCapacityUnits' => write_capacity_units
-          },
+          'ProvisionedThroughput' => throughput_description,
           'TableName' => name,
-          'TableStatus' => status
-        }
+          'TableStatus' => status,
+          'ItemCount' => items.count,
+          'TableSizeBytes' => size_bytes
+        }.merge(local_secondary_indexes_description)
       }
     end
 
+    def throughput_description
+      result = {
+        'NumberOfDecreasesToday' => 0,
+        'ReadCapacityUnits' => read_capacity_units,
+        'WriteCapacityUnits' => write_capacity_units
+      }
+
+      if last_increased_time
+        result['LastIncreaseDateTime'] = @last_increased_time
+      end
+
+      if last_decreased_time
+        result['LastDecreaseDateTime'] = @last_decreased_time
+      end
+
+      result
+    end
+
+    def local_secondary_indexes_description
+      if local_secondary_indexes
+        { 'LocalSecondaryIndexes' => local_secondary_indexes.map(&:description) }
+      else
+        {}
+      end
+    end
+
     def create_table_data
       {
         'TableName' => name,
+        'AttributeDefinitions' => attribute_definitions.map(&:description),
         'KeySchema' => key_schema.description,
         'ProvisionedThroughput' => {
           'ReadCapacityUnits' => read_capacity_units,
@@ -45,13 +72,8 @@ module FakeDynamo
       }
     end
 
-    def size_description
-      { 'ItemCount' => items.count,
-        'TableSizeBytes' => size_bytes }
-    end
-
     def describe_table
-      { 'Table' => description['TableDescription'] }.merge(size_description)
+      { 'Table' => description['TableDescription'] }
     end
 
     def activate
@@ -78,16 +100,7 @@ module FakeDynamo
 
       @read_capacity_units, @write_capacity_units = read_capacity_units, write_capacity_units
 
-      response = description.merge(size_description)
-
-      if last_increased_time
-        response['TableDescription']['ProvisionedThroughput']['LastIncreaseDateTime'] = @last_increased_time
-      end
-
-      if last_decreased_time
-        response['TableDescription']['ProvisionedThroughput']['LastDecreaseDateTime'] = @last_decreased_time
-      end
-
+      response = description
       response['TableDescription']['TableStatus'] = 'UPDATING'
       response
     end
@@ -98,7 +111,7 @@ module FakeDynamo
       check_conditions(old_item, data['Expected'])
       @items[item.key] = item
 
-      consumed_capacity.merge(return_values(data, old_item))
+      return_values(data, old_item).merge(item.collection_metrics(data))
     end
 
     def batch_put_request(data)
@@ -110,7 +123,7 @@ module FakeDynamo
     end
 
     def get_item(data)
-      response = consumed_capacity
+      response = consumed_capacity(data)
       if item_hash = get_raw_item(data['Key'], data['AttributesToGet'])
         response.merge!('Item' => item_hash)
       end
@@ -142,7 +155,12 @@ module FakeDynamo
       check_conditions(item, data['Expected'])
 
       @items.delete(key) if item
-      consumed_capacity.merge(return_values(data, item))
+      if !item
+        item = Item.from_key(key)
+        consumed_capacity(data).merge(item.collection_metrics(data))
+      else
+        return_values(data, item).merge(consumed_capacity(data)).merge(item.collection_metrics(data))
+      end
     end
 
     def batch_delete_request(data)
@@ -159,10 +177,11 @@ module FakeDynamo
       check_conditions(item, data['Expected'])
 
       unless item
+        item = Item.from_key(key)
         if create_item?(data)
-          item = @items[key] = Item.from_key(key)
+          @items[key] = item
         else
-          return consumed_capacity
+          return consumed_capacity(data).merge(item.collection_metrics(data))
         end
         item_created = true
       end
@@ -170,8 +189,10 @@ module FakeDynamo
       old_item = deep_copy(item)
       begin
         old_hash = item.as_hash
-        data['AttributeUpdates'].each do |name, update_data|
-          item.update(name, update_data)
+        if attribute_updates = data['AttributeUpdates']
+          attribute_updates.each do |name, update_data|
+            item.update(name, update_data)
+          end
         end
       rescue => e
         if item_created
@@ -182,7 +203,7 @@ module FakeDynamo
         raise e
       end
 
-      consumed_capacity.merge(return_values(data, old_hash, item))
+      return_values(data, old_hash, item).merge(item.collection_metrics(data))
     end
 
     def deep_copy(x)
@@ -190,66 +211,100 @@ module FakeDynamo
     end
 
     def query(data)
-      unless key_schema.range_key
-        raise ValidationException, "Query can be performed only on a table with a HASH,RANGE key schema"
+      range_key_present
+      select_and_attributes_to_get_present(data)
+      validate_limit(data)
+
+      index = nil
+      if index_name = data['IndexName']
+        index = local_secondary_indexes.find { |i| i.name == index_name }
+        raise ValidationException, "The provided starting key is invalid" unless index
+        schema = index.key_schema
+      else
+        schema = key_schema
       end
 
-      count_and_attributes_to_get_present?(data)
-      validate_limit(data)
+      hash_condition = data['KeyConditions'][schema.hash_key.name]
+      validate_hash_condition(hash_condition)
 
-      hash_attribute = Attribute.from_hash(key_schema.hash_key.name, data['HashKeyValue'])
+      hash_attribute = Attribute.from_hash(schema.hash_key.name, hash_condition['AttributeValueList'].first)
       matched_items = get_items_by_hash_key(hash_attribute)
 
       forward = data.has_key?('ScanIndexForward') ? data['ScanIndexForward'] : true
-      matched_items = drop_till_start(matched_items, data['ExclusiveStartKey'], forward)
+      matched_items = drop_till_start(matched_items, data['ExclusiveStartKey'], forward, schema)
 
-      if data['RangeKeyCondition']
-        conditions = {key_schema.range_key.name => data['RangeKeyCondition']}
+      if !(range_condition = data['KeyConditions'].clone.tap { |h| h.delete(schema.hash_key.name) }).empty?
+        validate_range_condition(range_condition, schema)
+        conditions = range_condition
       else
         conditions = {}
       end
 
-      result, last_evaluated_item, _ = filter(matched_items, conditions, data['Limit'], true)
-
-      response = {
-        'Count' => result.size,
-        'ConsumedCapacityUnits' => 1 }
+      results, last_evaluated_item, _ = filter(matched_items, conditions, data['Limit'], true)
 
-      unless data['Count']
-        response['Items'] = result.map { |r| filter_attributes(r, data['AttributesToGet']) }
-      end
+      response = {'Count' => results.size}.merge(consumed_capacity(data))
+      merge_items(response, data, results, index)
 
       if last_evaluated_item
-        response['LastEvaluatedKey'] = last_evaluated_item.key.as_key_hash
+        response['LastEvaluatedKey'] = last_evaluated_item.key.as_hash
       end
       response
     end
 
     def scan(data)
-      count_and_attributes_to_get_present?(data)
+      select_and_attributes_to_get_present(data)
       validate_limit(data)
       conditions = data['ScanFilter'] || {}
-      all_items = drop_till_start(items.values, data['ExclusiveStartKey'], true)
-      result, last_evaluated_item, scaned_count = filter(all_items, conditions, data['Limit'], false)
+      all_items = drop_till_start(items.values, data['ExclusiveStartKey'], true, key_schema)
+      results, last_evaluated_item, scaned_count = filter(all_items, conditions, data['Limit'], false)
       response = {
-        'Count' => result.size,
-        'ScannedCount' => scaned_count,
-        'ConsumedCapacityUnits' => 1 }
+        'Count' => results.size,
+        'ScannedCount' => scaned_count}.merge(consumed_capacity(data))
 
-      unless data['Count']
-        response['Items'] = result.map { |r| filter_attributes(r, data['AttributesToGet']) }
-      end
+      merge_items(response, data, results)
 
       if last_evaluated_item
-        response['LastEvaluatedKey'] = last_evaluated_item.key.as_key_hash
+        response['LastEvaluatedKey'] = last_evaluated_item.key.as_hash
+      end
+
+      response
+    end
+
+    def merge_items(response, data, results, index = nil)
+      if data['Select'] != 'COUNT'
+        attributes_to_get = nil # select everything
+
+        if data['AttributesToGet']
+          attributes_to_get = data['AttributesToGet']
+        elsif data['Select'] == 'ALL_PROJECTED_ATTRIBUTES'
+          attributes_to_get = projected_attributes(index)
+        end
+
+        response['Items'] = results.map { |r| filter_attributes(r, attributes_to_get) }
       end
 
       response
     end
 
-    def count_and_attributes_to_get_present?(data)
-      if data['Count'] and data['AttributesToGet']
-        raise ValidationException, "Cannot specify the AttributesToGet when choosing to get only the Count"
+    def projected_attributes(index)
+      if !index
+        raise ValidationException, "ALL_PROJECTED_ATTRIBUTES can be used only when Querying using an IndexName"
+      else
+        case index.projection.type
+        when 'ALL'
+          nil
+        when 'KEYS_ONLY'
+          (key_schema.keys + index.key_schema.keys).uniq
+        when 'INCLUDE'
+          (key_schema.keys + index.key_schema.keys + index.projection.non_key_attributes).uniq
+        end
+      end
+    end
+
+    def select_and_attributes_to_get_present(data)
+      select = data['Select']
+      if select and data['AttributesToGet'] and (select != 'SPECIFIC_ATTRIBUTES')
+        raise ValidationException, "Cannot specify the AttributesToGet when choosing to get only the #{select}"
       end
     end
 
@@ -259,7 +314,7 @@ module FakeDynamo
       end
     end
 
-    def drop_till_start(all_items, start_key_hash, forward)
+    def drop_till_start(all_items, start_key_hash, forward, schema)
       all_items = all_items.sort_by { |item| item.key }
 
       unless forward
@@ -267,7 +322,7 @@ module FakeDynamo
       end
 
       if start_key_hash
-        start_key = Key.from_data(start_key_hash, key_schema)
+        start_key = Key.from_data(start_key_hash, schema)
         all_items.drop_while do |item|
           if forward
             item.key <= start_key
@@ -296,6 +351,8 @@ module FakeDynamo
           end
         end
 
+        scaned_count += 1
+
         if select
           result << item
           if (limit -= 1) == 0
@@ -303,8 +360,6 @@ module FakeDynamo
             break
           end
         end
-
-        scaned_count += 1
       end
       [result, last_evaluated_item, scaned_count]
     end
@@ -316,9 +371,13 @@ module FakeDynamo
     end
 
     def create_item?(data)
-      data['AttributeUpdates'].any? do |name, update_data|
-        action = update_data['Action']
-        ['PUT', 'ADD', nil].include? action
+      if attribute_updates = data['AttributeUpdates']
+        attribute_updates.any? do |name, update_data|
+          action = update_data['Action']
+          ['PUT', 'ADD', nil].include? action
+        end
+      else
+        true
       end
     end
 
@@ -352,17 +411,23 @@ module FakeDynamo
                  raise 'unknown return value'
                end
 
-      unless result.empty?
-        { 'Attributes' => result }
+      result = unless result.empty?
+                 { 'Attributes' => result }
+               else
+                 {}
+               end
+
+      result.merge(consumed_capacity(data))
+    end
+
+    def consumed_capacity(data)
+      if data['ReturnConsumedCapacity'] == 'TOTAL'
+        {'ConsumedCapacity' => { 'CapacityUnits' => 1, 'TableName' => @name }}
       else
         {}
       end
     end
 
-    def consumed_capacity
-      { 'ConsumedCapacityUnits' => 1 }
-    end
-
     def check_conditions(old_item, conditions)
       return unless conditions
 
@@ -403,8 +468,12 @@ module FakeDynamo
 
     def extract_values(data)
       @name = data['TableName']
-      @key_schema = KeySchema.new(data['KeySchema'])
+      @key_schema = KeySchema.new(data['KeySchema'], data['AttributeDefinitions'])
+      set_local_secondary_indexes(data)
+      @attribute_definitions = data['AttributeDefinitions'].map(&Attribute.method(:from_data))
       set_throughput(data['ProvisionedThroughput'])
+
+      validate_attribute_definitions
     end
 
     def set_throughput(throughput)
@@ -412,5 +481,34 @@ module FakeDynamo
       @write_capacity_units = throughput['WriteCapacityUnits']
     end
 
+    def set_local_secondary_indexes(data)
+      if indexes_data = data['LocalSecondaryIndexes']
+        @local_secondary_indexes = indexes_data.map do |index|
+          LocalSecondaryIndex.from_data(index, data['AttributeDefinitions'], @key_schema)
+        end
+        validate_range_key(key_schema)
+        validate_index_names(@local_secondary_indexes)
+      end
+    end
+
+    def validate_attribute_definitions
+      attribute_keys = @attribute_definitions.map(&:name)
+      used_keys = @key_schema.keys
+      if @local_secondary_indexes
+        used_keys += @local_secondary_indexes.map(&:key_schema).map(&:keys).flatten
+      end
+
+      used_keys.uniq!
+
+      if used_keys.uniq.size != attribute_keys.size
+        raise ValidationException, "Some AttributeDefinitions are not used AttributeDefinitions: #{attribute_keys.inspect}, keys used: #{used_keys.inspect}"
+      end
+    end
+
+    def range_key_present
+      unless key_schema.range_key
+        raise ValidationException, "Query can be performed only on a table with a HASH,RANGE key schema"
+      end
+    end
   end
 end