dynamoid 1.3.4 → 2.0.0

data/Rakefile

@@ -18,30 +18,6 @@ RSpec::Core::RakeTask.new(:spec) do |spec|
   spec.pattern = FileList["spec/**/*_spec.rb"]
 end
 
-RSpec::Core::RakeTask.new(:rcov) do |spec|
-  spec.pattern = "spec/**/*_spec.rb"
-  spec.rcov = true
-end
-
-desc "Start DynamoDBLocal, run tests, clean up"
-task :unattended_spec do |t|
-
-  if system("bin/start_dynamodblocal")
-    puts "DynamoDBLocal started; proceeding with specs."
-  else
-    raise "Unable to start DynamoDBLocal.  Cannot run unattended specs."
-  end
-
-  #Cleanup
-  at_exit do
-    unless system("bin/stop_dynamodblocal")
-      $stderr.puts "Unable to cleanly stop DynamoDBLocal."
-    end
-  end
-
-  Rake::Task["spec"].invoke
-end
-
 require "yard"
 YARD::Rake::YardocTask.new do |t|
   t.files   = ["lib/**/*.rb", "README", "LICENSE"]   # optional

data/docker-compose.yml

@@ -0,0 +1,7 @@
+version: '2'
+
+services:
+  dynamodb:
+    image: deangiberson/aws-dynamodb-local
+    ports:
+      - 8000:8000

data/dynamoid.gemspec

@@ -50,13 +50,12 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency(%q<activesupport>, [">= 4"])
   spec.add_runtime_dependency(%q<aws-sdk-resources>, ["~> 2"])
   spec.add_runtime_dependency(%q<concurrent-ruby>, [">= 1.0"])
-  spec.add_development_dependency "pry", "~> 0.10"
+  spec.add_development_dependency "pry"
   spec.add_development_dependency "bundler", "~> 1.14"
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "appraisal", "~> 2.1"
   spec.add_development_dependency "wwtd", "~> 1.3"
   spec.add_development_dependency(%q<yard>, [">= 0"])
-  spec.add_development_dependency(%q<coveralls>, [">= 0"])
-  spec.add_development_dependency(%q<rspec-retry>, [">= 0"])
+  spec.add_development_dependency "coveralls", "~> 0.8"
 end

data/gemfiles/rails_4_0.gemfile

@@ -3,7 +3,6 @@
 source "https://rubygems.org"
 
 gem "rails", "~> 4.0.0"
-# Still tested against Ruby 2.0.0, which can't install nokogiri 1.7+
-gem "nokogiri", "~> 1.6.8.1"
+gem "nokogiri", "~> 1.6.8"
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_4_1.gemfile

@@ -3,7 +3,6 @@
 source "https://rubygems.org"
 
 gem "rails", "~> 4.1.0"
-# Still tested against Ruby 2.0.0, which can't install nokogiri 1.7+
-gem "nokogiri", "~> 1.6.8.1"
+gem "nokogiri", "~> 1.6.8"
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_4_2.gemfile

@@ -3,7 +3,6 @@
 source "https://rubygems.org"
 
 gem "rails", "~> 4.2.0"
-# Still tested against Ruby 2.0.0, which can't install nokogiri 1.7+
-gem "nokogiri", "~> 1.6.8.1"
+gem "nokogiri", "~> 1.6.8"
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_5_0.gemfile

@@ -4,4 +4,4 @@ source "https://rubygems.org"
 
 gem "rails", "~> 5.0.0"
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_5_1.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "rails", "~> 5.1.0"
+
+gemspec path: "../"

data/lib/dynamoid.rb

@@ -1,36 +1,36 @@
-require "delegate"
-require "time"
-require "securerandom"
-require "active_support"
-require "active_support/core_ext"
-require "active_support/json"
-require "active_support/inflector"
-require "active_support/lazy_load_hooks"
-require "active_support/time_with_zone"
-require "active_model"
-
-require "dynamoid/version"
-require "dynamoid/errors"
-require "dynamoid/fields"
-require "dynamoid/indexes"
-require "dynamoid/associations"
-require "dynamoid/persistence"
-require "dynamoid/dirty"
-require "dynamoid/validations"
-require "dynamoid/criteria"
-require "dynamoid/finders"
-require "dynamoid/identity_map"
-require "dynamoid/config"
-require "dynamoid/components"
-require "dynamoid/document"
-require "dynamoid/adapter"
-
-require "dynamoid/tasks/database"
-
-require "dynamoid/middleware/identity_map"
+require 'delegate'
+require 'time'
+require 'securerandom'
+require 'active_support'
+require 'active_support/core_ext'
+require 'active_support/json'
+require 'active_support/inflector'
+require 'active_support/lazy_load_hooks'
+require 'active_support/time_with_zone'
+require 'active_model'
+
+require 'dynamoid/version'
+require 'dynamoid/errors'
+require 'dynamoid/fields'
+require 'dynamoid/indexes'
+require 'dynamoid/associations'
+require 'dynamoid/persistence'
+require 'dynamoid/dirty'
+require 'dynamoid/validations'
+require 'dynamoid/criteria'
+require 'dynamoid/finders'
+require 'dynamoid/identity_map'
+require 'dynamoid/config'
+require 'dynamoid/components'
+require 'dynamoid/document'
+require 'dynamoid/adapter'
+
+require 'dynamoid/tasks/database'
+
+require 'dynamoid/middleware/identity_map'
 
 if defined?(Rails)
-  require "dynamoid/railtie"
+  require 'dynamoid/railtie'
 end
 
 module Dynamoid

data/lib/dynamoid/adapter.rb

@@ -99,13 +99,13 @@ module Dynamoid
     # @param [Array] range_key of the record to delete, can also be a string of just one range_key
     #
     def delete(table, ids, options = {})
-      range_key = options[:range_key] #array of range keys that matches the ids passed in
+      range_key = options[:range_key] # array of range keys that matches the ids passed in
       if ids.respond_to?(:each)
         if range_key.respond_to?(:each)
-          #turn ids into array of arrays each element being hash_key, range_key
-          ids = ids.each_with_index.map{|id,i| [id,range_key[i]]}
+          # turn ids into array of arrays each element being hash_key, range_key
+          ids = ids.each_with_index.map{|id, i| [id, range_key[i]]}
         else
-          ids = range_key ? [[ids, range_key]] : ids
+          ids = range_key ? ids.map { |id| [id, range_key] } : ids
         end
 
         batch_delete_item(table => ids)
@@ -120,7 +120,7 @@ module Dynamoid
     # @param [Hash] scan_hash a hash of attributes: matching records will be returned by the scan
     #
     # @since 0.2.0
-    def scan(table, query, opts = {})
+    def scan(table, query = {}, opts = {})
       benchmark('Scan', table, query) {adapter.scan(table, query, opts)}
     end
 

data/lib/dynamoid/adapter_plugin/aws_sdk_v2.rb

@@ -3,7 +3,7 @@ module Dynamoid
 
     # The AwsSdkV2 adapter provides support for the aws-sdk version 2 for ruby.
     class AwsSdkV2
-      EQ = "EQ".freeze
+      EQ = 'EQ'.freeze
       RANGE_MAP = {
           range_greater_than: 'GT',
           range_less_than:    'LT',
@@ -28,16 +28,16 @@ module Dynamoid
           contains:     'CONTAINS',
           not_contains: 'NOT_CONTAINS'
       }
-      HASH_KEY  = "HASH".freeze
-      RANGE_KEY = "RANGE".freeze
-      STRING_TYPE  = "S".freeze
-      NUM_TYPE     = "N".freeze
-      BINARY_TYPE  = "B".freeze
+      HASH_KEY  = 'HASH'.freeze
+      RANGE_KEY = 'RANGE'.freeze
+      STRING_TYPE  = 'S'.freeze
+      NUM_TYPE     = 'N'.freeze
+      BINARY_TYPE  = 'B'.freeze
       TABLE_STATUSES = {
-          creating: "CREATING",
-          updating: "UPDATING",
-          deleting: "DELETING",
-          active: "ACTIVE"
+          creating: 'CREATING',
+          updating: 'UPDATING',
+          deleting: 'DELETING',
+          active: 'ACTIVE'
       }.freeze
       PARSE_TABLE_STATUS = ->(resp, lookup = :table) {
         # lookup is table for describe_table API
@@ -45,6 +45,8 @@ module Dynamoid
         #   because Amazon, damnit.
         resp.send(lookup).table_status
       }
+      BATCH_WRITE_ITEM_REQUESTS_LIMIT = 25
+
       attr_reader :table_cache
 
       # Establish the connection to DynamoDB.
@@ -87,24 +89,31 @@ module Dynamoid
       # @param [Array]  items to be processed
       # @param [Hash]   additional options
       #
-      #See: http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method
+      # See:
+      # * http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
+      # * http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method
+      #
+      # TODO handle rejections because of exceeding limit for the whole request - 16 MB,
+      # item size limit - 400 KB or because provisioned throughput is exceeded
       def batch_write_item table_name, objects, options = {}
-        request_items = []
-        options ||= {}
-        objects.each do |o|
-          request_items << { "put_request" => { item: o } }
+        requests = []
+
+        objects.each_slice(BATCH_WRITE_ITEM_REQUESTS_LIMIT) do |os|
+          requests << os.map { |o| { put_request: { item: o } } }
         end
 
         begin
-          client.batch_write_item(
-            {
-              request_items: {
-                table_name => request_items,
-              },
-              return_consumed_capacity: "TOTAL",
-              return_item_collection_metrics: "SIZE"
-            }.merge!(options)
-          )
+          requests.each do |request_items|
+            client.batch_write_item(
+              {
+                request_items: {
+                  table_name => request_items,
+                },
+                return_consumed_capacity: 'TOTAL',
+                return_item_collection_metrics: 'SIZE'
+              }.merge!(options)
+            )
+          end
         rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException => e
           raise Dynamoid::Errors::ConditionalCheckFailedException, e
         end
@@ -139,7 +148,7 @@ module Dynamoid
             request_items = Hash.new{|h, k| h[k] = []}
 
             keys = if rng.present?
-              Array(ids).map do |h,r|
+              Array(ids).map do |h, r|
                 { hk => h, rng => r }
               end
             else
@@ -169,22 +178,41 @@ module Dynamoid
       #
       # @example Delete IDs 1 and 2 from the table testtable
       #   Dynamoid::AdapterPlugin::AwsSdk.batch_delete_item('table1' => ['1', '2'])
-      #or
+      # or
       #   Dynamoid::AdapterPlugin::AwsSdkV2.batch_delete_item('table1' => [['hk1', 'rk2'], ['hk1', 'rk2']]]))
       #
       # @param [Hash] options the hash of tables and IDs to delete
       #
-      # @return nil
+      # See:
+      # * http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
+      # * http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method
       #
-      # @todo: Provide support for passing options to underlying delete_item http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#delete_item-instance_method
+      # TODO handle rejections because of internal processing failures
       def batch_delete_item(options)
+        requests = []
+
         options.each_pair do |table_name, ids|
           table = describe_table(table_name)
-          ids.each do |id|
-            client.delete_item(table_name: table_name, key: key_stanza(table, *id))
+
+          ids.each_slice(BATCH_WRITE_ITEM_REQUESTS_LIMIT) do |sliced_ids|
+            delete_requests = sliced_ids.map { |id|
+              {delete_request: {key: key_stanza(table, *id)}}
+            }
+
+            requests << {table_name => delete_requests}
+          end
+        end
+
+        begin
+          requests.map do |request_items|
+            client.batch_write_item(
+              request_items: request_items,
+              return_consumed_capacity: 'TOTAL',
+              return_item_collection_metrics: 'SIZE')
           end
+        rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException => e
+          raise Dynamoid::Errors::ConditionalCheckFailedException, e
         end
-        nil
       end
 
       # Create a table on DynamoDB. This usually takes a long time to complete.
@@ -210,8 +238,8 @@ module Dynamoid
         gs_indexes = options[:global_secondary_indexes]
 
         key_schema = {
-          :hash_key_schema => { key => (options[:hash_key_type] || :string) },
-          :range_key_schema => options[:range_key]
+          hash_key_schema: { key => (options[:hash_key_type] || :string) },
+          range_key_schema: options[:range_key]
         }
         attribute_definitions = build_all_attribute_definitions(
           key_schema,
@@ -367,7 +395,7 @@ module Dynamoid
             key: key_stanza(table, key, range_key),
             attribute_updates: iu.to_h,
             expected: expected_stanza(conditions),
-            return_values: "ALL_NEW"
+            return_values: 'ALL_NEW'
           )
           result_item_to_hash(result[:attributes])
         rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException => e
@@ -489,7 +517,7 @@ module Dynamoid
         end
 
         query_filter = {}
-        opts.reject {|k,_| k.in? RANGE_MAP.keys}.each do |attr, hash|
+        opts.reject {|k, _| k.in? RANGE_MAP.keys}.each do |attr, hash|
           query_filter[attr] = {
             comparison_operator: FIELD_MAP[hash.keys[0]],
             attribute_value_list: [
@@ -557,7 +585,7 @@ module Dynamoid
       # @since 1.0.0
       #
       # @todo: Provide support for various options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#scan-instance_method
-      def scan(table_name, scan_hash, select_opts = {})
+      def scan(table_name, scan_hash = {}, select_opts = {})
         request = { table_name: table_name }
         request[:consistent_read] = true if select_opts.delete(:consistent_read)
 
@@ -661,7 +689,7 @@ module Dynamoid
           check = {again: true}
           while check[:again]
             sleep Dynamoid::Config.sync_retry_wait_seconds
-            resp = client.describe_table({ table_name: table_name })
+            resp = client.describe_table(table_name: table_name)
             check = check_table_status?(counter, resp, status)
             Dynamoid.logger.info "Checked table status for #{table_name} (check #{check.inspect})"
             counter += 1
@@ -689,7 +717,7 @@ module Dynamoid
         end
       end
 
-      #Converts from symbol to the API string for the given data type
+      # Converts from symbol to the API string for the given data type
       # E.g. :number -> 'N'
       def api_type(type)
         case(type)
@@ -714,13 +742,13 @@ module Dynamoid
       # @return an Expected stanza for the given conditions hash
       #
       def expected_stanza(conditions = nil)
-        expected = Hash.new { |h,k| h[k] = {} }
+        expected = Hash.new { |h, k| h[k] = {} }
         return expected unless conditions
 
         conditions.delete(:unless_exists).try(:each) do |col|
           expected[col.to_s][:exists] = false
         end
-        conditions.delete(:if).try(:each) do |col,val|
+        conditions.delete(:if).try(:each) do |col, val|
           expected[col.to_s][:value] = val
         end
 
@@ -741,7 +769,7 @@ module Dynamoid
       #
       def result_item_to_hash(item)
         {}.tap do |r|
-          item.each { |k,v| r[k.to_sym] = v }
+          item.each { |k, v| r[k.to_sym] = v }
         end
       end
 
@@ -770,10 +798,10 @@ module Dynamoid
         key_schema = aws_key_schema(index.hash_key_schema, index.range_key_schema)
 
         hash = {
-          :index_name => index.name,
-          :key_schema => key_schema,
-          :projection => {
-            :projection_type => index.projection_type.to_s.upcase
+          index_name: index.name,
+          key_schema: key_schema,
+          projection: {
+            projection_type: index.projection_type.to_s.upcase
           }
         }
 
@@ -785,8 +813,8 @@ module Dynamoid
         # Only global secondary indexes have a separate throughput.
         if index.type == :global_secondary
           hash[:provisioned_throughput] = {
-            :read_capacity_units => index.read_capacity,
-            :write_capacity_units => index.write_capacity
+            read_capacity_units: index.read_capacity,
+            write_capacity_units: index.write_capacity
           }
         end
         hash
@@ -856,7 +884,6 @@ module Dynamoid
         attribute_definitions
       end
 
-
       # Builds an attribute definitions based on hash key and range key
       # @params [Hash] hash_key_schema - eg: {:id => :string}
       # @params [Hash] range_key_schema - eg: {:created_at => :datetime}
@@ -887,8 +914,8 @@ module Dynamoid
         aws_type = api_type(dynamoid_type)
 
         {
-          :attribute_name => name.to_s,
-          :attribute_type => aws_type
+          attribute_name: name.to_s,
+          attribute_type: aws_type
         }
       end
 
@@ -913,7 +940,7 @@ module Dynamoid
         def range_type
           range_type ||= schema[:attribute_definitions].find { |d|
             d[:attribute_name] == range_key
-          }.try(:fetch,:attribute_type, nil)
+          }.try(:fetch, :attribute_type, nil)
         end
 
         def hash_key
@@ -982,19 +1009,19 @@ module Dynamoid
         def to_h
           ret = {}
 
-          @additions.each do |k,v|
+          @additions.each do |k, v|
             ret[k.to_s] = {
               action: ADD,
               value: v
             }
           end
-          @deletions.each do |k,v|
+          @deletions.each do |k, v|
             ret[k.to_s] = {
               action: DELETE,
               value: v
             }
           end
-          @updates.each do |k,v|
+          @updates.each do |k, v|
             ret[k.to_s] = {
               action: PUT,
               value: v
@@ -1004,9 +1031,9 @@ module Dynamoid
           ret
         end
 
-        ADD    = "ADD".freeze
-        DELETE = "DELETE".freeze
-        PUT    = "PUT".freeze
+        ADD    = 'ADD'.freeze
+        DELETE = 'DELETE'.freeze
+        PUT    = 'PUT'.freeze
       end
     end
   end