dynamoid 3.2.0 → 3.6.0

data/lib/dynamoid.rb

@@ -30,6 +30,7 @@ require 'dynamoid/criteria'
 require 'dynamoid/finders'
 require 'dynamoid/identity_map'
 require 'dynamoid/config'
+require 'dynamoid/loadable'
 require 'dynamoid/components'
 require 'dynamoid/document'
 require 'dynamoid/adapter'
@@ -56,6 +57,7 @@ module Dynamoid
     @included_models ||= []
   end
 
+  # @private
   def adapter
     @adapter ||= Adapter.new
   end

data/lib/dynamoid/adapter.rb

@@ -3,6 +3,7 @@
 # require only 'concurrent/atom' once this issue is resolved:
 #   https://github.com/ruby-concurrency/concurrent-ruby/pull/377
 require 'concurrent'
+require 'dynamoid/adapter_plugin/aws_sdk_v3'
 
 # encoding: utf-8
 module Dynamoid
@@ -10,6 +11,7 @@ module Dynamoid
   # 1) For the rest of Dynamoid, the gateway to DynamoDB.
   # 2) Allows switching `config.adapter` to ease development of a new adapter.
   # 3) Caches the list of tables Dynamoid knows about.
+  # @private
   class Adapter
     def initialize
       @adapter_ = Concurrent::Atom.new(nil)
@@ -29,7 +31,7 @@ module Dynamoid
     def adapter
       unless @adapter_.value
         adapter = self.class.adapter_plugin_class.new
-        adapter.connect! if adapter.respond_to?(:connect!)
+        adapter.connect!
         @adapter_.compare_and_set(nil, adapter)
         clear_cache!
       end
@@ -77,8 +79,8 @@ module Dynamoid
     #
     # @param [String] table the name of the table to write the object to
     # @param [Array] ids to fetch, can also be a string of just one id
-    # @param [Hash] options: Passed to the underlying query. The :range_key option is required whenever the table has a range key,
-    #                        unless multiple ids are passed in.
+    # @param [Hash] options Passed to the underlying query. The :range_key option is required whenever the table has a range key,
+    #                       unless multiple ids are passed in.
     #
     # @since 0.2.0
     def read(table, ids, options = {}, &blk)
@@ -93,7 +95,9 @@ module Dynamoid
     #
     # @param [String] table the name of the table to write the object to
     # @param [Array] ids to delete, can also be a string of just one id
-    # @param [Array] range_key of the record to delete, can also be a string of just one range_key
+    # @param [Hash] options allowed only +range_key+ - range key or array of
+    #                       range keys of the record to delete, can also be
+    #                       a string of just one range_key, and +conditions+
     #
     def delete(table, ids, options = {})
       range_key = options[:range_key] # array of range keys that matches the ids passed in
@@ -114,7 +118,7 @@ module Dynamoid
     # Scans a table. Generally quite slow; try to avoid using scan if at all possible.
     #
     # @param [String] table the name of the table to write the object to
-    # @param [Hash] scan_hash a hash of attributes: matching records will be returned by the scan
+    # @param [Hash] query a hash of attributes: matching records will be returned by the scan
     #
     # @since 0.2.0
     def scan(table, query = {}, opts = {})
@@ -123,8 +127,12 @@ module Dynamoid
 
     def create_table(table_name, key, options = {})
       unless tables.include?(table_name)
-        benchmark('Create Table') { adapter.create_table(table_name, key, options) }
+        result = nil
+        benchmark('Create Table') { result = adapter.create_table(table_name, key, options) }
         tables << table_name
+        result
+      else
+        false
       end
     end
 
@@ -142,11 +150,7 @@ module Dynamoid
       #
       # @since 0.2.0
       define_method(m) do |*args, &blk|
-        if blk.present?
-          benchmark(m.to_s, *args) { adapter.send(m, *args, &blk) }
-        else
-          benchmark(m.to_s, *args) { adapter.send(m, *args) }
-        end
+        benchmark(m.to_s, *args) { adapter.send(m, *args, &blk) }
       end
     end
 
@@ -179,10 +183,6 @@ module Dynamoid
     end
 
     def self.adapter_plugin_class
-      unless Dynamoid.const_defined?(:AdapterPlugin) && Dynamoid::AdapterPlugin.const_defined?(Dynamoid::Config.adapter.camelcase)
-        require "dynamoid/adapter_plugin/#{Dynamoid::Config.adapter}"
-      end
-
       Dynamoid::AdapterPlugin.const_get(Dynamoid::Config.adapter.camelcase)
     end
   end

data/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb

@@ -3,11 +3,13 @@
 require_relative 'aws_sdk_v3/query'
 require_relative 'aws_sdk_v3/scan'
 require_relative 'aws_sdk_v3/create_table'
+require_relative 'aws_sdk_v3/batch_get_item'
 require_relative 'aws_sdk_v3/item_updater'
 require_relative 'aws_sdk_v3/table'
 require_relative 'aws_sdk_v3/until_past_table_status'
 
 module Dynamoid
+  # @private
   module AdapterPlugin
     # The AwsSdkV3 adapter provides support for the aws-sdk version 2 for ruby.
     class AwsSdkV3
@@ -22,8 +24,6 @@ module Dynamoid
         range_eq:           'EQ'
       }.freeze
 
-      # Don't implement NULL and NOT_NULL because it doesn't make seanse -
-      # we declare schema in models
       FIELD_MAP = {
         eq:           'EQ',
         ne:           'NE',
@@ -35,7 +35,9 @@ module Dynamoid
         between:      'BETWEEN',
         in:           'IN',
         contains:     'CONTAINS',
-        not_contains: 'NOT_CONTAINS'
+        not_contains: 'NOT_CONTAINS',
+        null:         'NULL',
+        not_null:     'NOT_NULL',
       }.freeze
       HASH_KEY  = 'HASH'
       RANGE_KEY = 'RANGE'
@@ -60,6 +62,25 @@ module Dynamoid
 
       attr_reader :table_cache
 
+      # Build an array of values for Condition
+      # Is used in ScanFilter and QueryFilter
+      # https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Condition.html
+      # @param [String] operator value of RANGE_MAP or FIELD_MAP hash, e.g. "EQ", "LT" etc
+      # @param [Object] value scalar value or array/set
+      def self.attribute_value_list(operator, value)
+        # For BETWEEN and IN operators we should keep value as is (it should be already an array)
+        # NULL and NOT_NULL require absence of attribute list
+        # For all the other operators we wrap the value with array
+        # https://docs.aws.amazon.com/en_us/amazondynamodb/latest/developerguide/LegacyConditionalParameters.Conditions.html
+        if %w[BETWEEN IN].include?(operator)
+          [value].flatten
+        elsif %w[NULL NOT_NULL].include?(operator)
+          nil
+        else
+          [value]
+        end
+      end
+
       # Establish the connection to DynamoDB.
       #
       # @return [Aws::DynamoDB::Client] the DynamoDB connection
@@ -74,19 +95,28 @@ module Dynamoid
         (Dynamoid::Config.settings.compact.keys & CONNECTION_CONFIG_OPTIONS).each do |option|
           @connection_hash[option] = Dynamoid::Config.send(option)
         end
-        if Dynamoid::Config.access_key?
-          @connection_hash[:access_key_id] = Dynamoid::Config.access_key
-        end
-        if Dynamoid::Config.secret_key?
-          @connection_hash[:secret_access_key] = Dynamoid::Config.secret_key
+
+        # if credentials are passed, they already contain access key & secret key
+        if Dynamoid::Config.credentials?
+          @connection_hash[:credentials] = Dynamoid::Config.credentials
+        else
+          # otherwise, pass access key & secret key for credentials creation
+          if Dynamoid::Config.access_key?
+            @connection_hash[:access_key_id] = Dynamoid::Config.access_key
+          end
+          if Dynamoid::Config.secret_key?
+            @connection_hash[:secret_access_key] = Dynamoid::Config.secret_key
+          end
         end
 
-        # https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-core/lib/aws-sdk-core/plugins/logging.rb
-        # https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-core/lib/aws-sdk-core/log/formatter.rb
-        formatter = Aws::Log::Formatter.new(':operation | Request :http_request_body | Response :http_response_body')
         @connection_hash[:logger] = Dynamoid::Config.logger
         @connection_hash[:log_level] = :debug
-        @connection_hash[:log_formatter] = formatter
+
+        # https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-core/lib/aws-sdk-core/plugins/logging.rb
+        # https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-core/lib/aws-sdk-core/log/formatter.rb
+        if Dynamoid::Config.log_formatter
+          @connection_hash[:log_formatter] = Dynamoid::Config.log_formatter
+        end
 
         @connection_hash
       end
@@ -114,9 +144,9 @@ module Dynamoid
       #   end
       #
       # @param [String] table_name the name of the table
-      # @param [Array]  items to be processed
-      # @param [Hash]   additional options
-      # @param [Proc]   optional block
+      # @param [Array]  objects to be processed
+      # @param [Hash]   options additional options
+      # @yield [true|false] invokes an optional block with argument - whether there are unprocessed items
       #
       # See:
       # * http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
@@ -171,9 +201,9 @@ module Dynamoid
       #     end
       #   end
       #
-      # @param [Hash] table_ids the hash of tables and IDs to retrieve
+      # @param [Hash] table_names_with_ids the hash of tables and IDs to retrieve
       # @param [Hash] options to be passed to underlying BatchGet call
-      # @param [Proc] optional block can be passed to handle each batch of items
+      # @param [Proc] block optional block can be passed to handle each batch of items
       #
       # @return [Hash] a hash where keys are the table names and the values are the retrieved items
       #
@@ -183,65 +213,11 @@ module Dynamoid
       # @since 1.0.0
       #
       # @todo: Provide support for passing options to underlying batch_get_item
-      def batch_get_item(table_ids, options = {})
-        request_items = Hash.new { |h, k| h[k] = [] }
-        return request_items if table_ids.all? { |_k, v| v.blank? }
-
-        ret = Hash.new([].freeze) # Default for tables where no rows are returned
-
-        table_ids.each do |t, ids|
-          next if ids.blank?
-
-          ids = Array(ids).dup
-          tbl = describe_table(t)
-          hk  = tbl.hash_key.to_s
-          rng = tbl.range_key.to_s
-
-          while ids.present?
-            batch = ids.shift(Dynamoid::Config.batch_size)
-
-            request_items = Hash.new { |h, k| h[k] = [] }
-
-            keys = if rng.present?
-                     Array(batch).map do |h, r|
-                       { hk => h, rng => r }
-                     end
-                   else
-                     Array(batch).map do |id|
-                       { hk => id }
-                     end
-                   end
-
-            request_items[t] = {
-              keys: keys,
-              consistent_read: options[:consistent_read]
-            }
-
-            results = client.batch_get_item(
-              request_items: request_items
-            )
-
-            if block_given?
-              batch_results = Hash.new([].freeze)
-
-              results.data[:responses].each do |table, rows|
-                batch_results[table] += rows.collect { |r| result_item_to_hash(r) }
-              end
-
-              yield(batch_results, results.unprocessed_keys.present?)
-            else
-              results.data[:responses].each do |table, rows|
-                ret[table] += rows.collect { |r| result_item_to_hash(r) }
-              end
-            end
-
-            if results.unprocessed_keys.present?
-              ids += results.unprocessed_keys[t].keys.map { |h| h[hk] }
-            end
-          end
+      def batch_get_item(table_names_with_ids, options = {}, &block)
+        tables_with_ids = table_names_with_ids.transform_keys do |name|
+          describe_table(name)
         end
-
-        ret unless block_given?
+        BatchGetItem.new(client, tables_with_ids, options).call(&block)
       end
 
       # Delete many items at once from DynamoDB. More efficient than delete each item individually.
@@ -299,8 +275,22 @@ module Dynamoid
       def create_table(table_name, key = :id, options = {})
         Dynamoid.logger.info "Creating #{table_name} table. This could take a while."
         CreateTable.new(client, table_name, key, options).call
+        true
       rescue Aws::DynamoDB::Errors::ResourceInUseException => e
         Dynamoid.logger.error "Table #{table_name} cannot be created as it already exists"
+        false
+      end
+
+      def update_time_to_live(table_name:, attribute:)
+        request = {
+          table_name: table_name,
+          time_to_live_specification: {
+            attribute_name: attribute,
+            enabled: true,
+          }
+        }
+
+        client.update_time_to_live(request)
       end
 
       # Create a table on DynamoDB *synchronously*.
@@ -356,9 +346,14 @@ module Dynamoid
       # @since 1.0.0
       def delete_table(table_name, options = {})
         resp = client.delete_table(table_name: table_name)
-        UntilPastTableStatus.new(table_name, :deleting).call if options[:sync] &&
-                                                                (status = PARSE_TABLE_STATUS.call(resp, :table_description)) &&
-                                                                status == TABLE_STATUSES[:deleting]
+
+        if options[:sync]
+          status = PARSE_TABLE_STATUS.call(resp, :table_description)
+          if status == TABLE_STATUSES[:deleting]
+            UntilPastTableStatus.new(client, table_name, :deleting).call
+          end
+        end
+
         table_cache.delete(table_name)
       rescue Aws::DynamoDB::Errors::ResourceInUseException => e
         Dynamoid.logger.error "Table #{table_name} cannot be deleted as it is in use"
@@ -552,11 +547,11 @@ module Dynamoid
         hk    = table.hash_key
         rk    = table.range_key
 
-        scan(table_name, {}, {}).flat_map{ |i| i }.each do |attributes|
-          opts = {}
-          opts[:range_key] = attributes[rk.to_sym] if rk
-          delete_item(table_name, attributes[hk], opts)
+        ids = scan(table_name, {}, {}).flat_map { |i| i }.map do |attributes|
+          rk ? [attributes[hk], attributes[rk.to_sym]] : attributes[hk]
         end
+
+        batch_delete_item(table_name => ids)
       end
 
       def count(table_name)
@@ -614,28 +609,13 @@ module Dynamoid
         end
       end
 
-      # Build an array of values for Condition
-      # Is used in ScanFilter and QueryFilter
-      # https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Condition.html
-      # @params [String] operator: value of RANGE_MAP or FIELD_MAP hash, e.g. "EQ", "LT" etc
-      # @params [Object] value: scalar value or array/set
-      def attribute_value_list(operator, value)
-        self.class.attribute_value_list(operator, value)
-      end
-
-      def self.attribute_value_list(operator, value)
-        # For BETWEEN and IN operators we should keep value as is (it should be already an array)
-        # For all the other operators we wrap the value with array
-        if %w[BETWEEN IN].include?(operator)
-          [value].flatten
-        else
-          [value]
-        end
-      end
-
       def sanitize_item(attributes)
+        config_value = Dynamoid.config.store_attribute_with_nil_value
+        store_attribute_with_nil_value = config_value.nil? ? false : !!config_value
+
         attributes.reject do |_, v|
-          v.nil? || ((v.is_a?(Set) || v.is_a?(String)) && v.empty?)
+          ((v.is_a?(Set) || v.is_a?(String)) && v.empty?) ||
+            (!store_attribute_with_nil_value && v.nil?)
         end.transform_values do |v|
           v.is_a?(Hash) ? v.stringify_keys : v
         end

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/batch_get_item.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Dynamoid
+  # @private
+  module AdapterPlugin
+    class AwsSdkV3
+      # Documentation
+      # https://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_get_item-instance_method
+      class BatchGetItem
+        attr_reader :client, :tables_with_ids, :options
+
+        def initialize(client, tables_with_ids, options = {})
+          @client = client
+          @tables_with_ids = tables_with_ids
+          @options = options
+        end
+
+        def call
+          results = {}
+
+          tables_with_ids.each do |table, ids|
+            if ids.blank?
+              results[table.name] = []
+              next
+            end
+
+            ids = Array(ids).dup
+
+            while ids.present?
+              batch = ids.shift(Dynamoid::Config.batch_size)
+              request = build_request(table, batch)
+              api_response = client.batch_get_item(request)
+              response = Response.new(api_response)
+
+              if block_given?
+                # return batch items as a result
+                batch_results = Hash.new([].freeze)
+                batch_results.update(response.items_grouped_by_table)
+
+                yield(batch_results, response.successful_partially?)
+              else
+                # collect all the batches to return at the end
+                results.update(response.items_grouped_by_table) { |_, its1, its2| its1 + its2 }
+              end
+
+              if response.successful_partially?
+                ids += response.unprocessed_ids(table)
+              end
+            end
+          end
+
+          results unless block_given?
+        end
+
+        private
+
+        def build_request(table, ids)
+          ids = Array(ids)
+
+          keys = if table.range_key.nil?
+                   ids.map { |hk| { table.hash_key => hk } }
+                 else
+                   ids.map { |hk, rk| { table.hash_key => hk, table.range_key => rk } }
+                 end
+
+          {
+            request_items: {
+              table.name => {
+                keys: keys,
+                consistent_read: options[:consistent_read]
+              }
+            }
+          }
+        end
+
+        # Helper class to work with response
+        class Response
+          def initialize(api_response)
+            @api_response = api_response
+          end
+
+          def successful_partially?
+            @api_response.unprocessed_keys.present?
+          end
+
+          def unprocessed_ids(table)
+            # unprocessed_keys Hash contains as values instances of
+            # Aws::DynamoDB::Types::KeysAndAttributes
+            @api_response.unprocessed_keys[table.name].keys.map { |h| h[table.hash_key.to_s] }
+          end
+
+          def items_grouped_by_table
+            # data[:responses] is a Hash[table_name -> items]
+            @api_response.data[:responses].transform_values do |items|
+              items.map(&method(:item_to_hash))
+            end
+          end
+
+          private
+
+          def item_to_hash(item)
+            item.symbolize_keys
+          end
+        end
+      end
+    end
+  end
+end