dynamoid 3.2.0 → 3.3.0

data/gemfiles/rails_4_2.gemfile

@@ -1,11 +1,9 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'activemodel', '~> 4.2.0'
-gem 'nokogiri', '~> 1.6.8'
-gem 'pry-byebug', platforms: :ruby
+gem "pry-byebug", platforms: :ruby
+gem "activemodel", "~> 4.2.0"
+gem "nokogiri", "~> 1.6.8"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_5_0.gemfile

@@ -1,10 +1,8 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'activemodel', '~> 5.0.0'
-gem 'pry-byebug', platforms: :ruby
+gem "pry-byebug", platforms: :ruby
+gem "activemodel", "~> 5.0.0"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_5_1.gemfile

@@ -1,10 +1,8 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'activemodel', '~> 5.1.0'
-gem 'pry-byebug', platforms: :ruby
+gem "pry-byebug", platforms: :ruby
+gem "activemodel", "~> 5.1.0"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_5_2.gemfile

@@ -1,10 +1,8 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'activemodel', '~> 5.2.0'
-gem 'pry-byebug', platforms: :ruby
+gem "pry-byebug", platforms: :ruby
+gem "activemodel", "~> 5.2.0"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_6_0.gemfile

@@ -0,0 +1,8 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "pry-byebug", platforms: :ruby
+gem "activemodel", "6.0.0"
+
+gemspec path: "../"

data/lib/dynamoid.rb

@@ -30,6 +30,7 @@ require 'dynamoid/criteria'
 require 'dynamoid/finders'
 require 'dynamoid/identity_map'
 require 'dynamoid/config'
+require 'dynamoid/loadable'
 require 'dynamoid/components'
 require 'dynamoid/document'
 require 'dynamoid/adapter'

data/lib/dynamoid/adapter.rb

@@ -3,6 +3,7 @@
 # require only 'concurrent/atom' once this issue is resolved:
 #   https://github.com/ruby-concurrency/concurrent-ruby/pull/377
 require 'concurrent'
+require "dynamoid/adapter_plugin/aws_sdk_v3"
 
 # encoding: utf-8
 module Dynamoid
@@ -29,7 +30,7 @@ module Dynamoid
     def adapter
       unless @adapter_.value
         adapter = self.class.adapter_plugin_class.new
-        adapter.connect! if adapter.respond_to?(:connect!)
+        adapter.connect!
         @adapter_.compare_and_set(nil, adapter)
         clear_cache!
       end
@@ -142,11 +143,7 @@ module Dynamoid
       #
       # @since 0.2.0
       define_method(m) do |*args, &blk|
-        if blk.present?
-          benchmark(m.to_s, *args) { adapter.send(m, *args, &blk) }
-        else
-          benchmark(m.to_s, *args) { adapter.send(m, *args) }
-        end
+        benchmark(m.to_s, *args) { adapter.send(m, *args, &blk) }
       end
     end
 
@@ -179,10 +176,6 @@ module Dynamoid
     end
 
     def self.adapter_plugin_class
-      unless Dynamoid.const_defined?(:AdapterPlugin) && Dynamoid::AdapterPlugin.const_defined?(Dynamoid::Config.adapter.camelcase)
-        require "dynamoid/adapter_plugin/#{Dynamoid::Config.adapter}"
-      end
-
       Dynamoid::AdapterPlugin.const_get(Dynamoid::Config.adapter.camelcase)
     end
   end

data/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb

@@ -3,6 +3,7 @@
 require_relative 'aws_sdk_v3/query'
 require_relative 'aws_sdk_v3/scan'
 require_relative 'aws_sdk_v3/create_table'
+require_relative 'aws_sdk_v3/batch_get_item'
 require_relative 'aws_sdk_v3/item_updater'
 require_relative 'aws_sdk_v3/table'
 require_relative 'aws_sdk_v3/until_past_table_status'
@@ -22,8 +23,6 @@ module Dynamoid
         range_eq:           'EQ'
       }.freeze
 
-      # Don't implement NULL and NOT_NULL because it doesn't make seanse -
-      # we declare schema in models
       FIELD_MAP = {
         eq:           'EQ',
         ne:           'NE',
@@ -35,7 +34,9 @@ module Dynamoid
         between:      'BETWEEN',
         in:           'IN',
         contains:     'CONTAINS',
-        not_contains: 'NOT_CONTAINS'
+        not_contains: 'NOT_CONTAINS',
+        null:         'NULL',
+        not_null:     'NOT_NULL',
       }.freeze
       HASH_KEY  = 'HASH'
       RANGE_KEY = 'RANGE'
@@ -183,65 +184,11 @@ module Dynamoid
       # @since 1.0.0
       #
       # @todo: Provide support for passing options to underlying batch_get_item
-      def batch_get_item(table_ids, options = {})
-        request_items = Hash.new { |h, k| h[k] = [] }
-        return request_items if table_ids.all? { |_k, v| v.blank? }
-
-        ret = Hash.new([].freeze) # Default for tables where no rows are returned
-
-        table_ids.each do |t, ids|
-          next if ids.blank?
-
-          ids = Array(ids).dup
-          tbl = describe_table(t)
-          hk  = tbl.hash_key.to_s
-          rng = tbl.range_key.to_s
-
-          while ids.present?
-            batch = ids.shift(Dynamoid::Config.batch_size)
-
-            request_items = Hash.new { |h, k| h[k] = [] }
-
-            keys = if rng.present?
-                     Array(batch).map do |h, r|
-                       { hk => h, rng => r }
-                     end
-                   else
-                     Array(batch).map do |id|
-                       { hk => id }
-                     end
-                   end
-
-            request_items[t] = {
-              keys: keys,
-              consistent_read: options[:consistent_read]
-            }
-
-            results = client.batch_get_item(
-              request_items: request_items
-            )
-
-            if block_given?
-              batch_results = Hash.new([].freeze)
-
-              results.data[:responses].each do |table, rows|
-                batch_results[table] += rows.collect { |r| result_item_to_hash(r) }
-              end
-
-              yield(batch_results, results.unprocessed_keys.present?)
-            else
-              results.data[:responses].each do |table, rows|
-                ret[table] += rows.collect { |r| result_item_to_hash(r) }
-              end
-            end
-
-            if results.unprocessed_keys.present?
-              ids += results.unprocessed_keys[t].keys.map { |h| h[hk] }
-            end
-          end
+      def batch_get_item(table_names_with_ids, options = {}, &block)
+        tables_with_ids = table_names_with_ids.transform_keys do |name|
+          describe_table(name)
         end
-
-        ret unless block_given?
+        BatchGetItem.new(client, tables_with_ids, options).call(&block)
       end
 
       # Delete many items at once from DynamoDB. More efficient than delete each item individually.
@@ -356,9 +303,14 @@ module Dynamoid
       # @since 1.0.0
       def delete_table(table_name, options = {})
         resp = client.delete_table(table_name: table_name)
-        UntilPastTableStatus.new(table_name, :deleting).call if options[:sync] &&
-                                                                (status = PARSE_TABLE_STATUS.call(resp, :table_description)) &&
-                                                                status == TABLE_STATUSES[:deleting]
+
+        if options[:sync]
+          status = PARSE_TABLE_STATUS.call(resp, :table_description)
+          if status == TABLE_STATUSES[:deleting]
+            UntilPastTableStatus.new(client, table_name, :deleting).call
+          end
+        end
+
         table_cache.delete(table_name)
       rescue Aws::DynamoDB::Errors::ResourceInUseException => e
         Dynamoid.logger.error "Table #{table_name} cannot be deleted as it is in use"
@@ -619,23 +571,27 @@ module Dynamoid
       # https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Condition.html
       # @params [String] operator: value of RANGE_MAP or FIELD_MAP hash, e.g. "EQ", "LT" etc
       # @params [Object] value: scalar value or array/set
-      def attribute_value_list(operator, value)
-        self.class.attribute_value_list(operator, value)
-      end
-
       def self.attribute_value_list(operator, value)
         # For BETWEEN and IN operators we should keep value as is (it should be already an array)
+        # NULL and NOT_NULL require absence of attribute list
         # For all the other operators we wrap the value with array
+        # https://docs.aws.amazon.com/en_us/amazondynamodb/latest/developerguide/LegacyConditionalParameters.Conditions.html
         if %w[BETWEEN IN].include?(operator)
           [value].flatten
+        elsif %w[NULL NOT_NULL].include?(operator)
+          nil
         else
           [value]
         end
       end
 
       def sanitize_item(attributes)
+        config_value = Dynamoid.config.store_attribute_with_nil_value
+        store_attribute_with_nil_value = config_value.nil? ? false : !!config_value
+
         attributes.reject do |_, v|
-          v.nil? || ((v.is_a?(Set) || v.is_a?(String)) && v.empty?)
+          ((v.is_a?(Set) || v.is_a?(String)) && v.empty?) ||
+            (!store_attribute_with_nil_value && v.nil?)
         end.transform_values do |v|
           v.is_a?(Hash) ? v.stringify_keys : v
         end

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/batch_get_item.rb

@@ -0,0 +1,105 @@
+module Dynamoid
+  module AdapterPlugin
+    class AwsSdkV3
+      # Documentation
+      # https://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#batch_get_item-instance_method
+      class BatchGetItem
+        attr_reader :client, :tables_with_ids, :options
+
+        def initialize(client, tables_with_ids, options = {})
+          @client = client
+          @tables_with_ids = tables_with_ids
+          @options = options
+        end
+
+        def call
+          results = {}
+
+          tables_with_ids.each do |table, ids|
+            if ids.blank?
+              results[table.name] = []
+              next
+            end
+
+            ids = Array(ids).dup
+
+            while ids.present?
+              batch = ids.shift(Dynamoid::Config.batch_size)
+              request = build_request(table, batch)
+              api_response = client.batch_get_item(request)
+              response = Response.new(api_response)
+
+              if block_given?
+                # return batch items as a result
+                batch_results = Hash.new([].freeze)
+                batch_results.update(response.items_grouped_by_table)
+
+                yield(batch_results, response.successful_partially?)
+              else
+                # collect all the batches to return at the end
+                results.update(response.items_grouped_by_table) { |_, its1, its2| its1 + its2 }
+              end
+
+              if response.successful_partially?
+                ids += response.unprocessed_ids(table)
+              end
+            end
+          end
+
+          results unless block_given?
+        end
+
+        private
+
+        def build_request(table, ids)
+          ids = Array(ids)
+
+          keys = if table.range_key.nil?
+                   ids.map { |hk| { table.hash_key => hk } }
+                 else
+                   ids.map { |hk, rk| { table.hash_key => hk, table.range_key => rk } }
+                 end
+
+          {
+            request_items: {
+              table.name => {
+                keys: keys,
+                consistent_read: options[:consistent_read]
+              }
+            }
+          }
+        end
+
+        # Helper class to work with response
+        class Response
+          def initialize(api_response)
+            @api_response = api_response
+          end
+
+          def successful_partially?
+            @api_response.unprocessed_keys.present?
+          end
+
+          def unprocessed_ids(table)
+            # unprocessed_keys Hash contains as values instances of
+            # Aws::DynamoDB::Types::KeysAndAttributes
+            @api_response.unprocessed_keys[table.name].keys.map { |h| h[table.hash_key.to_s] }
+          end
+
+          def items_grouped_by_table
+            # data[:responses] is a Hash[table_name -> items]
+            @api_response.data[:responses].transform_values do |items|
+              items.map(&method(:item_to_hash))
+            end
+          end
+
+          private
+
+          def item_to_hash(item)
+            item.symbolize_keys
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/create_table.rb

@@ -62,11 +62,16 @@ module Dynamoid
           end
           resp = client.create_table(client_opts)
           options[:sync] = true if !options.key?(:sync) && ls_indexes.present? || gs_indexes.present?
-          UntilPastTableStatus.new(table_name, :creating).call if options[:sync] &&
-                                                                  (status = PARSE_TABLE_STATUS.call(resp, :table_description)) &&
-                                                                  status == TABLE_STATUSES[:creating]
+
+          if options[:sync]
+            status = PARSE_TABLE_STATUS.call(resp, :table_description)
+            if status == TABLE_STATUSES[:creating]
+              UntilPastTableStatus.new(client, table_name, :creating).call
+            end
+          end
+
           # Response to original create_table, which, if options[:sync]
-          #   may have an outdated table_description.table_status of "CREATING"
+          # may have an outdated table_description.table_status of "CREATING"
           resp
         end
 

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb

@@ -11,6 +11,7 @@ module Dynamoid
         OPTIONS_KEYS = %i[
           limit hash_key hash_value range_key consistent_read scan_index_forward
           select index_name batch_size exclusive_start_key record_limit scan_limit
+          project
         ].freeze
 
         attr_reader :client, :table, :options, :conditions
@@ -63,10 +64,11 @@ module Dynamoid
           batch_size = options[:batch_size]
           limit = [record_limit, scan_limit, batch_size].compact.min
 
-          request[:limit]          = limit if limit
-          request[:table_name]     = table.name
-          request[:key_conditions] = key_conditions
-          request[:query_filter]   = query_filter
+          request[:limit]             = limit if limit
+          request[:table_name]        = table.name
+          request[:key_conditions]    = key_conditions
+          request[:query_filter]      = query_filter
+          request[:attributes_to_get] = attributes_to_get
 
           request
         end
@@ -117,6 +119,11 @@ module Dynamoid
             result
           end
         end
+
+        def attributes_to_get
+          return if options[:project].nil?
+          options[:project].map(&:to_s)
+        end
       end
     end
   end

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/scan.rb

@@ -54,9 +54,10 @@ module Dynamoid
           batch_size = options[:batch_size]
           limit = [record_limit, scan_limit, batch_size].compact.min
 
-          request[:limit]       = limit if limit
-          request[:table_name]  = table.name
-          request[:scan_filter] = scan_filter
+          request[:limit]             = limit if limit
+          request[:table_name]        = table.name
+          request[:scan_filter]       = scan_filter
+          request[:attributes_to_get] = attributes_to_get
 
           request
         end
@@ -75,10 +76,17 @@ module Dynamoid
               comparison_operator: AwsSdkV3::FIELD_MAP[cond.keys[0]],
               attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::FIELD_MAP[cond.keys[0]], cond.values[0].freeze)
             }
+            # nil means operator doesn't require attribute value list
+            conditions.delete(:attribute_value_list) if conditions[:attribute_value_list].nil?
             result[attr] = condition
             result
           end
         end
+
+        def attributes_to_get
+          return if options[:project].nil?
+          options[:project].map(&:to_s)
+        end
       end
     end
   end