dynamoid 3.10.0 → 3.12.1

data/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb

@@ -8,6 +8,7 @@ require_relative 'aws_sdk_v3/batch_get_item'
 require_relative 'aws_sdk_v3/item_updater'
 require_relative 'aws_sdk_v3/table'
 require_relative 'aws_sdk_v3/until_past_table_status'
+require_relative 'aws_sdk_v3/transact'
 
 module Dynamoid
   # @private
@@ -289,6 +290,18 @@ module Dynamoid
         raise Dynamoid::Errors::ConditionalCheckFailedException, e
       end
 
+      def transact_write_items(items)
+        Transact.new(client).transact_write_items(items)
+      end
+
+      def transact_read_items(items)
+        request = {
+          transact_items: items,
+          return_consumed_capacity: 'TOTAL',
+        }
+        client.transact_get_items(request)
+      end
+
       # Create a table on DynamoDB. This usually takes a long time to complete.
       #
       # @param [String] table_name the name of the table to create
@@ -512,7 +525,7 @@ module Dynamoid
       # @since 1.0.0
       #
       # @todo Provide support for various other options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#query-instance_method
-      def query(table_name, key_conditions, non_key_conditions = {}, options = {})
+      def query(table_name, key_conditions, non_key_conditions = [], options = {})
         Enumerator.new do |yielder|
           table = describe_table(table_name)
 
@@ -545,7 +558,7 @@ module Dynamoid
       # @since 1.0.0
       #
       # @todo: Provide support for various options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#scan-instance_method
-      def scan(table_name, conditions = {}, options = {})
+      def scan(table_name, conditions = [], options = {})
         Enumerator.new do |yielder|
           table = describe_table(table_name)
 
@@ -558,7 +571,7 @@ module Dynamoid
         end
       end
 
-      def scan_count(table_name, conditions = {}, options = {})
+      def scan_count(table_name, conditions = [], options = {})
         table = describe_table(table_name)
         options[:select] = 'COUNT'
 
@@ -662,8 +675,7 @@ module Dynamoid
         store_attribute_with_nil_value = config_value.nil? ? false : !!config_value
 
         attributes.reject do |_, v|
-          ((v.is_a?(Set) || v.is_a?(String)) && v.empty?) ||
-            (!store_attribute_with_nil_value && v.nil?)
+          !store_attribute_with_nil_value && v.nil?
         end.transform_values do |v|
           v.is_a?(Hash) ? v.stringify_keys : v
         end

data/lib/dynamoid/components.rb

@@ -13,6 +13,7 @@ module Dynamoid
 
       define_model_callbacks :create, :save, :destroy, :update
       define_model_callbacks :initialize, :find, :touch, only: :after
+      define_model_callbacks :commit, :rollback, only: :after
 
       before_save :set_expires_field
       after_initialize :set_inheritance_field

data/lib/dynamoid/config.rb

@@ -36,6 +36,7 @@ module Dynamoid
     option :read_capacity, default: 100
     option :write_capacity, default: 20
     option :warn_on_scan, default: true
+    option :error_on_scan, default: false
     option :endpoint, default: nil
     option :identity_map, default: false
     option :timestamps, default: true
@@ -48,7 +49,9 @@ module Dynamoid
     option :dynamodb_timezone, default: :utc # available values - :utc, :local, time zone name like "Hawaii"
     option :store_datetime_as_string, default: false # store Time fields in ISO 8601 string format
     option :store_date_as_string, default: false # store Date fields in ISO 8601 string format
+    option :store_empty_string_as_nil, default: true # store attribute's empty String value as null
     option :store_boolean_as_native, default: true
+    option :store_binary_as_native, default: false
     option :backoff, default: nil # callable object to handle exceeding of table throughput limit
     option :backoff_strategies, default: {
       constant: BackoffStrategies::ConstantBackoff,

data/lib/dynamoid/criteria/chain.rb

@@ -95,20 +95,27 @@ module Dynamoid
       #
       # Internally +where+ performs either +Scan+ or +Query+ operation.
       #
+      # Conditions can be specified as an expression as well:
+      #
+      #   Post.where('links_count = :v', v: 2)
+      #
+      # This way complex expressions can be constructed (e.g. with AND, OR, and NOT
+      # keyword):
+      #
+      #   Address.where('city = :c AND (post_code = :pc1 OR post_code = :pc2)', city: 'A', pc1: '001', pc2: '002')
+      #
+      # See documentation for condition expression's syntax and examples:
+      # - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.OperatorsAndFunctions.html
+      # - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.FilterExpression.html
+      #
       # @return [Dynamoid::Criteria::Chain]
       # @since 0.2.0
-      def where(args)
-        detector = NonexistentFieldsDetector.new(args, @source)
-        if detector.found?
-          Dynamoid.logger.warn(detector.warning_message)
+      def where(conditions, placeholders = nil)
+        if conditions.is_a?(Hash)
+          where_with_hash(conditions)
+        else
+          where_with_string(conditions, placeholders)
         end
-
-        @where_conditions.update(args.symbolize_keys)
-
-        # we should re-initialize keys detector every time we change @where_conditions
-        @key_fields_detector = KeyFieldsDetector.new(@where_conditions, @source, forced_index_name: @forced_index_name)
-
-        self
       end
 
       # Turns on strongly consistent reads.
@@ -500,6 +507,29 @@ module Dynamoid
 
       private
 
+      def where_with_hash(conditions)
+        detector = NonexistentFieldsDetector.new(conditions, @source)
+        if detector.found?
+          Dynamoid.logger.warn(detector.warning_message)
+        end
+
+        @where_conditions.update_with_hash(conditions.symbolize_keys)
+
+        # we should re-initialize keys detector every time we change @where_conditions
+        @key_fields_detector = KeyFieldsDetector.new(@where_conditions, @source, forced_index_name: @forced_index_name)
+
+        self
+      end
+
+      def where_with_string(query, placeholders)
+        @where_conditions.update_with_string(query, placeholders)
+
+        # we should re-initialize keys detector every time we change @where_conditions
+        @key_fields_detector = KeyFieldsDetector.new(@where_conditions, @source, forced_index_name: @forced_index_name)
+
+        self
+      end
+
       # The actual records referenced by the association.
       #
       # @return [Enumerator] an iterator of the found records.
@@ -533,7 +563,7 @@ module Dynamoid
         if @key_fields_detector.key_present?
           raw_pages_via_query
         else
-          issue_scan_warning if Dynamoid::Config.warn_on_scan && !@where_conditions.empty?
+          validate_scan_conditions
           raw_pages_via_scan
         end
       end
@@ -568,6 +598,12 @@ module Dynamoid
         end
       end
 
+      def validate_scan_conditions
+        raise Dynamoid::Errors::ScanProhibited if Dynamoid::Config.error_on_scan && !@where_conditions.empty?
+
+        issue_scan_warning if Dynamoid::Config.warn_on_scan && !@where_conditions.empty?
+      end
+
       def issue_scan_warning
         Dynamoid.logger.warn 'Queries without an index are forced to use scan and are generally much slower than indexed queries!'
         Dynamoid.logger.warn "You can index this query by adding index declaration to #{source.to_s.underscore}.rb:"
@@ -635,12 +671,12 @@ module Dynamoid
       end
 
       def query_non_key_conditions
-        opts = {}
+        hash_conditions = {}
 
         # Honor STI and :type field if it presents
         if @source.attributes.key?(@source.inheritance_field) &&
            @key_fields_detector.hash_key.to_sym != @source.inheritance_field.to_sym
-          @where_conditions.update(sti_condition)
+          @where_conditions.update_with_hash(sti_condition)
         end
 
         # TODO: Separate key conditions and non-key conditions properly:
@@ -650,22 +686,30 @@ module Dynamoid
           .reject { |k, _| k.to_s =~ /^#{@key_fields_detector.range_key}\./ }
         keys.each do |key|
           name, condition = field_condition(key, @where_conditions[key])
-          opts[name] ||= []
-          opts[name] << condition
+          hash_conditions[name] ||= []
+          hash_conditions[name] << condition
         end
 
-        opts
+        string_conditions = []
+        @where_conditions.string_conditions.each do |query, placeholders|
+          placeholders ||= {}
+          string_conditions << [query, placeholders]
+        end
+
+        [hash_conditions] + string_conditions
       end
 
       # TODO: casting should be operator aware
       # e.g. for NULL operator value should be boolean
       # and isn't related to an attribute own type
       def type_cast_condition_parameter(key, value)
-        return value if %i[array set].include?(source.attributes[key.to_sym][:type])
+        field_type = source.attributes[key.to_sym][:type]
+
+        return value if %i[array set].include?(field_type)
 
         if [true, false].include?(value) # Support argument for null/not_null operators
           value
-        elsif !value.respond_to?(:to_ary)
+        elsif !value.respond_to?(:to_ary) || field_type == :serialized
           options = source.attributes[key.to_sym]
           value_casted = TypeCasting.cast_field(value, options)
           Dumping.dump_field(value_casted, options)
@@ -721,16 +765,25 @@ module Dynamoid
       def scan_conditions
         # Honor STI and :type field if it presents
         if sti_condition
-          @where_conditions.update(sti_condition)
+          @where_conditions.update_with_hash(sti_condition)
         end
 
-        {}.tap do |opts|
+        hash_conditions = {}
+        hash_conditions.tap do |opts|
           @where_conditions.keys.map(&:to_sym).each do |key|
             name, condition = field_condition(key, @where_conditions[key])
             opts[name] ||= []
             opts[name] << condition
           end
         end
+
+        string_conditions = []
+        @where_conditions.string_conditions.each do |query, placeholders|
+          placeholders ||= {}
+          string_conditions << [query, placeholders]
+        end
+
+        [hash_conditions] + string_conditions
       end
 
       def scan_options

data/lib/dynamoid/criteria/where_conditions.rb

@@ -4,24 +4,31 @@ module Dynamoid
   module Criteria
     # @private
     class WhereConditions
+      attr_reader :string_conditions
+
       def initialize
-        @conditions = []
+        @hash_conditions = []
+        @string_conditions = []
+      end
+
+      def update_with_hash(hash)
+        @hash_conditions << hash.symbolize_keys
       end
 
-      def update(hash)
-        @conditions << hash.symbolize_keys
+      def update_with_string(query, placeholders)
+        @string_conditions << [query, placeholders]
       end
 
       def keys
-        @conditions.flat_map(&:keys)
+        @hash_conditions.flat_map(&:keys)
       end
 
       def empty?
-        @conditions.empty?
+        @hash_conditions.empty? && @string_conditions.empty?
       end
 
       def [](key)
-        hash = @conditions.find { |h| h.key?(key) }
+        hash = @hash_conditions.find { |h| h.key?(key) }
         hash[key] if hash
       end
     end

data/lib/dynamoid/dirty.rb

@@ -36,8 +36,11 @@ module Dynamoid
         end
       end
 
-      def from_database(*)
-        super.tap(&:clear_changes_information)
+      def from_database(attributes_from_database)
+        super.tap do |model|
+          model.clear_changes_information
+          model.assign_attributes_from_database(DeepDupper.dup_attributes(model.attributes, model.class))
+        end
       end
     end
 
@@ -108,7 +111,7 @@ module Dynamoid
     #
     # @return [ActiveSupport::HashWithIndifferentAccess]
     def changes
-      ActiveSupport::HashWithIndifferentAccess[changed.map { |name| [name, attribute_change(name)] }]
+      ActiveSupport::HashWithIndifferentAccess[changed_attributes.map { |name, old_value| [name, [old_value, read_attribute(name)]] }]
     end
 
     # Returns a hash of attributes that were changed before the model was saved.
@@ -132,19 +135,21 @@ module Dynamoid
     #
     # @return [ActiveSupport::HashWithIndifferentAccess]
     def changed_attributes
-      @changed_attributes ||= ActiveSupport::HashWithIndifferentAccess.new
+      attributes_changed_by_setter.merge(attributes_changed_in_place)
     end
 
     # Clear all dirty data: current changes and previous changes.
     def clear_changes_information
       @previously_changed = ActiveSupport::HashWithIndifferentAccess.new
-      @changed_attributes = ActiveSupport::HashWithIndifferentAccess.new
+      @attributes_changed_by_setter = ActiveSupport::HashWithIndifferentAccess.new
+      @attributes_from_database = HashWithIndifferentAccess.new(DeepDupper.dup_attributes(@attributes, self.class))
     end
 
     # Clears dirty data and moves +changes+ to +previous_changes+.
     def changes_applied
       @previously_changed = changes
-      @changed_attributes = ActiveSupport::HashWithIndifferentAccess.new
+      @attributes_changed_by_setter = ActiveSupport::HashWithIndifferentAccess.new
+      @attributes_from_database = HashWithIndifferentAccess.new(DeepDupper.dup_attributes(@attributes, self.class))
     end
 
     # Remove changes information for the provided attributes.
@@ -152,6 +157,9 @@ module Dynamoid
     # @param attributes [Array[String]] - a list of attributes to clear changes for
     def clear_attribute_changes(names)
       attributes_changed_by_setter.except!(*names)
+
+      slice = HashWithIndifferentAccess.new(@attributes).slice(*names)
+      attributes_from_database.merge!(DeepDupper.dup_attributes(slice, self.class))
     end
 
     # Handle <tt>*_changed?</tt> for +method_missing+.
@@ -220,12 +228,16 @@ module Dynamoid
       previous_changes[name] if attribute_previously_changed?(name)
     end
 
+    # @private
+    def assign_attributes_from_database(attributes_from_database)
+      @attributes_from_database = HashWithIndifferentAccess.new(attributes_from_database)
+    end
+
     private
 
     def changes_include?(name)
-      attributes_changed_by_setter.include?(name)
+      attribute_changed_by_setter?(name) || attribute_changed_in_place?(name)
     end
-    alias attribute_changed_by_setter? changes_include?
 
     # Handle <tt>*_change</tt> for +method_missing+.
     def attribute_change(name)
@@ -259,13 +271,87 @@ module Dynamoid
       previous_changes.include?(name)
     end
 
-    # This is necessary because `changed_attributes` might be overridden in
-    # other implemntations (e.g. in `ActiveRecord`)
-    alias attributes_changed_by_setter changed_attributes
+    def attributes_changed_by_setter
+      @attributes_changed_by_setter ||= ActiveSupport::HashWithIndifferentAccess.new
+    end
+
+    def attribute_changed_by_setter?(name)
+      attributes_changed_by_setter.include?(name)
+    end
+
+    def attributes_from_database
+      @attributes_from_database ||= ActiveSupport::HashWithIndifferentAccess.new
+    end
 
     # Force an attribute to have a particular "before" value
     def set_attribute_was(name, old_value)
       attributes_changed_by_setter[name] = old_value
     end
+
+    def attributes_changed_in_place
+      attributes_from_database.select do |name, _|
+        attribute_changed_in_place?(name)
+      end
+    end
+
+    def attribute_changed_in_place?(name)
+      return false if attribute_changed_by_setter?(name)
+
+      value_from_database = attributes_from_database[name]
+      return false if value_from_database.nil?
+
+      value = read_attribute(name)
+      type_options = self.class.attributes[name.to_sym]
+
+      unless type_options[:type].is_a?(Class) && !type_options[:comparable]
+        # common case
+        value != value_from_database
+      else
+        # objects of a custom type that does not implement its own `#==` method
+        # (that's declared by `comparable: false` or just not specifying the
+        # option `comparable`) are compared by comparing their dumps
+        dump = Dumping.dump_field(value, type_options)
+        dump_from_database = Dumping.dump_field(value_from_database, type_options)
+        dump != dump_from_database
+      end
+    end
+
+    module DeepDupper
+      def self.dup_attributes(attributes, klass)
+        attributes.map do |name, value|
+          type_options = klass.attributes[name.to_sym]
+          value_duplicate = dup_attribute(value, type_options)
+          [name, value_duplicate]
+        end.to_h
+      end
+
+      def self.dup_attribute(value, type_options)
+        type, of = type_options.values_at(:type, :of)
+
+        case value
+        when NilClass, TrueClass, FalseClass, Numeric, Symbol, IO
+          # Till Ruby 2.4 these immutable objects could not be duplicated.
+          # IO objects (used for the binary type) cannot be duplicated as well.
+          value
+        when Array
+          if of.is_a? Class
+            # custom type
+            value.map { |e| dup_attribute(e, type: of) }
+          else
+            value.deep_dup
+          end
+        when Set
+          Set.new(value.map { |e| dup_attribute(e, type: of) })
+        else
+          if type.is_a? Class
+            # custom type
+            dump = Dumping.dump_field(value, type_options)
+            Undumping.undump_field(dump.deep_dup, type_options)
+          else
+            value.deep_dup
+          end
+        end
+      end
+    end
   end
 end

data/lib/dynamoid/dumping.rb

@@ -70,7 +70,8 @@ module Dynamoid
       end
 
       def invalid_value?(value)
-        (value.is_a?(Set) || value.is_a?(String)) && value.empty?
+        (value.is_a?(Set) && value.empty?) ||
+          (value.is_a?(String) && value.empty? && Config.store_empty_string_as_nil)
       end
     end
 
@@ -86,6 +87,12 @@ module Dynamoid
 
     # string -> string
     class StringDumper < Base
+      def process(string)
+        return nil if string.nil?
+        return nil if string.empty? && Config.store_empty_string_as_nil
+
+        string
+      end
     end
 
     # integer -> number
@@ -101,6 +108,8 @@ module Dynamoid
       ALLOWED_TYPES = %i[string integer number date datetime serialized].freeze
 
       def process(set)
+        return nil if set.is_a?(Set) && set.empty?
+
         if @options.key?(:of)
           process_typed_collection(set)
         else
@@ -112,13 +121,13 @@ module Dynamoid
 
       def process_typed_collection(set)
         if allowed_type?
-          dumper = Dumping.find_dumper(element_options)
-          result = set.map { |el| dumper.process(el) }
-
-          if element_type == :string
-            result.reject!(&:empty?)
+          # StringDumper may replace "" with nil so we cannot distinguish it from an explicit nil
+          if element_type == :string && Config.store_empty_string_as_nil
+            set.reject! { |s| s && s.empty? }
           end
 
+          dumper = Dumping.find_dumper(element_options)
+          result = set.map { |el| dumper.process(el) }
           result.to_set
         else
           raise ArgumentError, "Set element type #{element_type} isn't supported"
@@ -164,14 +173,13 @@ module Dynamoid
 
       def process_typed_collection(array)
         if allowed_type?
-          dumper = Dumping.find_dumper(element_options)
-          result = array.map { |el| dumper.process(el) }
-
-          if element_type == :string
-            result.reject!(&:empty?)
+          # StringDumper may replace "" with nil so we cannot distinguish it from an explicit nil
+          if element_type == :string && Config.store_empty_string_as_nil
+            array.reject! { |s| s && s.empty? }
           end
 
-          result
+          dumper = Dumping.find_dumper(element_options)
+          array.map { |el| dumper.process(el) }
         else
           raise ArgumentError, "Array element type #{element_type} isn't supported"
         end
@@ -289,10 +297,24 @@ module Dynamoid
       end
     end
 
-    # string -> string
+    # string -> StringIO
     class BinaryDumper < Base
       def process(value)
-        Base64.strict_encode64(value)
+        store_as_binary = if @options[:store_as_native_binary].nil?
+                            Dynamoid.config.store_binary_as_native
+                          else
+                            @options[:store_as_native_binary]
+                          end
+
+        if store_as_binary
+          if value.is_a?(StringIO) || value.is_a?(IO)
+            value
+          else
+            StringIO.new(value)
+          end
+        else
+          Base64.strict_encode64(value)
+        end
       end
     end
 
@@ -301,10 +323,10 @@ module Dynamoid
       def process(value)
         field_class = @options[:type]
 
-        if value.respond_to?(:dynamoid_dump)
-          value.dynamoid_dump
-        elsif field_class.respond_to?(:dynamoid_dump)
+        if field_class.respond_to?(:dynamoid_dump)
           field_class.dynamoid_dump(value)
+        elsif value.respond_to?(:dynamoid_dump)
+          value.dynamoid_dump
         else
           raise ArgumentError, "Neither #{field_class} nor #{value} supports serialization for Dynamoid."
         end

data/lib/dynamoid/errors.rb

@@ -6,6 +6,7 @@ module Dynamoid
     # Generic Dynamoid error
     class Error < StandardError; end
 
+    class MissingHashKey < Error; end
     class MissingRangeKey < Error; end
 
     class MissingIndex < Error; end
@@ -15,18 +16,27 @@ module Dynamoid
     class InvalidIndex < Error
       def initialize(item)
         if item.is_a? String
-          super(item)
+          super
         else
           super("Validation failed: #{item.errors.full_messages.join(', ')}")
         end
       end
     end
 
+    class RecordNotSaved < Error
+      attr_reader :record
+
+      def initialize(record)
+        super('Failed to save the item')
+        @record = record
+      end
+    end
+
     class RecordNotDestroyed < Error
       attr_reader :record
 
       def initialize(record)
-        super('Failed to destroy item')
+        super('Failed to destroy the item')
         @record = record
       end
     end
@@ -76,8 +86,25 @@ module Dynamoid
 
     class UnsupportedKeyType < Error; end
 
-    class UnknownAttribute < Error; end
+    class UnknownAttribute < Error
+      attr_reader :model_class, :attribute_name
+
+      def initialize(model_class, attribute_name)
+        super("Attribute #{attribute_name} does not exist in #{model_class}")
+
+        @model_class = model_class
+        @attribute_name = attribute_name
+      end
+    end
 
     class SubclassNotFound < Error; end
+
+    class Rollback < Error; end
+
+    class ScanProhibited < Error
+      def initialize(_msg = nil)
+        super('Scan operations prohibited. Modify Dynamoid::Config.error_on_scan to change this behavior.')
+      end
+    end
   end
 end

data/lib/dynamoid/fields.rb

@@ -197,9 +197,19 @@ module Dynamoid
       #     field :id, :integer
       #   end
       #
+      # To declare a new attribute with not-default type as a table hash key a
+      # :key_type option can be used:
+      #
+      #   class User
+      #     include Dynamoid::Document
+      #
+      #     table key: :user_id, key_type: :integer
+      #   end
+      #
       # @param options [Hash] options to override default table settings
       # @option options [Symbol] :name name of a table
       # @option options [Symbol] :key name of a hash key attribute
+      # @option options [Symbol] :key_type type of a hash key attribute
       # @option options [Symbol] :inheritance_field name of an attribute used for STI
       # @option options [Symbol] :capacity_mode table billing mode - either +provisioned+ or +on_demand+
       # @option options [Integer] :write_capacity table write capacity units
@@ -210,11 +220,11 @@ module Dynamoid
       # @since 0.4.0
       def table(options)
         self.options = options
-
         # a default 'id' column is created when Dynamoid::Document is included
         unless attributes.key? hash_key
           remove_field :id
-          field(hash_key)
+          key_type = options[:key_type] || :string
+          field(hash_key, key_type)
         end
 
         # The created_at/updated_at fields are declared in the `included` callback first.
@@ -291,7 +301,7 @@ module Dynamoid
       old_value = read_attribute(name)
 
       unless attribute_is_present_on_model?(name)
-        raise Dynamoid::Errors::UnknownAttribute, "Attribute #{name} is not part of the model"
+        raise Dynamoid::Errors::UnknownAttribute.new(self.class, name)
       end
 
       if association = @associations[name]