dynamoid 3.0.0 → 3.1.0

data/lib/dynamoid/adapter_plugin/query.rb

@@ -0,0 +1,144 @@
+module Dynamoid
+  module AdapterPlugin
+    class Query
+      OPTIONS_KEYS = [
+        :limit, :hash_key, :hash_value, :range_key, :consistent_read, :scan_index_forward,
+        :select, :index_name, :batch_size, :exclusive_start_key, :record_limit, :scan_limit
+      ]
+
+      attr_reader :client, :table, :options, :conditions
+
+      def initialize(client, table, opts = {})
+        @client = client
+        @table = table
+
+        opts = opts.symbolize_keys
+        @options = opts.slice(*OPTIONS_KEYS)
+        @conditions = opts.except(*OPTIONS_KEYS)
+      end
+
+      def call
+        request = build_request
+
+        Enumerator.new do |yielder|
+          record_count = 0
+          scan_count = 0
+
+          backoff = Dynamoid.config.backoff ? Dynamoid.config.build_backoff : nil
+
+          loop do
+            # Adjust the limit down if the remaining record and/or scan limit are
+            # lower to obey limits. We can assume the difference won't be
+            # negative due to break statements below but choose smaller limit
+            # which is why we have 2 separate if statements.
+            # NOTE: Adjusting based on record_limit can cause many HTTP requests
+            # being made. We may want to change this behavior, but it affects
+            # filtering on data with potentially large gaps.
+            # Example:
+            #    User.where('created_at.gte' => 1.day.ago).record_limit(1000)
+            #    Records 1-999 User's that fit criteria
+            #    Records 1000-2000 Users's that do not fit criteria
+            #    Record 2001 fits criteria
+            # The underlying implementation will have 1 page for records 1-999
+            # then will request with limit 1 for records 1000-2000 (making 1000
+            # requests of limit 1) until hit record 2001.
+            if request[:limit] && record_limit && record_limit - record_count < request[:limit]
+              request[:limit] = record_limit - record_count
+            end
+            if request[:limit] && scan_limit && scan_limit - scan_count < request[:limit]
+              request[:limit] = scan_limit - scan_count
+            end
+
+            response = client.query(request)
+
+            yielder << response
+
+            record_count += response.count
+            break if record_limit && record_count >= record_limit
+
+            scan_count += response.scanned_count
+            break if scan_limit && scan_count >= scan_limit
+
+            if response.last_evaluated_key
+              request[:exclusive_start_key] = response.last_evaluated_key
+            else
+              break
+            end
+
+            backoff.call if backoff
+          end
+        end
+      end
+
+      private
+
+      def build_request
+        request = options.slice(
+          :consistent_read,
+          :scan_index_forward,
+          :select,
+          :index_name,
+          :exclusive_start_key
+        ).compact
+
+        # Deal with various limits and batching
+        batch_size = options[:batch_size]
+        limit = [record_limit, scan_limit, batch_size].compact.min
+
+        request[:limit]          = limit if limit
+        request[:table_name]     = table.name
+        request[:key_conditions] = key_conditions
+        request[:query_filter]   = query_filter
+
+        request
+      end
+
+      def record_limit
+        options[:record_limit]
+      end
+
+      def scan_limit
+        options[:scan_limit]
+      end
+
+      def hash_key_name
+        (options[:hash_key] || table.hash_key)
+      end
+
+      def range_key_name
+        (options[:range_key] || table.range_key)
+      end
+
+      def key_conditions
+        result = {
+          hash_key_name => {
+            comparison_operator: AwsSdkV3::EQ,
+            attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::EQ, options[:hash_value].freeze)
+          }
+        }
+
+        conditions.slice(*AwsSdkV3::RANGE_MAP.keys).each do |k, _v|
+          op = AwsSdkV3::RANGE_MAP[k]
+
+          result[range_key_name] = {
+            comparison_operator: op,
+            attribute_value_list: AwsSdkV3.attribute_value_list(op, conditions[k].freeze)
+          }
+        end
+
+        result
+      end
+
+      def query_filter
+        conditions.except(*AwsSdkV3::RANGE_MAP.keys).reduce({}) do |result, (attr, cond)|
+          condition = {
+            comparison_operator: AwsSdkV3::FIELD_MAP[cond.keys[0]],
+            attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::FIELD_MAP[cond.keys[0]], cond.values[0].freeze)
+          }
+          result[attr] = condition
+          result
+        end
+      end
+    end
+  end
+end

data/lib/dynamoid/adapter_plugin/scan.rb

@@ -0,0 +1,107 @@
+module Dynamoid
+  module AdapterPlugin
+    class Scan
+      attr_reader :client, :table, :conditions, :options
+
+      def initialize(client, table, conditions = {}, options = {})
+        @client = client
+        @table = table
+        @conditions = conditions
+        @options = options
+      end
+
+      def call
+        request = build_request
+
+        Enumerator.new do |yielder|
+          record_count = 0
+          scan_count = 0
+
+          backoff = Dynamoid.config.backoff ? Dynamoid.config.build_backoff : nil
+
+          loop do
+            # Adjust the limit down if the remaining record and/or scan limit are
+            # lower to obey limits. We can assume the difference won't be
+            # negative due to break statements below but choose smaller limit
+            # which is why we have 2 separate if statements.
+            # NOTE: Adjusting based on record_limit can cause many HTTP requests
+            # being made. We may want to change this behavior, but it affects
+            # filtering on data with potentially large gaps.
+            # Example:
+            #    User.where('created_at.gte' => 1.day.ago).record_limit(1000)
+            #    Records 1-999 User's that fit criteria
+            #    Records 1000-2000 Users's that do not fit criteria
+            #    Record 2001 fits criteria
+            # The underlying implementation will have 1 page for records 1-999
+            # then will request with limit 1 for records 1000-2000 (making 1000
+            # requests of limit 1) until hit record 2001.
+            if request[:limit] && record_limit && record_limit - record_count < request[:limit]
+              request[:limit] = record_limit - record_count
+            end
+            if request[:limit] && scan_limit && scan_limit - scan_count < request[:limit]
+              request[:limit] = scan_limit - scan_count
+            end
+
+            response = client.scan(request)
+
+            yielder << response
+
+            record_count += response.count
+            break if record_limit && record_count >= record_limit
+
+            scan_count += response.scanned_count
+            break if scan_limit && scan_count >= scan_limit
+
+            # Keep pulling if we haven't finished paging in all data
+            if response.last_evaluated_key
+              request[:exclusive_start_key] = response.last_evaluated_key
+            else
+              break
+            end
+
+            backoff.call if backoff
+          end
+        end
+      end
+
+      private
+
+      def build_request
+        request = options.slice(
+          :consistent_read,
+          :exclusive_start_key,
+          :select
+        ).compact
+
+        # Deal with various limits and batching
+        batch_size = options[:batch_size]
+        limit = [record_limit, scan_limit, batch_size].compact.min
+
+        request[:limit]       = limit if limit
+        request[:table_name]  = table.name
+        request[:scan_filter] = scan_filter
+
+        request
+      end
+
+      def record_limit
+        options[:record_limit]
+      end
+
+      def scan_limit
+        options[:scan_limit]
+      end
+
+      def scan_filter
+        conditions.reduce({}) do |result, (attr, cond)|
+          condition = {
+            comparison_operator: AwsSdkV3::FIELD_MAP[cond.keys[0]],
+            attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::FIELD_MAP[cond.keys[0]], cond.values[0].freeze)
+          }
+          result[attr] = condition
+          result
+        end
+      end
+    end
+  end
+end

data/lib/dynamoid/components.rb

@@ -14,7 +14,7 @@ module Dynamoid
 
       before_create :set_created_at
       before_save :set_updated_at
-      after_initialize :set_type
+      after_initialize :set_inheritance_field
     end
 
     include ActiveModel::AttributeMethods

data/lib/dynamoid/config.rb

@@ -2,6 +2,7 @@
 
 require 'uri'
 require 'logger'
+require 'null_logger'
 require 'dynamoid/config/options'
 require 'dynamoid/config/backoff_strategies/constant_backoff'
 require 'dynamoid/config/backoff_strategies/exponential_backoff'
@@ -41,6 +42,10 @@ module Dynamoid
       constant: BackoffStrategies::ConstantBackoff,
       exponential: BackoffStrategies::ExponentialBackoff
     }
+    option :http_continue_timeout, default: nil # specify if you'd like to overwrite Aws Configure - default: 1
+    option :http_idle_timeout, default: nil     #                                                  - default: 5
+    option :http_open_timeout, default: nil     #                                                  - default: 15
+    option :http_read_timeout, default: nil     #                                                  - default: 60
 
     # The default logger for Dynamoid: either the Rails logger or just stdout.
     #
@@ -61,7 +66,7 @@ module Dynamoid
     # @since 0.2.0
     def logger=(logger)
       case logger
-      when false, nil then @logger = Logger.new('/dev/null')
+      when false, nil then @logger = NullLogger.new
       when true then @logger = default_logger
       else
         @logger = logger if logger.respond_to?(:info)

data/lib/dynamoid/criteria/chain.rb

@@ -18,8 +18,9 @@ module Dynamoid #:nodoc:
         @scan_index_forward = true
 
         # Honor STI and :type field if it presents
-        if @source.attributes.key?(:type)
-          @query[:'type.in'] = @source.deep_subclasses.map(&:name) << @source.name
+        type = @source.inheritance_field
+        if @source.attributes.key?(type)
+          @query[:"#{type}.in"] = @source.deep_subclasses.map(&:name) << @source.name
         end
       end
 
@@ -51,6 +52,14 @@ module Dynamoid #:nodoc:
         records
       end
 
+      def count
+        if key_present?
+          count_via_query
+        else
+          count_via_scan
+        end
+      end
+
       # Returns the last fetched record matched the criteria
       # Enumerable doesn't implement `last`, only `first`
       # So we have to implement it ourselves
@@ -163,6 +172,14 @@ module Dynamoid #:nodoc:
         end
       end
 
+      def count_via_query
+        Dynamoid.adapter.query_count(source.table_name, range_query)
+      end
+
+      def count_via_scan
+        Dynamoid.adapter.scan_count(source.table_name, scan_query, scan_opts)
+      end
+
       def range_hash(key)
         name, operation = key.to_s.split('.')
         val = type_cast_condition_parameter(name, query[key])

data/lib/dynamoid/document.rb

@@ -50,6 +50,11 @@ module Dynamoid #:nodoc:
         options[:write_capacity] || Dynamoid::Config.write_capacity
       end
 
+      # Returns the field name used to support STI for this table.
+      def inheritance_field
+        options[:inheritance_field] || :type
+      end
+
       # Returns the id field for this class.
       #
       # @since 0.4.0
@@ -102,7 +107,7 @@ module Dynamoid #:nodoc:
       #
       # @since 0.2.0
       def build(attrs = {})
-        attrs[:type] ? attrs[:type].constantize.new(attrs) : new(attrs)
+        choose_right_class(attrs).new(attrs)
       end
 
       # Does this object exist?
@@ -251,18 +256,43 @@ module Dynamoid #:nodoc:
             attrs.each do |k, v|
               value_casted = TypeCasting.cast_field(v, attributes[k])
               value_dumped = Dumping.dump_field(value_casted, attributes[k])
+
               t.set(k => value_dumped)
             end
           end
+
           attrs_undumped = Undumping.undump_attributes(new_attrs, attributes)
           new(attrs_undumped)
         rescue Dynamoid::Errors::ConditionalCheckFailedException
         end
       end
 
+      def inc(hash_key_value, range_key_value=nil, counters)
+        options = if range_key
+                    value_casted = TypeCasting.cast_field(range_key_value, attributes[range_key])
+                    value_dumped = Dumping.dump_field(value_casted, attributes[range_key])
+                    { range_key: value_dumped }
+                  else
+                    {}
+                  end
+
+        Dynamoid.adapter.update_item(table_name, hash_key_value, options) do |t|
+          counters.each do |k, v|
+            value_casted = TypeCasting.cast_field(v, attributes[k])
+            value_dumped = Dumping.dump_field(value_casted, attributes[k])
+
+            t.add(k => value_dumped)
+          end
+        end
+      end
+
       def deep_subclasses
         subclasses + subclasses.map(&:deep_subclasses).flatten
       end
+
+      def choose_right_class(attrs)
+        attrs[inheritance_field] ? attrs[inheritance_field].constantize : self
+      end
     end
 
     # Initialize a new object.
@@ -281,6 +311,7 @@ module Dynamoid #:nodoc:
         @new_record = true
         @attributes ||= {}
         @associations ||= {}
+        @attributes_before_type_cast ||= {}
 
         self.class.attributes.each do |_, options|
           if options[:type].is_a?(Class) && options[:default]

data/lib/dynamoid/dumping.rb

@@ -11,7 +11,9 @@ module Dynamoid
     end
 
     def self.dump_field(value, options)
-      dumper = field_dumper(options)
+      return nil if value.nil?
+
+      dumper = find_dumper(options)
 
       if dumper.nil?
         raise ArgumentError, "Unknown type #{options[:type]}"
@@ -20,7 +22,7 @@ module Dynamoid
       dumper.process(value)
     end
 
-    def self.field_dumper(options)
+    def self.find_dumper(options)
       dumper_class = case options[:type]
                      when :string     then StringDumper
                      when :integer    then IntegerDumper
@@ -64,10 +66,106 @@ module Dynamoid
 
     # set -> set
     class SetDumper < Base
+      ALLOWED_TYPES = [:string, :integer, :number, :date, :datetime, :serialized]
+
+      def process(set)
+        if @options.key?(:of)
+          process_typed_collection(set)
+        else
+          set
+        end
+      end
+
+      private
+
+      def process_typed_collection(set)
+        if allowed_type?
+          dumper = Dumping.find_dumper(element_options)
+          result = set.map { |el| dumper.process(el) }
+
+          if element_type == :string
+            result.reject!(&:empty?)
+          end
+
+          result.to_set
+        else
+          raise ArgumentError, "Set element type #{element_type} isn't supported"
+        end
+      end
+
+      def allowed_type?
+        ALLOWED_TYPES.include?(element_type) || element_type.is_a?(Class)
+      end
+
+      def element_type
+        unless @options[:of].is_a?(Hash)
+          @options[:of]
+        else
+          @options[:of].keys.first
+        end
+      end
+
+      def element_options
+        unless @options[:of].is_a?(Hash)
+          { type: element_type }
+        else
+          @options[:of][element_type].dup.tap do |options|
+            options[:type] = element_type
+          end
+        end
+      end
     end
 
     # array -> array
     class ArrayDumper < Base
+      ALLOWED_TYPES = [:string, :integer, :number, :date, :datetime, :serialized]
+
+      def process(array)
+        if @options.key?(:of)
+          process_typed_collection(array)
+        else
+          array
+        end
+      end
+
+      private
+
+      def process_typed_collection(array)
+        if allowed_type?
+          dumper = Dumping.find_dumper(element_options)
+          result = array.map { |el| dumper.process(el) }
+
+          if element_type == :string
+            result.reject!(&:empty?)
+          end
+
+          result
+        else
+          raise ArgumentError, "Array element type #{element_type} isn't supported"
+        end
+      end
+
+      def allowed_type?
+        ALLOWED_TYPES.include?(element_type) || element_type.is_a?(Class)
+      end
+
+      def element_type
+        unless @options[:of].is_a?(Hash)
+          @options[:of]
+        else
+          @options[:of].keys.first
+        end
+      end
+
+      def element_options
+        unless @options[:of].is_a?(Hash)
+          { type: element_type }
+        else
+          @options[:of][element_type].dup.tap do |options|
+            options[:type] = element_type
+          end
+        end
+      end
     end
 
     # datetime -> integer/string
@@ -122,6 +220,34 @@ module Dynamoid
 
     # any standard Ruby object -> self
     class RawDumper < Base
+      def process(value)
+        deep_sanitize(value)
+      end
+
+      private
+
+      def deep_sanitize(el)
+        case el
+        when Hash
+          sanitize_hash(el).transform_values { |v| deep_sanitize(v) }
+        when Array
+          sanitize_array(el).map { |v| deep_sanitize(v) }
+        else
+          el
+        end
+      end
+
+      def sanitize_hash(h)
+        h.transform_values { |v| invalid_value?(v) ? nil : v }
+      end
+
+      def sanitize_array(a)
+        a.map { |v| invalid_value?(v) ? nil : v }
+      end
+
+      def invalid_value?(v)
+        (v.is_a?(Set) || v.is_a?(String)) && v.empty?
+      end
     end
 
     # object -> string