dynamoid 3.1.0 → 3.2.0

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/scan.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require_relative 'middleware/backoff'
+require_relative 'middleware/limit'
+require_relative 'middleware/start_key'
+
+module Dynamoid
+  module AdapterPlugin
+    class AwsSdkV3
+      class Scan
+        attr_reader :client, :table, :conditions, :options
+
+        def initialize(client, table, conditions = {}, options = {})
+          @client = client
+          @table = table
+          @conditions = conditions
+          @options = options
+        end
+
+        def call
+          request = build_request
+
+          Enumerator.new do |yielder|
+            api_call = -> (request) do
+              client.scan(request).tap do |response|
+                yielder << response
+              end
+            end
+
+            middlewares = Middleware::Backoff.new(
+              Middleware::StartKey.new(
+                Middleware::Limit.new(api_call, record_limit: record_limit, scan_limit: scan_limit)
+              )
+            )
+
+            catch :stop_pagination do
+              loop do
+                middlewares.call(request)
+              end
+            end
+          end
+        end
+
+        private
+
+        def build_request
+          request = options.slice(
+            :consistent_read,
+            :exclusive_start_key,
+            :select
+          ).compact
+
+          # Deal with various limits and batching
+          batch_size = options[:batch_size]
+          limit = [record_limit, scan_limit, batch_size].compact.min
+
+          request[:limit]       = limit if limit
+          request[:table_name]  = table.name
+          request[:scan_filter] = scan_filter
+
+          request
+        end
+
+        def record_limit
+          options[:record_limit]
+        end
+
+        def scan_limit
+          options[:scan_limit]
+        end
+
+        def scan_filter
+          conditions.reduce({}) do |result, (attr, cond)|
+            condition = {
+              comparison_operator: AwsSdkV3::FIELD_MAP[cond.keys[0]],
+              attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::FIELD_MAP[cond.keys[0]], cond.values[0].freeze)
+            }
+            result[attr] = condition
+            result
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/table.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Dynamoid
+  module AdapterPlugin
+    class AwsSdkV3
+      # Represents a table. Exposes data from the "DescribeTable" API call, and also
+      # provides methods for coercing values to the proper types based on the table's schema data
+      class Table
+        attr_reader :schema
+
+        #
+        # @param [Hash] schema Data returns from a "DescribeTable" call
+        #
+        def initialize(schema)
+          @schema = schema[:table]
+        end
+
+        def range_key
+          @range_key ||= schema[:key_schema].find { |d| d[:key_type] == RANGE_KEY }.try(:attribute_name)
+        end
+
+        def range_type
+          range_type ||= schema[:attribute_definitions].find do |d|
+            d[:attribute_name] == range_key
+          end.try(:fetch, :attribute_type, nil)
+        end
+
+        def hash_key
+          @hash_key ||= schema[:key_schema].find { |d| d[:key_type] == HASH_KEY }.try(:attribute_name).to_sym
+        end
+
+        #
+        # Returns the API type (e.g. "N", "S") for the given column, if the schema defines it,
+        # nil otherwise
+        #
+        def col_type(col)
+          col = col.to_s
+          col_def = schema[:attribute_definitions].find { |d| d[:attribute_name] == col.to_s }
+          col_def && col_def[:attribute_type]
+        end
+
+        def item_count
+          schema[:item_count]
+        end
+
+        def name
+          schema[:table_name]
+        end
+      end
+    end
+  end
+end

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/until_past_table_status.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Dynamoid
+  module AdapterPlugin
+    class AwsSdkV3
+      class UntilPastTableStatus
+        attr_reader :table_name, :status
+
+        def initialize(table_name, status = :creating)
+          @table_name = table_name
+          @status = status
+        end
+
+        def call
+          counter = 0
+          resp = nil
+          begin
+            check = { again: true }
+            while check[:again]
+              sleep Dynamoid::Config.sync_retry_wait_seconds
+              resp = client.describe_table(table_name: table_name)
+              check = check_table_status?(counter, resp, status)
+              Dynamoid.logger.info "Checked table status for #{table_name} (check #{check.inspect})"
+              counter += 1
+            end
+          # If you issue a DescribeTable request immediately after a CreateTable
+          #   request, DynamoDB might return a ResourceNotFoundException.
+          # This is because DescribeTable uses an eventually consistent query,
+          #   and the metadata for your table might not be available at that moment.
+          # Wait for a few seconds, and then try the DescribeTable request again.
+          # See: http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#describe_table-instance_method
+          rescue Aws::DynamoDB::Errors::ResourceNotFoundException => e
+            case status
+            when :creating then
+              if counter >= Dynamoid::Config.sync_retry_max_times
+                Dynamoid.logger.warn "Waiting on table metadata for #{table_name} (check #{counter})"
+                retry # start over at first line of begin, does not reset counter
+              else
+                Dynamoid.logger.error "Exhausted max retries (Dynamoid::Config.sync_retry_max_times) waiting on table metadata for #{table_name} (check #{counter})"
+                raise e
+              end
+            else
+              # When deleting a table, "not found" is the goal.
+              Dynamoid.logger.info "Checked table status for #{table_name}: Not Found (check #{check.inspect})"
+            end
+          end
+        end
+
+        private
+
+        def check_table_status?(counter, resp, expect_status)
+          status = PARSE_TABLE_STATUS.call(resp)
+          again = counter < Dynamoid::Config.sync_retry_max_times &&
+                  status == TABLE_STATUSES[expect_status]
+          { again: again, status: status, counter: counter }
+        end
+      end
+    end
+  end
+end

data/lib/dynamoid/associations/has_and_belongs_to_many.rb

@@ -16,6 +16,7 @@ module Dynamoid #:nodoc:
         key_name = options[:inverse_of] || source.class.to_s.pluralize.underscore.to_sym
         guess = target_class.associations[key_name]
         return nil if guess.nil? || guess[:type] != :has_and_belongs_to_many
+
         key_name
       end
     end

data/lib/dynamoid/associations/has_many.rb

@@ -16,6 +16,7 @@ module Dynamoid #:nodoc:
         key_name = options[:inverse_of] || source.class.to_s.singularize.underscore.to_sym
         guess = target_class.associations[key_name]
         return nil if guess.nil? || guess[:type] != :belongs_to
+
         key_name
       end
     end

data/lib/dynamoid/associations/has_one.rb

@@ -17,6 +17,7 @@ module Dynamoid #:nodoc:
         key_name = options[:inverse_of] || source.class.to_s.singularize.underscore.to_sym
         guess = target_class.associations[key_name]
         return nil if guess.nil? || guess[:type] != :belongs_to
+
         key_name
       end
     end

data/lib/dynamoid/associations/single_association.rb

@@ -81,6 +81,7 @@ module Dynamoid #:nodoc:
       # @since 0.2.0
       def find_target
         return if source_ids.empty?
+
         target_class.find(source_ids.first)
       end
 

data/lib/dynamoid/criteria.rb

@@ -8,17 +8,17 @@ module Dynamoid
     extend ActiveSupport::Concern
 
     module ClassMethods
-      %i[where all first last each record_limit scan_limit batch start scan_index_forward].each do |meth|
+      %i[where all first last each record_limit scan_limit batch start scan_index_forward find_by_pages].each do |meth|
         # Return a criteria chain in response to a method that will begin or end a chain. For more information,
         # see Dynamoid::Criteria::Chain.
         #
         # @since 0.2.0
-        define_method(meth) do |*args|
+        define_method(meth) do |*args, &blk|
           chain = Dynamoid::Criteria::Chain.new(self)
           if args
-            chain.send(meth, *args)
+            chain.send(meth, *args, &blk)
           else
-            chain.send(meth)
+            chain.send(meth, &blk)
           end
         end
       end

data/lib/dynamoid/criteria/chain.rb

@@ -1,12 +1,17 @@
 # frozen_string_literal: true
 
-module Dynamoid #:nodoc:
+require_relative 'key_fields_detector'
+require_relative 'ignored_conditions_detector'
+require_relative 'overwritten_conditions_detector'
+require_relative 'nonexistent_fields_detector'
+
+module Dynamoid
   module Criteria
     # The criteria chain is equivalent to an ActiveRecord relation (and realistically I should change the name from
     # chain to relation). It is a chainable object that builds up a query and eventually executes it by a Query or Scan.
     class Chain
-      attr_accessor :query, :source, :values, :consistent_read
-      attr_reader :hash_key, :range_key, :index_name
+      attr_reader :query, :source, :consistent_read, :key_fields_detector
+
       include Enumerable
       # Create a new criteria chain.
       #
@@ -22,6 +27,9 @@ module Dynamoid #:nodoc:
         if @source.attributes.key?(type)
           @query[:"#{type}.in"] = @source.deep_subclasses.map(&:name) << @source.name
         end
+
+        # we should re-initialize keys detector every time we change query
+        @key_fields_detector = KeyFieldsDetector.new(@query, @source)
       end
 
       # The workhorse method of the criteria chain. Each key in the passed in hash will become another criteria that the
@@ -36,7 +44,26 @@ module Dynamoid #:nodoc:
       #
       # @since 0.2.0
       def where(args)
-        query.update(args.dup.symbolize_keys)
+        detector = IgnoredConditionsDetector.new(args)
+        if detector.found?
+          Dynamoid.logger.warn(detector.warning_message)
+        end
+
+        detector = OverwrittenConditionsDetector.new(@query, args)
+        if detector.found?
+          Dynamoid.logger.warn(detector.warning_message)
+        end
+
+        detector = NonexistentFieldsDetector.new(args, @source)
+        if detector.found?
+          Dynamoid.logger.warn(detector.warning_message)
+        end
+
+        query.update(args.symbolize_keys)
+
+        # we should re-initialize keys detector every time we change query
+        @key_fields_detector = KeyFieldsDetector.new(@query, @source)
+
         self
       end
 
@@ -53,7 +80,7 @@ module Dynamoid #:nodoc:
       end
 
       def count
-        if key_present?
+        if @key_fields_detector.key_present?
           count_via_query
         else
           count_via_scan
@@ -74,13 +101,13 @@ module Dynamoid #:nodoc:
         ids = []
         ranges = []
 
-        if key_present?
-          Dynamoid.adapter.query(source.table_name, range_query).collect do |hash|
+        if @key_fields_detector.key_present?
+          Dynamoid.adapter.query(source.table_name, range_query).flat_map{ |i| i }.collect do |hash|
             ids << hash[source.hash_key.to_sym]
             ranges << hash[source.range_key.to_sym] if source.range_key
           end
         else
-          Dynamoid.adapter.scan(source.table_name, scan_query, scan_opts).collect do |hash|
+          Dynamoid.adapter.scan(source.table_name, scan_query, scan_opts).flat_map{ |i| i }.collect do |hash|
             ids << hash[source.hash_key.to_sym]
             ranges << hash[source.range_key.to_sym] if source.range_key
           end
@@ -128,6 +155,10 @@ module Dynamoid #:nodoc:
         records.each(&block)
       end
 
+      def find_by_pages(&block)
+        pages.each(&block)
+      end
+
       private
 
       # The actual records referenced by the association.
@@ -136,42 +167,57 @@ module Dynamoid #:nodoc:
       #
       # @since 0.2.0
       def records
-        if key_present?
-          records_via_query
+        pages.lazy.flat_map { |i| i }
+      end
+
+      # Arrays of records, sized based on the actual pages produced by DynamoDB
+      #
+      # @return [Enumerator] an iterator of the found records.
+      #
+      # @since 3.1.0
+      def pages
+        if @key_fields_detector.key_present?
+          pages_via_query
         else
-          records_via_scan
+          issue_scan_warning if Dynamoid::Config.warn_on_scan && query.present?
+          pages_via_scan
         end
       end
 
-      def records_via_query
+      # If the query matches an index, we'll query the associated table to find results.
+      #
+      # @return [Enumerator] an iterator of the found pages. An array of records
+      #
+      # @since 3.1.0
+      def pages_via_query
         Enumerator.new do |yielder|
-          Dynamoid.adapter.query(source.table_name, range_query).each do |hash|
-            yielder.yield source.from_database(hash)
+          Dynamoid.adapter.query(source.table_name, range_query).each do |items, metadata|
+            yielder.yield items.map { |hash| source.from_database(hash) }, metadata.slice(:last_evaluated_key)
           end
         end
       end
 
       # If the query does not match an index, we'll manually scan the associated table to find results.
       #
-      # @return [Enumerator] an iterator of the found records.
+      # @return [Enumerator] an iterator of the found pages. An array of records
       #
-      # @since 0.2.0
-      def records_via_scan
-        if Dynamoid::Config.warn_on_scan && query.present?
-          Dynamoid.logger.warn 'Queries without an index are forced to use scan and are generally much slower than indexed queries!'
-          Dynamoid.logger.warn "You can index this query by adding index declaration to #{source.to_s.downcase}.rb:"
-          Dynamoid.logger.warn "* global_secondary_index hash_key: 'some-name', range_key: 'some-another-name'"
-          Dynamoid.logger.warn "* local_secondary_indexe range_key: 'some-name'"
-          Dynamoid.logger.warn "Not indexed attributes: #{query.keys.sort.collect { |name| ":#{name}" }.join(', ')}"
-        end
-
+      # @since 3.1.0
+      def pages_via_scan
         Enumerator.new do |yielder|
-          Dynamoid.adapter.scan(source.table_name, scan_query, scan_opts).each do |hash|
-            yielder.yield source.from_database(hash)
+          Dynamoid.adapter.scan(source.table_name, scan_query, scan_opts).each do |items, metadata|
+            yielder.yield(items.map { |hash| source.from_database(hash) }, metadata.slice(:last_evaluated_key))
           end
         end
       end
 
+      def issue_scan_warning
+        Dynamoid.logger.warn 'Queries without an index are forced to use scan and are generally much slower than indexed queries!'
+        Dynamoid.logger.warn "You can index this query by adding index declaration to #{source.to_s.downcase}.rb:"
+        Dynamoid.logger.warn "* global_secondary_index hash_key: 'some-name', range_key: 'some-another-name'"
+        Dynamoid.logger.warn "* local_secondary_index range_key: 'some-name'"
+        Dynamoid.logger.warn "Not indexed attributes: #{query.keys.sort.collect { |name| ":#{name}" }.join(', ')}"
+      end
+
       def count_via_query
         Dynamoid.adapter.query_count(source.table_name, range_query)
       end
@@ -238,24 +284,24 @@ module Dynamoid #:nodoc:
         opts = {}
 
         # Add hash key
-        opts[:hash_key] = @hash_key
-        opts[:hash_value] = type_cast_condition_parameter(@hash_key, query[@hash_key])
+        opts[:hash_key] = @key_fields_detector.hash_key
+        opts[:hash_value] = type_cast_condition_parameter(@key_fields_detector.hash_key, query[@key_fields_detector.hash_key])
 
         # Add range key
-        if @range_key
-          opts[:range_key] = @range_key
-          if query[@range_key].present?
-            value = type_cast_condition_parameter(@range_key, query[@range_key])
+        if @key_fields_detector.range_key
+          opts[:range_key] = @key_fields_detector.range_key
+          if query[@key_fields_detector.range_key].present?
+            value = type_cast_condition_parameter(@key_fields_detector.range_key, query[@key_fields_detector.range_key])
             opts.update(range_eq: value)
           end
 
-          query.keys.select { |k| k.to_s =~ /^#{@range_key}\./ }.each do |key|
+          query.keys.select { |k| k.to_s =~ /^#{@key_fields_detector.range_key}\./ }.each do |key|
             opts.merge!(range_hash(key))
           end
         end
 
-        (query.keys.map(&:to_sym) - [@hash_key.to_sym, @range_key.try(:to_sym)])
-          .reject { |k, _| k.to_s =~ /^#{@range_key}\./ }
+        (query.keys.map(&:to_sym) - [@key_fields_detector.hash_key.to_sym, @key_fields_detector.range_key.try(:to_sym)])
+          .reject { |k, _| k.to_s =~ /^#{@key_fields_detector.range_key}\./ }
           .each do |key|
           if key.to_s.include?('.')
             opts.update(field_hash(key))
@@ -284,53 +330,14 @@ module Dynamoid #:nodoc:
         end
       end
 
-      def key_present?
-        query_keys = query.keys.collect { |k| k.to_s.split('.').first }
-
-        # See if querying based on table hash key
-        if query.keys.map(&:to_s).include?(source.hash_key.to_s)
-          @hash_key = source.hash_key
-
-          # Use table's default range key
-          if query_keys.include?(source.range_key.to_s)
-            @range_key = source.range_key
-            return true
-          end
-
-          # See if can use any local secondary index range key
-          # Chooses the first LSI found that can be utilized for the query
-          source.local_secondary_indexes.each do |_, lsi|
-            next unless query_keys.include?(lsi.range_key.to_s)
-            @range_key = lsi.range_key
-            @index_name = lsi.name
-          end
-
-          return true
-        end
-
-        # See if can use any global secondary index
-        # Chooses the first GSI found that can be utilized for the query
-        # But only do so if projects ALL attributes otherwise we won't
-        # get back full data
-        source.global_secondary_indexes.each do |_, gsi|
-          next unless query.keys.map(&:to_s).include?(gsi.hash_key.to_s) && gsi.projected_attributes == :all
-          @hash_key = gsi.hash_key
-          @range_key = gsi.range_key
-          @index_name = gsi.name
-          return true
-        end
-
-        # Could not utilize any indices so we'll have to scan
-        false
-      end
-
       # Start key needs to be set up based on the index utilized
       # If using a secondary index then we must include the index's composite key
       # as well as the tables composite key.
       def start_key
         return @start if @start.is_a?(Hash)
-        hash_key = @hash_key || source.hash_key
-        range_key = @range_key || source.range_key
+
+        hash_key = @key_fields_detector.hash_key || source.hash_key
+        range_key = @key_fields_detector.range_key || source.range_key
 
         key = {}
         key[hash_key] = type_cast_condition_parameter(hash_key, @start.send(hash_key))
@@ -349,7 +356,7 @@ module Dynamoid #:nodoc:
 
       def query_opts
         opts = {}
-        opts[:index_name] = @index_name if @index_name
+        opts[:index_name] = @key_fields_detector.index_name if @key_fields_detector.index_name
         opts[:select] = 'ALL_ATTRIBUTES'
         opts[:record_limit] = @record_limit if @record_limit
         opts[:scan_limit] = @scan_limit if @scan_limit