dynamoid 3.1.0 → 3.2.0

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/create_table.rb

@@ -0,0 +1,234 @@
+# frozen_string_literal: true
+
+require_relative 'until_past_table_status'
+
+module Dynamoid
+  module AdapterPlugin
+    class AwsSdkV3
+      class CreateTable
+        attr_reader :client, :table_name, :key, :options
+
+        def initialize(client, table_name, key, options)
+          @client = client
+          @table_name = table_name
+          @key = key
+          @options = options
+        end
+
+        def call
+          read_capacity = options[:read_capacity] || Dynamoid::Config.read_capacity
+          write_capacity = options[:write_capacity] || Dynamoid::Config.write_capacity
+
+          secondary_indexes = options.slice(
+            :local_secondary_indexes,
+            :global_secondary_indexes
+          )
+          ls_indexes = options[:local_secondary_indexes]
+          gs_indexes = options[:global_secondary_indexes]
+
+          key_schema = {
+            hash_key_schema: { key => (options[:hash_key_type] || :string) },
+            range_key_schema: options[:range_key]
+          }
+          attribute_definitions = build_all_attribute_definitions(
+            key_schema,
+            secondary_indexes
+          )
+          key_schema = aws_key_schema(
+            key_schema[:hash_key_schema],
+            key_schema[:range_key_schema]
+          )
+
+          client_opts = {
+            table_name: table_name,
+            provisioned_throughput: {
+              read_capacity_units: read_capacity,
+              write_capacity_units: write_capacity
+            },
+            key_schema: key_schema,
+            attribute_definitions: attribute_definitions
+          }
+
+          if ls_indexes.present?
+            client_opts[:local_secondary_indexes] = ls_indexes.map do |index|
+              index_to_aws_hash(index)
+            end
+          end
+
+          if gs_indexes.present?
+            client_opts[:global_secondary_indexes] = gs_indexes.map do |index|
+              index_to_aws_hash(index)
+            end
+          end
+          resp = client.create_table(client_opts)
+          options[:sync] = true if !options.key?(:sync) && ls_indexes.present? || gs_indexes.present?
+          UntilPastTableStatus.new(table_name, :creating).call if options[:sync] &&
+                                                                  (status = PARSE_TABLE_STATUS.call(resp, :table_description)) &&
+                                                                  status == TABLE_STATUSES[:creating]
+          # Response to original create_table, which, if options[:sync]
+          #   may have an outdated table_description.table_status of "CREATING"
+          resp
+        end
+
+        private
+
+        # Builds aws attributes definitions based off of primary hash/range and
+        # secondary indexes
+        #
+        # @param key_data
+        # @option key_data [Hash] hash_key_schema - eg: {:id => :string}
+        # @option key_data [Hash] range_key_schema - eg: {:created_at => :number}
+        # @param [Hash] secondary_indexes
+        # @option secondary_indexes [Array<Dynamoid::Indexes::Index>] :local_secondary_indexes
+        # @option secondary_indexes [Array<Dynamoid::Indexes::Index>] :global_secondary_indexes
+        def build_all_attribute_definitions(key_schema, secondary_indexes = {})
+          ls_indexes = secondary_indexes[:local_secondary_indexes]
+          gs_indexes = secondary_indexes[:global_secondary_indexes]
+
+          attribute_definitions = []
+
+          attribute_definitions << build_attribute_definitions(
+            key_schema[:hash_key_schema],
+            key_schema[:range_key_schema]
+          )
+
+          if ls_indexes.present?
+            ls_indexes.map do |index|
+              attribute_definitions << build_attribute_definitions(
+                index.hash_key_schema,
+                index.range_key_schema
+              )
+            end
+          end
+
+          if gs_indexes.present?
+            gs_indexes.map do |index|
+              attribute_definitions << build_attribute_definitions(
+                index.hash_key_schema,
+                index.range_key_schema
+              )
+            end
+          end
+
+          attribute_definitions.flatten!
+          # uniq these definitions because range keys might be common between
+          # primary and secondary indexes
+          attribute_definitions.uniq!
+          attribute_definitions
+        end
+
+        # Builds an attribute definitions based on hash key and range key
+        # @params [Hash] hash_key_schema - eg: {:id => :string}
+        # @params [Hash] range_key_schema - eg: {:created_at => :datetime}
+        # @return [Array]
+        def build_attribute_definitions(hash_key_schema, range_key_schema = nil)
+          attrs = []
+
+          attrs << attribute_definition_element(
+            hash_key_schema.keys.first,
+            hash_key_schema.values.first
+          )
+
+          if range_key_schema.present?
+            attrs << attribute_definition_element(
+              range_key_schema.keys.first,
+              range_key_schema.values.first
+            )
+          end
+
+          attrs
+        end
+
+        # Builds an aws attribute definition based on name and dynamoid type
+        # @params [Symbol] name - eg: :id
+        # @params [Symbol] dynamoid_type - eg: :string
+        # @return [Hash]
+        def attribute_definition_element(name, dynamoid_type)
+          aws_type = api_type(dynamoid_type)
+
+          {
+            attribute_name: name.to_s,
+            attribute_type: aws_type
+          }
+        end
+
+        # Converts from symbol to the API string for the given data type
+        # E.g. :number -> 'N'
+        def api_type(type)
+          case type
+          when :string then STRING_TYPE
+          when :number then NUM_TYPE
+          when :binary then BINARY_TYPE
+          else raise "Unknown type: #{type}"
+          end
+        end
+
+        # Converts a Dynamoid::Indexes::Index to an AWS API-compatible hash.
+        # This resulting hash is of the form:
+        #
+        #   {
+        #     index_name: String
+        #     keys: {
+        #       hash_key: aws_key_schema (hash)
+        #       range_key: aws_key_schema (hash)
+        #     }
+        #     projection: {
+        #       projection_type: (ALL, KEYS_ONLY, INCLUDE) String
+        #       non_key_attributes: (optional) Array
+        #     }
+        #     provisioned_throughput: {
+        #       read_capacity_units: Integer
+        #       write_capacity_units: Integer
+        #     }
+        #   }
+        #
+        # @param [Dynamoid::Indexes::Index] index the index.
+        # @return [Hash] hash representing an AWS Index definition.
+        def index_to_aws_hash(index)
+          key_schema = aws_key_schema(index.hash_key_schema, index.range_key_schema)
+
+          hash = {
+            index_name: index.name,
+            key_schema: key_schema,
+            projection: {
+              projection_type: index.projection_type.to_s.upcase
+            }
+          }
+
+          # If the projection type is include, specify the non key attributes
+          if index.projection_type == :include
+            hash[:projection][:non_key_attributes] = index.projected_attributes
+          end
+
+          # Only global secondary indexes have a separate throughput.
+          if index.type == :global_secondary
+            hash[:provisioned_throughput] = {
+              read_capacity_units: index.read_capacity,
+              write_capacity_units: index.write_capacity
+            }
+          end
+          hash
+        end
+
+        # Converts hash_key_schema and range_key_schema to aws_key_schema
+        # @param [Hash] hash_key_schema eg: {:id => :string}
+        # @param [Hash] range_key_schema eg: {:created_at => :number}
+        # @return [Array]
+        def aws_key_schema(hash_key_schema, range_key_schema)
+          schema = [{
+            attribute_name: hash_key_schema.keys.first.to_s,
+            key_type: HASH_KEY
+          }]
+
+          if range_key_schema.present?
+            schema << {
+              attribute_name: range_key_schema.keys.first.to_s,
+              key_type: RANGE_KEY
+            }
+          end
+          schema
+        end
+      end
+    end
+  end
+end

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/item_updater.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Dynamoid
+  module AdapterPlugin
+    class AwsSdkV3
+      # Mimics behavior of the yielded object on DynamoDB's update_item API (high level).
+      class ItemUpdater
+        attr_reader :table, :key, :range_key
+
+        def initialize(table, key, range_key = nil)
+          @table = table
+          @key = key
+          @range_key = range_key
+          @additions = {}
+          @deletions = {}
+          @updates   = {}
+        end
+
+        #
+        # Adds the given values to the values already stored in the corresponding columns.
+        # The column must contain a Set or a number.
+        #
+        # @param [Hash] vals keys of the hash are the columns to update, vals are the values to
+        #               add. values must be a Set, Array, or Numeric
+        #
+        def add(values)
+          @additions.merge!(sanitize_attributes(values))
+        end
+
+        #
+        # Removes values from the sets of the given columns
+        #
+        # @param [Hash] values keys of the hash are the columns, values are Arrays/Sets of items
+        #               to remove
+        #
+        def delete(values)
+          @deletions.merge!(sanitize_attributes(values))
+        end
+
+        #
+        # Replaces the values of one or more attributes
+        #
+        def set(values)
+          @updates.merge!(sanitize_attributes(values))
+        end
+
+        #
+        # Returns an AttributeUpdates hash suitable for passing to the V2 Client API
+        #
+        def to_h
+          ret = {}
+
+          @additions.each do |k, v|
+            ret[k.to_s] = {
+              action: ADD,
+              value: v
+            }
+          end
+          @deletions.each do |k, v|
+            ret[k.to_s] = {
+              action: DELETE,
+              value: v
+            }
+          end
+          @updates.each do |k, v|
+            ret[k.to_s] = {
+              action: PUT,
+              value: v
+            }
+          end
+
+          ret
+        end
+
+        private
+
+        def sanitize_attributes(attributes)
+          attributes.transform_values do |v|
+            v.is_a?(Hash) ? v.stringify_keys : v
+          end
+        end
+
+        ADD    = 'ADD'
+        DELETE = 'DELETE'
+        PUT    = 'PUT'
+      end
+    end
+  end
+end

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/middleware/backoff.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Dynamoid
+  module AdapterPlugin
+    class AwsSdkV3
+      module Middleware
+        class Backoff
+          def initialize(next_chain)
+            @next_chain = next_chain
+            @backoff = Dynamoid.config.backoff ? Dynamoid.config.build_backoff : nil
+          end
+
+          def call(request)
+            response = @next_chain.call(request)
+            @backoff.call if @backoff
+
+            return response
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/middleware/limit.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Dynamoid
+  module AdapterPlugin
+    class AwsSdkV3
+      module Middleware
+        class Limit
+          def initialize(next_chain, record_limit: nil, scan_limit: nil)
+            @next_chain = next_chain
+
+            @record_limit = record_limit
+            @scan_limit = scan_limit
+
+            @record_count = 0
+            @scan_count = 0
+          end
+
+          def call(request)
+            # Adjust the limit down if the remaining record and/or scan limit are
+            # lower to obey limits. We can assume the difference won't be
+            # negative due to break statements below but choose smaller limit
+            # which is why we have 2 separate if statements.
+            # NOTE: Adjusting based on record_limit can cause many HTTP requests
+            # being made. We may want to change this behavior, but it affects
+            # filtering on data with potentially large gaps.
+            # Example:
+            #    User.where('created_at.gte' => 1.day.ago).record_limit(1000)
+            #    Records 1-999 User's that fit criteria
+            #    Records 1000-2000 Users's that do not fit criteria
+            #    Record 2001 fits criteria
+            # The underlying implementation will have 1 page for records 1-999
+            # then will request with limit 1 for records 1000-2000 (making 1000
+            # requests of limit 1) until hit record 2001.
+            if request[:limit] && @record_limit && @record_limit - @record_count < request[:limit]
+              request[:limit] = @record_limit - @record_count
+            end
+            if request[:limit] && @scan_limit && @scan_limit - @scan_count < request[:limit]
+              request[:limit] = @scan_limit - @scan_count
+            end
+
+            response = @next_chain.call(request)
+
+            @record_count += response.count
+            throw :stop_pagination if @record_limit && @record_count >= @record_limit
+
+            @scan_count += response.scanned_count
+            throw :stop_pagination if @scan_limit && @scan_count >= @scan_limit
+
+            return response
+          end
+        end
+
+      end
+    end
+  end
+end
+

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/middleware/start_key.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Dynamoid
+  module AdapterPlugin
+    class AwsSdkV3
+      module Middleware
+        class StartKey
+          def initialize(next_chain)
+            @next_chain = next_chain
+          end
+
+          def call(request)
+            response = @next_chain.call(request)
+
+            if response.last_evaluated_key
+              request[:exclusive_start_key] = response.last_evaluated_key
+            else
+              throw :stop_pagination
+            end
+
+            return response
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require_relative 'middleware/backoff'
+require_relative 'middleware/limit'
+require_relative 'middleware/start_key'
+
+module Dynamoid
+  module AdapterPlugin
+    class AwsSdkV3
+      class Query
+        OPTIONS_KEYS = %i[
+          limit hash_key hash_value range_key consistent_read scan_index_forward
+          select index_name batch_size exclusive_start_key record_limit scan_limit
+        ].freeze
+
+        attr_reader :client, :table, :options, :conditions
+
+        def initialize(client, table, opts = {})
+          @client = client
+          @table = table
+
+          opts = opts.symbolize_keys
+          @options = opts.slice(*OPTIONS_KEYS)
+          @conditions = opts.except(*OPTIONS_KEYS)
+        end
+
+        def call
+          request = build_request
+
+          Enumerator.new do |yielder|
+            api_call = -> (request) do
+              client.query(request).tap do |response|
+                yielder << response
+              end
+            end
+
+            middlewares = Middleware::Backoff.new(
+              Middleware::StartKey.new(
+                Middleware::Limit.new(api_call, record_limit: record_limit, scan_limit: scan_limit)
+              )
+            )
+
+            catch :stop_pagination do
+              loop do
+                middlewares.call(request)
+              end
+            end
+          end
+        end
+
+        private
+
+        def build_request
+          request = options.slice(
+            :consistent_read,
+            :scan_index_forward,
+            :select,
+            :index_name,
+            :exclusive_start_key
+          ).compact
+
+          # Deal with various limits and batching
+          batch_size = options[:batch_size]
+          limit = [record_limit, scan_limit, batch_size].compact.min
+
+          request[:limit]          = limit if limit
+          request[:table_name]     = table.name
+          request[:key_conditions] = key_conditions
+          request[:query_filter]   = query_filter
+
+          request
+        end
+
+        def record_limit
+          options[:record_limit]
+        end
+
+        def scan_limit
+          options[:scan_limit]
+        end
+
+        def hash_key_name
+          (options[:hash_key] || table.hash_key)
+        end
+
+        def range_key_name
+          (options[:range_key] || table.range_key)
+        end
+
+        def key_conditions
+          result = {
+            hash_key_name => {
+              comparison_operator: AwsSdkV3::EQ,
+              attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::EQ, options[:hash_value].freeze)
+            }
+          }
+
+          conditions.slice(*AwsSdkV3::RANGE_MAP.keys).each do |k, _v|
+            op = AwsSdkV3::RANGE_MAP[k]
+
+            result[range_key_name] = {
+              comparison_operator: op,
+              attribute_value_list: AwsSdkV3.attribute_value_list(op, conditions[k].freeze)
+            }
+          end
+
+          result
+        end
+
+        def query_filter
+          conditions.except(*AwsSdkV3::RANGE_MAP.keys).reduce({}) do |result, (attr, cond)|
+            condition = {
+              comparison_operator: AwsSdkV3::FIELD_MAP[cond.keys[0]],
+              attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::FIELD_MAP[cond.keys[0]], cond.values[0].freeze)
+            }
+            result[attr] = condition
+            result
+          end
+        end
+      end
+    end
+  end
+end