dynomite 1.2.7 → 2.0.0

data/lib/dynomite/item/query/relation/comparision_expression.rb

@@ -0,0 +1,21 @@
+class Dynomite::Item::Query::Relation
+  class ComparisionExpression
+    def initialize(where_group, comparisions)
+      @where_group, @comparisions = where_group, comparisions
+    end
+
+    def or?
+      @where_group.or?
+    end
+
+    def build
+      # join @comparisions with AND if there are more than one
+      expression = []
+      expression << 'NOT' if @where_group.not?
+      expression << '(' if @comparisions.size > 1
+      expression << @comparisions.join(' AND ') # always AND within a group
+      expression << ')' if @comparisions.size > 1
+      expression.join(' ')
+    end
+  end
+end

data/lib/dynomite/item/query/relation/comparision_map.rb

@@ -0,0 +1,19 @@
+class Dynomite::Item::Query::Relation
+  module ComparisionMap
+    COMPARISION_MAP = {
+      'eq' => '=',
+      'gt' => '>',
+      'gte' => '>=',
+      'lt' => '<',
+      'lte' => '<=',
+    }
+
+    def comparision_for(operator)
+      COMPARISION_MAP[operator] || operator
+    end
+
+    def comparision_operators
+      COMPARISION_MAP.keys + COMPARISION_MAP.values
+    end
+  end
+end

data/lib/dynomite/item/query/relation/delete.rb

@@ -0,0 +1,38 @@
+class Dynomite::Item::Query::Relation
+  module Delete
+    def delete_all
+      # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method
+      # A single call to BatchWriteItem can transmit up to 16MB of data over the network,
+      # consisting of up to 25 item put or delete operations.
+      batch_limit = 25 # max batch size for batch_write_item
+      each_page.each do |page|
+        page.each_slice(batch_limit) do |slice|
+          primary_keys = slice.map(&:primary_key)
+          delete_requests = primary_keys.map do |primary_key|
+            {
+              delete_request: {
+                key: primary_key,
+              },
+            }
+          end
+          request_items = { @source.table_name => delete_requests }
+          client.batch_write_item(request_items: request_items)
+        end
+      end
+    end
+
+    def destroy_all
+      each(&:destroy)
+    end
+
+    # require args
+    def delete_by(args)
+      where(args).delete_all
+    end
+
+    # require args
+    def destroy_by(args)
+      where(args).destroy_all
+    end
+  end
+end

data/lib/dynomite/item/query/relation/ids.rb

@@ -0,0 +1,21 @@
+class Dynomite::Item::Query::Relation
+  module Ids
+    def pluck(*names)
+      project(*names)
+      super # provided by Ruby Enumerable
+    end
+
+    def ids
+      project(:id).each.map(&:id).to_a
+    end
+
+    def exists?(args={})
+      !!limit(1).first
+    end
+
+    # Surprisingly, Enumberable does not provide empty?
+    def empty?
+      !exists?
+    end
+  end
+end

data/lib/dynomite/item/query/relation/math.rb

@@ -0,0 +1,19 @@
+class Dynomite::Item::Query::Relation
+  module Math
+    def average(field)
+      map(&field).sum.to_f / count
+    end
+
+    def min(field)
+      map(&field).min
+    end
+
+    def max(field)
+      map(&field).max
+    end
+
+    def sum(field)
+      map(&field).sum
+    end
+  end
+end

data/lib/dynomite/item/query/relation/where_field.rb

@@ -0,0 +1,32 @@
+class Dynomite::Item::Query::Relation
+  class WhereField
+    attr_reader :full_field, :value, :index
+    def initialize(full_field, value, index)
+      @full_field, @value, @index = full_field.to_s, value, index
+    end
+
+    def field
+      @full_field.split('.').first
+    end
+
+    def operator
+      if raw_operator
+        not? ? raw_operator[4..-1] : raw_operator
+      end
+    end
+
+    def not?
+      raw_operator.match(/^not_/) if raw_operator
+    end
+
+    def raw_operator
+      _, operator = @full_field.split('.')
+      operator.downcase if operator
+    end
+
+    # Example: price_1, price_2
+    def reference
+      "#{field}_#{@index}"
+    end
+  end
+end

data/lib/dynomite/item/query/relation/where_group.rb

@@ -0,0 +1,78 @@
+class Dynomite::Item::Query::Relation
+  class WhereGroup
+    delegate :keys, :size, to: :hash
+    include ComparisionMap
+
+    attr_accessor :hash, :meta
+    def initialize(relation, hash={}, meta={})
+      @relation = relation
+      @hash = hash
+      @meta = meta
+    end
+
+    def to_s
+      "#<#{self.class.name} @hash=#{@hash} @meta=#{@meta}>"
+    end
+
+    def each
+      @hash.each do |full_field, value|
+        where_field = WhereField.new(full_field, value, @relation.index)
+        yield where_field
+        @relation.index += 1
+      end
+    end
+
+    def fields
+      @hash.map do |full_field, value|
+        full_field.to_s.split('.').first
+      end
+    end
+
+    def not?
+      @meta[:not]
+    end
+
+    def or?
+      @meta[:or]
+    end
+
+    # Method helps remove duplication and DRY up building of compare expression.
+    # It a bit confusing but unsure how to make it clearer.
+    def build_compare_expression_if
+      comparisions = []
+      each do |where_field|
+        field = where_field.field
+        next unless yield(field) # only build if condition is true
+        comparisions << build_compare(where_field)
+      end
+      unless comparisions.empty?
+        ComparisionExpression.new(self, comparisions) # to pass in where_group for or? and not?
+      end
+    end
+
+    def build_compare(where_field)
+      reference = where_field.reference
+      operator = where_field.operator
+      expression = case operator
+      when 'in'
+        # ProductStatus in (:avail, :back, :disc)
+        array = Array(where_field.value)
+        list = array.map.with_index { |v, i| ":#{reference}_#{i}" }.join(', ') # values
+        "##{reference} in (#{list})"
+      when 'between'
+        # sortKeyName between :sortkeyval1 AND :sortkeyval2
+        "##{reference} between :#{reference}_0 AND :#{reference}_1"
+      when 'begins_with'
+        # begins_with ( sortKeyName, :sortkeyval )
+        "begins_with(##{reference}, :#{reference})"
+      when *comparision_operators # eq, gt, gte, lt, lte, =, >, >=, <, <=
+        comparision = comparision_for(operator)
+        "##{reference} #{comparision} :#{reference}"
+      else
+        "##{reference} = :#{reference}"
+      end
+      expression = "NOT (#{expression})" if where_field.not?
+      expression
+    end
+  end
+end

data/lib/dynomite/item/query/relation.rb

@@ -0,0 +1,127 @@
+module Dynomite::Item::Query
+  # Builds up the query with methods like where and eventually executes Query or Scan.
+  class Relation
+    extend Memoist
+    include Dynomite::Client
+    include Enumerable
+    include Chain
+    include Math
+    include Ids
+    include Delete
+
+    attr_accessor :index, :query, :source
+    def initialize(source)
+      @source = source # source is the model class. IE: Post User etc
+      @query = {
+        where: [],
+        attribute_exists: [],
+        attribute_not_exists: [],
+        attribute_type: [],
+        begins_with: [],
+        contains: [],
+        size_fn: [],
+      }
+      @index = 0
+    end
+
+    # Enumerable provides .to_a, add force so it's more like lazy Enumerable.
+    # Also, not using: `alias load to_a` because load is Enumerable private method
+    # Instead user should use force or eager. Docs:
+    # https://docs.ruby-lang.org/en/master/Enumerator/Lazy.html
+    # https://www.rubydoc.info/stdlib/core/Enumerator/Lazy
+    alias size count
+    alias force to_a
+
+    def each(&block)
+      items.each(&block)
+    end
+
+    def items
+      pages.flat_map { |i| i } # flat_map flattens the Lazy Enumerator since yielding pages
+    end
+    private :items
+
+    def pages
+      raw_pages.map do |raw_page|
+        raw_page.items.map.map(&method(:build_item))
+      end
+    end
+    alias each_page pages
+
+    def raw_pages
+      params = to_params
+      # total_limit is the total limit across all pages
+      # For the AWS API call itself use the default limit and allow AWS to scan 1MB for page
+      total_limit = params.delete(:limit)
+      total_count = 0
+      Enumerator.new do |y|
+        last_evaluated_key = :start
+        while last_evaluated_key
+          if last_evaluated_key && last_evaluated_key != :start
+            params[:exclusive_start_key] = last_evaluated_key
+          end
+
+          meth = params[:key_condition_expression] ? :query : :scan
+          log_debug(params)
+          raw_warn_scan if meth == :scan
+          response = client.send(meth, params) # scan or query
+          records = response.items.map { |i| build_item(i, run_callback: false) }
+          y.yield(response, records)
+
+          # Track total_count across pages. If limit is set, then stop when we reach it.
+          # Since limit can be greater than each API response paged size.
+          total_count += response.items.size
+          break if total_limit && total_count >= total_limit
+
+          last_evaluated_key = response.last_evaluated_key
+        end
+      end.lazy
+    end
+    alias each_raw_page raw_pages
+
+    def build_item(i, run_callback: true)
+      item = @source.new(i) # IE: Post.new(i)
+      item.new_record = false
+      item.run_callbacks :find if run_callback
+      item
+    end
+
+    # Enumerable provides .first but does not provide .last
+    # Note, cannot use:
+    #     scan_index_forward(false).limit(1).first
+    # Since that will not work for queries that do not have a sort key.
+    # Users need to use query directly if they want to find the last item more efficiently.
+    def last
+      warn_scan <<~EOL
+        WARN: Dynomite::Item::Query::Relation#last is slow.
+        Consider using query directly if you have a primary key that as a sort key.
+      EOL
+      to_a.last # force load of lazy enumerator. slow
+    end
+
+    def to_params
+      Params.new(self, @source).to_h
+    end
+
+    # Allows all to chain itself. This allows.
+    #
+    #   Post.where(category: "Electronics").all
+    #   Post.limit(1).all
+    #   Post.all.all         # also works, side effect
+    #
+    def all
+      self
+    end
+
+    def raw_warn_scan
+      warn_on_scan = @warn_on_scan.nil? ? Dynomite.config.warn_on_scan : @warn_on_scan
+      return unless warn_on_scan
+      warn_scan <<~EOL
+        WARN: Scanning detected. It's recommended to not use scan. It can be slow.
+        Scanning table: #{@source.table_name}
+        Try creating a LSI or GSI index so dynomite can use query instead.
+        Docs: https://rubyonjets.com/docs/database/dynamodb/indexing/
+      EOL
+    end
+  end
+end

data/lib/dynomite/item/query.rb

@@ -0,0 +1,7 @@
+class Dynomite::Item
+  module Query
+    extend ActiveSupport::Concern
+    include Partiql
+    include Delegates
+  end
+end

data/lib/dynomite/item/read/find.rb

@@ -0,0 +1,196 @@
+module Dynomite::Item::Read
+  module Find
+    extend ActiveSupport::Concern
+
+    # Note: options are merged into get_item params.
+    def find(id, options={})
+      self.class.find(id, options)
+    end
+
+    class_methods do
+      def find_by(attrs, options={})
+        # Note: ActionController::Parameters does not have .any? Unsure if should use .to_h on it
+        # A blank attrs will break find_by since DynamoDB doesnt support blank attribute values
+        # Guard blank attrs like and return new return so validation error surfaces in a standard CRUD scaffold
+        attrs = attrs.to_h
+        return nil if attrs.any? { |k,v| v.blank? }
+
+        primary_key_attrs = attrs.stringify_keys.slice(partition_key_field, sort_key_field).symbolize_keys
+        if primary_key_attrs.size == primary_key_fields.size
+          find(primary_key_attrs, options.merge(raise_error: false))
+        else
+          where(attrs).first # possible scan
+        end
+      end
+
+      # Examples with out the args are received.
+      #
+      #   Post.find("ae3ae")
+      #     args ["ae3ae"]
+      #   Post.find(id: "ae3ae")
+      #     args [{:id=>"ae3ae"}]
+      #   Post.find("ae3ae", consistent_read: true)
+      #     args ["ae3ae", {:consistent_read=>true}]
+      #   Post.find({id: "ae3ae"}, consistent_read: true)
+      #     args [{:id=>"ae3ae"}, {:consistent_read=>true}]
+      #   Product.find(category: "Electronics", product_id: 101)
+      #     args [{:category=>"Electronics", :product_id=>101}]
+      #   Product.find({category: "Electronics", product_id: 101}, consistent_read: true)
+      #     args [{:category=>"Electronics", :product_id=>101}, {:consistent_read=>true}]
+      #
+      #   Product.find(id1,id2,id3)
+      #     args [id1,id2,id3]
+      #   Product.find([id1,id2,id3])
+      #     args [[id1,id2,id3]]
+      #   Product.find([id1,id2,id3], consistent_read: true)
+      #     args [[id1,id2,id3], {:consistent_read=>true}]
+      #
+      # Note: options are merged into get_item params.
+      def find(*args)
+        options = {}
+        key_schema = if args.size == 1
+                       # args is an array of one element:
+                       # ["f74de472"]
+                       # [{:id=>"f74de472"}]
+                       # [{:category=>"Electronics", :product_id=>101}]
+                       # [[id1, id2, id3]]
+                       get_key_schema_from_one_arg(args.first)
+                     else
+                       # ["f74de472", {:consistent_read=>true}]
+                       # [{:id=>"f74de472"}, {:consistent_read=>true}]
+                       # [{:category=>"Electronics", :product_id=>101}, {:consistent_read=>true}]
+                       # [[id1, id2, id3], {:consistent_read=>true}] # HERE
+                       options = args.extract_options!
+                       if args.size >= 2 # still at least 2 after extracting options
+                         get_key_schema_from_one_arg(args) # [id1, id2, id3]
+                       else
+                         get_key_schema_from_one_arg(args.first)
+                       end
+                     end
+
+        raise_error = options.delete(:raise_error) # clean for params
+        raise_error = raise_error.nil? ? true : raise_error
+
+        # Early return if ids the provided arg. find(ids)
+        # List of ids: IE: [id1, id2, id3]
+        if key_schema.is_a?(Array)
+          keys = key_schema # [{"id"=>"post-1"}, {"id"=>"post-2"}]
+          ids = keys.map(&:values).flatten # ["post-1", "post-2"]
+          items = batch_get_items(keys, options)
+          result = items
+          result = nil if items.size != keys.size # some missing items
+
+          if result.nil?
+            if raise_error
+              looking, found = ids, items.map(&:id)
+              missing = ids - found
+              message = "Couldn't find all #{self.name.pluralize} with '#{partition_key_field}': (#{looking.join(', ')}) (found #{found.size} results, but was looking for #{looking.size})"
+              message << ". Missing: #{missing.sort.join(', ')}" if found.size > 0
+              raise Dynomite::Error::RecordNotFound.new(message)
+            else
+              return items # return early
+            end
+          else
+            return items
+          end
+        end
+
+        params = {
+          table_name: table_name,
+          key: key_schema,
+        }
+        params.merge!(options)
+
+        log_debug(params)
+        attrs = client.get_item(params).item # unwraps the item's attrs
+
+        # Mimic ActiveRecord::RecordNotFound behavior
+        raise Dynomite::Error::RecordNotFound if attrs.nil? && raise_error != false
+
+        build_item(attrs)
+      end
+
+      def batch_get_items(*args)
+        options = args.extract_options!
+        retries = options.delete(:retries)|| 0
+        keys = args.flatten
+        items = []
+        unprocessed_keys = []
+
+        # exponential backoff to handle unprocessed keys
+        delay = 2 ** retries
+        if retries > 0
+          logger.debug "batch_get_items: sleeping for #{delay} seconds and will retry. retries: #{retries}"
+          sleep(delay)
+          if retries >= 3 # 2 + 4 + 8 = 14s total of retries
+            raise "ERROR: Exceeded max retries: #{retries}. Unable to batch_get_items for keys: #{keys.inspect}"
+          end
+        end
+
+        max_batch_size = 100
+        keys.each_slice(max_batch_size).each do |slice|
+          # Note: client.batch_get_items will silently not return items if any of the
+          # keys are not found. So we do not have to do a compact and remove nils.
+          # Merge options to allow passing in consistent_read: true
+          params = options.merge(keys: slice)
+          resp = client.batch_get_item(
+            request_items: {
+              table_name => params
+            }
+          )
+          resp[:responses][table_name].each do |item|
+            items << build_item(item)
+          end
+          unprocessed_keys += resp[:unprocessed_keys][table_name] if resp[:unprocessed_keys][table_name]
+        end
+
+        # Recursively call batch_get_items if there are unprocessed_keys
+        # Increase the retries by 1 for exponential backoff
+        items += batch_get_items(unprocessed_keys, options.merge(retries: retries+1)) if unprocessed_keys.any?
+
+        items
+      end
+
+      def build_item(attrs)
+        return unless attrs # is nil when no item found: client.get_item(params).item
+        item = self.new(attrs)
+        item.new_record = false
+        item.run_callbacks :find # find and find_by leads to build_item. so we can run callbacks here
+        item
+      end
+
+      # Examples of id:
+      #
+      #   "f74de472"
+      #   {:id=>"f74de472"}
+      #   {:category=>"Electronics", :product_id=>101}
+      #   [id1, id2, id3]
+      #
+      # Returns hash of key_schema
+      #
+      #   { id: "f74de472" }
+      #   { category: "Electronics", product_id: 101 }
+      #
+      def get_key_schema_from_one_arg(id)
+        # standardize key structure
+        case id
+        when Integer, String, Symbol # "f74de472"
+          if id.is_a?(String) && id.starts_with?(id_prefix)
+            { id: id }
+          else
+            if sort_key_field
+              raise "ERROR: You must provide both partition and sort key for class: #{self.name} partition_key_field: #{partition_key_field} sort_key_field: #{sort_key_field}"
+            end
+            { partition_key_field => id }
+          end
+        when Hash  # {:id=>"f74de472"} or {:category=>"Electronics", :product_id=>101}
+          id # User needs to provide both partition and sort key
+        when Array # [id1, id2, id3]
+          id.map do |i|
+            i.is_a?(Hash) ? i : { partition_key_field => i }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynomite/item/read/find_with_event.rb

@@ -0,0 +1,42 @@
+module Dynomite::Item::Read
+  module FindWithEvent
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def find_all_with_stream_event(event, options={})
+        # For event payload structure see
+        # https://rubyonjets.com/docs/events/dynamodb/#event-payload
+        # Keys structure:
+        # {
+        #   "Records": [
+        #     "dynamodb": {
+        #       "Keys": {
+        #         "id": {
+        #           "S": "post-1"
+        #         }
+        #       },
+        event = JSON.load(event) if event.is_a?(String)
+        event = event.deep_symbolize_keys
+        # raw_keys: { "id": { "S": "post-1" } }
+        raw_keys = event[:Records].map do |record|
+          record[:dynamodb][:Keys]
+        end
+        # keys: { id: "post-1" }
+        keys = get_key_schema_from_raw_keys(raw_keys)
+        items = find(keys, options) # find can return single item or Array of items
+        Array(items) # ensure Array is returned
+      end
+
+      def get_key_schema_from_raw_keys(raw_keys)
+        # raw_keys: { "id": { "S": "post-1" } }
+        # keys: { id: "post-1" }
+        # Note: raw_keys can have duplicates.
+        # IE: [{ "id": { "S": "post-1" }, "id": { "S": "post-1" } }]
+        keys = raw_keys.uniq.map do |hash|
+          hash.transform_values { |value| value.values.first }
+        end
+        get_key_schema_from_one_arg(keys)
+      end
+    end
+  end
+end