dynomite 1.2.7 → 2.0.1

data/lib/dynomite/item/query/params/function/attribute_type.rb

@@ -0,0 +1,30 @@
+module Dynomite::Item::Query::Params::Function
+  class AttributeType < Base
+    def filter_expression
+      filter_expression = []
+      @query[:attribute_type].each do |attribute_type|
+        path, type = attribute_type[:path], attribute_type[:type]
+        path = normalize_expression_path(path)
+        type = type_map(type)
+        filter_expression << "attribute_type(#{path}, :#{type})"
+      end
+      filter_expression
+    end
+
+    def attribute_names
+      paths = @query[:attribute_type].map { |attribute_type| attribute_type[:path] }
+      build_attribute_names_with_dot_paths(paths)
+    end
+
+    def attribute_values
+      values = {}
+      @query[:attribute_type].each do |attribute_type|
+        type = attribute_type[:type]
+        type = type_map(type)
+        values[":#{type}"] = type
+      end
+      values
+    end
+  end
+end
+

data/lib/dynomite/item/query/params/function/base.rb

@@ -0,0 +1,33 @@
+module Dynomite::Item::Query::Params::Function
+  class Base
+    include Dynomite::Item::Query::Params::Helpers
+    include Dynomite::Types
+
+    def initialize(query)
+      @query = query
+    end
+
+    def build_attribute_names_with_dot_paths(paths)
+      attribute_names = {}
+      paths.each do |path|
+        fields = path.split('.')
+        fields.each do |field|
+          if field.starts_with?('#')
+            key = field
+            value = field[1..-1]
+          else
+            key = "##{field}"
+            value = field
+          end
+          attribute_names[key] = value
+        end
+      end
+      attribute_names
+    end
+
+    def attribute_values
+      {}
+    end
+  end
+end
+

data/lib/dynomite/item/query/params/function/begins_with.rb

@@ -0,0 +1,32 @@
+module Dynomite::Item::Query::Params::Function
+  class BeginsWith < Base
+    def filter_expression
+      filter_expression = []
+      @query[query_key].each do |begins_with|
+        path, substr = begins_with[:path], begins_with[:substr]
+        path = normalize_expression_path(path)
+        filter_expression << "#{query_key}(#{path}, :#{substr})"
+      end
+      filter_expression
+    end
+
+    def attribute_names
+      paths = @query[query_key].map { |begins_with| begins_with[:path] }
+      build_attribute_names_with_dot_paths(paths)
+    end
+
+    def attribute_values
+      values = {}
+      @query[query_key].each do |begins_with|
+        path, substr = begins_with[:path], begins_with[:substr]
+        values[":#{substr}"] = substr
+      end
+      values
+    end
+
+    # interface method so Contains < BeginsWith can override
+    def query_key
+      :begins_with # must be a symbol
+    end
+  end
+end

data/lib/dynomite/item/query/params/function/contains.rb

@@ -0,0 +1,7 @@
+module Dynomite::Item::Query::Params::Function
+  class Contains < BeginsWith
+    def query_key
+      :contains # must be symbol
+    end
+  end
+end

data/lib/dynomite/item/query/params/function/size_fn.rb

@@ -0,0 +1,37 @@
+module Dynomite::Item::Query::Params::Function
+  class SizeFn < Base
+    include Dynomite::Item::Query::Relation::ComparisionMap
+
+    # Product.size_fn("category.gt", 100)
+    def filter_expression
+      filter_expression = []
+      @query[:size_fn].each_with_index do |size_fn, index|
+        path, size = size_fn[:path], size_fn[:size]
+        elements = path.split('.')
+        operator = elements.pop # remove last element
+        path = elements.join('.') # path no longer has operator
+        comparision = comparision_for(operator)
+        path = normalize_expression_path(path)
+        filter_expression << "size(#{path}) #{comparision} :size_value#{index}"
+      end
+      filter_expression
+    end
+
+    def attribute_names
+      paths = @query[:size_fn].map do |size_fn|
+        path = size_fn[:path]
+        path.split('.')[0..-2].join('.') # remove last element: comparision operator
+      end
+      build_attribute_names_with_dot_paths(paths)
+    end
+
+    def attribute_values
+      values = {}
+      @query[:size_fn].each_with_index do |size_fn, index|
+        size = size_fn[:size]
+        values[":size_value#{index}"] = size
+      end
+      values
+    end
+  end
+end

data/lib/dynomite/item/query/params/helpers.rb

@@ -0,0 +1,94 @@
+class Dynomite::Item::Query::Params
+  module Helpers
+    # Important to reset relation index for each relation chain so that
+    # attribute name references are correct.
+    # Using `with_where_groups` when interating ensures index is reset.
+    def with_where_groups
+      @relation.index = 0
+      @relation.query[:where].each do |where_group|
+        yield(where_group)
+      end
+    end
+
+    def query
+      @relation.query
+    end
+
+    # Certain queries require a scan, so we can't use the key condition
+    def scan_required?(index)
+      return true if index.nil? # first check
+      return true if query[:force_scan]
+      return true if disable_index_for_any_or?
+      return true if disable_index_for_not?(index)
+      return true if disable_index_for_consistent_read?(index)
+
+      all_where_fields.find do |full_field|
+        field, operator = full_field.split('.')
+        index.fields.include?(field) && !operator.nil?
+      end
+    end
+
+    # Always run scan when any or in chain
+    # For dynomite, `or` expressions will always result in a scan operation.
+    # This is because `key_condition_expression` does not support OR expressions.
+    # Nor does it make sense to use query with an index in the first pass with
+    # `key_condition_expression` and use `filter_expression` in the second pass.
+    # The `key_condition_expression` and `filter_expression` are AND with each other,
+    # so it would not be possible to do an OR without a scan.
+    def disable_index_for_any_or?
+      disable = query[:where].any? { |where_group| where_group.or? }
+      logger.info "Disabling index since an or was used" if disable && ENV['DYNOMITE_DEBUG']
+      disable
+    end
+
+    def disable_index_for_not?(index)
+      disable = query[:where].any? do |where_group|
+        x = where_group.fields & index.fields
+        !x.empty? && where_group.not?
+      end
+      logger.info "Disabling index since a not was used for the index" if disable && ENV['DYNOMITE_DEBUG']
+      disable
+    end
+
+    def disable_index_for_consistent_read?(index)
+      if query.key?(:consistent_read)
+        if index.nil?
+          true # must use scan for consistent read
+        elsif index.primary?
+          false # can use index for consistent ready for the primary key index only
+        else
+          true # must use scan for GSI indexes
+        end
+      else
+        false
+      end
+    end
+
+    # full field names with operator
+    def all_where_fields
+      query[:where].map(&:keys).flatten.map(&:to_s)
+    end
+
+    def all_where_field_names
+      all_where_fields.map { |k| k.split('.').first }
+    end
+
+    def normalize_expression_path(path)
+      path.split('.').map do |field|
+        field.starts_with?('#') ? field : field.prepend('#')
+      end.join('.')
+    end
+
+    def normalize_project_expression(args)
+      project_expression = []
+      args.map do |element|
+        if element.is_a?(String)
+          project_expression += element.split(',').map(&:strip)
+        else
+          project_expression << element.to_s
+        end
+      end
+      project_expression
+    end
+  end
+end

data/lib/dynomite/item/query/params/key_condition.rb

@@ -0,0 +1,34 @@
+class Dynomite::Item::Query::Params
+  class KeyCondition < Base
+    def initialize(relation, index, partition_key_field, sort_key_field)
+      @relation, @index, @partition_key_field, @sort_key_field = relation, index, partition_key_field, sort_key_field
+      @expressions = []
+    end
+
+    def expression
+      build
+      join_expressions
+    end
+
+    def build
+      with_where_groups do |where_group|
+        expression = where_group.build_compare_expression_if do |field|
+          @index.fields.include?(field)
+        end
+        next unless expression
+        @expressions << expression
+      end
+    end
+    memoize :build
+
+    def full_primary_key_in_query?
+      field_names = all_where_field_names
+      if @sort_key_field
+        field_names.include?(@sort_key_field) && field_names.include?(@partition_key_field)
+      else
+        field_names.include?(@partition_key_field)
+      end
+    end
+  end
+end
+

data/lib/dynomite/item/query/params.rb

@@ -0,0 +1,115 @@
+module Dynomite::Item::Query
+  class Params
+    extend Memoist
+    include Dynomite::Types
+    include Helpers
+
+    attr_reader :source
+    delegate :partition_key_field, :sort_key_field, :table_name, to: :source
+
+    def initialize(relation, source)
+      @relation, @source = relation, source
+      @query = relation.query
+    end
+
+    # key condition
+    # 1. primary key highest precedence
+    # 2. index
+    # filter expression
+    # 1. if field used in key condition
+    # 2. then don’t use in filter expression
+    # attributes
+    # 1. all values will be mapped over
+    # 2. will be in key condition or filter expression
+    # index name
+    # 1. if key condition set
+    # 2. unless primary key
+    def to_h
+      # set @index first. used throughout class
+      @index = index_finder.find(@query[:index_name])
+      @index = nil if scan_required?(@index) # IE: NOT operator on where field
+
+      # must build in this order
+      build_key_condition_expression if @index # must build first
+      build_filter_expression
+      build_attributes                         # must build last
+
+      @params = {
+        expression_attribute_names: @expression_attribute_names,    # both scan and query
+        expression_attribute_values: @expression_attribute_values,  # both scan and query
+        table_name: table_name,
+      }
+
+      @params[:filter_expression] = @filter_expression # both scan and query
+      @params[:key_condition_expression] = @key_condition_expression # query only. required
+
+      # primary index does not have a name but they are added to the @key_condition_expression
+      @params[:index_name] = @index.index_name if @index && !@index.primary? # both scan and query can use index
+
+      @params.reject! { |k,v| v.blank? }
+
+      # scan_index_forward after reject! so it's not removed
+      @params[:scan_index_forward] = !!@query[:scan_index_forward] if @query.key?(:scan_index_forward)
+      @params[:limit] = @query[:limit] if @query.key?(:limit)
+      @params[:projection_expression] = projection_expression if projection_expression
+      @params[:consistent_read] = @query[:consistent_read] if @query.key?(:consistent_read)
+      @params[:exclusive_start_key] = @query[:exclusive_start_key] if @query.key?(:exclusive_start_key)
+
+      log_index_info
+      @params
+    end
+
+    def projection_expression
+      return if @query[:projection_expression].nil?
+      projection_expression = normalize_project_expression(@query[:projection_expression])
+      projection_expression.map do |field|
+        '#'+field
+      end.join(", ")
+    end
+
+    # key_condition_expression is the most restrictive way to query.
+    # It requires the primary key and sort key.
+    # It also requires either the primary key or an index.
+    # Otherwise, we do not set it at all.
+    #
+    # So we'll build this first and then add the other expressions to it.
+    #
+    # key condition
+    # 1. primary key highest precedence
+    # 2. index
+    # 3. track fields used
+    def build_key_condition_expression
+      key_condition = KeyCondition.new(@relation, @index, partition_key_field, sort_key_field)
+      @key_condition_expression = key_condition.expression
+    end
+
+    def build_filter_expression
+      filter = Filter.new(@relation, @index)
+      @filter_expression = filter.expression
+    end
+
+    def build_attributes
+      expression_attribute = ExpressionAttribute.new(@relation)
+      @expression_attribute_names = expression_attribute.names
+      @expression_attribute_values = expression_attribute.values
+    end
+
+    def log_index_info
+      return unless ENV['DYNOMITE_DEBUG']
+
+      if @index
+        Dynomite.logger.info "Index used #{@index.index_name}"
+      elsif @params[:@expression_attribute_names]
+        attributes_list = @params[:@expression_attribute_names].values.join(", ")
+        Dynomite.logger.info "Not using index. None found for the attributes: #{attributes_list}"
+      else
+        Dynomite.logger.info "Not using index. @params #{@params.inspect}"
+      end
+    end
+
+    def index_finder
+      Dynomite::Item::Indexes::Finder.new(@source, @query)
+    end
+    memoize :index_finder
+  end
+end

data/lib/dynomite/item/query/partiql/executer.rb

@@ -0,0 +1,72 @@
+module Dynomite::Item::Query::Partiql
+  class Executer
+    include Dynomite::Client
+
+    def initialize(source)
+      @source = source # source is the model class. IE: Post User etc
+    end
+
+    # Execute PartiQL query
+    #
+    # AWS Docs:
+    # - https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#execute_statement-instance_method
+    # - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.select.html
+    #
+    # resp = client.execute_statement({
+    #   statement: "PartiQLStatement", # required
+    #   parameters: ["value"], # value <Hash,Array,String,Numeric,Boolean,IO,Set,nil>
+    #   consistent_read: false,
+    #   next_token: "PartiQLNextToken",
+    #   return_consumed_capacity: "INDEXES", # accepts INDEXES, TOTAL, NONE
+    #   limit: 1,
+    #   return_values_on_condition_check_failure: "ALL_OLD", # accepts ALL_OLD, NONE
+    # })
+    def call(statement, parameters = {}, options = {})
+      total_count = 0
+      # total_limit is the total limit across all pages
+      # For the AWS API call itself use the default limit and allow AWS to scan 1MB for page
+      total_limit = parameters.delete(:limit)
+      enumerator = Enumerator.new do |y|
+        next_token = :start
+        while next_token
+          if next_token && next_token != :start
+            options[:next_token] = next_token
+          end
+
+          params = { statement: statement }
+          params[:parameters] = parameters unless parameters.empty?
+          raw = options.delete(:raw)
+          params.merge!(options)
+          log_debug(params)
+          resp = client.execute_statement(params)
+          if raw
+            y.yield(resp.items)
+          else
+            page = resp.items.map { |i| build_item(i) }
+            y.yield(page)
+          end
+
+          # Track total_count across pages. If limit is set, then stop when we reach it.
+          # Remember the limit is per page for each API call, not total.
+          total_count += page.size
+          break if total_limit && total_count >= total_limit
+
+          next_token = resp.next_token
+        end
+      end
+      if statement =~ /^SELECT/i
+        enumerator.lazy.flat_map { |i| i } # lazy.flat_map flattens the array since yielding pages
+        # Returns a lazy enumerator: #<Enumerator::Lazy: ...>
+      else
+        # For non-SELECT statements: INSERT, UPDATE, DELETE
+        enumerator.first # call first to execute the query immediately
+      end
+    end
+
+    def build_item(i)
+      item = @source.new(i) # IE: Post.new(i)
+      item.new_record = false
+      item
+    end
+  end
+end

data/lib/dynomite/item/query/partiql.rb

@@ -0,0 +1,67 @@
+module Dynomite::Item::Query
+  module Partiql
+    extend ActiveSupport::Concern
+    class_methods do
+      # Example:
+      #
+      #   Product.execute_pql('SELECT * FROM "demo-dev_products" WHERE name = ?', ['Laptop'])
+      #
+      # Note WHERE is required
+      def execute_pql(statement, parameters = {}, options = {})
+        Executer.new(self).call(statement, parameters, options)
+      end
+
+      # Example:
+      #
+      #   Product.execute_pql('SELECT * FROM "demo-dev_products" WHERE name = ?', ['Laptop'])
+      #   Product.find_by_pql('name = ?', ['Laptop'])
+      #
+      # Returns [Item, Item, ...] (lazy)
+      def find_by_pql(where, parameters = {}, options = {})
+        select_all(where, parameters, options.merge(raw: false))
+      end
+
+      # Example:
+      #
+      #   Product.execute_pql('SELECT * FROM "demo-dev_products" WHERE name = ?', ['Laptop'])
+      #   Product.select_all('name = ?', ['Laptop'])
+      #
+      # Returns [Hash, Hash, ...] (lazy)
+      def select_all(where, parameters = {}, options = {})
+        options[:raw] = true unless options.key?(:raw)
+        statement = %Q|SELECT * FROM "#{table_name}" WHERE #{where}|
+        execute_pql(statement, parameters, options)
+      end
+
+      # Example:
+      #
+      #   Post.execute_pql('UPDATE "demo-dev_posts" SET title = ? WHERE id = ?', ['post 1b', 'post-gRssngpbm5OfDIwr'])
+      #   Post.update_pql('SET title = ? WHERE id = ?', ['post 1c', 'post-gRssngpbm5OfDIwr'])
+      #
+      def update_pql(set_where, parameters, options = {})
+        statement = %Q|UPDATE "#{table_name}" #{set_where}|
+        execute_pql(statement, parameters, options).to_a # to_a to force the lazy Enumerator to execute
+      end
+
+      # Example:
+      #
+      #   Post.execute_pql('DELETE FROM "demo-dev_posts" WHERE id = ?', ['post-2QXlmfHCKcPDsnJC'])
+      #   Post.delete_pql('id = ?', ['post-2QXlmfHCKcPDsnJC'])
+      #
+      def delete_pql(where, parameters = {}, options = {})
+        statement = %Q|DELETE FROM "#{table_name}" WHERE #{where}|
+        execute_pql(statement, parameters, options).to_a # to_a to force the lazy Enumerator to execute
+      end
+
+      # Example:
+      #
+      #   Post.execute_pql(%Q|INSERT INTO "demo-dev_posts" VALUE {'id': ?, 'title': ?}|, ['post-1', 'post 1'])
+      #   Post.insert_pql("{'id': ?, 'title': ?}", ['post-3', 'post 3'])
+      #
+      def insert_pql(values, parameters = {}, options = {})
+        statement = %Q|INSERT INTO "#{table_name}" VALUE #{values}|
+        execute_pql(statement, parameters, options).to_a # to_a to force the lazy Enumerator to execute
+      end
+    end
+  end
+end

data/lib/dynomite/item/query/relation/chain.rb

@@ -0,0 +1,125 @@
+class Dynomite::Item::Query::Relation
+  # Builds up the query with methods like where and eventually executes Query or Scan.
+  module Chain
+    def where(args={})
+      @query[:where] << WhereGroup.new(self, args)
+      self
+    end
+    alias :and :where
+
+    def or(args={})
+      @query[:where] << WhereGroup.new(self, args, or: true)
+      self
+    end
+
+    def not(args={})
+      @query[:where] << WhereGroup.new(self, args, not: true)
+      self
+    end
+
+    def excluding(*args)
+      ids = args.map do |object|
+        object.is_a?(Dynomite::Item) ? object.id : object
+      end
+      self.not("id.in": ids)
+    end
+
+    def scan_index_forward(value=true)
+      @query[:scan_index_forward] = value
+      self
+    end
+
+    def scan_index_backward(value=true)
+      scan_index_forward(!value)
+    end
+
+    # The default limit for both Scan and Query in Amazon DynamoDB is 1 MB of data read.
+    # It'll stop per api call regardless of the limit you set once it hits 1MB.
+    def limit(value)
+      @query[:limit] = value
+      self
+    end
+
+    # Product.where(category: "Electronics").project("id, category").first
+    def project(*fields)
+      @query[:projection_expression] = fields
+      self
+    end
+    alias projection_expression project
+
+    # Disable use of index and query method. Force a scan method
+    def force_scan
+      @query[:force_scan] = true
+      self
+    end
+
+    # Note consistent read it supported with GSI.
+    # You may want to use force_scan if really need a consistent read.
+    def consistent_read(value=true)
+      @query[:consistent_read] = value
+      self
+    end
+    alias consistent consistent_read
+
+    def exclusive_start_key(hash)
+      @query[:exclusive_start_key] = hash
+      self
+    end
+    alias start_from exclusive_start_key
+    alias start_at exclusive_start_key
+    alias start exclusive_start_key
+
+    # Could add some magically behavior to strip off the -index if the index name is 3 characters long.
+    # but think that's even more obscure.
+    #
+    # suffix allows for shorter syntax:
+    #
+    #    index_name('created_at') vs index_name('created_at-index')
+    #
+    # Note: Tried using the shorter index method name but it seems to conflict an index method.
+    # Even though Enumerable has an index method, it doesn't seem to be the one thats conflicting.
+    # It's somewhere else.
+    def index_name(name, suffix: 'index')
+      name = [name, suffix].compact.join('-') if !name.ends_with?('index') && suffix
+      @query[:index_name] = name.to_s
+      self
+    end
+
+    def warn_on_scan(value=true)
+      @warn_on_scan = value
+      self
+    end
+
+    def attribute_exists(path)
+      @query[:attribute_exists] << path
+      self
+    end
+
+    def attribute_not_exists(path)
+      @query[:attribute_not_exists] << path
+      self
+    end
+
+    def attribute_type(path, type)
+      @query[:attribute_type] << {path: path, type: type}
+      self
+    end
+
+    # This is a function that accepts a path and a value.
+    # This is different from the comparision operator. IE: ""
+    def begins_with(path, substr)
+      @query[:begins_with] << {path: path, substr: substr}
+      self
+    end
+
+    def contains(path, substr)
+      @query[:contains] << {path: path, substr: substr}
+      self
+    end
+
+    def size_fn(path, size)
+      @query[:size_fn] << {path: path, size: size}
+      self
+    end
+  end
+end

data/lib/dynomite/item/query/relation/comparision_expression.rb

@@ -0,0 +1,21 @@
+class Dynomite::Item::Query::Relation
+  class ComparisionExpression
+    def initialize(where_group, comparisions)
+      @where_group, @comparisions = where_group, comparisions
+    end
+
+    def or?
+      @where_group.or?
+    end
+
+    def build
+      # join @comparisions with AND if there are more than one
+      expression = []
+      expression << 'NOT' if @where_group.not?
+      expression << '(' if @comparisions.size > 1
+      expression << @comparisions.join(' AND ') # always AND within a group
+      expression << ')' if @comparisions.size > 1
+      expression.join(' ')
+    end
+  end
+end

data/lib/dynomite/item/query/relation/comparision_map.rb

@@ -0,0 +1,19 @@
+class Dynomite::Item::Query::Relation
+  module ComparisionMap
+    COMPARISION_MAP = {
+      'eq' => '=',
+      'gt' => '>',
+      'gte' => '>=',
+      'lt' => '<',
+      'lte' => '<=',
+    }
+
+    def comparision_for(operator)
+      COMPARISION_MAP[operator] || operator
+    end
+
+    def comparision_operators
+      COMPARISION_MAP.keys + COMPARISION_MAP.values
+    end
+  end
+end