redis-memo 0.0.0.beta → 0.0.0.beta.5

data/lib/redis_memo/memoize_query/cached_select/bind_params.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+class RedisMemo::MemoizeQuery::CachedSelect
+  class BindParams
+    def params
+      #
+      # Bind params is hash of sets: each key is a model class, each value is a
+      # set of hashes for memoized column conditions. Example:
+      #
+      #   {
+      #      Site => [
+      #        {name: 'a', city: 'b'},
+      #        {name: 'a', city: 'c'},
+      #        {name: 'b', city: 'b'},
+      #        {name: 'b', city: 'c'},
+      #      ],
+      #   }
+      #
+      @params ||= Hash.new do |models, model|
+        models[model] = []
+      end
+    end
+
+    def union(other)
+      return unless other
+
+      # The tree is almost always right-heavy. Merge into the right node for better
+      # performance.
+      other.params.merge!(params) do |_, other_attrs_set, attrs_set|
+        if other_attrs_set.empty?
+          attrs_set
+        elsif attrs_set.empty?
+          other_attrs_set
+        else
+          attrs_set + other_attrs_set
+        end
+      end
+
+      other
+    end
+
+    def product(other)
+      #  Example:
+      #
+      #  and(
+      #    [{a: 1}, {a: 2}],
+      #    [{b: 1}, {b: 2}],
+      #  )
+      #
+      #  =>
+      #
+      #  [
+      #    {a: 1, b: 1},
+      #    {a: 1, b: 2},
+      #    {a: 2, b: 1},
+      #    {a: 2, b: 2},
+      #  ]
+      return unless other
+
+      # The tree is almost always right-heavy. Merge into the right node for better
+      # performance.
+      params.each do |model, attrs_set|
+        next if attrs_set.empty?
+
+        # The other model does not have any conditions so far: carry the
+        # attributes over to the other node
+        if other.params[model].empty?
+          other.params[model] = attrs_set
+          next
+        end
+
+        # Distribute the current attrs into the other
+        other_attrs_set_size = other.params[model].size
+        other_attrs_set = other.params[model]
+        merged_attrs_set = Array.new(other_attrs_set_size * attrs_set.size)
+
+        attrs_set.each_with_index do |attrs, i|
+          other_attrs_set.each_with_index do |other_attrs, j|
+            k = i * other_attrs_set_size + j
+            merged_attrs = merged_attrs_set[k] = other_attrs.dup
+            attrs.each do |name, val|
+              # Conflict detected. For example:
+              #
+              #   (a = 1 or b = 1) and (a = 2 or b = 2)
+              #
+              #  Keep:     a = 1 and b = 2, a = 2 and b = 1
+              #  Discard:  a = 1 and a = 2, b = 1 and b = 2
+              if merged_attrs.include?(name) && merged_attrs[name] != val
+                merged_attrs_set[k] = nil
+                break
+              end
+
+              merged_attrs[name] = val
+            end
+          end
+        end
+
+        merged_attrs_set.compact!
+        other.params[model] = merged_attrs_set
+      end
+
+      other
+    end
+
+    def uniq!
+      params.each do |_, attrs_set|
+        attrs_set.uniq!
+      end
+    end
+
+    def memoizable?
+      return false if params.empty?
+
+      params.each do |model, attrs_set|
+        return false if attrs_set.empty?
+
+        attrs_set.each do |attrs|
+          return false unless RedisMemo::MemoizeQuery
+            .memoized_columns(model)
+            .include?(attrs.keys.sort)
+        end
+      end
+
+      true
+    end
+  end
+end

data/lib/redis_memo/memoize_query/cached_select/connection_adapter.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class RedisMemo::MemoizeQuery::CachedSelect
+  module ConnectionAdapter
+    def cacheable_query(*args)
+      query, binds = super(*args)
+
+      # Persist the arel object to StatementCache#execute
+      query.instance_variable_set(:@__redis_memo_memoize_query_memoize_query_arel, args.last)
+
+      [query, binds]
+    end
+
+    def exec_query(*args)
+      # An Arel AST in Thread local is set prior to supported query methods
+      if !RedisMemo.without_memo? &&
+          RedisMemo::MemoizeQuery::CachedSelect.extract_bind_params(args[0])
+        # [Reids $model Load] $sql $binds
+        RedisMemo::DefaultOptions.logger&.info(
+          "[Redis] \u001b[36;1m#{args[1]} \u001b[34;1m#{args[0]}\u001b[0m #{
+            args[2].map { |bind| [bind.name, bind.value_for_database]}
+          }"
+        )
+
+        super(*args)
+      else
+        RedisMemo.without_memo { super(*args) }
+      end
+    end
+
+    def select_all(*args)
+      if args[0].is_a?(Arel::SelectManager)
+        RedisMemo::MemoizeQuery::CachedSelect.current_query = args[0]
+      end
+
+      super(*args)
+    ensure
+      RedisMemo::MemoizeQuery::CachedSelect.reset_current_query
+    end
+  end
+end

data/lib/redis_memo/memoize_query/cached_select/statement_cache.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class RedisMemo::MemoizeQuery::CachedSelect
+  module StatementCache
+    def execute(*args)
+      arel = query_builder.instance_variable_get(:@__redis_memo_memoize_query_memoize_query_arel)
+      RedisMemo::MemoizeQuery::CachedSelect.current_query = arel
+      RedisMemo::MemoizeQuery::CachedSelect.current_substitutes =
+        bind_map.map_substitutes(args[0])
+
+      super(*args)
+    ensure
+      RedisMemo::MemoizeQuery::CachedSelect.reset_current_query
+    end
+  end
+end

data/lib/redis_memo/memoize_query/invalidation.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+#
+# Automatically invalidate memoizable when modifying ActiveRecords objects.
+# You still need to invalidate memos when you are using SQL queries to perform
+# update / delete (does not trigger record callbacks)
+#
+class RedisMemo::MemoizeQuery::Invalidation
+  def self.install(model_class)
+    var_name = :@@__redis_memo_memoize_query_invalidation_installed__
+    return if model_class.class_variable_defined?(var_name)
+
+    model_class.class_eval do
+      # A memory-persistent memoizable used for invalidating all queries of a
+      # particular model
+      def self.redis_memo_class_memoizable
+        @redis_memo_class_memoizable ||= RedisMemo::MemoizeQuery.create_memo(self)
+      end
+
+      %i(delete decrement! increment!).each do |method_name|
+        alias_method :"without_redis_memo_invalidation_#{method_name}", method_name
+
+        define_method method_name do |*args|
+          result = send(:"without_redis_memo_invalidation_#{method_name}", *args)
+
+          RedisMemo::MemoizeQuery.invalidate(self)
+
+          result
+        end
+      end
+    end
+
+    # Methods that won't trigger model callbacks
+    # https://guides.rubyonrails.org/active_record_callbacks.html#skipping-callbacks
+    %i(
+      decrement_counter
+      delete_all delete_by
+      increment_counter
+      insert insert! insert_all insert_all!
+      touch_all
+      update_column update_columns update_all update_counters
+      upsert upsert_all
+    ).each do |method_name|
+      # Example: Model.update_all
+      rewrite_default_method(
+        model_class,
+        model_class,
+        method_name,
+        class_method: true,
+      )
+
+      # Example: Model.where(...).update_all
+      rewrite_default_method(
+        model_class,
+        model_class.const_get(:ActiveRecord_Relation),
+        method_name,
+        class_method: false,
+      )
+    end
+
+    %i(
+      import import!
+    ).each do |method_name|
+      rewrite_import_method(
+        model_class,
+        method_name,
+      )
+    end
+
+    model_class.class_variable_set(var_name, true)
+  end
+
+  private
+
+  #
+  # There’s no good way to perform fine-grind cache invalidation when
+  # operations are bulk update operations such as update_all, and delete_all
+  # witout fetching additional data from the database, which might lead to
+  # performance degradation. Thus, by default, we simply invalidate all
+  # existing cached records after each bulk_updates.
+  #
+  def self.rewrite_default_method(model_class, klass, method_name, class_method:)
+    methods = class_method ? :methods : :instance_methods
+    return unless klass.send(methods).include?(method_name)
+
+    klass = klass.singleton_class if class_method
+    klass.class_eval do
+      alias_method :"#{method_name}_without_redis_memo_invalidation", method_name
+
+      define_method method_name do |*args|
+        result = send(:"#{method_name}_without_redis_memo_invalidation", *args)
+        RedisMemo::MemoizeQuery.invalidate_all(model_class)
+        result
+      end
+    end
+  end
+
+  def self.rewrite_import_method(model_class, method_name)
+    # This optimization to avoid over-invalidation only works on postgres
+    unless ActiveRecord::Base.connection.is_a?(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter)
+      rewrite_default_method(model_class, model_class, method_name, class_method: true)
+      return
+    end
+
+    model_class.singleton_class.class_eval do
+      alias_method :"#{method_name}_without_redis_memo_invalidation", method_name
+
+      # For the args format, see
+      # https://github.com/zdennis/activerecord-import/blob/master/lib/activerecord-import/import.rb#L128
+      define_method method_name do |*args, &blk|
+        options = args.last.is_a?(Hash) ? args.last : {}
+        records = args[args.last.is_a?(Hash) ? -2 : -1]
+        columns_to_update = options[:on_duplicate_key_update]
+        if columns_to_update.is_a?(Hash)
+          columns_to_update = columns_to_update[:columns]
+        end
+
+        if records.last.is_a?(Hash)
+          records.map! { |hash| model_class.new(hash) }
+        end
+
+        # Invalidate the records before and after the import to resolve
+        # - default values filled by the database
+        # - updates on conflict conditions
+        records_to_invalidate =
+          if columns_to_update
+            RedisMemo::MemoizeQuery::Invalidation.send(
+              :select_by_columns,
+              records,
+              columns_to_update,
+            )
+          else
+            []
+          end
+
+        result = send(:"#{method_name}_without_redis_memo_invalidation", *args, &blk)
+
+        # Offload the records to invalidate while selecting the next set of
+        # records to invalidate
+        case records_to_invalidate
+        when Array
+          RedisMemo::MemoizeQuery.invalidate(*records_to_invalidate) unless records_to_invalidate.empty?
+
+          RedisMemo::MemoizeQuery.invalidate(*RedisMemo::MemoizeQuery::Invalidation.send(
+            :select_by_id,
+            model_class,
+            # Not all databases support "RETURNING", which is useful when
+            # invaldating records after bulk creation
+            result.ids,
+          ))
+        else
+          RedisMemo::MemoizeQuery.invalidate_all(model_class)
+        end
+
+        result
+      end
+    end
+  end
+
+  def self.select_by_columns(records, columns_to_update)
+    model_class = records.first.class
+    or_chain = nil
+    columns_to_select = columns_to_update & RedisMemo::MemoizeQuery
+      .memoized_columns(model_class)
+      .to_a.flatten.uniq
+
+    # Nothing to invalidate here
+    return [] if columns_to_select.empty?
+
+    RedisMemo::Tracer.trace(
+      'redis_memo.memoize_query.invalidation',
+      "#{__method__}##{model_class.name}",
+    ) do
+      records.each do |record|
+        conditions = {}
+        columns_to_select.each do |column|
+          conditions[column] = record.send(column)
+        end
+        if or_chain
+          or_chain = or_chain.or(model_class.where(conditions))
+        else
+          or_chain = model_class.where(conditions)
+        end
+      end
+
+      record_count = RedisMemo.without_memo { or_chain.count }
+      if record_count > bulk_operations_invalidation_limit
+        nil
+      else
+        RedisMemo.without_memo { or_chain.to_a }
+      end
+    end
+  end
+
+  def self.select_by_id(model_class, ids)
+    RedisMemo::Tracer.trace(
+      'redis_memo.memoize_query.invalidation',
+      "#{__method__}##{model_class.name}",
+    ) do
+      RedisMemo.without_memo do
+        model_class.where(model_class.primary_key => ids).to_a
+      end
+    end
+  end
+
+  def self.bulk_operations_invalidation_limit
+    ENV['REDIS_MEMO_BULK_OPERATIONS_INVALIDATION_LIMIT']&.to_i ||
+      RedisMemo::DefaultOptions.bulk_operations_invalidation_limit ||
+      10000
+  end
+end

data/lib/redis_memo/memoize_query/memoize_table_column.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+require_relative 'memoize_method'
+
+module RedisMemo::MemoizeQuery
+end

data/lib/redis_memo/{memoize_records → memoize_query}/model_callback.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class RedisMemo::MemoizeRecords::ModelCallback
+class RedisMemo::MemoizeQuery::ModelCallback
   def self.install(model_class)
     var_name = :@@__redis_memo_memoize_record_after_save_callback_installed__
     return if model_class.class_variable_defined?(var_name)
@@ -12,10 +12,10 @@ class RedisMemo::MemoizeRecords::ModelCallback
   end
 
   def after_save(record)
-    RedisMemo::MemoizeRecords.invalidate(record)
+    RedisMemo::MemoizeQuery.invalidate(record)
   end
 
   def after_destroy(record)
-    RedisMemo::MemoizeRecords.invalidate(record)
+    RedisMemo::MemoizeQuery.invalidate(record)
   end
 end

data/lib/redis_memo/options.rb

@@ -13,7 +13,7 @@ class RedisMemo::Options
   )
     @compress = compress.nil? ? true : compress
     @compress_threshold = compress_threshold || 1.kilobyte
-    @redis = redis
+    @redis_config = redis
     @redis_client = nil
     @redis_error_handler = redis_error_handler
     @tracer = tracer
@@ -22,20 +22,18 @@ class RedisMemo::Options
     @expires_in = expires_in
   end
 
-  def redis(&blk)
-    if blk.nil?
-      return @redis_client if @redis_client.is_a?(RedisMemo::Redis)
+  def redis
+    @redis_client ||= RedisMemo::Redis.new(redis_config)
+  end
 
-      if @redis.respond_to?(:call)
-        @redis_client = RedisMemo::Redis.new(@redis.call)
-      elsif @redis
-        @redis_client = RedisMemo::Redis.new(@redis)
-      else
-        @redis_client = RedisMemo::Redis.new
-      end
-    else
-      @redis = blk
-    end
+  def redis_config
+    @redis_config || {}
+  end
+
+  def redis=(config)
+    @redis_config = config
+    @redis_client = nil
+    redis
   end
 
   def tracer(&blk)
@@ -74,15 +72,16 @@ class RedisMemo::Options
   end
 
   attr_accessor :async
+  attr_accessor :bulk_operations_invalidation_limit
+  attr_accessor :cache_out_of_date_handler
+  attr_accessor :cache_validation_sampler
   attr_accessor :compress
   attr_accessor :compress_threshold
-  attr_accessor :redis_error_handler
+  attr_accessor :connection_pool
   attr_accessor :expires_in
-  attr_accessor :cache_validation_sampler
-  attr_accessor :cache_out_of_date_handler
+  attr_accessor :redis_error_handler
 
   attr_writer :global_cache_key_version
-  attr_writer :redis
   attr_writer :tracer
   attr_writer :logger
 end