redis-memo 0.0.0.beta.4 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 72bcf0679b811825222a6b38de0fcdadf4546640d2f61b18ebc827e37137dc41
-  data.tar.gz: 7a55c7bbd084a24afe085e85bc5cd5c486cf240094b64ca09d48e9e0ce126384
+  metadata.gz: 976988ff54686693396ffb07c47b9e4326646837b3be53264891c886f03771c6
+  data.tar.gz: 9e4a3d4dca78cb9389af8e1728addb3aa8d04493365bcfda2ea236e20c657971
 SHA512:
-  metadata.gz: 5a3af887512e4b6571829fcfda719e43abb917eb4773407bd27788b0b671a42263ec31c2f6286f0a1279a354851c8c8cb48eb9701cba59863e500ca47419080e
-  data.tar.gz: feeb72b177ec07953b839e6e0759cbdf42dfd4eddd19f89599e721589462e849e7685a659b03128504d670764223f1805402cb3ee8a96411b834f229ba837d99
+  metadata.gz: 118d801be413e14616becfb3a9c4e4715a1e7b7cb6722ac48bd8b1c63e6b1c8f565c8a148909b4bd7b4b538f89d1d79145ad137403139eb7158cad61c9d7a847
+  data.tar.gz: 52bdf762fc8bf22a6b65fc158512e001dd12490904db9766cd9f3ff1aee74e61f23f218bfa46cdf57fdb67f9f0d2508649fedc2379d6224d0d34416ffbafb902

data/lib/redis_memo/memoizable/dependency.rb

@@ -26,14 +26,18 @@ class RedisMemo::Memoizable::Dependency
         instance_exec(&memo.depends_on)
       end
     when ActiveRecord::Relation
-      extracted = extract_dependencies_for_relation(dependency)
+      extracted = self.class.extract_from_relation(dependency)
       nodes.merge!(extracted.nodes)
-    when UsingActiveRecord
-      [
-        dependency.redis_memo_class_memoizable,
-        RedisMemo::MemoizeQuery.create_memo(dependency, **conditions),
-      ].each do |memo|
+    when RedisMemo::MemoizeQuery::CachedSelect::BindParams
+      # A private API
+      dependency.params.each do |model, attrs_set|
+        memo = model.redis_memo_class_memoizable
         nodes[memo.cache_key] = memo
+
+        attrs_set.each do |attrs|
+          memo = RedisMemo::MemoizeQuery.create_memo(model, **attrs)
+          nodes[memo.cache_key] = memo
+        end
       end
     else
       raise(
@@ -43,24 +47,21 @@ class RedisMemo::Memoizable::Dependency
     end
   end
 
-  def extract_dependencies_for_relation(relation)
+  private
+
+  def self.extract_from_relation(relation)
     # Extract the dependent memos of an Arel without calling exec_query to actually execute the query
     RedisMemo::MemoizeQuery::CachedSelect.with_new_query_context do
       connection = ActiveRecord::Base.connection
       query, binds, _ = connection.send(:to_sql_and_binds, relation.arel)
       RedisMemo::MemoizeQuery::CachedSelect.current_query = relation.arel
       is_query_cached = RedisMemo::MemoizeQuery::CachedSelect.extract_bind_params(query)
-        raise(
-          RedisMemo::WithoutMemoization,
-          "Arel query is not cached using RedisMemo."
-        ) unless is_query_cached
-        extracted_dependency = connection.dependency_of(:exec_query, query, nil, binds)
-    end
-  end
 
-  class UsingActiveRecord
-    def self.===(dependency)
-      RedisMemo::MemoizeQuery.using_active_record?(dependency)
+      unless is_query_cached
+        raise RedisMemo::WithoutMemoization, 'Arel query is not cached using RedisMemo'
+      end
+
+      connection.dependency_of(:exec_query, query, nil, binds)
     end
   end
 end

data/lib/redis_memo/memoize_method.rb

@@ -81,14 +81,88 @@ module RedisMemo::MemoizeMethod
 
   def self.get_or_extract_dependencies(ref, *method_args, &depends_on)
     if RedisMemo::Cache.local_dependency_cache
-      RedisMemo::Cache.local_dependency_cache[ref] ||= {}
-      RedisMemo::Cache.local_dependency_cache[ref][depends_on] ||= {}
-      RedisMemo::Cache.local_dependency_cache[ref][depends_on][method_args] ||= extract_dependencies(ref, *method_args, &depends_on)
+      RedisMemo::Cache.local_dependency_cache[ref.class] ||= {}
+      RedisMemo::Cache.local_dependency_cache[ref.class][depends_on] ||= {}
+      named_args = exclude_anonymous_args(depends_on, ref, method_args)
+      RedisMemo::Cache.local_dependency_cache[ref.class][depends_on][named_args] ||= extract_dependencies(ref, *method_args, &depends_on)
     else
       extract_dependencies(ref, *method_args, &depends_on)
     end
   end
 
+  # We only look at named method parameters in the dependency block in order to define its dependent
+  # memos and ignore anonymous parameters, following the convention that nil or :_ is an anonymous parameter.
+  # Example:
+  # ```
+  #    def method(param1, param2)
+  #    end
+  #
+  #    memoize_method :method do |_, _, param2|`
+  #      depends_on RedisMemo::Memoizable.new(param2: param2)
+  #    end
+  # ```
+  #  `exclude_anonymous_args(depends_on, ref, [1, 2])` returns [2]
+  def self.exclude_anonymous_args(depends_on, ref, args)
+    return [] if depends_on.parameters.empty? or args.empty?
+
+    positional_args = []
+    kwargs = {}
+    depends_on_args = [ref] + args
+    options = depends_on_args.extract_options!
+
+    # Keep track of the splat start index, and the number of positional args before and after the splat,
+    # so we can map which args belong to positional args and which args belong to the splat.
+    named_splat = false
+    splat_index = nil
+    num_positional_args_after_splat = 0
+    num_positional_args_before_splat = 0
+
+    depends_on.parameters.each_with_index do |param, i|
+      # Defined by https://github.com/ruby/ruby/blob/22b8ddfd1049c3fd1e368684c4fd03bceb041b3a/proc.c#L3048-L3059
+      case param.first
+      when :opt, :req
+        if splat_index
+          num_positional_args_after_splat += 1
+        else
+          num_positional_args_before_splat += 1
+        end
+      when :rest
+        named_splat = is_named?(param)
+        splat_index = i
+      when :key, :keyreq
+        kwargs[param.last] = options[param.last] if is_named?(param)
+      when :keyrest
+        kwargs.merge!(options) if is_named?(param)
+      else
+        raise(RedisMemo::ArgumentError, "#{param.first} argument isn't supported in the dependency block")
+      end
+    end
+
+    # Determine the named positional and splat arguments after we know the # of pos. arguments before and after splat
+    after_splat_index = depends_on_args.size - num_positional_args_after_splat
+    depends_on_args.each_with_index do |arg, i|
+      # if the index is within the splat
+      if i >= num_positional_args_before_splat && i < after_splat_index
+        positional_args << arg if named_splat
+      else
+        j = i < num_positional_args_before_splat ? i : i - (after_splat_index - splat_index) - 1
+        positional_args << arg if is_named?(depends_on.parameters[j])
+      end
+    end
+
+    if !kwargs.empty?
+      positional_args + [kwargs]
+    elsif named_splat && !options.empty?
+      positional_args + [options]
+    else
+      positional_args
+    end
+  end
+private
+  def self.is_named?(param)
+    param.size == 2 && param.last != :_
+  end
+
   def self.method_cache_keys(future_contexts)
     memos = Array.new(future_contexts.size)
     future_contexts.each_with_index do |(_, _, dependent_memos), i|

data/lib/redis_memo/memoize_query/cached_select.rb

@@ -111,15 +111,8 @@ class RedisMemo::MemoizeQuery::CachedSelect
           sql.gsub(/(\$\d+)/, '?')      # $1 -> ?
              .gsub(/((, *)*\?)+/, '?')  # (?, ?, ? ...) -> (?)
         end,
-      ) do |_, sql, name, binds, **kwargs|
-        RedisMemo::MemoizeQuery::CachedSelect
-          .current_query_bind_params
-          .params
-          .each do |model, attrs_set|
-            attrs_set.each do |attrs|
-              depends_on model, **attrs
-            end
-          end
+      ) do |_, sql, _, binds, **|
+        depends_on RedisMemo::MemoizeQuery::CachedSelect.current_query_bind_params
 
         depends_on RedisMemo::Memoizable.new(
           __redis_memo_memoize_query_memoize_query_sql__: sql,
@@ -274,9 +267,6 @@ class RedisMemo::MemoizeQuery::CachedSelect
 
       bind_params
     when Arel::Nodes::SelectStatement
-      # No OREDER BY
-      return unless node.orders.empty?
-
       node.cores.each do |core|
         # We don't support JOINs
         return unless core.source.right.empty?

data/lib/redis_memo/memoize_query/invalidation.rb

@@ -36,10 +36,8 @@ class RedisMemo::MemoizeQuery::Invalidation
       decrement_counter
       delete_all delete_by
       increment_counter
-      insert insert! insert_all insert_all!
       touch_all
       update_column update_columns update_all update_counters
-      upsert upsert_all
     ).each do |method_name|
       # Example: Model.update_all
       rewrite_default_method(
@@ -58,6 +56,24 @@ class RedisMemo::MemoizeQuery::Invalidation
       )
     end
 
+    %i(
+      insert insert! insert_all insert_all!
+    ).each do |method_name|
+      rewrite_insert_method(
+        model_class,
+        method_name,
+      )
+    end
+
+    %i(
+      upsert upsert_all
+    ).each do |method_name|
+      rewrite_upsert_method(
+        model_class,
+        method_name,
+      )
+    end
+
     %i(
       import import!
     ).each do |method_name|
@@ -70,6 +86,36 @@ class RedisMemo::MemoizeQuery::Invalidation
     model_class.class_variable_set(var_name, true)
   end
 
+  def self.invalidate_new_records(model_class, &blk)
+    current_id = model_class.maximum(model_class.primary_key)
+    result = blk.call
+    records = select_by_new_ids(model_class, current_id)
+    RedisMemo::MemoizeQuery.invalidate(*records) unless records.empty?
+    result
+  end
+
+  def self.invalidate_records_by_conflict_target(model_class, records:, conflict_target: nil, &blk)
+    if conflict_target.nil?
+      # When the conflict_target is not set, we are basically inserting new
+      # records since duplicate rows are simply skipped
+      return invalidate_new_records(model_class, &blk)
+    end
+
+    relation = build_relation_by_conflict_target(model_class, records, conflict_target)
+    # Invalidate records before updating
+    records = select_by_conflict_target_relation(model_class, relation)
+    RedisMemo::MemoizeQuery.invalidate(*records) unless records.empty?
+
+    # Perform updating
+    result = blk.call
+
+    # Invalidate records after updating
+    records = select_by_conflict_target_relation(model_class, relation)
+    RedisMemo::MemoizeQuery.invalidate(*records) unless records.empty?
+
+    result
+  end
+
   private
 
   #
@@ -95,12 +141,48 @@ class RedisMemo::MemoizeQuery::Invalidation
     end
   end
 
-  def self.rewrite_import_method(model_class, method_name)
-    # This optimization to avoid over-invalidation only works on postgres
-    unless ActiveRecord::Base.connection.is_a?(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter)
-      rewrite_default_method(model_class, model_class, method_name, class_method: true)
-      return
+  def self.rewrite_insert_method(model_class, method_name)
+    return unless model_class.respond_to?(method_name)
+
+    model_class.singleton_class.class_eval do
+      alias_method :"#{method_name}_without_redis_memo_invalidation", method_name
+
+      define_method method_name do |*args, &blk|
+        RedisMemo::MemoizeQuery::Invalidation.invalidate_new_records(model_class) do
+          send(:"#{method_name}_without_redis_memo_invalidation", *args, &blk)
+        end
+      end
     end
+  end
+
+  def self.rewrite_upsert_method(model_class, method_name)
+    return unless model_class.respond_to?(method_name)
+
+    model_class.singleton_class.class_eval do
+      alias_method :"#{method_name}_without_redis_memo_invalidation", method_name
+
+      define_method method_name do |attributes, unique_by: nil, **kwargs, &blk|
+        RedisMemo::MemoizeQuery::Invalidation.invalidate_records_by_conflict_target(
+          model_class,
+          records: nil, # not used
+          # upsert does not support on_duplicate_key_update yet at activerecord
+          # HEAD (6.1.3)
+          conflict_target: nil,
+        ) do
+          send(
+            :"#{method_name}_without_redis_memo_invalidation",
+            attributes,
+            unique_by: unique_by,
+            **kwargs,
+            &blk
+          )
+        end
+      end
+    end
+  end
+
+  def self.rewrite_import_method(model_class, method_name)
+    return unless model_class.respond_to?(method_name)
 
     model_class.singleton_class.class_eval do
       alias_method :"#{method_name}_without_redis_memo_invalidation", method_name
@@ -110,102 +192,75 @@ class RedisMemo::MemoizeQuery::Invalidation
       define_method method_name do |*args, &blk|
         options = args.last.is_a?(Hash) ? args.last : {}
         records = args[args.last.is_a?(Hash) ? -2 : -1]
-        columns_to_update = options[:on_duplicate_key_update]
-        if columns_to_update.is_a?(Hash)
-          columns_to_update = columns_to_update[:columns]
-        end
-
-        if records.last.is_a?(Hash)
-          records.map! { |hash| model_class.new(hash) }
-        end
-
-        # Invalidate the records before and after the import to resolve
-        # - default values filled by the database
-        # - updates on conflict conditions
-        records_to_invalidate =
-          if columns_to_update
-            RedisMemo::MemoizeQuery::Invalidation.send(
-              :select_by_columns,
-              records,
-              columns_to_update,
-            )
+        on_duplicate_key_update = options[:on_duplicate_key_update]
+        conflict_target =
+          case on_duplicate_key_update
+          when Hash
+            # The conflict_target option is only supported in PostgreSQL. In
+            # MySQL, the primary_key is used as the conflict_target
+            on_duplicate_key_update[:conflict_target] || [model_class.primary_key.to_sym]
+          when Array
+            # The default conflict_target is just the primary_key
+            [model_class.primary_key.to_sym]
           else
-            []
+            # Ignore duplicate rows
+            nil
           end
 
-        result = send(:"#{method_name}_without_redis_memo_invalidation", *args, &blk)
-
-        # Offload the records to invalidate while selecting the next set of
-        # records to invalidate
-        case records_to_invalidate
-        when Array
-          RedisMemo::MemoizeQuery.invalidate(*records_to_invalidate) unless records_to_invalidate.empty?
-
-          RedisMemo::MemoizeQuery.invalidate(*RedisMemo::MemoizeQuery::Invalidation.send(
-            :select_by_id,
-            model_class,
-            # Not all databases support "RETURNING", which is useful when
-            # invaldating records after bulk creation
-            result.ids,
-          ))
-        else
-          RedisMemo::MemoizeQuery.invalidate_all(model_class)
+        if conflict_target && records.last.is_a?(Hash)
+          records.map! { |hash| model_class.new(hash) }
         end
 
-        result
+        RedisMemo::MemoizeQuery::Invalidation.invalidate_records_by_conflict_target(
+          model_class,
+          records: records,
+          conflict_target: conflict_target,
+        ) do
+          send(:"#{method_name}_without_redis_memo_invalidation", *args, &blk)
+        end
       end
     end
   end
 
-  def self.select_by_columns(records, columns_to_update)
-    model_class = records.first.class
+  def self.build_relation_by_conflict_target(model_class, records, conflict_target)
     or_chain = nil
-    columns_to_select = columns_to_update & RedisMemo::MemoizeQuery
-      .memoized_columns(model_class)
-      .to_a.flatten.uniq
-
-    # Nothing to invalidate here
-    return [] if columns_to_select.empty?
 
-    RedisMemo::Tracer.trace(
-      'redis_memo.memoize_query.invalidation',
-      "#{__method__}##{model_class.name}",
-    ) do
-      records.each do |record|
-        conditions = {}
-        columns_to_select.each do |column|
-          conditions[column] = record.send(column)
-        end
-        if or_chain
-          or_chain = or_chain.or(model_class.where(conditions))
-        else
-          or_chain = model_class.where(conditions)
-        end
+    records.each do |record|
+      conditions = {}
+      conflict_target.each do |column|
+        conditions[column] = record.send(column)
       end
-
-      record_count = RedisMemo.without_memo { or_chain.count }
-      if record_count > bulk_operations_invalidation_limit
-        nil
+      if or_chain
+        or_chain = or_chain.or(model_class.where(conditions))
       else
-        RedisMemo.without_memo { or_chain.to_a }
+        or_chain = model_class.where(conditions)
       end
     end
+
+    or_chain
   end
 
-  def self.select_by_id(model_class, ids)
+  def self.select_by_new_ids(model_class, target_id)
     RedisMemo::Tracer.trace(
       'redis_memo.memoize_query.invalidation',
       "#{__method__}##{model_class.name}",
     ) do
       RedisMemo.without_memo do
-        model_class.where(model_class.primary_key => ids).to_a
+        model_class.where(
+          model_class.arel_table[model_class.primary_key].gt(target_id),
+        ).to_a
       end
     end
   end
 
-  def self.bulk_operations_invalidation_limit
-    ENV['REDIS_MEMO_BULK_OPERATIONS_INVALIDATION_LIMIT']&.to_i ||
-      RedisMemo::DefaultOptions.bulk_operations_invalidation_limit ||
-      10000
+  def self.select_by_conflict_target_relation(model_class, relation)
+    return [] unless relation
+
+    RedisMemo::Tracer.trace(
+      'redis_memo.memoize_query.invalidation',
+      "#{__method__}##{model_class.name}",
+    ) do
+      RedisMemo.without_memo { relation.reload }
+    end
   end
 end

data/lib/redis_memo/options.rb

@@ -72,7 +72,6 @@ class RedisMemo::Options
   end
 
   attr_accessor :async
-  attr_accessor :bulk_operations_invalidation_limit
   attr_accessor :cache_out_of_date_handler
   attr_accessor :cache_validation_sampler
   attr_accessor :compress

data/lib/redis_memo/redis.rb

@@ -31,7 +31,8 @@ class RedisMemo::Redis < Redis::Distributed
   end
 
   class WithReplicas < ::Redis
-    def initialize(options)
+    def initialize(orig_options)
+      options = orig_options.dup
       primary_option = options.shift
       @replicas = options.map do |option|
         option[:logger] ||= RedisMemo::DefaultOptions.logger

data/lib/redis_memo/testing.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# Redis memo can be flaky due to transient network errors (e.g. Redis connection errors), or when
+# used with async handlers. This class allows users to override the default redis-memo behavior
+# to be more robust when testing their code that uses redis-memo.
+module RedisMemo
+  class Testing
+
+    def self.__test_mode
+      @__test_mode
+    end
+
+    def self.__test_mode=(mode)
+      @__test_mode = mode
+    end
+
+    def self.enable_test_mode(&blk)
+      __set_test_mode(true, &blk)
+    end
+
+    def self.disable_test_mode(&blk)
+      __set_test_mode(false, &blk)
+    end
+
+    def self.enabled?
+      __test_mode
+    end
+
+    private
+
+    def self.__set_test_mode(mode, &blk)
+      if blk.nil?
+        __test_mode = mode
+      else
+        prev_mode = __test_mode
+        begin
+          __test_mode = mode
+          yield
+        ensure
+          __test_mode = prev_mode
+        end
+      end
+    end
+  end
+
+  module TestOverrides
+    def without_memo?
+      if RedisMemo::Testing.enabled? && !RedisMemo::Memoizable::Invalidation.class_variable_get(:@@invalidation_queue).empty?
+        return true
+      end
+      super
+    end
+  end
+  singleton_class.prepend(TestOverrides)
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis-memo
 version: !ruby/object:Gem::Version
-  version: 0.0.0.beta.4
+  version: 0.1.2
 platform: ruby
 authors:
 - Chan Zuckerberg Initiative
@@ -192,6 +192,7 @@ files:
 - lib/redis_memo/middleware.rb
 - lib/redis_memo/options.rb
 - lib/redis_memo/redis.rb
+- lib/redis_memo/testing.rb
 - lib/redis_memo/tracer.rb
 homepage: https://github.com/chanzuckerberg/redis-memo
 licenses:
@@ -208,9 +209,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.0.8
 signing_key: