redis-memo 0.0.0.beta → 0.0.0.beta.5

data/lib/redis_memo/memoize_method.rb

@@ -6,7 +6,7 @@ require_relative 'middleware'
 require_relative 'options'
 
 module RedisMemo::MemoizeMethod
-  def memoize_method(method_name, **options, &depends_on)
+  def memoize_method(method_name, method_id: nil, **options, &depends_on)
     method_name_without_memo = :"_redis_memo_#{method_name}_without_memo"
     method_name_with_memo = :"_redis_memo_#{method_name}_with_memo"
 
@@ -15,11 +15,24 @@ module RedisMemo::MemoizeMethod
     define_method method_name_with_memo do |*args|
       return send(method_name_without_memo, *args) if RedisMemo.without_memo?
 
+      dependent_memos = nil
+      if depends_on
+        dependency = RedisMemo::MemoizeMethod.get_or_extract_dependencies(self, *args, &depends_on)
+        dependent_memos = dependency.memos
+      end
+
       future = RedisMemo::Future.new(
         self,
-        RedisMemo::MemoizeMethod.method_id(self, method_name),
+        case method_id
+        when NilClass
+          RedisMemo::MemoizeMethod.method_id(self, method_name)
+        when String, Symbol
+          method_id
+        else
+          method_id.call(self, *args)
+        end,
         args,
-        depends_on,
+        dependent_memos,
         options,
         method_name_without_memo,
       )
@@ -30,9 +43,25 @@ module RedisMemo::MemoizeMethod
       end
 
       future.execute
+    rescue RedisMemo::WithoutMemoization
+      send(method_name_without_memo, *args)
     end
 
     alias_method method_name, method_name_with_memo
+
+    @__redis_memo_method_dependencies ||= Hash.new
+    @__redis_memo_method_dependencies[method_name] = depends_on
+
+    define_method :dependency_of do |method_name, *method_args|
+      method_depends_on = self.class.instance_variable_get(:@__redis_memo_method_dependencies)[method_name]
+      unless method_depends_on
+        raise(
+          RedisMemo::ArgumentError,
+          "#{method_name} is not a memoized method"
+        )
+      end
+      RedisMemo::MemoizeMethod.get_or_extract_dependencies(self, *method_args, &method_depends_on)
+    end
   end
 
   def self.method_id(ref, method_name)
@@ -42,23 +71,34 @@ module RedisMemo::MemoizeMethod
     "#{class_name}#{is_class_method ? '::' : '#'}#{method_name}"
   end
 
-  def self.method_cache_keys(future_contexts)
-    memos = Array.new(future_contexts.size)
-    future_contexts.each_with_index do |(ref, _, method_args, depends_on), i|
-      if depends_on
-        dependency = RedisMemo::Memoizable::Dependency.new
+  def self.extract_dependencies(ref, *method_args, &depends_on)
+    dependency = RedisMemo::Memoizable::Dependency.new
 
-        # Resolve the dependency recursively
-        dependency.instance_exec(ref, *method_args, &depends_on)
+    # Resolve the dependency recursively
+    dependency.instance_exec(ref, *method_args, &depends_on)
+    dependency
+  end
 
-        memos[i] = dependency.memos
-      end
+  def self.get_or_extract_dependencies(ref, *method_args, &depends_on)
+    if RedisMemo::Cache.local_dependency_cache
+      RedisMemo::Cache.local_dependency_cache[ref] ||= {}
+      RedisMemo::Cache.local_dependency_cache[ref][depends_on] ||= {}
+      RedisMemo::Cache.local_dependency_cache[ref][depends_on][method_args] ||= extract_dependencies(ref, *method_args, &depends_on)
+    else
+      extract_dependencies(ref, *method_args, &depends_on)
+    end
+  end
+
+  def self.method_cache_keys(future_contexts)
+    memos = Array.new(future_contexts.size)
+    future_contexts.each_with_index do |(_, _, dependent_memos), i|
+      memos[i] = dependent_memos
     end
 
     j = 0
     memo_checksums = RedisMemo::Memoizable.checksums(memos.compact)
     method_cache_key_versions = Array.new(future_contexts.size)
-    future_contexts.each_with_index do |(_, method_id, method_args, _), i|
+    future_contexts.each_with_index do |(method_id, method_args, _), i|
       if memos[i]
         method_cache_key_versions[i] = [method_id, memo_checksums[j]]
         j += 1

data/lib/redis_memo/memoize_query.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+require_relative 'memoize_method'
+
+if defined?(ActiveRecord)
+  # Hook into ActiveRecord to cache SQL queries and perform auto cache
+  # invalidation
+  module RedisMemo::MemoizeQuery
+    require_relative 'memoize_query/cached_select'
+    require_relative 'memoize_query/invalidation'
+    require_relative 'memoize_query/model_callback'
+
+    # Only editable columns will be used to create memos that are invalidatable
+    # after each record save
+    def memoize_table_column(*raw_columns, editable: true)
+      RedisMemo::MemoizeQuery.using_active_record!(self)
+      return if ENV["REDIS_MEMO_DISABLE_#{self.table_name.upcase}"] == 'true'
+
+      columns = raw_columns.map(&:to_sym).sort
+
+      RedisMemo::MemoizeQuery.memoized_columns(self, editable_only: true) << columns if editable
+      RedisMemo::MemoizeQuery.memoized_columns(self, editable_only: false) << columns
+
+      RedisMemo::MemoizeQuery::ModelCallback.install(self)
+      RedisMemo::MemoizeQuery::Invalidation.install(self)
+
+      if ENV['REDIS_MEMO_DISABLE_CACHED_SELECT'] != 'true'
+        RedisMemo::MemoizeQuery::CachedSelect.install(ActiveRecord::Base.connection)
+      end
+
+      # The code below might fail due to missing DB/table errors
+      columns.each do |column|
+        unless self.columns_hash.include?(column.to_s)
+          raise(
+            RedisMemo::ArgumentError,
+            "'#{self.name}' does not contain column '#{column}'",
+          )
+        end
+      end
+
+      unless ENV["REDIS_MEMO_DISABLE_QUERY_#{self.table_name.upcase}"] == 'true'
+        RedisMemo::MemoizeQuery::CachedSelect.enabled_models[self.table_name] = self
+      end
+    rescue ActiveRecord::NoDatabaseError, ActiveRecord::StatementInvalid
+      # no-opts: models with memoize_table_column decleared might be loaded in
+      # rake tasks that are used to create databases
+    end
+
+    def self.using_active_record!(model_class)
+      unless using_active_record?(model_class)
+        raise RedisMemo::ArgumentError, "'#{model_class.name}' does not use ActiveRecord"
+      end
+    end
+
+    def self.using_active_record?(model_class)
+      model_class.respond_to?(:<) && model_class < ActiveRecord::Base
+    end
+
+    @@memoized_columns = Hash.new { |h, k| h[k] = [Set.new, Set.new] }
+
+    def self.memoized_columns(model_or_table, editable_only: false)
+      table = model_or_table.is_a?(Class) ? model_or_table.table_name : model_or_table
+      @@memoized_columns[table.to_sym][editable_only ? 1 : 0]
+    end
+
+    # extra_props are considered as AND conditions on the model class
+    def self.create_memo(model_class, **extra_props)
+      using_active_record!(model_class)
+
+      keys = extra_props.keys.sort
+      if !keys.empty? && !memoized_columns(model_class).include?(keys)
+        raise(
+          RedisMemo::ArgumentError,
+          "'#{model_class.name}' has not memoized columns: #{keys}",
+        )
+      end
+
+      extra_props.each do |key, value|
+        # The data type is ensured by the database, thus we don't need to cast
+        # types here for better performance
+        column_name = key.to_s
+        extra_props[key] =
+          if model_class.defined_enums.include?(column_name)
+            enum_mapping = model_class.defined_enums[column_name]
+            # Assume a value is a converted enum if it does not exist in the
+            # enum mapping
+            (enum_mapping[value.to_s] || value).to_s
+          else
+            value.to_s
+          end
+      end
+
+      RedisMemo::Memoizable.new(
+        __redis_memo_memoize_query_table_name__: model_class.table_name,
+        **extra_props,
+      )
+    end
+
+    def self.invalidate_all(model_class)
+      RedisMemo::Tracer.trace(
+        'redis_memo.memoizable.invalidate_all',
+        model_class.name,
+      ) do
+        RedisMemo::Memoizable.invalidate([model_class.redis_memo_class_memoizable])
+      end
+    end
+
+    def self.invalidate(*records)
+      RedisMemo::Memoizable.invalidate(
+        records.map { |record| to_memos(record) }.flatten,
+      )
+    end
+
+    def self.to_memos(record)
+      # Invalidate memos with current values
+      memos_to_invalidate = memoized_columns(record.class).map do |columns|
+        props = {}
+        columns.each do |column|
+          props[column] = record.send(column)
+        end
+
+        create_memo(record.class, **props)
+      end
+
+      # Create memos with previous values if
+      #  - there are saved changes
+      #  - this is not creating a new record
+      if !record.saved_changes.empty? && !record.saved_changes.include?(record.class.primary_key)
+        previous_values = {}
+        record.saved_changes.each do |column, (previous_value, _)|
+          previous_values[column.to_sym] = previous_value
+        end
+
+        memoized_columns(record.class, editable_only: true).each do |columns|
+          props = previous_values.slice(*columns)
+          next if props.empty?
+
+          # Fill the column values that have not changed
+          columns.each do |column|
+            next if props.include?(column)
+
+            props[column] = record.send(column)
+          end
+
+          memos_to_invalidate << create_memo(record.class, **props)
+        end
+      end
+
+      memos_to_invalidate
+    end
+  end
+end

data/lib/redis_memo/{memoize_records → memoize_query}/cached_select.rb

@@ -38,7 +38,7 @@
 # the `memoize_table_column` declaration on the model class.
 #
 #     class MyRecord < ApplicationRecord
-#       extend RedisMemo::MemoizeRecords
+#       extend RedisMemo::MemoizeQuery
 #       memoize_table_column :value
 #     end
 #
@@ -87,8 +87,17 @@
 #
 # See +extract_bind_params+ for the precise detection logic.
 #
-class RedisMemo::MemoizeRecords::CachedSelect
-  # TODO: merge this into RedisMemo::MemoizeQuery
+class RedisMemo::MemoizeQuery::CachedSelect
+  require_relative 'cached_select/bind_params'
+  require_relative 'cached_select/connection_adapter'
+  require_relative 'cached_select/statement_cache'
+
+  @@enabled_models = {}
+
+  def self.enabled_models
+    @@enabled_models
+  end
+
   def self.install(connection)
     klass = connection.class
     return if klass.singleton_class < RedisMemo::MemoizeMethod
@@ -96,8 +105,14 @@ class RedisMemo::MemoizeRecords::CachedSelect
     klass.class_eval do
       extend RedisMemo::MemoizeMethod
 
-      memoize_method :exec_query do |_, sql, name, binds, **kwargs|
-        RedisMemo::MemoizeRecords::CachedSelect
+      memoize_method(
+        :exec_query,
+        method_id: proc do |_, sql, *args|
+          sql.gsub(/(\$\d+)/, '?')      # $1 -> ?
+             .gsub(/((, *)*\?)+/, '?')  # (?, ?, ? ...) -> (?)
+        end,
+      ) do |_, sql, name, binds, **kwargs|
+        RedisMemo::MemoizeQuery::CachedSelect
           .current_query_bind_params
           .params
           .each do |model, attrs_set|
@@ -107,8 +122,15 @@ class RedisMemo::MemoizeRecords::CachedSelect
           end
 
         depends_on RedisMemo::Memoizable.new(
-          __redis_memo_memoize_records_memoize_query_sql__: sql,
-          __redis_memo_memoize_records_memoize_query_binds__: binds.map(&:value_for_database),
+          __redis_memo_memoize_query_memoize_query_sql__: sql,
+          __redis_memo_memoize_query_memoize_query_binds__: binds.map do |bind|
+            if bind.respond_to?(:value_for_database)
+              bind.value_for_database
+            else
+              # In activerecord >= 6, a bind could be an actual database value
+              bind
+            end
+          end
         )
       end
     end
@@ -137,64 +159,6 @@ class RedisMemo::MemoizeRecords::CachedSelect
     end
   end
 
-  module ConnectionAdapter
-    def cacheable_query(*args)
-      query, binds = super(*args)
-
-      # Persist the arel object to StatementCache#execute
-      query.instance_variable_set(:@__redis_memo_memoize_records_memoize_query_arel, args.last)
-
-      [query, binds]
-    end
-
-    def exec_query(*args)
-      # An Arel AST in Thread local is set prior to supported query methods
-      if !RedisMemo.without_memo? &&
-          RedisMemo::MemoizeRecords::CachedSelect.extract_bind_params(args[0])
-        # [Reids $model Load] $sql $binds
-        RedisMemo::DefaultOptions.logger&.info(
-          "[Redis] \u001b[36;1m#{args[1]} \u001b[34;1m#{args[0]}\u001b[0m #{
-            args[2].map { |bind| [bind.name, bind.value_for_database]}
-          }"
-        )
-
-        RedisMemo::Tracer.trace(
-          'redis_memo.memoize_query',
-          args[0]
-            .gsub(/(\$\d+)/, '?') # $1 -> ?
-            .gsub(/((, *)*\?)+/, '?'), # (?, ?, ? ...) -> (?)
-        ) do
-          super(*args)
-        end
-      else
-        RedisMemo.without_memo { super(*args) }
-      end
-    end
-
-    def select_all(*args)
-      if args[0].is_a?(Arel::SelectManager)
-        RedisMemo::MemoizeRecords::CachedSelect.current_query = args[0]
-      end
-
-      super(*args)
-    ensure
-      RedisMemo::MemoizeRecords::CachedSelect.reset_current_query
-    end
-  end
-
-  module StatementCache
-    def execute(*args)
-      arel = query_builder.instance_variable_get(:@__redis_memo_memoize_records_memoize_query_arel)
-      RedisMemo::MemoizeRecords::CachedSelect.current_query = arel
-      RedisMemo::MemoizeRecords::CachedSelect.current_substitutes =
-        bind_map.map_substitutes(args[0])
-
-      super(*args)
-    ensure
-      RedisMemo::MemoizeRecords::CachedSelect.reset_current_query
-    end
-  end
-
   def self.extract_bind_params(sql)
     ast = Thread.current[THREAD_KEY_AREL]&.ast
     return false unless ast.is_a?(Arel::Nodes::SelectStatement)
@@ -230,6 +194,19 @@ class RedisMemo::MemoizeRecords::CachedSelect
     Thread.current[THREAD_KEY_AREL_BIND_PARAMS] = nil
   end
 
+  def self.with_new_query_context
+    prev_arel = Thread.current[THREAD_KEY_AREL]
+    prev_substitutes = Thread.current[THREAD_KEY_SUBSTITUTES]
+    prev_bind_params = Thread.current[THREAD_KEY_AREL_BIND_PARAMS]
+    RedisMemo::MemoizeQuery::CachedSelect.reset_current_query
+
+    yield
+  ensure
+    Thread.current[THREAD_KEY_AREL] = prev_arel
+    Thread.current[THREAD_KEY_SUBSTITUTES] = prev_substitutes
+    Thread.current[THREAD_KEY_AREL_BIND_PARAMS] = prev_bind_params
+  end
+
   private
 
   # A pre-order Depth First Search
@@ -240,7 +217,7 @@ class RedisMemo::MemoizeRecords::CachedSelect
     bind_params = BindParams.new
 
     case node
-    when Arel::Nodes::Equality, Arel::Nodes::In
+    when NodeHasFilterCondition
       attr_node = node.left
       return unless attr_node.is_a?(Arel::Attributes::Attribute)
 
@@ -255,16 +232,8 @@ class RedisMemo::MemoizeRecords::CachedSelect
           return
         end
 
-      type_caster = table_node.send(:type_caster)
-      binding_relation =
-        case type_caster
-        when ActiveRecord::TypeCaster::Map
-          type_caster.send(:types)
-        when ActiveRecord::TypeCaster::Connection
-          type_caster.instance_variable_get(:@klass)
-        else
-          return
-        end
+      binding_relation = extract_binding_relation(table_node)
+      return unless binding_relation
 
       rights = node.right.is_a?(Array) ? node.right : [node.right]
       substitutes = Thread.current[THREAD_KEY_SUBSTITUTES]
@@ -285,7 +254,13 @@ class RedisMemo::MemoizeRecords::CachedSelect
           }
         when Arel::Nodes::Casted
           bind_params.params[binding_relation] << {
-            right.attribute.name.to_sym => right.val,
+            right.attribute.name.to_sym =>
+              if right.respond_to?(:val)
+                right.val
+              else
+                # activerecord >= 6
+                right.value
+              end,
           }
         else
           bind_params = bind_params.union(extract_bind_params_recurse(right))
@@ -303,11 +278,17 @@ class RedisMemo::MemoizeRecords::CachedSelect
       return unless node.orders.empty?
 
       node.cores.each do |core|
+        # We don't support JOINs
+        return unless core.source.right.empty?
+
         # Should have a WHERE if directly selecting from a table
         source_node = core.source.left
+        binding_relation = nil
         case source_node
         when Arel::Table
-          return if core.wheres.empty?
+          binding_relation = extract_binding_relation(source_node)
+
+          return if core.wheres.empty? || binding_relation.nil?
         when Arel::Nodes::TableAlias
           bind_params = bind_params.union(
             extract_bind_params_recurse(source_node.left)
@@ -334,6 +315,9 @@ class RedisMemo::MemoizeRecords::CachedSelect
 
           return unless bind_params
         end
+
+        # Reject any unbound select queries
+        return if binding_relation && bind_params.params[binding_relation].empty?
       end
 
       bind_params
@@ -352,7 +336,7 @@ class RedisMemo::MemoizeRecords::CachedSelect
       end
 
       bind_params
-    when Arel::Nodes::Join, Arel::Nodes::Union, Arel::Nodes::Or
+    when Arel::Nodes::Union, Arel::Nodes::Or
       [node.left, node.right].each do |child|
         bind_params = bind_params.union(
           extract_bind_params_recurse(child)
@@ -368,132 +352,29 @@ class RedisMemo::MemoizeRecords::CachedSelect
     end
   end
 
-  class BindParams
-    def params
-      #
-      # Bind params is hash of sets: each key is a model class, each value is a
-      # set of hashes for memoized column conditions. Example:
-      #
-      #   {
-      #      Site => [
-      #        {name: 'a', city: 'b'},
-      #        {name: 'a', city: 'c'},
-      #        {name: 'b', city: 'b'},
-      #        {name: 'b', city: 'c'},
-      #      ],
-      #   }
-      #
-      @params ||= Hash.new do |models, model|
-        models[model] = []
-      end
-    end
-
-    def union(other)
-      return unless other
+  def self.extract_binding_relation(table_node)
+    enabled_models[table_node.try(:name)]
+  end
 
-      # The tree is almost always right-heavy. Merge into the right node for better
-      # performance.
-      other.params.merge!(params) do |_, other_attrs_set, attrs_set|
-        if other_attrs_set.empty?
-          attrs_set
-        elsif attrs_set.empty?
-          other_attrs_set
+  class NodeHasFilterCondition
+    def self.===(node)
+      case node
+      when Arel::Nodes::Equality, Arel::Nodes::In
+        true
+      else
+        # In activerecord >= 6, a new arel node HomogeneousIn is introduced
+        if defined?(Arel::Nodes::HomogeneousIn) &&
+           node.is_a?(Arel::Nodes::HomogeneousIn)
+          true
         else
-          attrs_set + other_attrs_set
-        end
-      end
-
-      other
-    end
-
-    def product(other)
-      #  Example:
-      #
-      #  and(
-      #    [{a: 1}, {a: 2}],
-      #    [{b: 1}, {b: 2}],
-      #  )
-      #
-      #  =>
-      #
-      #  [
-      #    {a: 1, b: 1},
-      #    {a: 1, b: 2},
-      #    {a: 2, b: 1},
-      #    {a: 2, b: 2},
-      #  ]
-      return unless other
-
-      # The tree is almost always right-heavy. Merge into the right node for better
-      # performance.
-      params.each do |model, attrs_set|
-        next if attrs_set.empty?
-
-        # The other model does not have any conditions so far: carry the
-        # attributes over to the other node
-        if other.params[model].empty?
-          other.params[model] = attrs_set
-          next
-        end
-
-        # Distribute the current attrs into the other
-        other_attrs_set_size = other.params[model].size
-        other_attrs_set = other.params[model]
-        merged_attrs_set = Array.new(other_attrs_set_size * attrs_set.size)
-
-        attrs_set.each_with_index do |attrs, i|
-          other_attrs_set.each_with_index do |other_attrs, j|
-            k = i * other_attrs_set_size + j
-            merged_attrs = merged_attrs_set[k] = other_attrs.dup
-            attrs.each do |name, val|
-              # Conflict detected. For example:
-              #
-              #   (a = 1 or b = 1) and (a = 2 or b = 2)
-              #
-              #  Keep:     a = 1 and b = 2, a = 2 and b = 1
-              #  Discard:  a = 1 and a = 2, b = 1 and b = 2
-              if merged_attrs.include?(name) && merged_attrs[name] != val
-                merged_attrs_set[k] = nil
-                break
-              end
-
-              merged_attrs[name] = val
-            end
-          end
+          false
         end
-
-        merged_attrs_set.compact!
-        other.params[model] = merged_attrs_set
-      end
-
-      other
-    end
-
-    def uniq!
-      params.each do |_, attrs_set|
-        attrs_set.uniq!
       end
     end
-
-    def memoizable?
-      return false if params.empty?
-
-      params.each do |model, attrs_set|
-        return false if attrs_set.empty?
-
-        attrs_set.each do |attrs|
-          return false unless RedisMemo::MemoizeRecords
-            .memoized_columns(model)
-            .include?(attrs.keys.sort)
-        end
-      end
-
-      true
-    end
   end
 
   # Thread locals to exchange information between RedisMemo and ActiveRecord
-  THREAD_KEY_AREL = :__redis_memo_memoize_records_cached_select_arel__
-  THREAD_KEY_SUBSTITUTES = :__redis_memo_memoize_records_cached_select_substitues__
-  THREAD_KEY_AREL_BIND_PARAMS = :__redis_memo_memoize_records_cached_select_arel_bind_params__
+  THREAD_KEY_AREL = :__redis_memo_memoize_query_cached_select_arel__
+  THREAD_KEY_SUBSTITUTES = :__redis_memo_memoize_query_cached_select_substitues__
+  THREAD_KEY_AREL_BIND_PARAMS = :__redis_memo_memoize_query_cached_select_arel_bind_params__
 end