redis-memo 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b3cb72a8a4bf3dcd6d30bb112e387710c65d09d60c3dc687e10db649da1e91a
-  data.tar.gz: b2452e1d4b2b7a3d588c13822eaffc1fb4336c3a4666f8a4ca473dbbd961578f
+  metadata.gz: 4e4b5e95c92879677df377b10736e7467efa86237d90ad41254bdd7a732a8260
+  data.tar.gz: 9dfb62913a6cfd96e114d6526f5876696e86d1fafb6256d2454d2723449e61c1
 SHA512:
-  metadata.gz: 4d9193ddb53419db1e14a5eda4d4ab73b4a96620f592998c764abe3a118fb266761b04a929ff49d15194a9bcf959f9cb7c82f45f783387a85f4e1be2b1315ada
-  data.tar.gz: 383b2b053917ceeb3991de1c7c399fa83cd8e2bc326f735a0d0924767cea5d6157cf2f3e4a07b3d5d6524b54cae7e62276fb0723daad6a66ebaa6d0b6da64745
+  metadata.gz: 62eaf4aedb48325cc90c10cd27a3877de85a436960c78d7e1205c5dbbbe77bdb6c50ac82ea1af11d7dbd7219e7fa96cdc270b37038f1674c1fa491dffb8acec3
+  data.tar.gz: 2315c83ce0848d5b2f36d60e37fe60bcece24d80505ff2d7c38b1a9c0d33b8785888325410bae695abc0746a8b4663e037daec07ea83f32b6aaeafd4bf955693

data/lib/redis_memo/memoizable/dependency.rb

@@ -30,7 +30,6 @@ class RedisMemo::Memoizable::Dependency
       extracted = self.class.extract_from_relation(dependency)
       nodes.merge!(extracted.nodes)
     when RedisMemo::MemoizeQuery::CachedSelect::BindParams
-      # A private API
       dependency.params.each do |model, attrs_set|
         memo = model.redis_memo_class_memoizable
         nodes[memo.cache_key] = memo

data/lib/redis_memo/memoize_query/cached_select.rb

@@ -113,12 +113,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
 
       memoize_method(
         :exec_query,
-        method_id: proc do |_, sql, *_args|
-          # replace $1 with ?,
-          # and (?, ?, ? ...) with (?)
-          sql.gsub(/(\$\d+)/, '?')
-             .gsub(/((, *)*\?)+/, '?')
-        end,
+        method_id: proc { |_, sql, *| RedisMemo::Util.tagify_parameterized_sql(sql) },
       ) do |_, sql, _, binds, **|
         depends_on RedisMemo::MemoizeQuery::CachedSelect.current_query_bind_params
 
@@ -161,26 +156,30 @@ class RedisMemo::MemoizeQuery::CachedSelect
   end
 
   # Extract bind params from the query by inspecting the SQL's AST recursively
-  # The bind params will be passed into the local thread variables
-  # See +extract_bind_params_recurse+ for how to extract binding params recursively
+  # The bind params will be passed into the local thread variables. See
+  # +construct_bind_params_recurse+ for how to construct binding params
+  # recursively.
   #
   # @param sql [String] SQL query
   # @return [Boolean] indicating whether a query should be cached
   def self.extract_bind_params(sql)
-    ast = RedisMemo::ThreadLocalVar.arel&.ast
-    return false unless ast.is_a?(Arel::Nodes::SelectStatement)
-    return false unless ast.to_sql == sql
-
-    RedisMemo::ThreadLocalVar.substitues ||= {}
-    # Iterate through the Arel AST in a Depth First Search
-    bind_params = extract_bind_params_recurse(ast)
-    return false unless bind_params
-
-    bind_params.uniq!
-    return false unless bind_params.memoizable?
-
-    RedisMemo::ThreadLocalVar.arel_bind_params = bind_params
-    true
+    RedisMemo::Tracer.trace(
+      'redis_memo.memoize_query.extract_bind_params',
+      RedisMemo::Util.tagify_parameterized_sql(sql),
+    ) do
+      ast = RedisMemo::ThreadLocalVar.arel&.ast
+      return false unless ast.is_a?(Arel::Nodes::SelectStatement)
+      return false unless ast.to_sql == sql
+
+      RedisMemo::ThreadLocalVar.substitues ||= {}
+      # Iterate through the Arel AST in a Depth First Search
+      bind_params = construct_bind_params_recurse(ast)
+      return false unless bind_params&.should_cache?
+
+      bind_params.extract!
+      RedisMemo::ThreadLocalVar.arel_bind_params = bind_params
+      true
+    end
   end
 
   def self.current_query_bind_params
@@ -222,7 +221,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
   # @param node [Arel::Nodes::Node]
   #
   # @return [RedisMemo::MemoizeQuery::CachedSelect::BindParams]
-  def self.extract_bind_params_recurse(node)
+  def self.construct_bind_params_recurse(node)
     # rubocop: disable Lint/NonLocalExitFromIterator
     bind_params = BindParams.new
 
@@ -273,7 +272,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
               end,
           }
         else
-          bind_params = bind_params.union(extract_bind_params_recurse(right))
+          bind_params = bind_params.union(construct_bind_params_recurse(right))
           return if !bind_params
         end
       end
@@ -294,7 +293,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
           return if core.wheres.empty? || binding_relation.nil?
         when Arel::Nodes::TableAlias
           bind_params = bind_params.union(
-            extract_bind_params_recurse(source_node.left),
+            construct_bind_params_recurse(source_node.left),
           )
 
           return unless bind_params
@@ -305,7 +304,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
         # Binds wheres before havings
         core.wheres.each do |where|
           bind_params = bind_params.union(
-            extract_bind_params_recurse(where),
+            construct_bind_params_recurse(where),
           )
 
           return unless bind_params
@@ -313,26 +312,23 @@ class RedisMemo::MemoizeQuery::CachedSelect
 
         core.havings.each do |having|
           bind_params = bind_params.union(
-            extract_bind_params_recurse(having),
+            construct_bind_params_recurse(having),
           )
 
           return unless bind_params
         end
-
-        # Reject any unbound select queries
-        return if binding_relation && bind_params.params[binding_relation].empty?
       end
 
       bind_params
     when Arel::Nodes::Grouping
       # Inline SQL
-      extract_bind_params_recurse(node.expr)
+      construct_bind_params_recurse(node.expr)
     when Arel::Nodes::LessThan, Arel::Nodes::LessThanOrEqual, Arel::Nodes::GreaterThan, Arel::Nodes::GreaterThanOrEqual, Arel::Nodes::NotEqual
       bind_params
     when Arel::Nodes::And
       node.children.each do |child|
         bind_params = bind_params.product(
-          extract_bind_params_recurse(child),
+          construct_bind_params_recurse(child),
         )
 
         return unless bind_params
@@ -342,7 +338,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
     when Arel::Nodes::Union, Arel::Nodes::Or
       [node.left, node.right].each do |child|
         bind_params = bind_params.union(
-          extract_bind_params_recurse(child),
+          construct_bind_params_recurse(child),
         )
 
         return unless bind_params

data/lib/redis_memo/memoize_query/cached_select/bind_params.rb

@@ -2,47 +2,122 @@
 
 class RedisMemo::MemoizeQuery::CachedSelect
   class BindParams
-    def params
-      #
-      # Bind params is hash of sets: each key is a model class, each value is a
-      # set of hashes for memoized column conditions. Example:
-      #
-      #   {
-      #      Site => [
-      #        {name: 'a', city: 'b'},
-      #        {name: 'a', city: 'c'},
-      #        {name: 'b', city: 'b'},
-      #        {name: 'b', city: 'c'},
-      #      ],
-      #   }
-      #
-      @params ||= Hash.new do |models, model|
-        models[model] = []
-      end
+    def initialize(left = nil, right = nil, operator = nil)
+      @left = left
+      @right = right
+      @operator = operator
     end
 
     def union(other)
       return unless other
 
-      # The tree is almost always right-heavy. Merge into the right node for better
-      # performance.
-      other.params.merge!(params) do |_, other_attrs_set, attrs_set|
-        if other_attrs_set.empty?
-          attrs_set
-        elsif attrs_set.empty?
-          other_attrs_set
-        else
-          attrs_set + other_attrs_set
+      self.class.new(self, other, __method__)
+    end
+
+    def product(other)
+      return unless other
+
+      self.class.new(self, other, __method__)
+    end
+
+    def should_cache?
+      plan!
+
+      if plan.model_attrs.empty? || plan.dependency_size_estimation.to_i > RedisMemo::DefaultOptions.max_query_dependency_size
+        return false
+      end
+
+      plan.model_attrs.each do |model, attrs_set|
+        return false if attrs_set.empty?
+
+        attrs_set.each do |attrs|
+          return false unless RedisMemo::MemoizeQuery
+            .memoized_columns(model)
+            .include?(attrs.keys.sort)
         end
       end
 
-      other
+      true
     end
 
-    def product(other)
+    #
+    # Extracted bind params is hash of sets: each key is a model class, each
+    # value is a set of hashes for memoized column conditions. Example:
+    #
+    #   {
+    #      Site => [
+    #        {name: 'a', city: 'b'},
+    #        {name: 'a', city: 'c'},
+    #        {name: 'b', city: 'b'},
+    #        {name: 'b', city: 'c'},
+    #      ],
+    #   }
+    #
+    def extract!
+      return if operator.nil?
+
+      left.extract!
+      right.extract!
+      __send__(:"#{operator}!")
+    end
+
+    def params
+      @params ||= Hash.new do |models, model|
+        models[model] = Set.new
+      end
+    end
+
+    protected
+
+    # BindParams is built recursively when iterating through the Arel AST
+    # nodes. BindParams represents a binary tree. Query parameters are added to
+    # the leaf nodes of the tree, and the leaf nodes are connected by
+    # operators, such as `union` (or conditions) or `product` (and conditions).
+    attr_accessor :left
+    attr_accessor :right
+    attr_accessor :operator
+    attr_accessor :plan
+
+    def plan!
+      self.plan = Plan.new(self)
+      return if operator.nil?
+
+      left.plan!
+      right.plan!
+      __send__(:"plan_#{operator}")
+    end
+
+    def plan_union
+      plan.dependency_size_estimation = left.plan.dependency_size_estimation + right.plan.dependency_size_estimation
+      plan.model_attrs = union_attrs_set(left.plan.model_attrs, right.plan.model_attrs)
+    end
+
+    def plan_product
+      plan.dependency_size_estimation = left.plan.dependency_size_estimation * right.plan.dependency_size_estimation
+      plan.model_attrs = product_attrs_set(left.plan.model_attrs, right.plan.model_attrs)
+    end
+
+    def union!
+      @params = union_attrs_set(left.params, right.params)
+    end
+
+    def product!
+      @params = product_attrs_set(left.params, right.params)
+    end
+
+    def union_attrs_set(left, right)
+      left.merge(right) do |_, attrs_set, other_attrs_set|
+        next attrs_set if other_attrs_set.empty?
+        next other_attrs_set if attrs_set.empty?
+
+        attrs_set + other_attrs_set
+      end
+    end
+
+    def product_attrs_set(left, right)
       #  Example:
       #
-      #  and(
+      #  product(
       #    [{a: 1}, {a: 2}],
       #    [{b: 1}, {b: 2}],
       #  )
@@ -55,29 +130,16 @@ class RedisMemo::MemoizeQuery::CachedSelect
       #    {a: 2, b: 1},
       #    {a: 2, b: 2},
       #  ]
-      return unless other
-
-      # The tree is almost always right-heavy. Merge into the right node for better
-      # performance.
-      params.each do |model, attrs_set|
-        next if attrs_set.empty?
-
-        # The other model does not have any conditions so far: carry the
-        # attributes over to the other node
-        if other.params[model].empty?
-          other.params[model] = attrs_set
-          next
-        end
-
-        # Distribute the current attrs into the other
-        other_attrs_set_size = other.params[model].size
-        other_attrs_set = other.params[model]
-        merged_attrs_set = Array.new(other_attrs_set_size * attrs_set.size)
+      left.merge(right) do |_, attrs_set, other_attrs_set|
+        next attrs_set if other_attrs_set.empty?
+        next other_attrs_set if attrs_set.empty?
 
-        attrs_set.each_with_index do |attrs, i|
-          other_attrs_set.each_with_index do |other_attrs, j|
-            k = i * other_attrs_set_size + j
-            merged_attrs = merged_attrs_set[k] = other_attrs.dup
+        # distribute the current attrs into the other
+        merged_attrs_set = Set.new
+        attrs_set.each do |attrs|
+          other_attrs_set.each do |other_attrs|
+            merged_attrs = other_attrs.dup
+            should_add_attrs = true
             attrs.each do |name, val|
               # Conflict detected. For example:
               #
@@ -86,42 +148,112 @@ class RedisMemo::MemoizeQuery::CachedSelect
               #  Keep:     a = 1 and b = 2, a = 2 and b = 1
               #  Discard:  a = 1 and a = 2, b = 1 and b = 2
               if merged_attrs.include?(name) && merged_attrs[name] != val
-                merged_attrs_set[k] = nil
+                should_add_attrs = false
                 break
               end
 
               merged_attrs[name] = val
             end
+            merged_attrs_set << merged_attrs if should_add_attrs
           end
         end
 
-        merged_attrs_set.compact!
-        other.params[model] = merged_attrs_set
+        merged_attrs_set
       end
-
-      other
     end
 
-    def uniq!
-      params.each do |_, attrs_set|
-        attrs_set.uniq!
-      end
-    end
+    # Prior to actually extracting the bind parameters, we first quickly
+    # estimate if it makes sense to do so. If a query contains too many
+    # dependencies, or contains dependencies that have not been memoized, then
+    # the query itself cannot be cached correctly/efficiently, so there’s no
+    # point to actually extract.
+    #
+    # The planning phase is similar to the extraction phase. Though in the
+    # planning phase, we can ignore all the actual attribute values and only
+    # look at the attribute names. This way, we can precompute the dependency
+    # size without populating their actual values.
+    #
+    # For example, in the planning phase,
+    #
+    #   {a:nil} x {b: nil} => {a: nil, b: nil}
+    #   {a:nil, b:nil} x {a: nil: b: nil} => {a: nil, b: nil}
+    #
+    # and in the extraction phase, that's where the # of dependency can
+    # actually grow significantly:
+    #
+    #   {a: [1,2,3]} x {b: [1,2,3]} => [{a: 1, b: 1}, ....]
+    #   {a:[1,2], b:[1,2]} x {a: [1,2,3]: b: [1,2,3]} => [{a: 1, b: 1}, ...]
+    #
+    class Plan
+      class DependencySizeEstimation
+        def initialize(hash = nil)
+          @hash = hash
+        end
 
-    def memoizable?
-      return false if params.empty?
+        def +(other)
+          merged_hash = hash.dup
+          other.hash.each do |k, v|
+            merged_hash[k] += v
+          end
+          self.class.new(merged_hash)
+        end
 
-      params.each do |model, attrs_set|
-        return false if attrs_set.empty?
+        def *(other)
+          merged_hash = hash.dup
+          other.hash.each do |k, v|
+            if merged_hash.include?(k)
+              merged_hash[k] *= v
+            else
+              merged_hash[k] = v
+            end
+          end
+          self.class.new(merged_hash)
+        end
 
-        attrs_set.each do |attrs|
-          return false unless RedisMemo::MemoizeQuery
-            .memoized_columns(model)
-            .include?(attrs.keys.sort)
+        def [](key)
+          hash[key]
+        end
+
+        def []=(key, val)
+          hash[key] = val
+        end
+
+        def to_i
+          ret = 0
+          hash.each do |_, v|
+            ret += v
+          end
+          ret
+        end
+
+        protected
+
+        def hash
+          @hash ||= Hash.new(0)
         end
       end
 
-      true
+      attr_accessor :dependency_size_estimation
+      attr_accessor :model_attrs
+
+      def initialize(bind_params)
+        @dependency_size_estimation = DependencySizeEstimation.new
+        @model_attrs = Hash.new do |models, model|
+          models[model] = Set.new
+        end
+
+        # An aggregated bind_params node can only obtain params by combining
+        # its children nodes
+        return if !bind_params.__send__(:operator).nil?
+
+        bind_params.params.each do |model, attrs_set|
+          @dependency_size_estimation[model] += attrs_set.size
+          attrs_set.each do |attrs|
+            # [k, nil]: Ignore the attr value and keep the name only
+            @model_attrs[model] << attrs.keys.map { |k| [k, nil] }.to_h
+          end
+        end
+      end
     end
   end
 end

data/lib/redis_memo/memoize_query/cached_select/connection_adapter.rb

@@ -15,14 +15,23 @@ class RedisMemo::MemoizeQuery::CachedSelect
       # An Arel AST in Thread local is set prior to supported query methods
       if !RedisMemo.without_memoization? &&
           RedisMemo::MemoizeQuery::CachedSelect.extract_bind_params(args[0])
+
+        time_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        ret = super(*args)
+        time_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
         # [Reids $model Load] $sql $binds
-        RedisMemo::DefaultOptions.logger&.info(
-          "[Redis] \u001b[36;1m#{args[1]} \u001b[34;1m#{args[0]}\u001b[0m #{
-            args[2].map { |bind| [bind.name, bind.value_for_database] }
+        RedisMemo::DefaultOptions.logger&.debug(
+          "[Redis] \u001b[36;1m#{
+            args[1] || 'SQL' # model name
+          } (#{format('%.1f', (time_end - time_start) * 1000.0)}ms)  \u001b[34;1m#{
+            args[0] # sql
+          }\u001b[0m #{
+            args[2].map { |bind| [bind.name, bind.value_for_database] } # binds
           }",
         )
 
-        super(*args)
+        ret
       else
         RedisMemo.without_memoization { super(*args) }
       end

data/lib/redis_memo/options.rb

@@ -19,6 +19,7 @@ class RedisMemo::Options
     global_cache_key_version: nil,
     expires_in: nil,
     max_connection_attempts: nil,
+    max_query_dependency_size: 5000,
     disable_all: false,
     disable_cached_select: false,
     disabled_models: Set.new
@@ -34,6 +35,7 @@ class RedisMemo::Options
     @global_cache_key_version = global_cache_key_version
     @expires_in = expires_in
     @max_connection_attempts = ENV['REDIS_MEMO_MAX_ATTEMPTS_PER_REQUEST']&.to_i || max_connection_attempts
+    @max_query_dependency_size = ENV['REDIS_MEMO_MAX_QUERY_DEPENDENCY_SIZE']&.to_i || max_query_dependency_size
     @disable_all = ENV['REDIS_MEMO_DISABLE_ALL'] == 'true' || disable_all
     @disable_cached_select = ENV['REDIS_MEMO_DISABLE_CACHED_SELECT'] == 'true' || disable_cached_select
     @disabled_models = disabled_models
@@ -162,6 +164,9 @@ class RedisMemo::Options
   # an issue with the Redis cluster itself.
   attr_accessor :max_connection_attempts
 
+  # Only cache a SQL query when the max number of dependency is smaller or equal to this number. Configurable via an ENV var REDIS_MEMO_MAX_QUERY_DEPENDENCY_SIZE. Default at 5000.
+  attr_accessor :max_query_dependency_size
+
   # Passed along to the Rails {RedisCacheStore}[https://api.rubyonrails.org/classes/ActiveSupport/Cache/RedisCacheStore.html], the error handler called for Redis related errors.
   attr_accessor :redis_error_handler
 

data/lib/redis_memo/redis.rb

@@ -17,12 +17,10 @@ class RedisMemo::Redis < Redis::Distributed
           if option.is_a?(Array)
             RedisMemo::Redis::WithReplicas.new(option)
           else
-            option[:logger] ||= RedisMemo::DefaultOptions.logger
             ::Redis.new(option)
           end
         end
       else
-        options[:logger] ||= RedisMemo::DefaultOptions.logger
         [::Redis.new(options)]
       end
 
@@ -49,11 +47,9 @@ class RedisMemo::Redis < Redis::Distributed
       options = orig_options.dup
       primary_option = options.shift
       @replicas = options.map do |option|
-        option[:logger] ||= RedisMemo::DefaultOptions.logger
         ::Redis.new(option)
       end
 
-      primary_option[:logger] ||= RedisMemo::DefaultOptions.logger
       super(primary_option)
     end
 

data/lib/redis_memo/util.rb

@@ -13,6 +13,12 @@ module RedisMemo::Util
     end
   end
 
+  def self.tagify_parameterized_sql(sql)
+    # replace $1 with ?,
+    # and (?, ?, ? ...) with (?)
+    sql.gsub(/(\$\d+)/, '?').gsub(/((, *)*\?)+/, '?')
+  end
+
   def self.uuid
     SecureRandom.uuid
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis-memo
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Chan Zuckerberg Initiative