redis-memo 1.0.0 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8b3cb72a8a4bf3dcd6d30bb112e387710c65d09d60c3dc687e10db649da1e91a
4
- data.tar.gz: b2452e1d4b2b7a3d588c13822eaffc1fb4336c3a4666f8a4ca473dbbd961578f
3
+ metadata.gz: 4e4b5e95c92879677df377b10736e7467efa86237d90ad41254bdd7a732a8260
4
+ data.tar.gz: 9dfb62913a6cfd96e114d6526f5876696e86d1fafb6256d2454d2723449e61c1
5
5
  SHA512:
6
- metadata.gz: 4d9193ddb53419db1e14a5eda4d4ab73b4a96620f592998c764abe3a118fb266761b04a929ff49d15194a9bcf959f9cb7c82f45f783387a85f4e1be2b1315ada
7
- data.tar.gz: 383b2b053917ceeb3991de1c7c399fa83cd8e2bc326f735a0d0924767cea5d6157cf2f3e4a07b3d5d6524b54cae7e62276fb0723daad6a66ebaa6d0b6da64745
6
+ metadata.gz: 62eaf4aedb48325cc90c10cd27a3877de85a436960c78d7e1205c5dbbbe77bdb6c50ac82ea1af11d7dbd7219e7fa96cdc270b37038f1674c1fa491dffb8acec3
7
+ data.tar.gz: 2315c83ce0848d5b2f36d60e37fe60bcece24d80505ff2d7c38b1a9c0d33b8785888325410bae695abc0746a8b4663e037daec07ea83f32b6aaeafd4bf955693
@@ -30,7 +30,6 @@ class RedisMemo::Memoizable::Dependency
30
30
  extracted = self.class.extract_from_relation(dependency)
31
31
  nodes.merge!(extracted.nodes)
32
32
  when RedisMemo::MemoizeQuery::CachedSelect::BindParams
33
- # A private API
34
33
  dependency.params.each do |model, attrs_set|
35
34
  memo = model.redis_memo_class_memoizable
36
35
  nodes[memo.cache_key] = memo
@@ -113,12 +113,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
113
113
 
114
114
  memoize_method(
115
115
  :exec_query,
116
- method_id: proc do |_, sql, *_args|
117
- # replace $1 with ?,
118
- # and (?, ?, ? ...) with (?)
119
- sql.gsub(/(\$\d+)/, '?')
120
- .gsub(/((, *)*\?)+/, '?')
121
- end,
116
+ method_id: proc { |_, sql, *| RedisMemo::Util.tagify_parameterized_sql(sql) },
122
117
  ) do |_, sql, _, binds, **|
123
118
  depends_on RedisMemo::MemoizeQuery::CachedSelect.current_query_bind_params
124
119
 
@@ -161,26 +156,30 @@ class RedisMemo::MemoizeQuery::CachedSelect
161
156
  end
162
157
 
163
158
  # Extract bind params from the query by inspecting the SQL's AST recursively
164
- # The bind params will be passed into the local thread variables
165
- # See +extract_bind_params_recurse+ for how to extract binding params recursively
159
+ # The bind params will be passed into the local thread variables. See
160
+ # +construct_bind_params_recurse+ for how to construct binding params
161
+ # recursively.
166
162
  #
167
163
  # @param sql [String] SQL query
168
164
  # @return [Boolean] indicating whether a query should be cached
169
165
  def self.extract_bind_params(sql)
170
- ast = RedisMemo::ThreadLocalVar.arel&.ast
171
- return false unless ast.is_a?(Arel::Nodes::SelectStatement)
172
- return false unless ast.to_sql == sql
173
-
174
- RedisMemo::ThreadLocalVar.substitues ||= {}
175
- # Iterate through the Arel AST in a Depth First Search
176
- bind_params = extract_bind_params_recurse(ast)
177
- return false unless bind_params
178
-
179
- bind_params.uniq!
180
- return false unless bind_params.memoizable?
181
-
182
- RedisMemo::ThreadLocalVar.arel_bind_params = bind_params
183
- true
166
+ RedisMemo::Tracer.trace(
167
+ 'redis_memo.memoize_query.extract_bind_params',
168
+ RedisMemo::Util.tagify_parameterized_sql(sql),
169
+ ) do
170
+ ast = RedisMemo::ThreadLocalVar.arel&.ast
171
+ return false unless ast.is_a?(Arel::Nodes::SelectStatement)
172
+ return false unless ast.to_sql == sql
173
+
174
+ RedisMemo::ThreadLocalVar.substitues ||= {}
175
+ # Iterate through the Arel AST in a Depth First Search
176
+ bind_params = construct_bind_params_recurse(ast)
177
+ return false unless bind_params&.should_cache?
178
+
179
+ bind_params.extract!
180
+ RedisMemo::ThreadLocalVar.arel_bind_params = bind_params
181
+ true
182
+ end
184
183
  end
185
184
 
186
185
  def self.current_query_bind_params
@@ -222,7 +221,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
222
221
  # @param node [Arel::Nodes::Node]
223
222
  #
224
223
  # @return [RedisMemo::MemoizeQuery::CachedSelect::BindParams]
225
- def self.extract_bind_params_recurse(node)
224
+ def self.construct_bind_params_recurse(node)
226
225
  # rubocop: disable Lint/NonLocalExitFromIterator
227
226
  bind_params = BindParams.new
228
227
 
@@ -273,7 +272,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
273
272
  end,
274
273
  }
275
274
  else
276
- bind_params = bind_params.union(extract_bind_params_recurse(right))
275
+ bind_params = bind_params.union(construct_bind_params_recurse(right))
277
276
  return if !bind_params
278
277
  end
279
278
  end
@@ -294,7 +293,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
294
293
  return if core.wheres.empty? || binding_relation.nil?
295
294
  when Arel::Nodes::TableAlias
296
295
  bind_params = bind_params.union(
297
- extract_bind_params_recurse(source_node.left),
296
+ construct_bind_params_recurse(source_node.left),
298
297
  )
299
298
 
300
299
  return unless bind_params
@@ -305,7 +304,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
305
304
  # Binds wheres before havings
306
305
  core.wheres.each do |where|
307
306
  bind_params = bind_params.union(
308
- extract_bind_params_recurse(where),
307
+ construct_bind_params_recurse(where),
309
308
  )
310
309
 
311
310
  return unless bind_params
@@ -313,26 +312,23 @@ class RedisMemo::MemoizeQuery::CachedSelect
313
312
 
314
313
  core.havings.each do |having|
315
314
  bind_params = bind_params.union(
316
- extract_bind_params_recurse(having),
315
+ construct_bind_params_recurse(having),
317
316
  )
318
317
 
319
318
  return unless bind_params
320
319
  end
321
-
322
- # Reject any unbound select queries
323
- return if binding_relation && bind_params.params[binding_relation].empty?
324
320
  end
325
321
 
326
322
  bind_params
327
323
  when Arel::Nodes::Grouping
328
324
  # Inline SQL
329
- extract_bind_params_recurse(node.expr)
325
+ construct_bind_params_recurse(node.expr)
330
326
  when Arel::Nodes::LessThan, Arel::Nodes::LessThanOrEqual, Arel::Nodes::GreaterThan, Arel::Nodes::GreaterThanOrEqual, Arel::Nodes::NotEqual
331
327
  bind_params
332
328
  when Arel::Nodes::And
333
329
  node.children.each do |child|
334
330
  bind_params = bind_params.product(
335
- extract_bind_params_recurse(child),
331
+ construct_bind_params_recurse(child),
336
332
  )
337
333
 
338
334
  return unless bind_params
@@ -342,7 +338,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
342
338
  when Arel::Nodes::Union, Arel::Nodes::Or
343
339
  [node.left, node.right].each do |child|
344
340
  bind_params = bind_params.union(
345
- extract_bind_params_recurse(child),
341
+ construct_bind_params_recurse(child),
346
342
  )
347
343
 
348
344
  return unless bind_params
@@ -2,47 +2,122 @@
2
2
 
3
3
  class RedisMemo::MemoizeQuery::CachedSelect
4
4
  class BindParams
5
- def params
6
- #
7
- # Bind params is hash of sets: each key is a model class, each value is a
8
- # set of hashes for memoized column conditions. Example:
9
- #
10
- # {
11
- # Site => [
12
- # {name: 'a', city: 'b'},
13
- # {name: 'a', city: 'c'},
14
- # {name: 'b', city: 'b'},
15
- # {name: 'b', city: 'c'},
16
- # ],
17
- # }
18
- #
19
- @params ||= Hash.new do |models, model|
20
- models[model] = []
21
- end
5
+ def initialize(left = nil, right = nil, operator = nil)
6
+ @left = left
7
+ @right = right
8
+ @operator = operator
22
9
  end
23
10
 
24
11
  def union(other)
25
12
  return unless other
26
13
 
27
- # The tree is almost always right-heavy. Merge into the right node for better
28
- # performance.
29
- other.params.merge!(params) do |_, other_attrs_set, attrs_set|
30
- if other_attrs_set.empty?
31
- attrs_set
32
- elsif attrs_set.empty?
33
- other_attrs_set
34
- else
35
- attrs_set + other_attrs_set
14
+ self.class.new(self, other, __method__)
15
+ end
16
+
17
+ def product(other)
18
+ return unless other
19
+
20
+ self.class.new(self, other, __method__)
21
+ end
22
+
23
+ def should_cache?
24
+ plan!
25
+
26
+ if plan.model_attrs.empty? || plan.dependency_size_estimation.to_i > RedisMemo::DefaultOptions.max_query_dependency_size
27
+ return false
28
+ end
29
+
30
+ plan.model_attrs.each do |model, attrs_set|
31
+ return false if attrs_set.empty?
32
+
33
+ attrs_set.each do |attrs|
34
+ return false unless RedisMemo::MemoizeQuery
35
+ .memoized_columns(model)
36
+ .include?(attrs.keys.sort)
36
37
  end
37
38
  end
38
39
 
39
- other
40
+ true
40
41
  end
41
42
 
42
- def product(other)
43
+ #
44
+ # Extracted bind params is hash of sets: each key is a model class, each
45
+ # value is a set of hashes for memoized column conditions. Example:
46
+ #
47
+ # {
48
+ # Site => [
49
+ # {name: 'a', city: 'b'},
50
+ # {name: 'a', city: 'c'},
51
+ # {name: 'b', city: 'b'},
52
+ # {name: 'b', city: 'c'},
53
+ # ],
54
+ # }
55
+ #
56
+ def extract!
57
+ return if operator.nil?
58
+
59
+ left.extract!
60
+ right.extract!
61
+ __send__(:"#{operator}!")
62
+ end
63
+
64
+ def params
65
+ @params ||= Hash.new do |models, model|
66
+ models[model] = Set.new
67
+ end
68
+ end
69
+
70
+ protected
71
+
72
+ # BindParams is built recursively when iterating through the Arel AST
73
+ # nodes. BindParams represents a binary tree. Query parameters are added to
74
+ # the leaf nodes of the tree, and the leaf nodes are connected by
75
+ # operators, such as `union` (or conditions) or `product` (and conditions).
76
+ attr_accessor :left
77
+ attr_accessor :right
78
+ attr_accessor :operator
79
+ attr_accessor :plan
80
+
81
+ def plan!
82
+ self.plan = Plan.new(self)
83
+ return if operator.nil?
84
+
85
+ left.plan!
86
+ right.plan!
87
+ __send__(:"plan_#{operator}")
88
+ end
89
+
90
+ def plan_union
91
+ plan.dependency_size_estimation = left.plan.dependency_size_estimation + right.plan.dependency_size_estimation
92
+ plan.model_attrs = union_attrs_set(left.plan.model_attrs, right.plan.model_attrs)
93
+ end
94
+
95
+ def plan_product
96
+ plan.dependency_size_estimation = left.plan.dependency_size_estimation * right.plan.dependency_size_estimation
97
+ plan.model_attrs = product_attrs_set(left.plan.model_attrs, right.plan.model_attrs)
98
+ end
99
+
100
+ def union!
101
+ @params = union_attrs_set(left.params, right.params)
102
+ end
103
+
104
+ def product!
105
+ @params = product_attrs_set(left.params, right.params)
106
+ end
107
+
108
+ def union_attrs_set(left, right)
109
+ left.merge(right) do |_, attrs_set, other_attrs_set|
110
+ next attrs_set if other_attrs_set.empty?
111
+ next other_attrs_set if attrs_set.empty?
112
+
113
+ attrs_set + other_attrs_set
114
+ end
115
+ end
116
+
117
+ def product_attrs_set(left, right)
43
118
  # Example:
44
119
  #
45
- # and(
120
+ # product(
46
121
  # [{a: 1}, {a: 2}],
47
122
  # [{b: 1}, {b: 2}],
48
123
  # )
@@ -55,29 +130,16 @@ class RedisMemo::MemoizeQuery::CachedSelect
55
130
  # {a: 2, b: 1},
56
131
  # {a: 2, b: 2},
57
132
  # ]
58
- return unless other
59
-
60
- # The tree is almost always right-heavy. Merge into the right node for better
61
- # performance.
62
- params.each do |model, attrs_set|
63
- next if attrs_set.empty?
64
-
65
- # The other model does not have any conditions so far: carry the
66
- # attributes over to the other node
67
- if other.params[model].empty?
68
- other.params[model] = attrs_set
69
- next
70
- end
71
-
72
- # Distribute the current attrs into the other
73
- other_attrs_set_size = other.params[model].size
74
- other_attrs_set = other.params[model]
75
- merged_attrs_set = Array.new(other_attrs_set_size * attrs_set.size)
133
+ left.merge(right) do |_, attrs_set, other_attrs_set|
134
+ next attrs_set if other_attrs_set.empty?
135
+ next other_attrs_set if attrs_set.empty?
76
136
 
77
- attrs_set.each_with_index do |attrs, i|
78
- other_attrs_set.each_with_index do |other_attrs, j|
79
- k = i * other_attrs_set_size + j
80
- merged_attrs = merged_attrs_set[k] = other_attrs.dup
137
+ # distribute the current attrs into the other
138
+ merged_attrs_set = Set.new
139
+ attrs_set.each do |attrs|
140
+ other_attrs_set.each do |other_attrs|
141
+ merged_attrs = other_attrs.dup
142
+ should_add_attrs = true
81
143
  attrs.each do |name, val|
82
144
  # Conflict detected. For example:
83
145
  #
@@ -86,42 +148,112 @@ class RedisMemo::MemoizeQuery::CachedSelect
86
148
  # Keep: a = 1 and b = 2, a = 2 and b = 1
87
149
  # Discard: a = 1 and a = 2, b = 1 and b = 2
88
150
  if merged_attrs.include?(name) && merged_attrs[name] != val
89
- merged_attrs_set[k] = nil
151
+ should_add_attrs = false
90
152
  break
91
153
  end
92
154
 
93
155
  merged_attrs[name] = val
94
156
  end
157
+ merged_attrs_set << merged_attrs if should_add_attrs
95
158
  end
96
159
  end
97
160
 
98
- merged_attrs_set.compact!
99
- other.params[model] = merged_attrs_set
161
+ merged_attrs_set
100
162
  end
101
-
102
- other
103
163
  end
104
164
 
105
- def uniq!
106
- params.each do |_, attrs_set|
107
- attrs_set.uniq!
108
- end
109
- end
165
+ # Prior to actually extracting the bind parameters, we first quickly
166
+ # estimate if it makes sense to do so. If a query contains too many
167
+ # dependencies, or contains dependencies that have not been memoized, then
168
+ # the query itself cannot be cached correctly/efficiently, so there’s no
169
+ # point to actually extract.
170
+ #
171
+ # The planning phase is similar to the extraction phase. Though in the
172
+ # planning phase, we can ignore all the actual attribute values and only
173
+ # look at the attribute names. This way, we can precompute the dependency
174
+ # size without populating their actual values.
175
+ #
176
+ # For example, in the planning phase,
177
+ #
178
+ # {a:nil} x {b: nil} => {a: nil, b: nil}
179
+ # {a:nil, b:nil} x {a: nil: b: nil} => {a: nil, b: nil}
180
+ #
181
+ # and in the extraction phase, that's where the # of dependency can
182
+ # actually grow significantly:
183
+ #
184
+ # {a: [1,2,3]} x {b: [1,2,3]} => [{a: 1, b: 1}, ....]
185
+ # {a:[1,2], b:[1,2]} x {a: [1,2,3]: b: [1,2,3]} => [{a: 1, b: 1}, ...]
186
+ #
187
+ class Plan
188
+ class DependencySizeEstimation
189
+ def initialize(hash = nil)
190
+ @hash = hash
191
+ end
110
192
 
111
- def memoizable?
112
- return false if params.empty?
193
+ def +(other)
194
+ merged_hash = hash.dup
195
+ other.hash.each do |k, v|
196
+ merged_hash[k] += v
197
+ end
198
+ self.class.new(merged_hash)
199
+ end
113
200
 
114
- params.each do |model, attrs_set|
115
- return false if attrs_set.empty?
201
+ def *(other)
202
+ merged_hash = hash.dup
203
+ other.hash.each do |k, v|
204
+ if merged_hash.include?(k)
205
+ merged_hash[k] *= v
206
+ else
207
+ merged_hash[k] = v
208
+ end
209
+ end
210
+ self.class.new(merged_hash)
211
+ end
116
212
 
117
- attrs_set.each do |attrs|
118
- return false unless RedisMemo::MemoizeQuery
119
- .memoized_columns(model)
120
- .include?(attrs.keys.sort)
213
+ def [](key)
214
+ hash[key]
215
+ end
216
+
217
+ def []=(key, val)
218
+ hash[key] = val
219
+ end
220
+
221
+ def to_i
222
+ ret = 0
223
+ hash.each do |_, v|
224
+ ret += v
225
+ end
226
+ ret
227
+ end
228
+
229
+ protected
230
+
231
+ def hash
232
+ @hash ||= Hash.new(0)
121
233
  end
122
234
  end
123
235
 
124
- true
236
+ attr_accessor :dependency_size_estimation
237
+ attr_accessor :model_attrs
238
+
239
+ def initialize(bind_params)
240
+ @dependency_size_estimation = DependencySizeEstimation.new
241
+ @model_attrs = Hash.new do |models, model|
242
+ models[model] = Set.new
243
+ end
244
+
245
+ # An aggregated bind_params node can only obtain params by combining
246
+ # its children nodes
247
+ return if !bind_params.__send__(:operator).nil?
248
+
249
+ bind_params.params.each do |model, attrs_set|
250
+ @dependency_size_estimation[model] += attrs_set.size
251
+ attrs_set.each do |attrs|
252
+ # [k, nil]: Ignore the attr value and keep the name only
253
+ @model_attrs[model] << attrs.keys.map { |k| [k, nil] }.to_h
254
+ end
255
+ end
256
+ end
125
257
  end
126
258
  end
127
259
  end
@@ -15,14 +15,23 @@ class RedisMemo::MemoizeQuery::CachedSelect
15
15
  # An Arel AST in Thread local is set prior to supported query methods
16
16
  if !RedisMemo.without_memoization? &&
17
17
  RedisMemo::MemoizeQuery::CachedSelect.extract_bind_params(args[0])
18
+
19
+ time_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
20
+ ret = super(*args)
21
+ time_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
22
+
18
23
  # [Reids $model Load] $sql $binds
19
- RedisMemo::DefaultOptions.logger&.info(
20
- "[Redis] \u001b[36;1m#{args[1]} \u001b[34;1m#{args[0]}\u001b[0m #{
21
- args[2].map { |bind| [bind.name, bind.value_for_database] }
24
+ RedisMemo::DefaultOptions.logger&.debug(
25
+ "[Redis] \u001b[36;1m#{
26
+ args[1] || 'SQL' # model name
27
+ } (#{format('%.1f', (time_end - time_start) * 1000.0)}ms) \u001b[34;1m#{
28
+ args[0] # sql
29
+ }\u001b[0m #{
30
+ args[2].map { |bind| [bind.name, bind.value_for_database] } # binds
22
31
  }",
23
32
  )
24
33
 
25
- super(*args)
34
+ ret
26
35
  else
27
36
  RedisMemo.without_memoization { super(*args) }
28
37
  end
@@ -19,6 +19,7 @@ class RedisMemo::Options
19
19
  global_cache_key_version: nil,
20
20
  expires_in: nil,
21
21
  max_connection_attempts: nil,
22
+ max_query_dependency_size: 5000,
22
23
  disable_all: false,
23
24
  disable_cached_select: false,
24
25
  disabled_models: Set.new
@@ -34,6 +35,7 @@ class RedisMemo::Options
34
35
  @global_cache_key_version = global_cache_key_version
35
36
  @expires_in = expires_in
36
37
  @max_connection_attempts = ENV['REDIS_MEMO_MAX_ATTEMPTS_PER_REQUEST']&.to_i || max_connection_attempts
38
+ @max_query_dependency_size = ENV['REDIS_MEMO_MAX_QUERY_DEPENDENCY_SIZE']&.to_i || max_query_dependency_size
37
39
  @disable_all = ENV['REDIS_MEMO_DISABLE_ALL'] == 'true' || disable_all
38
40
  @disable_cached_select = ENV['REDIS_MEMO_DISABLE_CACHED_SELECT'] == 'true' || disable_cached_select
39
41
  @disabled_models = disabled_models
@@ -162,6 +164,9 @@ class RedisMemo::Options
162
164
  # an issue with the Redis cluster itself.
163
165
  attr_accessor :max_connection_attempts
164
166
 
167
+ # Only cache a SQL query when the max number of dependency is smaller or equal to this number. Configurable via an ENV var REDIS_MEMO_MAX_QUERY_DEPENDENCY_SIZE. Default at 5000.
168
+ attr_accessor :max_query_dependency_size
169
+
165
170
  # Passed along to the Rails {RedisCacheStore}[https://api.rubyonrails.org/classes/ActiveSupport/Cache/RedisCacheStore.html], the error handler called for Redis related errors.
166
171
  attr_accessor :redis_error_handler
167
172
 
@@ -17,12 +17,10 @@ class RedisMemo::Redis < Redis::Distributed
17
17
  if option.is_a?(Array)
18
18
  RedisMemo::Redis::WithReplicas.new(option)
19
19
  else
20
- option[:logger] ||= RedisMemo::DefaultOptions.logger
21
20
  ::Redis.new(option)
22
21
  end
23
22
  end
24
23
  else
25
- options[:logger] ||= RedisMemo::DefaultOptions.logger
26
24
  [::Redis.new(options)]
27
25
  end
28
26
 
@@ -49,11 +47,9 @@ class RedisMemo::Redis < Redis::Distributed
49
47
  options = orig_options.dup
50
48
  primary_option = options.shift
51
49
  @replicas = options.map do |option|
52
- option[:logger] ||= RedisMemo::DefaultOptions.logger
53
50
  ::Redis.new(option)
54
51
  end
55
52
 
56
- primary_option[:logger] ||= RedisMemo::DefaultOptions.logger
57
53
  super(primary_option)
58
54
  end
59
55
 
@@ -13,6 +13,12 @@ module RedisMemo::Util
13
13
  end
14
14
  end
15
15
 
16
+ def self.tagify_parameterized_sql(sql)
17
+ # replace $1 with ?,
18
+ # and (?, ?, ? ...) with (?)
19
+ sql.gsub(/(\$\d+)/, '?').gsub(/((, *)*\?)+/, '?')
20
+ end
21
+
16
22
  def self.uuid
17
23
  SecureRandom.uuid
18
24
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: redis-memo
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chan Zuckerberg Initiative