redis-memo 0.1.4 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/redis_memo.rb +38 -57
- data/lib/redis_memo/after_commit.rb +2 -2
- data/lib/redis_memo/batch.rb +36 -11
- data/lib/redis_memo/cache.rb +27 -22
- data/lib/redis_memo/connection_pool.rb +4 -3
- data/lib/redis_memo/errors.rb +9 -0
- data/lib/redis_memo/future.rb +22 -13
- data/lib/redis_memo/memoizable.rb +109 -72
- data/lib/redis_memo/memoizable/bump_version.lua +39 -0
- data/lib/redis_memo/memoizable/dependency.rb +7 -10
- data/lib/redis_memo/memoizable/invalidation.rb +67 -68
- data/lib/redis_memo/memoize_method.rb +169 -131
- data/lib/redis_memo/memoize_query.rb +135 -93
- data/lib/redis_memo/memoize_query/cached_select.rb +59 -46
- data/lib/redis_memo/memoize_query/cached_select/connection_adapter.rb +7 -7
- data/lib/redis_memo/memoize_query/invalidation.rb +22 -20
- data/lib/redis_memo/memoize_query/memoize_table_column.rb +1 -0
- data/lib/redis_memo/middleware.rb +1 -1
- data/lib/redis_memo/options.rb +106 -5
- data/lib/redis_memo/railtie.rb +11 -0
- data/lib/redis_memo/redis.rb +15 -1
- data/lib/redis_memo/testing.rb +4 -10
- data/lib/redis_memo/thread_local_var.rb +16 -0
- data/lib/redis_memo/tracer.rb +1 -0
- data/lib/redis_memo/util.rb +19 -0
- metadata +79 -4
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
##
|
4
4
|
# Inspect a SQL's AST to memoize SELECT statements
|
5
5
|
#
|
6
6
|
# As Rails applies additional logic on top of the rows returned from the
|
@@ -94,6 +94,12 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
94
94
|
|
95
95
|
@@enabled_models = {}
|
96
96
|
|
97
|
+
# Thread locals to exchange information between RedisMemo and ActiveRecord
|
98
|
+
RedisMemo::ThreadLocalVar.define :arel
|
99
|
+
RedisMemo::ThreadLocalVar.define :substitues
|
100
|
+
RedisMemo::ThreadLocalVar.define :arel_bind_params
|
101
|
+
|
102
|
+
# @return [Hash] models enabled for caching
|
97
103
|
def self.enabled_models
|
98
104
|
@@enabled_models
|
99
105
|
end
|
@@ -107,9 +113,11 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
107
113
|
|
108
114
|
memoize_method(
|
109
115
|
:exec_query,
|
110
|
-
method_id: proc do |_, sql, *
|
111
|
-
|
112
|
-
|
116
|
+
method_id: proc do |_, sql, *_args|
|
117
|
+
# replace $1 with ?,
|
118
|
+
# and (?, ?, ? ...) with (?)
|
119
|
+
sql.gsub(/(\$\d+)/, '?')
|
120
|
+
.gsub(/((, *)*\?)+/, '?')
|
113
121
|
end,
|
114
122
|
) do |_, sql, _, binds, **|
|
115
123
|
depends_on RedisMemo::MemoizeQuery::CachedSelect.current_query_bind_params
|
@@ -123,7 +131,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
123
131
|
# In activerecord >= 6, a bind could be an actual database value
|
124
132
|
bind
|
125
133
|
end
|
126
|
-
end
|
134
|
+
end,
|
127
135
|
)
|
128
136
|
end
|
129
137
|
end
|
@@ -152,12 +160,18 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
152
160
|
end
|
153
161
|
end
|
154
162
|
|
163
|
+
# Extract bind params from the query by inspecting the SQL's AST recursively
|
164
|
+
# The bind params will be passed into the local thread variables
|
165
|
+
# See +extract_bind_params_recurse+ for how to extract binding params recursively
|
166
|
+
#
|
167
|
+
# @param sql [String] SQL query
|
168
|
+
# @return [Boolean] indicating whether a query should be cached
|
155
169
|
def self.extract_bind_params(sql)
|
156
|
-
ast =
|
170
|
+
ast = RedisMemo::ThreadLocalVar.arel&.ast
|
157
171
|
return false unless ast.is_a?(Arel::Nodes::SelectStatement)
|
158
172
|
return false unless ast.to_sql == sql
|
159
173
|
|
160
|
-
|
174
|
+
RedisMemo::ThreadLocalVar.substitues ||= {}
|
161
175
|
# Iterate through the Arel AST in a Depth First Search
|
162
176
|
bind_params = extract_bind_params_recurse(ast)
|
163
177
|
return false unless bind_params
|
@@ -165,48 +179,51 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
165
179
|
bind_params.uniq!
|
166
180
|
return false unless bind_params.memoizable?
|
167
181
|
|
168
|
-
|
182
|
+
RedisMemo::ThreadLocalVar.arel_bind_params = bind_params
|
169
183
|
true
|
170
184
|
end
|
171
185
|
|
172
186
|
def self.current_query_bind_params
|
173
|
-
|
187
|
+
RedisMemo::ThreadLocalVar.arel_bind_params
|
174
188
|
end
|
175
189
|
|
176
190
|
def self.current_query=(arel)
|
177
|
-
|
191
|
+
RedisMemo::ThreadLocalVar.arel = arel
|
178
192
|
end
|
179
193
|
|
180
194
|
def self.current_substitutes=(substitutes)
|
181
|
-
|
195
|
+
RedisMemo::ThreadLocalVar.substitues = substitutes
|
182
196
|
end
|
183
197
|
|
184
198
|
def self.reset_current_query
|
185
|
-
|
186
|
-
|
187
|
-
|
199
|
+
RedisMemo::ThreadLocalVar.arel = nil
|
200
|
+
RedisMemo::ThreadLocalVar.substitues = nil
|
201
|
+
RedisMemo::ThreadLocalVar.arel_bind_params = nil
|
188
202
|
end
|
189
203
|
|
190
204
|
def self.with_new_query_context
|
191
|
-
prev_arel =
|
192
|
-
prev_substitutes =
|
193
|
-
prev_bind_params =
|
205
|
+
prev_arel = RedisMemo::ThreadLocalVar.arel
|
206
|
+
prev_substitutes = RedisMemo::ThreadLocalVar.substitues
|
207
|
+
prev_bind_params = RedisMemo::ThreadLocalVar.arel_bind_params
|
194
208
|
RedisMemo::MemoizeQuery::CachedSelect.reset_current_query
|
195
209
|
|
196
210
|
yield
|
197
211
|
ensure
|
198
|
-
|
199
|
-
|
200
|
-
|
212
|
+
RedisMemo::ThreadLocalVar.arel = prev_arel
|
213
|
+
RedisMemo::ThreadLocalVar.substitues = prev_substitutes
|
214
|
+
RedisMemo::ThreadLocalVar.arel_bind_params = prev_bind_params
|
201
215
|
end
|
202
216
|
|
203
|
-
private
|
204
|
-
|
205
217
|
# A pre-order Depth First Search
|
206
218
|
#
|
207
219
|
# Note: Arel::Nodes#each returns a list in post-order, and it does not step
|
208
220
|
# into Union nodes. So we're implementing our own DFS
|
221
|
+
#
|
222
|
+
# @param node [Arel::Nodes::Node]
|
223
|
+
#
|
224
|
+
# @return [RedisMemo::MemoizeQuery::CachedSelect::BindParams]
|
209
225
|
def self.extract_bind_params_recurse(node)
|
226
|
+
# rubocop: disable Lint/NonLocalExitFromIterator
|
210
227
|
bind_params = BindParams.new
|
211
228
|
|
212
229
|
case node
|
@@ -229,7 +246,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
229
246
|
return unless binding_relation
|
230
247
|
|
231
248
|
rights = node.right.is_a?(Array) ? node.right : [node.right]
|
232
|
-
substitutes =
|
249
|
+
substitutes = RedisMemo::ThreadLocalVar.substitues
|
233
250
|
|
234
251
|
rights.each do |right|
|
235
252
|
case right
|
@@ -257,11 +274,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
257
274
|
}
|
258
275
|
else
|
259
276
|
bind_params = bind_params.union(extract_bind_params_recurse(right))
|
260
|
-
if bind_params
|
261
|
-
next
|
262
|
-
else
|
263
|
-
return
|
264
|
-
end
|
277
|
+
return if !bind_params
|
265
278
|
end
|
266
279
|
end
|
267
280
|
|
@@ -281,7 +294,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
281
294
|
return if core.wheres.empty? || binding_relation.nil?
|
282
295
|
when Arel::Nodes::TableAlias
|
283
296
|
bind_params = bind_params.union(
|
284
|
-
extract_bind_params_recurse(source_node.left)
|
297
|
+
extract_bind_params_recurse(source_node.left),
|
285
298
|
)
|
286
299
|
|
287
300
|
return unless bind_params
|
@@ -292,7 +305,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
292
305
|
# Binds wheres before havings
|
293
306
|
core.wheres.each do |where|
|
294
307
|
bind_params = bind_params.union(
|
295
|
-
extract_bind_params_recurse(where)
|
308
|
+
extract_bind_params_recurse(where),
|
296
309
|
)
|
297
310
|
|
298
311
|
return unless bind_params
|
@@ -300,7 +313,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
300
313
|
|
301
314
|
core.havings.each do |having|
|
302
315
|
bind_params = bind_params.union(
|
303
|
-
extract_bind_params_recurse(having)
|
316
|
+
extract_bind_params_recurse(having),
|
304
317
|
)
|
305
318
|
|
306
319
|
return unless bind_params
|
@@ -313,13 +326,13 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
313
326
|
bind_params
|
314
327
|
when Arel::Nodes::Grouping
|
315
328
|
# Inline SQL
|
316
|
-
return if node.expr.is_a?(Arel::Nodes::SqlLiteral)
|
317
|
-
|
318
329
|
extract_bind_params_recurse(node.expr)
|
330
|
+
when Arel::Nodes::LessThan, Arel::Nodes::LessThanOrEqual, Arel::Nodes::GreaterThan, Arel::Nodes::GreaterThanOrEqual, Arel::Nodes::NotEqual
|
331
|
+
bind_params
|
319
332
|
when Arel::Nodes::And
|
320
333
|
node.children.each do |child|
|
321
334
|
bind_params = bind_params.product(
|
322
|
-
extract_bind_params_recurse(child)
|
335
|
+
extract_bind_params_recurse(child),
|
323
336
|
)
|
324
337
|
|
325
338
|
return unless bind_params
|
@@ -329,28 +342,33 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
329
342
|
when Arel::Nodes::Union, Arel::Nodes::Or
|
330
343
|
[node.left, node.right].each do |child|
|
331
344
|
bind_params = bind_params.union(
|
332
|
-
extract_bind_params_recurse(child)
|
345
|
+
extract_bind_params_recurse(child),
|
333
346
|
)
|
334
347
|
|
335
348
|
return unless bind_params
|
336
349
|
end
|
337
350
|
|
338
351
|
bind_params
|
339
|
-
|
340
|
-
when Arel::Nodes::NotEqual
|
341
|
-
# We don't cache based on NOT queries (where.not) because it is unbound
|
342
|
-
# but we memoize queries with NOT and other bound queries, so we return the original bind_params
|
343
|
-
return bind_params
|
344
352
|
else
|
345
353
|
# Not yet supported
|
346
|
-
|
354
|
+
nil
|
347
355
|
end
|
356
|
+
# rubocop: enable Lint/NonLocalExitFromIterator
|
348
357
|
end
|
349
358
|
|
359
|
+
# Retrieve the model info from the table node
|
360
|
+
# table node is an Arel::Table object, e.g. <Arel::Table @name="sites" ...>
|
361
|
+
# and we can retrieve the model info by inspecting thhe table name
|
362
|
+
# See +RedisMemo::MemoizeQuery::memoize_table_column+ for how to construct enabled_models
|
363
|
+
#
|
364
|
+
# @params table_node [Arel::Table]
|
350
365
|
def self.extract_binding_relation(table_node)
|
351
366
|
enabled_models[table_node.try(:name)]
|
352
367
|
end
|
353
368
|
|
369
|
+
#
|
370
|
+
# Identify whether the node has filter condition
|
371
|
+
#
|
354
372
|
class NodeHasFilterCondition
|
355
373
|
def self.===(node)
|
356
374
|
case node
|
@@ -367,9 +385,4 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
367
385
|
end
|
368
386
|
end
|
369
387
|
end
|
370
|
-
|
371
|
-
# Thread locals to exchange information between RedisMemo and ActiveRecord
|
372
|
-
THREAD_KEY_AREL = :__redis_memo_memoize_query_cached_select_arel__
|
373
|
-
THREAD_KEY_SUBSTITUTES = :__redis_memo_memoize_query_cached_select_substitues__
|
374
|
-
THREAD_KEY_AREL_BIND_PARAMS = :__redis_memo_memoize_query_cached_select_arel_bind_params__
|
375
388
|
end
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
class RedisMemo::MemoizeQuery::CachedSelect
|
4
4
|
module ConnectionAdapter
|
5
|
-
def cacheable_query(*args)
|
5
|
+
ruby2_keywords def cacheable_query(*args)
|
6
6
|
query, binds = super(*args)
|
7
7
|
|
8
8
|
# Persist the arel object to StatementCache#execute
|
@@ -11,24 +11,24 @@ class RedisMemo::MemoizeQuery::CachedSelect
|
|
11
11
|
[query, binds]
|
12
12
|
end
|
13
13
|
|
14
|
-
def exec_query(*args)
|
14
|
+
ruby2_keywords def exec_query(*args)
|
15
15
|
# An Arel AST in Thread local is set prior to supported query methods
|
16
|
-
if !RedisMemo.
|
16
|
+
if !RedisMemo.without_memoization? &&
|
17
17
|
RedisMemo::MemoizeQuery::CachedSelect.extract_bind_params(args[0])
|
18
18
|
# [Reids $model Load] $sql $binds
|
19
19
|
RedisMemo::DefaultOptions.logger&.info(
|
20
20
|
"[Redis] \u001b[36;1m#{args[1]} \u001b[34;1m#{args[0]}\u001b[0m #{
|
21
|
-
args[2].map { |bind| [bind.name, bind.value_for_database]}
|
22
|
-
}"
|
21
|
+
args[2].map { |bind| [bind.name, bind.value_for_database] }
|
22
|
+
}",
|
23
23
|
)
|
24
24
|
|
25
25
|
super(*args)
|
26
26
|
else
|
27
|
-
RedisMemo.
|
27
|
+
RedisMemo.without_memoization { super(*args) }
|
28
28
|
end
|
29
29
|
end
|
30
30
|
|
31
|
-
def select_all(*args)
|
31
|
+
ruby2_keywords def select_all(*args)
|
32
32
|
if args[0].is_a?(Arel::SelectManager)
|
33
33
|
RedisMemo::MemoizeQuery::CachedSelect.current_query = args[0]
|
34
34
|
end
|
@@ -17,28 +17,29 @@ class RedisMemo::MemoizeQuery::Invalidation
|
|
17
17
|
@redis_memo_class_memoizable ||= RedisMemo::MemoizeQuery.create_memo(self)
|
18
18
|
end
|
19
19
|
|
20
|
-
%i
|
20
|
+
%i[delete decrement! increment!].each do |method_name|
|
21
21
|
alias_method :"without_redis_memo_invalidation_#{method_name}", method_name
|
22
22
|
|
23
23
|
define_method method_name do |*args|
|
24
|
-
result =
|
24
|
+
result = __send__(:"without_redis_memo_invalidation_#{method_name}", *args)
|
25
25
|
|
26
26
|
RedisMemo::MemoizeQuery.invalidate(self)
|
27
27
|
|
28
28
|
result
|
29
29
|
end
|
30
|
+
ruby2_keywords method_name
|
30
31
|
end
|
31
32
|
end
|
32
33
|
|
33
34
|
# Methods that won't trigger model callbacks
|
34
35
|
# https://guides.rubyonrails.org/active_record_callbacks.html#skipping-callbacks
|
35
|
-
%i
|
36
|
+
%i[
|
36
37
|
decrement_counter
|
37
38
|
delete_all delete_by
|
38
39
|
increment_counter
|
39
40
|
touch_all
|
40
41
|
update_column update_columns update_all update_counters
|
41
|
-
|
42
|
+
].each do |method_name|
|
42
43
|
# Example: Model.update_all
|
43
44
|
rewrite_default_method(
|
44
45
|
model_class,
|
@@ -56,27 +57,27 @@ class RedisMemo::MemoizeQuery::Invalidation
|
|
56
57
|
)
|
57
58
|
end
|
58
59
|
|
59
|
-
%i
|
60
|
+
%i[
|
60
61
|
insert insert! insert_all insert_all!
|
61
|
-
|
62
|
+
].each do |method_name|
|
62
63
|
rewrite_insert_method(
|
63
64
|
model_class,
|
64
65
|
method_name,
|
65
66
|
)
|
66
67
|
end
|
67
68
|
|
68
|
-
%i
|
69
|
+
%i[
|
69
70
|
upsert upsert_all
|
70
|
-
|
71
|
+
].each do |method_name|
|
71
72
|
rewrite_upsert_method(
|
72
73
|
model_class,
|
73
74
|
method_name,
|
74
75
|
)
|
75
76
|
end
|
76
77
|
|
77
|
-
%i
|
78
|
+
%i[
|
78
79
|
import import!
|
79
|
-
|
80
|
+
].each do |method_name|
|
80
81
|
rewrite_import_method(
|
81
82
|
model_class,
|
82
83
|
method_name,
|
@@ -116,8 +117,6 @@ class RedisMemo::MemoizeQuery::Invalidation
|
|
116
117
|
result
|
117
118
|
end
|
118
119
|
|
119
|
-
private
|
120
|
-
|
121
120
|
#
|
122
121
|
# There’s no good way to perform fine-grind cache invalidation when
|
123
122
|
# operations are bulk update operations such as update_all, and delete_all
|
@@ -127,17 +126,18 @@ class RedisMemo::MemoizeQuery::Invalidation
|
|
127
126
|
#
|
128
127
|
def self.rewrite_default_method(model_class, klass, method_name, class_method:)
|
129
128
|
methods = class_method ? :methods : :instance_methods
|
130
|
-
return unless klass.
|
129
|
+
return unless klass.__send__(methods).include?(method_name)
|
131
130
|
|
132
131
|
klass = klass.singleton_class if class_method
|
133
132
|
klass.class_eval do
|
134
133
|
alias_method :"#{method_name}_without_redis_memo_invalidation", method_name
|
135
134
|
|
136
135
|
define_method method_name do |*args|
|
137
|
-
result =
|
136
|
+
result = __send__(:"#{method_name}_without_redis_memo_invalidation", *args)
|
138
137
|
RedisMemo::MemoizeQuery.invalidate_all(model_class)
|
139
138
|
result
|
140
139
|
end
|
140
|
+
ruby2_keywords method_name
|
141
141
|
end
|
142
142
|
end
|
143
143
|
|
@@ -149,9 +149,10 @@ class RedisMemo::MemoizeQuery::Invalidation
|
|
149
149
|
|
150
150
|
define_method method_name do |*args, &blk|
|
151
151
|
RedisMemo::MemoizeQuery::Invalidation.invalidate_new_records(model_class) do
|
152
|
-
|
152
|
+
__send__(:"#{method_name}_without_redis_memo_invalidation", *args, &blk)
|
153
153
|
end
|
154
154
|
end
|
155
|
+
ruby2_keywords method_name
|
155
156
|
end
|
156
157
|
end
|
157
158
|
|
@@ -169,7 +170,7 @@ class RedisMemo::MemoizeQuery::Invalidation
|
|
169
170
|
# HEAD (6.1.3)
|
170
171
|
conflict_target: nil,
|
171
172
|
) do
|
172
|
-
|
173
|
+
__send__(
|
173
174
|
:"#{method_name}_without_redis_memo_invalidation",
|
174
175
|
attributes,
|
175
176
|
unique_by: unique_by,
|
@@ -216,9 +217,10 @@ class RedisMemo::MemoizeQuery::Invalidation
|
|
216
217
|
records: records,
|
217
218
|
conflict_target: conflict_target,
|
218
219
|
) do
|
219
|
-
|
220
|
+
__send__(:"#{method_name}_without_redis_memo_invalidation", *args, &blk)
|
220
221
|
end
|
221
222
|
end
|
223
|
+
ruby2_keywords method_name
|
222
224
|
end
|
223
225
|
end
|
224
226
|
|
@@ -228,7 +230,7 @@ class RedisMemo::MemoizeQuery::Invalidation
|
|
228
230
|
records.each do |record|
|
229
231
|
conditions = {}
|
230
232
|
conflict_target.each do |column|
|
231
|
-
conditions[column] = record.
|
233
|
+
conditions[column] = record.__send__(column)
|
232
234
|
end
|
233
235
|
if or_chain
|
234
236
|
or_chain = or_chain.or(model_class.where(conditions))
|
@@ -245,7 +247,7 @@ class RedisMemo::MemoizeQuery::Invalidation
|
|
245
247
|
'redis_memo.memoize_query.invalidation',
|
246
248
|
"#{__method__}##{model_class.name}",
|
247
249
|
) do
|
248
|
-
RedisMemo.
|
250
|
+
RedisMemo.without_memoization do
|
249
251
|
model_class.where(
|
250
252
|
model_class.arel_table[model_class.primary_key].gt(target_id),
|
251
253
|
).to_a
|
@@ -260,7 +262,7 @@ class RedisMemo::MemoizeQuery::Invalidation
|
|
260
262
|
'redis_memo.memoize_query.invalidation',
|
261
263
|
"#{__method__}##{model_class.name}",
|
262
264
|
) do
|
263
|
-
RedisMemo.
|
265
|
+
RedisMemo.without_memoization { relation.reload }
|
264
266
|
end
|
265
267
|
end
|
266
268
|
end
|
@@ -9,7 +9,7 @@ class RedisMemo::Middleware
|
|
9
9
|
result = nil
|
10
10
|
|
11
11
|
RedisMemo::Cache.with_local_cache do
|
12
|
-
RedisMemo.with_max_connection_attempts(
|
12
|
+
RedisMemo.with_max_connection_attempts(RedisMemo::DefaultOptions.max_connection_attempts) do
|
13
13
|
result = @app.call(env)
|
14
14
|
end
|
15
15
|
end
|