redis-memo 0.1.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/redis_memo.rb +48 -36
- data/lib/redis_memo/after_commit.rb +2 -2
- data/lib/redis_memo/batch.rb +36 -11
- data/lib/redis_memo/cache.rb +36 -19
- data/lib/redis_memo/connection_pool.rb +4 -3
- data/lib/redis_memo/errors.rb +9 -0
- data/lib/redis_memo/future.rb +22 -13
- data/lib/redis_memo/memoizable.rb +109 -72
- data/lib/redis_memo/memoizable/bump_version.lua +39 -0
- data/lib/redis_memo/memoizable/dependency.rb +10 -11
- data/lib/redis_memo/memoizable/invalidation.rb +68 -66
- data/lib/redis_memo/memoize_method.rb +169 -131
- data/lib/redis_memo/memoize_query.rb +135 -92
- data/lib/redis_memo/memoize_query/cached_select.rb +73 -62
- data/lib/redis_memo/memoize_query/cached_select/bind_params.rb +202 -70
- data/lib/redis_memo/memoize_query/cached_select/connection_adapter.rb +19 -10
- data/lib/redis_memo/memoize_query/invalidation.rb +22 -20
- data/lib/redis_memo/memoize_query/memoize_table_column.rb +1 -0
- data/lib/redis_memo/middleware.rb +3 -1
- data/lib/redis_memo/options.rb +111 -5
- data/lib/redis_memo/railtie.rb +11 -0
- data/lib/redis_memo/redis.rb +15 -5
- data/lib/redis_memo/testing.rb +49 -0
- data/lib/redis_memo/thread_local_var.rb +16 -0
- data/lib/redis_memo/tracer.rb +1 -0
- data/lib/redis_memo/util.rb +25 -0
- metadata +80 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4e4b5e95c92879677df377b10736e7467efa86237d90ad41254bdd7a732a8260
|
4
|
+
data.tar.gz: 9dfb62913a6cfd96e114d6526f5876696e86d1fafb6256d2454d2723449e61c1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 62eaf4aedb48325cc90c10cd27a3877de85a436960c78d7e1205c5dbbbe77bdb6c50ac82ea1af11d7dbd7219e7fa96cdc270b37038f1674c1fa491dffb8acec3
|
7
|
+
data.tar.gz: 2315c83ce0848d5b2f36d60e37fe60bcece24d80505ff2d7c38b1a9c0d33b8785888325410bae695abc0746a8b4663e037daec07ea83f32b6aaeafd4bf955693
|
data/lib/redis_memo.rb
CHANGED
@@ -3,11 +3,20 @@
|
|
3
3
|
require 'active_support/all'
|
4
4
|
require 'digest'
|
5
5
|
require 'json'
|
6
|
+
require 'ruby2_keywords'
|
6
7
|
require 'securerandom'
|
7
8
|
|
8
9
|
module RedisMemo
|
10
|
+
require 'redis_memo/thread_local_var'
|
11
|
+
|
12
|
+
ThreadLocalVar.define :without_memoization
|
13
|
+
ThreadLocalVar.define :connection_attempts_count
|
14
|
+
ThreadLocalVar.define :max_connection_attempts
|
15
|
+
|
16
|
+
require 'redis_memo/errors'
|
9
17
|
require 'redis_memo/memoize_method'
|
10
|
-
require 'redis_memo/memoize_query'
|
18
|
+
require 'redis_memo/memoize_query' if defined?(ActiveRecord)
|
19
|
+
require 'redis_memo/railtie' if defined?(Rails) && defined?(Rails::Railtie)
|
11
20
|
|
12
21
|
# A process-level +RedisMemo::Options+ instance that stores the global
|
13
22
|
# options. This object can be modified by +RedisMemo.configure+.
|
@@ -17,9 +26,6 @@ module RedisMemo
|
|
17
26
|
# +DefaultOptions+ as the default value.
|
18
27
|
DefaultOptions = RedisMemo::Options.new
|
19
28
|
|
20
|
-
# @todo Move thread keys to +RedisMemo::ThreadKey+
|
21
|
-
THREAD_KEY_WITHOUT_MEMO = :__redis_memo_without_memo__
|
22
|
-
|
23
29
|
# Configure global-level default options. Those options will be used unless
|
24
30
|
# some options specified at +memoize_method+ callsite level. See
|
25
31
|
# +RedisMemo::Options+ for all the possible options.
|
@@ -35,10 +41,10 @@ module RedisMemo
|
|
35
41
|
# to Redis.
|
36
42
|
# - Batches cannot be nested
|
37
43
|
# - When a batch is still open (while still in the +RedisMemo.batch+ block)
|
38
|
-
#
|
39
|
-
#
|
44
|
+
# the return value of any memoized method is a +RedisMemo::Future+ instead of
|
45
|
+
# the actual method value
|
40
46
|
# - The actual method values are returned as a list, in the same order as
|
41
|
-
#
|
47
|
+
# invoking, after exiting the block
|
42
48
|
#
|
43
49
|
# @example
|
44
50
|
# results = RedisMemo.batch do
|
@@ -56,46 +62,52 @@ module RedisMemo
|
|
56
62
|
RedisMemo::Batch.close
|
57
63
|
end
|
58
64
|
|
59
|
-
# @todo Move this method out of the top namespace
|
60
|
-
def self.checksum(serialized)
|
61
|
-
Digest::SHA1.base64digest(serialized)
|
62
|
-
end
|
63
|
-
|
64
|
-
# @todo Move this method out of the top namespace
|
65
|
-
def self.uuid
|
66
|
-
SecureRandom.uuid
|
67
|
-
end
|
68
|
-
|
69
|
-
# @todo Move this method out of the top namespace
|
70
|
-
def self.deep_sort_hash(orig_hash)
|
71
|
-
{}.tap do |new_hash|
|
72
|
-
orig_hash.sort.each do |k, v|
|
73
|
-
new_hash[k] = v.is_a?(Hash) ? deep_sort_hash(v) : v
|
74
|
-
end
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
65
|
# Whether the current execution context has been configured to skip
|
79
66
|
# memoization and use the uncached code path.
|
80
67
|
#
|
81
68
|
# @return [Boolean]
|
82
|
-
def self.
|
83
|
-
|
69
|
+
def self.without_memoization?
|
70
|
+
RedisMemo::DefaultOptions.disable_all || ThreadLocalVar.without_memoization == true
|
84
71
|
end
|
85
72
|
|
86
73
|
# Configure the wrapped code in the block to skip memoization.
|
87
74
|
#
|
88
75
|
# @yield [] no_args The block of code to skip memoization.
|
89
|
-
def self.
|
90
|
-
prev_value =
|
91
|
-
|
76
|
+
def self.without_memoization
|
77
|
+
prev_value = ThreadLocalVar.without_memoization
|
78
|
+
ThreadLocalVar.without_memoization = true
|
92
79
|
yield
|
93
80
|
ensure
|
94
|
-
|
81
|
+
ThreadLocalVar.without_memoization = prev_value
|
95
82
|
end
|
96
83
|
|
97
|
-
#
|
98
|
-
|
99
|
-
|
100
|
-
|
84
|
+
# Set the max connection attempts to Redis per code block. If we fail to
|
85
|
+
# connect to Redis more than `max_attempts` times, the rest of the code block
|
86
|
+
# will fall back to the uncached flow, `RedisMemo.without_memoization`.
|
87
|
+
#
|
88
|
+
# @param [Integer] The max number of connection attempts.
|
89
|
+
# @yield [] no_args the block of code to set the max attempts for.
|
90
|
+
def self.with_max_connection_attempts(max_attempts)
|
91
|
+
prev_value = ThreadLocalVar.without_memoization
|
92
|
+
ThreadLocalVar.connection_attempts_count = 0
|
93
|
+
ThreadLocalVar.max_connection_attempts = max_attempts
|
94
|
+
|
95
|
+
yield
|
96
|
+
ensure
|
97
|
+
ThreadLocalVar.without_memoization = prev_value
|
98
|
+
ThreadLocalVar.connection_attempts_count = nil
|
99
|
+
ThreadLocalVar.max_connection_attempts = nil
|
100
|
+
end
|
101
|
+
|
102
|
+
private_class_method def self.incr_connection_attempts # :nodoc:
|
103
|
+
return unless ThreadLocalVar.max_connection_attempts && ThreadLocalVar.connection_attempts_count
|
104
|
+
|
105
|
+
# The connection attempts count and max connection attempts are reset in
|
106
|
+
# RedisMemo.with_max_connection_attempts
|
107
|
+
ThreadLocalVar.connection_attempts_count += 1
|
108
|
+
return if ThreadLocalVar.connection_attempts_count <
|
109
|
+
ThreadLocalVar.max_connection_attempts
|
110
|
+
|
111
|
+
ThreadLocalVar.without_memoization = true
|
112
|
+
end
|
101
113
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
# TODO: -> RedisMemo::Memoizable::AfterCommit
|
3
4
|
|
4
5
|
class RedisMemo::AfterCommit
|
@@ -34,8 +35,6 @@ class RedisMemo::AfterCommit
|
|
34
35
|
connection.transaction_open? && connection.current_transaction.joinable?
|
35
36
|
end
|
36
37
|
|
37
|
-
private
|
38
|
-
|
39
38
|
def self.after_commit(&blk)
|
40
39
|
connection.add_transaction_record(
|
41
40
|
RedisMemo::AfterCommit::Callback.new(connection, committed: blk),
|
@@ -50,6 +49,7 @@ class RedisMemo::AfterCommit
|
|
50
49
|
|
51
50
|
def self.reset_after_transaction
|
52
51
|
return if @@callback_added
|
52
|
+
|
53
53
|
@@callback_added = true
|
54
54
|
|
55
55
|
after_commit do
|
data/lib/redis_memo/batch.rb
CHANGED
@@ -1,30 +1,54 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require_relative 'cache'
|
3
4
|
require_relative 'tracer'
|
4
5
|
|
6
|
+
##
|
7
|
+
# This class facilitates the batching of Redis calls triggered by +memoize_method+
|
8
|
+
# to minimize the number of round trips to Redis.
|
9
|
+
#
|
10
|
+
# - Batches cannot be nested
|
11
|
+
# - When a batch is still open (while still in the +RedisMemo.batch+ block)
|
12
|
+
# the return value of any memoized method is a +RedisMemo::Future+ instead of
|
13
|
+
# the actual method value
|
14
|
+
# - The actual method values are returned as a list, in the same order as
|
15
|
+
# invoking, after exiting the block
|
16
|
+
#
|
17
|
+
# @example
|
18
|
+
# results = RedisMemo.batch do
|
19
|
+
# 5.times { |i| memoized_calculation(i) }
|
20
|
+
# nil # Not the return value of the block
|
21
|
+
# end
|
22
|
+
# results # [1,2,3,4,5] (results from the memoized_calculation calls)
|
5
23
|
class RedisMemo::Batch
|
6
|
-
|
24
|
+
RedisMemo::ThreadLocalVar.define :batch
|
7
25
|
|
26
|
+
# Opens a new batch. If a batch is already open, raises an error
|
27
|
+
# to prevent nested batches.
|
8
28
|
def self.open
|
9
29
|
if current
|
10
|
-
raise RedisMemo::RuntimeError
|
30
|
+
raise RedisMemo::RuntimeError.new('Batch can not be nested')
|
11
31
|
end
|
12
32
|
|
13
|
-
|
33
|
+
RedisMemo::ThreadLocalVar.batch = []
|
14
34
|
end
|
15
35
|
|
36
|
+
# Closes the current batch, returning the futures in that batch.
|
16
37
|
def self.close
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
38
|
+
return unless current
|
39
|
+
|
40
|
+
futures = current
|
41
|
+
RedisMemo::ThreadLocalVar.batch = nil
|
42
|
+
futures
|
22
43
|
end
|
23
44
|
|
45
|
+
# Retrieves the current open batch.
|
24
46
|
def self.current
|
25
|
-
|
47
|
+
RedisMemo::ThreadLocalVar.batch
|
26
48
|
end
|
27
49
|
|
50
|
+
# Executes all the futures in the current batch using batched calls to
|
51
|
+
# Redis and closes it.
|
28
52
|
def self.execute
|
29
53
|
futures = close
|
30
54
|
return unless futures
|
@@ -33,7 +57,8 @@ class RedisMemo::Batch
|
|
33
57
|
method_cache_keys = nil
|
34
58
|
|
35
59
|
RedisMemo::Tracer.trace('redis_memo.cache.batch.read', nil) do
|
36
|
-
method_cache_keys = RedisMemo::MemoizeMethod.
|
60
|
+
method_cache_keys = RedisMemo::MemoizeMethod.__send__(
|
61
|
+
:method_cache_keys,
|
37
62
|
futures.map(&:context),
|
38
63
|
)
|
39
64
|
|
data/lib/redis_memo/cache.rb
CHANGED
@@ -1,14 +1,17 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require_relative 'options'
|
3
4
|
require_relative 'redis'
|
4
5
|
require_relative 'connection_pool'
|
5
6
|
|
6
7
|
class RedisMemo::Cache < ActiveSupport::Cache::RedisCacheStore
|
7
|
-
|
8
|
+
# This needs to be an Exception since RedisCacheStore rescues all
|
9
|
+
# RuntimeErrors
|
10
|
+
class Rescuable < Exception; end # rubocop:disable Lint/InheritException
|
8
11
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
+
RedisMemo::ThreadLocalVar.define :local_cache
|
13
|
+
RedisMemo::ThreadLocalVar.define :local_dependency_cache
|
14
|
+
RedisMemo::ThreadLocalVar.define :raise_error
|
12
15
|
|
13
16
|
@@redis = nil
|
14
17
|
@@redis_store = nil
|
@@ -17,11 +20,15 @@ class RedisMemo::Cache < ActiveSupport::Cache::RedisCacheStore
|
|
17
20
|
RedisMemo::DefaultOptions.redis_error_handler&.call(exception, method)
|
18
21
|
RedisMemo::DefaultOptions.logger&.warn(exception.full_message)
|
19
22
|
|
20
|
-
if
|
23
|
+
if exception.is_a?(Redis::BaseConnectionError)
|
24
|
+
RedisMemo.__send__(:incr_connection_attempts)
|
25
|
+
end
|
26
|
+
|
27
|
+
if RedisMemo::ThreadLocalVar.raise_error
|
21
28
|
raise RedisMemo::Cache::Rescuable
|
22
|
-
else
|
23
|
-
returning
|
24
29
|
end
|
30
|
+
|
31
|
+
returning
|
25
32
|
end
|
26
33
|
|
27
34
|
def self.redis
|
@@ -48,28 +55,38 @@ class RedisMemo::Cache < ActiveSupport::Cache::RedisCacheStore
|
|
48
55
|
# ActiveSupport::Cache::Entry object -- which is slower comparing to a simple
|
49
56
|
# hash storing object references
|
50
57
|
def self.local_cache
|
51
|
-
|
58
|
+
RedisMemo::ThreadLocalVar.local_cache
|
52
59
|
end
|
53
60
|
|
54
61
|
def self.local_dependency_cache
|
55
|
-
|
62
|
+
RedisMemo::ThreadLocalVar.local_dependency_cache
|
63
|
+
end
|
64
|
+
|
65
|
+
# See https://github.com/rails/rails/blob/fe76a95b0d252a2d7c25e69498b720c96b243ea2/activesupport/lib/active_support/cache/redis_cache_store.rb#L477
|
66
|
+
# We overwrite this private method so we can also rescue ConnectionPool::TimeoutErrors
|
67
|
+
def failsafe(method, returning: nil)
|
68
|
+
yield
|
69
|
+
rescue ::Redis::BaseError, ::ConnectionPool::TimeoutError => error
|
70
|
+
handle_exception exception: error, method: method, returning: returning
|
71
|
+
returning
|
56
72
|
end
|
73
|
+
private :failsafe
|
57
74
|
|
58
75
|
class << self
|
59
|
-
def with_local_cache
|
60
|
-
|
61
|
-
|
62
|
-
|
76
|
+
def with_local_cache
|
77
|
+
RedisMemo::ThreadLocalVar.local_cache = {}
|
78
|
+
RedisMemo::ThreadLocalVar.local_dependency_cache = {}
|
79
|
+
yield
|
63
80
|
ensure
|
64
|
-
|
65
|
-
|
81
|
+
RedisMemo::ThreadLocalVar.local_cache = nil
|
82
|
+
RedisMemo::ThreadLocalVar.local_dependency_cache = nil
|
66
83
|
end
|
67
84
|
|
68
85
|
# RedisCacheStore doesn't read from the local cache before reading from redis
|
69
86
|
def read_multi(*keys, raw: false, raise_error: false)
|
70
87
|
return {} if keys.empty?
|
71
88
|
|
72
|
-
|
89
|
+
RedisMemo::ThreadLocalVar.raise_error = raise_error
|
73
90
|
|
74
91
|
local_entries = local_cache&.slice(*keys) || {}
|
75
92
|
|
@@ -88,7 +105,7 @@ class RedisMemo::Cache < ActiveSupport::Cache::RedisCacheStore
|
|
88
105
|
end
|
89
106
|
|
90
107
|
def write(key, value, disable_async: false, raise_error: false, **options)
|
91
|
-
|
108
|
+
RedisMemo::ThreadLocalVar.raise_error = raise_error
|
92
109
|
|
93
110
|
if local_cache
|
94
111
|
local_cache[key] = value
|
@@ -99,7 +116,7 @@ class RedisMemo::Cache < ActiveSupport::Cache::RedisCacheStore
|
|
99
116
|
redis_store.write(key, value, **options)
|
100
117
|
else
|
101
118
|
async.call do
|
102
|
-
|
119
|
+
RedisMemo::ThreadLocalVar.raise_error = raise_error
|
103
120
|
redis_store.write(key, value, **options)
|
104
121
|
end
|
105
122
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'connection_pool'
|
3
4
|
require_relative 'redis'
|
4
5
|
|
@@ -11,17 +12,17 @@ class RedisMemo::ConnectionPool
|
|
11
12
|
end
|
12
13
|
|
13
14
|
# Avoid method_missing when possible for better performance
|
14
|
-
%i
|
15
|
+
%i[get mget mapped_mget set eval evalsha run_script].each do |method_name|
|
15
16
|
define_method method_name do |*args, &blk|
|
16
17
|
@connection_pool.with do |redis|
|
17
|
-
redis.
|
18
|
+
redis.__send__(method_name, *args, &blk)
|
18
19
|
end
|
19
20
|
end
|
20
21
|
end
|
21
22
|
|
22
23
|
def method_missing(method_name, *args, &blk)
|
23
24
|
@connection_pool.with do |redis|
|
24
|
-
redis.
|
25
|
+
redis.__send__(method_name, *args, &blk)
|
25
26
|
end
|
26
27
|
end
|
27
28
|
end
|
data/lib/redis_memo/future.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require_relative 'cache'
|
3
4
|
require_relative 'tracer'
|
4
5
|
|
@@ -11,14 +12,14 @@ class RedisMemo::Future
|
|
11
12
|
method_args,
|
12
13
|
dependent_memos,
|
13
14
|
cache_options,
|
14
|
-
|
15
|
+
method_name_without_memoization
|
15
16
|
)
|
16
17
|
@ref = ref
|
17
18
|
@method_id = method_id
|
18
19
|
@method_args = method_args
|
19
20
|
@dependent_memos = dependent_memos
|
20
21
|
@cache_options = cache_options
|
21
|
-
@
|
22
|
+
@method_name_without_memoization = method_name_without_memoization
|
22
23
|
@method_cache_key = nil
|
23
24
|
@cache_hit = false
|
24
25
|
@cached_result = nil
|
@@ -33,18 +34,26 @@ class RedisMemo::Future
|
|
33
34
|
|
34
35
|
def method_cache_key
|
35
36
|
@method_cache_key ||=
|
36
|
-
RedisMemo::MemoizeMethod.method_cache_keys
|
37
|
+
RedisMemo::MemoizeMethod.__send__(:method_cache_keys, [context])&.first || ''
|
37
38
|
end
|
38
39
|
|
39
|
-
def
|
40
|
+
def validate_cache_result
|
41
|
+
cache_validation_sample_percentage =
|
42
|
+
@cache_options[:cache_validation_sample_percentage] || RedisMemo::DefaultOptions.cache_validation_sample_percentage
|
43
|
+
|
44
|
+
if cache_validation_sample_percentage.nil?
|
45
|
+
false
|
46
|
+
else
|
47
|
+
cache_validation_sample_percentage > Random.rand(0...100)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def execute(cached_results = nil)
|
40
52
|
if RedisMemo::Batch.current
|
41
|
-
raise RedisMemo::RuntimeError
|
53
|
+
raise RedisMemo::RuntimeError.new('Cannot execute future when a batch is still open')
|
42
54
|
end
|
43
55
|
|
44
56
|
if cache_hit?(cached_results)
|
45
|
-
validate_cache_result =
|
46
|
-
RedisMemo::DefaultOptions.cache_validation_sampler&.call(@method_id)
|
47
|
-
|
48
57
|
if validate_cache_result && cached_result != fresh_result
|
49
58
|
RedisMemo::DefaultOptions.cache_out_of_date_handler&.call(
|
50
59
|
@ref,
|
@@ -64,7 +73,7 @@ class RedisMemo::Future
|
|
64
73
|
|
65
74
|
def result
|
66
75
|
unless @computed_cached_result
|
67
|
-
raise RedisMemo::RuntimeError
|
76
|
+
raise RedisMemo::RuntimeError.new('Future has not been executed')
|
68
77
|
end
|
69
78
|
|
70
79
|
@fresh_result || @cached_result
|
@@ -72,13 +81,13 @@ class RedisMemo::Future
|
|
72
81
|
|
73
82
|
private
|
74
83
|
|
75
|
-
def cache_hit?(cached_results=nil)
|
84
|
+
def cache_hit?(cached_results = nil)
|
76
85
|
cached_result(cached_results)
|
77
86
|
|
78
87
|
@cache_hit
|
79
88
|
end
|
80
89
|
|
81
|
-
def cached_result(cached_results=nil)
|
90
|
+
def cached_result(cached_results = nil)
|
82
91
|
return @cached_result if @computed_cached_result
|
83
92
|
|
84
93
|
@cache_hit = false
|
@@ -102,7 +111,7 @@ class RedisMemo::Future
|
|
102
111
|
|
103
112
|
RedisMemo::Tracer.trace('redis_memo.cache.write', @method_id) do
|
104
113
|
# cache miss
|
105
|
-
@fresh_result = @ref.
|
114
|
+
@fresh_result = @ref.__send__(@method_name_without_memoization, *@method_args)
|
106
115
|
if @cache_options.include?(:expires_in) && @cache_options[:expires_in].respond_to?(:call)
|
107
116
|
@cache_options[:expires_in] = @cache_options[:expires_in].call(@fresh_result)
|
108
117
|
end
|
@@ -114,7 +123,7 @@ class RedisMemo::Future
|
|
114
123
|
# result)
|
115
124
|
@cache_hit && @cached_result != @fresh_result
|
116
125
|
)
|
117
|
-
|
126
|
+
)
|
118
127
|
RedisMemo::Cache.write(method_cache_key, @fresh_result, **@cache_options)
|
119
128
|
end
|
120
129
|
end
|