redis-memo 0.0.0.beta.2 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: bfc639839800aa1290622d733d9fd23619de8ccd63f2129e4570eb41a295884b
4
- data.tar.gz: 26e2d86776244ba5cb0a8cebbc18e468ccf737888de9990c3a2bb9fc931bd4d5
3
+ metadata.gz: dd1d436b41e1f30d0d0910d086811f675b87b6bbe799a0087da7453b4c016b66
4
+ data.tar.gz: 7ca970165cb321e0016d6f70f6135868df19b68d27a180084f8053f94bf780c0
5
5
  SHA512:
6
- metadata.gz: 3c6950a588fbf448d4b88e126ee9b32753ef314225d3aa5f69430d7e386a9a1085b517a08f4f6de2725810f35b493c20e2af62729f547891bcc78aaf4ce73ef2
7
- data.tar.gz: 1e380d3ff6d457a6c93dcdebcbf9be072e919b2f5ff25377f169287ed8ac6c379f9a8db29a494d9174cff91e815e680201970ece43baba69b627678b628a3d77
6
+ metadata.gz: d79af32bc2a55a3755072cd0502d355745c7f060241d6a7f5e116eab8c0b7a45b107d02ef5d4bbcec0e6acb5bf435f1bb4d6b50084d13e410307610ef675f420
7
+ data.tar.gz: d9e0714a72c526be0043185dfc2c0dc3f6a6c94557da5b81e31905616b1a156cb6d2c89934548332802fff42a24ed53c1518a97b3912d2cc8beae41d9dffc7bf
data/lib/redis-memo.rb CHANGED
@@ -1 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Automatically require the main file of the redis-memo gem when adding it to
4
+ # the Gemfile
1
5
  require_relative 'redis_memo'
data/lib/redis_memo.rb CHANGED
@@ -1,18 +1,53 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  require 'active_support/all'
3
4
  require 'digest'
4
5
  require 'json'
6
+ require 'securerandom'
5
7
 
6
8
  module RedisMemo
7
9
  require 'redis_memo/memoize_method'
8
10
  require 'redis_memo/memoize_query'
9
11
 
12
+ # A process-level +RedisMemo::Options+ instance that stores the global
13
+ # options. This object can be modified by +RedisMemo.configure+.
14
+ #
15
+ # +memoize_method+ allows users to provide method-level configuration.
16
+ # When no callsite-level configuration specified we will use the values in
17
+ # +DefaultOptions+ as the default value.
10
18
  DefaultOptions = RedisMemo::Options.new
11
19
 
20
+ # @todo Move thread keys to +RedisMemo::ThreadKey+
21
+ THREAD_KEY_WITHOUT_MEMO = :__redis_memo_without_memo__
22
+
23
+ # Configure global-level default options. Those options will be used unless
24
+ # some options specified at +memoize_method+ callsite level. See
25
+ # +RedisMemo::Options+ for all the possible options.
26
+ #
27
+ # @yieldparam [RedisMemo::Options] default_options
28
+ # +RedisMemo::DefaultOptions+
29
+ # @return [void]
12
30
  def self.configure(&blk)
13
31
  blk.call(DefaultOptions)
14
32
  end
15
33
 
34
+ # Batch Redis calls triggered by +memoize_method+ to minimize the round trips
35
+ # to Redis.
36
+ # - Batches cannot be nested
37
+ # - When a batch is still open (while still in the +RedisMemo.batch+ block)
38
+ # the return value of any memoized method is a +RedisMemo::Future+ instead of
39
+ # the actual method value
40
+ # - The actual method values are returned as a list, in the same order as
41
+ # invoking, after exiting the block
42
+ #
43
+ # @example
44
+ # results = RedisMemo.batch do
45
+ # 5.times { |i| memoized_calculation(i) }
46
+ # nil # Not the return value of the block
47
+ # end
48
+ # results.size == 5 # true
49
+ #
50
+ # See +RedisMemo::Batch+ for more information.
16
51
  def self.batch(&blk)
17
52
  RedisMemo::Batch.open
18
53
  blk.call
@@ -21,10 +56,17 @@ module RedisMemo
21
56
  RedisMemo::Batch.close
22
57
  end
23
58
 
59
+ # @todo Move this method out of the top namespace
24
60
  def self.checksum(serialized)
25
61
  Digest::SHA1.base64digest(serialized)
26
62
  end
27
63
 
64
+ # @todo Move this method out of the top namespace
65
+ def self.uuid
66
+ SecureRandom.uuid
67
+ end
68
+
69
+ # @todo Move this method out of the top namespace
28
70
  def self.deep_sort_hash(orig_hash)
29
71
  {}.tap do |new_hash|
30
72
  orig_hash.sort.each do |k, v|
@@ -33,12 +75,17 @@ module RedisMemo
33
75
  end
34
76
  end
35
77
 
36
- THREAD_KEY_WITHOUT_MEMO = :__redis_memo_without_memo__
37
-
78
+ # Whether the current execution context has been configured to skip
79
+ # memoization and use the uncached code path.
80
+ #
81
+ # @return [Boolean]
38
82
  def self.without_memo?
39
83
  Thread.current[THREAD_KEY_WITHOUT_MEMO] == true
40
84
  end
41
85
 
86
+ # Configure the wrapped code in the block to skip memoization.
87
+ #
88
+ # @yield [] no_args The block of code to skip memoization.
42
89
  def self.without_memo
43
90
  prev_value = Thread.current[THREAD_KEY_WITHOUT_MEMO]
44
91
  Thread.current[THREAD_KEY_WITHOUT_MEMO] = true
@@ -47,6 +94,8 @@ module RedisMemo
47
94
  Thread.current[THREAD_KEY_WITHOUT_MEMO] = prev_value
48
95
  end
49
96
 
97
+ # @todo Move errors to a separate file errors.rb
50
98
  class ArgumentError < ::ArgumentError; end
51
99
  class RuntimeError < ::RuntimeError; end
100
+ class WithoutMemoization < Exception; end
52
101
  end
@@ -66,7 +66,12 @@ class RedisMemo::AfterCommit
66
66
  @@pending_memo_versions.each do |key, version|
67
67
  invalidation_queue =
68
68
  RedisMemo::Memoizable::Invalidation.class_variable_get(:@@invalidation_queue)
69
- invalidation_queue << [key, version, @@previous_memo_versions[key]]
69
+
70
+ invalidation_queue << RedisMemo::Memoizable::Invalidation::Task.new(
71
+ key,
72
+ version,
73
+ @@previous_memo_versions[key],
74
+ )
70
75
  end
71
76
 
72
77
  RedisMemo::Memoizable::Invalidation.drain_invalidation_queue
@@ -1,12 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
  require_relative 'options'
3
3
  require_relative 'redis'
4
+ require_relative 'connection_pool'
4
5
 
5
6
  class RedisMemo::Cache < ActiveSupport::Cache::RedisCacheStore
6
7
  class Rescuable < Exception; end
7
8
 
8
- THREAD_KEY_LOCAL_CACHE = :__redis_memo_cache_local_cache__
9
- THREAD_KEY_RAISE_ERROR = :__redis_memo_cache_raise_error__
9
+ THREAD_KEY_LOCAL_CACHE = :__redis_memo_cache_local_cache__
10
+ THREAD_KEY_LOCAL_DEPENDENCY_CACHE = :__redis_memo_local_cache_dependency_cache__
11
+ THREAD_KEY_RAISE_ERROR = :__redis_memo_cache_raise_error__
10
12
 
11
13
  @@redis = nil
12
14
  @@redis_store = nil
@@ -23,7 +25,12 @@ class RedisMemo::Cache < ActiveSupport::Cache::RedisCacheStore
23
25
  end
24
26
 
25
27
  def self.redis
26
- @@redis ||= RedisMemo::DefaultOptions.redis
28
+ @@redis ||=
29
+ if RedisMemo::DefaultOptions.connection_pool
30
+ RedisMemo::ConnectionPool.new(**RedisMemo::DefaultOptions.connection_pool)
31
+ else
32
+ RedisMemo::DefaultOptions.redis
33
+ end
27
34
  end
28
35
 
29
36
  def self.redis_store
@@ -44,16 +51,22 @@ class RedisMemo::Cache < ActiveSupport::Cache::RedisCacheStore
44
51
  Thread.current[THREAD_KEY_LOCAL_CACHE]
45
52
  end
46
53
 
54
+ def self.local_dependency_cache
55
+ Thread.current[THREAD_KEY_LOCAL_DEPENDENCY_CACHE]
56
+ end
57
+
47
58
  class << self
48
59
  def with_local_cache(&blk)
49
60
  Thread.current[THREAD_KEY_LOCAL_CACHE] = {}
61
+ Thread.current[THREAD_KEY_LOCAL_DEPENDENCY_CACHE] = {}
50
62
  blk.call
51
63
  ensure
52
64
  Thread.current[THREAD_KEY_LOCAL_CACHE] = nil
65
+ Thread.current[THREAD_KEY_LOCAL_DEPENDENCY_CACHE] = nil
53
66
  end
54
67
 
55
68
  # RedisCacheStore doesn't read from the local cache before reading from redis
56
- def read_multi(*keys, raise_error: false)
69
+ def read_multi(*keys, raw: false, raise_error: false)
57
70
  return {} if keys.empty?
58
71
 
59
72
  Thread.current[THREAD_KEY_RAISE_ERROR] = raise_error
@@ -64,7 +77,7 @@ class RedisMemo::Cache < ActiveSupport::Cache::RedisCacheStore
64
77
  keys_to_fetch -= local_entries.keys unless local_entries.empty?
65
78
  return local_entries if keys_to_fetch.empty?
66
79
 
67
- remote_entries = redis_store.read_multi(*keys_to_fetch)
80
+ remote_entries = redis_store.read_multi(*keys_to_fetch, raw: raw)
68
81
  local_cache&.merge!(remote_entries)
69
82
 
70
83
  if local_entries.empty?
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+ require 'connection_pool'
3
+ require_relative 'redis'
4
+
5
+ class RedisMemo::ConnectionPool
6
+ def initialize(**options)
7
+ @connection_pool = ::ConnectionPool.new(**options) do
8
+ # Construct a new client every time the block gets called
9
+ RedisMemo::Redis.new(RedisMemo::DefaultOptions.redis_config)
10
+ end
11
+ end
12
+
13
+ # Avoid method_missing when possible for better performance
14
+ %i(get mget mapped_mget set eval).each do |method_name|
15
+ define_method method_name do |*args, &blk|
16
+ @connection_pool.with do |redis|
17
+ redis.send(method_name, *args, &blk)
18
+ end
19
+ end
20
+ end
21
+
22
+ def method_missing(method_name, *args, &blk)
23
+ @connection_pool.with do |redis|
24
+ redis.send(method_name, *args, &blk)
25
+ end
26
+ end
27
+ end
@@ -9,14 +9,14 @@ class RedisMemo::Future
9
9
  ref,
10
10
  method_id,
11
11
  method_args,
12
- depends_on,
12
+ dependent_memos,
13
13
  cache_options,
14
14
  method_name_without_memo
15
15
  )
16
16
  @ref = ref
17
17
  @method_id = method_id
18
18
  @method_args = method_args
19
- @depends_on = depends_on
19
+ @dependent_memos = dependent_memos
20
20
  @cache_options = cache_options
21
21
  @method_name_without_memo = method_name_without_memo
22
22
  @method_cache_key = nil
@@ -28,7 +28,7 @@ class RedisMemo::Future
28
28
  end
29
29
 
30
30
  def context
31
- [@ref, @method_id, @method_args, @depends_on]
31
+ [@method_id, @method_args, @dependent_memos]
32
32
  end
33
33
 
34
34
  def method_cache_key
@@ -1,5 +1,4 @@
1
1
  # frozen_string_literal: true
2
- require 'securerandom'
3
2
 
4
3
  class RedisMemo::Memoizable
5
4
  require_relative 'memoizable/dependency'
@@ -52,7 +51,7 @@ class RedisMemo::Memoizable
52
51
  cache_key = instance.cache_key
53
52
  RedisMemo::Memoizable::Invalidation.bump_version_later(
54
53
  cache_key,
55
- SecureRandom.uuid,
54
+ RedisMemo.uuid,
56
55
  )
57
56
  end
58
57
 
@@ -83,7 +82,11 @@ class RedisMemo::Memoizable
83
82
  if keys_to_fetch.empty?
84
83
  {}
85
84
  else
86
- RedisMemo::Cache.read_multi(*keys_to_fetch, raise_error: true)
85
+ RedisMemo::Cache.read_multi(
86
+ *keys_to_fetch,
87
+ raw: true,
88
+ raise_error: true,
89
+ )
87
90
  end
88
91
  memo_versions.merge!(cached_versions) unless cached_versions.empty?
89
92
 
@@ -96,7 +99,7 @@ class RedisMemo::Memoizable
96
99
  # cached result.
97
100
  need_to_bump_versions = true
98
101
 
99
- new_version = SecureRandom.uuid
102
+ new_version = RedisMemo.uuid
100
103
  RedisMemo::Memoizable::Invalidation.bump_version_later(
101
104
  key,
102
105
  new_version,
@@ -25,12 +25,19 @@ class RedisMemo::Memoizable::Dependency
25
25
  # Extract dependencies from the current memoizable and recurse
26
26
  instance_exec(&memo.depends_on)
27
27
  end
28
- when UsingActiveRecord
29
- [
30
- dependency.redis_memo_class_memoizable,
31
- RedisMemo::MemoizeQuery.create_memo(dependency, **conditions),
32
- ].each do |memo|
28
+ when ActiveRecord::Relation
29
+ extracted = self.class.extract_from_relation(dependency)
30
+ nodes.merge!(extracted.nodes)
31
+ when RedisMemo::MemoizeQuery::CachedSelect::BindParams
32
+ # A private API
33
+ dependency.params.each do |model, attrs_set|
34
+ memo = model.redis_memo_class_memoizable
33
35
  nodes[memo.cache_key] = memo
36
+
37
+ attrs_set.each do |attrs|
38
+ memo = RedisMemo::MemoizeQuery.create_memo(model, **attrs)
39
+ nodes[memo.cache_key] = memo
40
+ end
34
41
  end
35
42
  else
36
43
  raise(
@@ -40,9 +47,21 @@ class RedisMemo::Memoizable::Dependency
40
47
  end
41
48
  end
42
49
 
43
- class UsingActiveRecord
44
- def self.===(dependency)
45
- RedisMemo::MemoizeQuery.using_active_record?(dependency)
50
+ private
51
+
52
+ def self.extract_from_relation(relation)
53
+ # Extract the dependent memos of an Arel without calling exec_query to actually execute the query
54
+ RedisMemo::MemoizeQuery::CachedSelect.with_new_query_context do
55
+ connection = ActiveRecord::Base.connection
56
+ query, binds, _ = connection.send(:to_sql_and_binds, relation.arel)
57
+ RedisMemo::MemoizeQuery::CachedSelect.current_query = relation.arel
58
+ is_query_cached = RedisMemo::MemoizeQuery::CachedSelect.extract_bind_params(query)
59
+
60
+ unless is_query_cached
61
+ raise RedisMemo::WithoutMemoization, 'Arel query is not cached using RedisMemo'
62
+ end
63
+
64
+ connection.dependency_of(:exec_query, query, nil, binds)
46
65
  end
47
66
  end
48
67
  end
@@ -3,12 +3,29 @@ require_relative '../after_commit'
3
3
  require_relative '../cache'
4
4
 
5
5
  module RedisMemo::Memoizable::Invalidation
6
- # This is a thread safe data structure
7
- #
8
- # Handle transient network errors during cache invalidation
9
- # Each item in the queue is a tuple:
10
- #
11
- # [key, version (a UUID), previous_version (a UUID)]
6
+ class Task
7
+ attr_reader :key
8
+ attr_reader :version
9
+ attr_reader :previous_version
10
+
11
+ def initialize(key, version, previous_version)
12
+ @key = key
13
+ @version = version
14
+ @previous_version = previous_version
15
+ @created_at = current_timestamp
16
+ end
17
+
18
+ def current_timestamp
19
+ Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond)
20
+ end
21
+
22
+ def duration
23
+ current_timestamp - @created_at
24
+ end
25
+ end
26
+
27
+ # This is a thread safe data structure to handle transient network errors
28
+ # during cache invalidation
12
29
  #
13
30
  # When an invalidation call arrives at Redis, we only bump to the specified
14
31
  # version (so the cached results using that version will become visible) if
@@ -28,8 +45,6 @@ module RedisMemo::Memoizable::Invalidation
28
45
  @@invalidation_queue = Queue.new
29
46
 
30
47
  def self.bump_version_later(key, version, previous_version: nil)
31
- RedisMemo::DefaultOptions.logger&.info("[received] Bump memo key #{key}")
32
-
33
48
  if RedisMemo::AfterCommit.in_transaction?
34
49
  previous_version ||= RedisMemo::AfterCommit.pending_memo_versions[key]
35
50
  end
@@ -41,7 +56,10 @@ module RedisMemo::Memoizable::Invalidation
41
56
  # Fill an expected previous version so the later calculation results
42
57
  # based on this version can still be rolled out if this version
43
58
  # does not change
44
- previous_version ||= RedisMemo::Cache.read_multi(key)[key]
59
+ previous_version ||= RedisMemo::Cache.read_multi(
60
+ key,
61
+ raw: true,
62
+ )[key]
45
63
  end
46
64
 
47
65
  local_cache&.send(:[]=, key, version)
@@ -52,7 +70,7 @@ module RedisMemo::Memoizable::Invalidation
52
70
  previous_version: previous_version,
53
71
  )
54
72
  else
55
- @@invalidation_queue << [key, version, previous_version]
73
+ @@invalidation_queue << Task.new(key, version, previous_version)
56
74
  end
57
75
  end
58
76
 
@@ -89,31 +107,32 @@ module RedisMemo::Memoizable::Invalidation
89
107
  return redis.call('set', key, new_version, unpack(px))
90
108
  LUA
91
109
 
92
- def self.bump_version(cache_key, version, previous_version:)
93
- RedisMemo::Tracer.trace('redis_memo.memoizable.bump_version', nil) do
110
+ def self.bump_version(task)
111
+ RedisMemo::Tracer.trace('redis_memo.memoizable.bump_version', task.key) do
94
112
  ttl = RedisMemo::DefaultOptions.expires_in
95
113
  ttl = (ttl * 1000.0).to_i if ttl
96
114
  RedisMemo::Cache.redis.eval(
97
115
  LUA_BUMP_VERSION,
98
- keys: [cache_key],
99
- argv: [previous_version, version, SecureRandom.uuid, ttl],
116
+ keys: [task.key],
117
+ argv: [task.previous_version, task.version, RedisMemo.uuid, ttl],
100
118
  )
119
+ RedisMemo::Tracer.set_tag(enqueue_to_finish: task.duration)
101
120
  end
102
- RedisMemo::DefaultOptions.logger&.info("[performed] Bump memo key #{cache_key}")
103
121
  end
104
122
 
105
123
  def self.drain_invalidation_queue_now
106
124
  retry_queue = []
107
125
  until @@invalidation_queue.empty?
108
- tuple = @@invalidation_queue.pop
126
+ task = @@invalidation_queue.pop
109
127
  begin
110
- bump_version(tuple[0], tuple[1], previous_version: tuple[2])
111
- rescue SignalException, Redis::BaseConnectionError
112
- retry_queue << tuple
128
+ bump_version(task)
129
+ rescue SignalException, Redis::BaseConnectionError,
130
+ ::ConnectionPool::TimeoutError
131
+ retry_queue << task
113
132
  end
114
133
  end
115
134
  ensure
116
- retry_queue.each { |t| @@invalidation_queue << t }
135
+ retry_queue.each { |task| @@invalidation_queue << task }
117
136
  end
118
137
 
119
138
  at_exit do
@@ -15,6 +15,12 @@ module RedisMemo::MemoizeMethod
15
15
  define_method method_name_with_memo do |*args|
16
16
  return send(method_name_without_memo, *args) if RedisMemo.without_memo?
17
17
 
18
+ dependent_memos = nil
19
+ if depends_on
20
+ dependency = RedisMemo::MemoizeMethod.get_or_extract_dependencies(self, *args, &depends_on)
21
+ dependent_memos = dependency.memos
22
+ end
23
+
18
24
  future = RedisMemo::Future.new(
19
25
  self,
20
26
  case method_id
@@ -26,7 +32,7 @@ module RedisMemo::MemoizeMethod
26
32
  method_id.call(self, *args)
27
33
  end,
28
34
  args,
29
- depends_on,
35
+ dependent_memos,
30
36
  options,
31
37
  method_name_without_memo,
32
38
  )
@@ -37,6 +43,8 @@ module RedisMemo::MemoizeMethod
37
43
  end
38
44
 
39
45
  future.execute
46
+ rescue RedisMemo::WithoutMemoization
47
+ send(method_name_without_memo, *args)
40
48
  end
41
49
 
42
50
  alias_method method_name, method_name_with_memo
@@ -52,7 +60,7 @@ module RedisMemo::MemoizeMethod
52
60
  "#{method_name} is not a memoized method"
53
61
  )
54
62
  end
55
- RedisMemo::MemoizeMethod.extract_dependencies(self, *method_args, &method_depends_on)
63
+ RedisMemo::MemoizeMethod.get_or_extract_dependencies(self, *method_args, &method_depends_on)
56
64
  end
57
65
  end
58
66
 
@@ -71,19 +79,100 @@ module RedisMemo::MemoizeMethod
71
79
  dependency
72
80
  end
73
81
 
82
+ def self.get_or_extract_dependencies(ref, *method_args, &depends_on)
83
+ if RedisMemo::Cache.local_dependency_cache
84
+ RedisMemo::Cache.local_dependency_cache[ref.class] ||= {}
85
+ RedisMemo::Cache.local_dependency_cache[ref.class][depends_on] ||= {}
86
+ named_args = exclude_anonymous_args(depends_on, ref, method_args)
87
+ RedisMemo::Cache.local_dependency_cache[ref.class][depends_on][named_args] ||= extract_dependencies(ref, *method_args, &depends_on)
88
+ else
89
+ extract_dependencies(ref, *method_args, &depends_on)
90
+ end
91
+ end
92
+
93
+ # We only look at named method parameters in the dependency block in order to define its dependent
94
+ # memos and ignore anonymous parameters, following the convention that nil or :_ is an anonymous parameter.
95
+ # Example:
96
+ # ```
97
+ # def method(param1, param2)
98
+ # end
99
+ #
100
+ # memoize_method :method do |_, _, param2|`
101
+ # depends_on RedisMemo::Memoizable.new(param2: param2)
102
+ # end
103
+ # ```
104
+ # `exclude_anonymous_args(depends_on, ref, [1, 2])` returns [2]
105
+ def self.exclude_anonymous_args(depends_on, ref, args)
106
+ return [] if depends_on.parameters.empty? or args.empty?
107
+
108
+ positional_args = []
109
+ kwargs = {}
110
+ depends_on_args = [ref] + args
111
+ options = depends_on_args.extract_options!
112
+
113
+ # Keep track of the splat start index, and the number of positional args before and after the splat,
114
+ # so we can map which args belong to positional args and which args belong to the splat.
115
+ named_splat = false
116
+ splat_index = nil
117
+ num_positional_args_after_splat = 0
118
+ num_positional_args_before_splat = 0
119
+
120
+ depends_on.parameters.each_with_index do |param, i|
121
+ # Defined by https://github.com/ruby/ruby/blob/22b8ddfd1049c3fd1e368684c4fd03bceb041b3a/proc.c#L3048-L3059
122
+ case param.first
123
+ when :opt, :req
124
+ if splat_index
125
+ num_positional_args_after_splat += 1
126
+ else
127
+ num_positional_args_before_splat += 1
128
+ end
129
+ when :rest
130
+ named_splat = is_named?(param)
131
+ splat_index = i
132
+ when :key, :keyreq
133
+ kwargs[param.last] = options[param.last] if is_named?(param)
134
+ when :keyrest
135
+ kwargs.merge!(options) if is_named?(param)
136
+ else
137
+ raise(RedisMemo::ArgumentError, "#{param.first} argument isn't supported in the dependency block")
138
+ end
139
+ end
140
+
141
+ # Determine the named positional and splat arguments after we know the # of pos. arguments before and after splat
142
+ after_splat_index = depends_on_args.size - num_positional_args_after_splat
143
+ depends_on_args.each_with_index do |arg, i|
144
+ # if the index is within the splat
145
+ if i >= num_positional_args_before_splat && i < after_splat_index
146
+ positional_args << arg if named_splat
147
+ else
148
+ j = i < num_positional_args_before_splat ? i : i - (after_splat_index - splat_index) - 1
149
+ positional_args << arg if is_named?(depends_on.parameters[j])
150
+ end
151
+ end
152
+
153
+ if !kwargs.empty?
154
+ positional_args + [kwargs]
155
+ elsif named_splat && !options.empty?
156
+ positional_args + [options]
157
+ else
158
+ positional_args
159
+ end
160
+ end
161
+ private
162
+ def self.is_named?(param)
163
+ param.size == 2 && param.last != :_
164
+ end
165
+
74
166
  def self.method_cache_keys(future_contexts)
75
167
  memos = Array.new(future_contexts.size)
76
- future_contexts.each_with_index do |(ref, _, method_args, depends_on), i|
77
- if depends_on
78
- dependency = extract_dependencies(ref, *method_args, &depends_on)
79
- memos[i] = dependency.memos
80
- end
168
+ future_contexts.each_with_index do |(_, _, dependent_memos), i|
169
+ memos[i] = dependent_memos
81
170
  end
82
171
 
83
172
  j = 0
84
173
  memo_checksums = RedisMemo::Memoizable.checksums(memos.compact)
85
174
  method_cache_key_versions = Array.new(future_contexts.size)
86
- future_contexts.each_with_index do |(_, method_id, method_args, _), i|
175
+ future_contexts.each_with_index do |(method_id, method_args, _), i|
87
176
  if memos[i]
88
177
  method_cache_key_versions[i] = [method_id, memo_checksums[j]]
89
178
  j += 1
@@ -13,6 +13,8 @@ if defined?(ActiveRecord)
13
13
  # after each record save
14
14
  def memoize_table_column(*raw_columns, editable: true)
15
15
  RedisMemo::MemoizeQuery.using_active_record!(self)
16
+ return if ENV["REDIS_MEMO_DISABLE_#{self.table_name.upcase}"] == 'true'
17
+
16
18
  columns = raw_columns.map(&:to_sym).sort
17
19
 
18
20
  RedisMemo::MemoizeQuery.memoized_columns(self, editable_only: true) << columns if editable
@@ -102,7 +104,13 @@ if defined?(ActiveRecord)
102
104
  end
103
105
  end
104
106
 
105
- def self.invalidate(record)
107
+ def self.invalidate(*records)
108
+ RedisMemo::Memoizable.invalidate(
109
+ records.map { |record| to_memos(record) }.flatten,
110
+ )
111
+ end
112
+
113
+ def self.to_memos(record)
106
114
  # Invalidate memos with current values
107
115
  memos_to_invalidate = memoized_columns(record.class).map do |columns|
108
116
  props = {}
@@ -137,7 +145,7 @@ if defined?(ActiveRecord)
137
145
  end
138
146
  end
139
147
 
140
- RedisMemo::Memoizable.invalidate(memos_to_invalidate)
148
+ memos_to_invalidate
141
149
  end
142
150
  end
143
151
  end
@@ -111,19 +111,19 @@ class RedisMemo::MemoizeQuery::CachedSelect
111
111
  sql.gsub(/(\$\d+)/, '?') # $1 -> ?
112
112
  .gsub(/((, *)*\?)+/, '?') # (?, ?, ? ...) -> (?)
113
113
  end,
114
- ) do |_, sql, name, binds, **kwargs|
115
- RedisMemo::MemoizeQuery::CachedSelect
116
- .current_query_bind_params
117
- .params
118
- .each do |model, attrs_set|
119
- attrs_set.each do |attrs|
120
- depends_on model, **attrs
121
- end
122
- end
114
+ ) do |_, sql, _, binds, **|
115
+ depends_on RedisMemo::MemoizeQuery::CachedSelect.current_query_bind_params
123
116
 
124
117
  depends_on RedisMemo::Memoizable.new(
125
118
  __redis_memo_memoize_query_memoize_query_sql__: sql,
126
- __redis_memo_memoize_query_memoize_query_binds__: binds.map(&:value_for_database),
119
+ __redis_memo_memoize_query_memoize_query_binds__: binds.map do |bind|
120
+ if bind.respond_to?(:value_for_database)
121
+ bind.value_for_database
122
+ else
123
+ # In activerecord >= 6, a bind could be an actual database value
124
+ bind
125
+ end
126
+ end
127
127
  )
128
128
  end
129
129
  end
@@ -187,6 +187,19 @@ class RedisMemo::MemoizeQuery::CachedSelect
187
187
  Thread.current[THREAD_KEY_AREL_BIND_PARAMS] = nil
188
188
  end
189
189
 
190
+ def self.with_new_query_context
191
+ prev_arel = Thread.current[THREAD_KEY_AREL]
192
+ prev_substitutes = Thread.current[THREAD_KEY_SUBSTITUTES]
193
+ prev_bind_params = Thread.current[THREAD_KEY_AREL_BIND_PARAMS]
194
+ RedisMemo::MemoizeQuery::CachedSelect.reset_current_query
195
+
196
+ yield
197
+ ensure
198
+ Thread.current[THREAD_KEY_AREL] = prev_arel
199
+ Thread.current[THREAD_KEY_SUBSTITUTES] = prev_substitutes
200
+ Thread.current[THREAD_KEY_AREL_BIND_PARAMS] = prev_bind_params
201
+ end
202
+
190
203
  private
191
204
 
192
205
  # A pre-order Depth First Search
@@ -197,7 +210,7 @@ class RedisMemo::MemoizeQuery::CachedSelect
197
210
  bind_params = BindParams.new
198
211
 
199
212
  case node
200
- when Arel::Nodes::Equality, Arel::Nodes::In
213
+ when NodeHasFilterCondition
201
214
  attr_node = node.left
202
215
  return unless attr_node.is_a?(Arel::Attributes::Attribute)
203
216
 
@@ -234,7 +247,13 @@ class RedisMemo::MemoizeQuery::CachedSelect
234
247
  }
235
248
  when Arel::Nodes::Casted
236
249
  bind_params.params[binding_relation] << {
237
- right.attribute.name.to_sym => right.val,
250
+ right.attribute.name.to_sym =>
251
+ if right.respond_to?(:val)
252
+ right.val
253
+ else
254
+ # activerecord >= 6
255
+ right.value
256
+ end,
238
257
  }
239
258
  else
240
259
  bind_params = bind_params.union(extract_bind_params_recurse(right))
@@ -330,6 +349,23 @@ class RedisMemo::MemoizeQuery::CachedSelect
330
349
  enabled_models[table_node.try(:name)]
331
350
  end
332
351
 
352
+ class NodeHasFilterCondition
353
+ def self.===(node)
354
+ case node
355
+ when Arel::Nodes::Equality, Arel::Nodes::In
356
+ true
357
+ else
358
+ # In activerecord >= 6, a new arel node HomogeneousIn is introduced
359
+ if defined?(Arel::Nodes::HomogeneousIn) &&
360
+ node.is_a?(Arel::Nodes::HomogeneousIn)
361
+ true
362
+ else
363
+ false
364
+ end
365
+ end
366
+ end
367
+ end
368
+
333
369
  # Thread locals to exchange information between RedisMemo and ActiveRecord
334
370
  THREAD_KEY_AREL = :__redis_memo_memoize_query_cached_select_arel__
335
371
  THREAD_KEY_SUBSTITUTES = :__redis_memo_memoize_query_cached_select_substitues__
@@ -33,17 +33,14 @@ class RedisMemo::MemoizeQuery::Invalidation
33
33
  # Methods that won't trigger model callbacks
34
34
  # https://guides.rubyonrails.org/active_record_callbacks.html#skipping-callbacks
35
35
  %i(
36
- import import!
37
36
  decrement_counter
38
37
  delete_all delete_by
39
38
  increment_counter
40
- insert insert! insert_all insert_all!
41
39
  touch_all
42
40
  update_column update_columns update_all update_counters
43
- upsert upsert_all
44
41
  ).each do |method_name|
45
42
  # Example: Model.update_all
46
- rewrite_bulk_update_method(
43
+ rewrite_default_method(
47
44
  model_class,
48
45
  model_class,
49
46
  method_name,
@@ -51,7 +48,7 @@ class RedisMemo::MemoizeQuery::Invalidation
51
48
  )
52
49
 
53
50
  # Example: Model.where(...).update_all
54
- rewrite_bulk_update_method(
51
+ rewrite_default_method(
55
52
  model_class,
56
53
  model_class.const_get(:ActiveRecord_Relation),
57
54
  method_name,
@@ -59,20 +56,76 @@ class RedisMemo::MemoizeQuery::Invalidation
59
56
  )
60
57
  end
61
58
 
59
+ %i(
60
+ insert insert! insert_all insert_all!
61
+ ).each do |method_name|
62
+ rewrite_insert_method(
63
+ model_class,
64
+ method_name,
65
+ )
66
+ end
67
+
68
+ %i(
69
+ upsert upsert_all
70
+ ).each do |method_name|
71
+ rewrite_upsert_method(
72
+ model_class,
73
+ method_name,
74
+ )
75
+ end
76
+
77
+ %i(
78
+ import import!
79
+ ).each do |method_name|
80
+ rewrite_import_method(
81
+ model_class,
82
+ method_name,
83
+ )
84
+ end
85
+
62
86
  model_class.class_variable_set(var_name, true)
63
87
  end
64
88
 
89
+ def self.invalidate_new_records(model_class, &blk)
90
+ current_id = model_class.maximum(model_class.primary_key)
91
+ result = blk.call
92
+ records = select_by_new_ids(model_class, current_id)
93
+ RedisMemo::MemoizeQuery.invalidate(*records) unless records.empty?
94
+ result
95
+ end
96
+
97
+ def self.invalidate_records_by_conflict_target(model_class, records:, conflict_target: nil, &blk)
98
+ if conflict_target.nil?
99
+ # When the conflict_target is not set, we are basically inserting new
100
+ # records since duplicate rows are simply skipped
101
+ return invalidate_new_records(model_class, &blk)
102
+ end
103
+
104
+ relation = build_relation_by_conflict_target(model_class, records, conflict_target)
105
+ # Invalidate records before updating
106
+ records = select_by_conflict_target_relation(model_class, relation)
107
+ RedisMemo::MemoizeQuery.invalidate(*records) unless records.empty?
108
+
109
+ # Perform updating
110
+ result = blk.call
111
+
112
+ # Invalidate records after updating
113
+ records = select_by_conflict_target_relation(model_class, relation)
114
+ RedisMemo::MemoizeQuery.invalidate(*records) unless records.empty?
115
+
116
+ result
117
+ end
118
+
65
119
  private
66
120
 
67
121
  #
68
- # There’s no good way to perform fine-grind cache invalidation when operations
69
- # are bulk update operations such as import, update_all, and destroy_all:
70
- # Performing fine-grind cache invalidation would require the applications to
71
- # fetch additional data from the database, which might lead to performance
72
- # degradation. Thus we simply invalidate all existing cached records after each
73
- # bulk_updates.
122
+ # There’s no good way to perform fine-grind cache invalidation when
123
+ # operations are bulk update operations such as update_all, and delete_all
124
+ # witout fetching additional data from the database, which might lead to
125
+ # performance degradation. Thus, by default, we simply invalidate all
126
+ # existing cached records after each bulk_updates.
74
127
  #
75
- def self.rewrite_bulk_update_method(model_class, klass, method_name, class_method:)
128
+ def self.rewrite_default_method(model_class, klass, method_name, class_method:)
76
129
  methods = class_method ? :methods : :instance_methods
77
130
  return unless klass.send(methods).include?(method_name)
78
131
 
@@ -87,4 +140,125 @@ class RedisMemo::MemoizeQuery::Invalidation
87
140
  end
88
141
  end
89
142
  end
143
+
144
+ def self.rewrite_insert_method(model_class, method_name)
145
+ return unless model_class.respond_to?(method_name)
146
+
147
+ model_class.singleton_class.class_eval do
148
+ alias_method :"#{method_name}_without_redis_memo_invalidation", method_name
149
+
150
+ define_method method_name do |*args, &blk|
151
+ RedisMemo::MemoizeQuery::Invalidation.invalidate_new_records(model_class) do
152
+ send(:"#{method_name}_without_redis_memo_invalidation", *args, &blk)
153
+ end
154
+ end
155
+ end
156
+ end
157
+
158
+ def self.rewrite_upsert_method(model_class, method_name)
159
+ return unless model_class.respond_to?(method_name)
160
+
161
+ model_class.singleton_class.class_eval do
162
+ alias_method :"#{method_name}_without_redis_memo_invalidation", method_name
163
+
164
+ define_method method_name do |attributes, unique_by: nil, **kwargs, &blk|
165
+ RedisMemo::MemoizeQuery::Invalidation.invalidate_records_by_conflict_target(
166
+ model_class,
167
+ records: nil, # not used
168
+ # upsert does not support on_duplicate_key_update yet at activerecord
169
+ # HEAD (6.1.3)
170
+ conflict_target: nil,
171
+ ) do
172
+ send(
173
+ :"#{method_name}_without_redis_memo_invalidation",
174
+ attributes,
175
+ unique_by: unique_by,
176
+ **kwargs,
177
+ &blk
178
+ )
179
+ end
180
+ end
181
+ end
182
+ end
183
+
184
+ def self.rewrite_import_method(model_class, method_name)
185
+ return unless model_class.respond_to?(method_name)
186
+
187
+ model_class.singleton_class.class_eval do
188
+ alias_method :"#{method_name}_without_redis_memo_invalidation", method_name
189
+
190
+ # For the args format, see
191
+ # https://github.com/zdennis/activerecord-import/blob/master/lib/activerecord-import/import.rb#L128
192
+ define_method method_name do |*args, &blk|
193
+ options = args.last.is_a?(Hash) ? args.last : {}
194
+ records = args[args.last.is_a?(Hash) ? -2 : -1]
195
+ on_duplicate_key_update = options[:on_duplicate_key_update]
196
+ conflict_target =
197
+ case on_duplicate_key_update
198
+ when Hash
199
+ # The conflict_target option is only supported in PostgreSQL. In
200
+ # MySQL, the primary_key is used as the conflict_target
201
+ on_duplicate_key_update[:conflict_target] || [model_class.primary_key.to_sym]
202
+ when Array
203
+ # The default conflict_target is just the primary_key
204
+ [model_class.primary_key.to_sym]
205
+ else
206
+ # Ignore duplicate rows
207
+ nil
208
+ end
209
+
210
+ if conflict_target && records.last.is_a?(Hash)
211
+ records.map! { |hash| model_class.new(hash) }
212
+ end
213
+
214
+ RedisMemo::MemoizeQuery::Invalidation.invalidate_records_by_conflict_target(
215
+ model_class,
216
+ records: records,
217
+ conflict_target: conflict_target,
218
+ ) do
219
+ send(:"#{method_name}_without_redis_memo_invalidation", *args, &blk)
220
+ end
221
+ end
222
+ end
223
+ end
224
+
225
+ def self.build_relation_by_conflict_target(model_class, records, conflict_target)
226
+ or_chain = nil
227
+
228
+ records.each do |record|
229
+ conditions = {}
230
+ conflict_target.each do |column|
231
+ conditions[column] = record.send(column)
232
+ end
233
+ if or_chain
234
+ or_chain = or_chain.or(model_class.where(conditions))
235
+ else
236
+ or_chain = model_class.where(conditions)
237
+ end
238
+ end
239
+
240
+ or_chain
241
+ end
242
+
243
+ def self.select_by_new_ids(model_class, target_id)
244
+ RedisMemo::Tracer.trace(
245
+ 'redis_memo.memoize_query.invalidation',
246
+ "#{__method__}##{model_class.name}",
247
+ ) do
248
+ RedisMemo.without_memo do
249
+ model_class.where(
250
+ model_class.arel_table[model_class.primary_key].gt(target_id),
251
+ ).to_a
252
+ end
253
+ end
254
+ end
255
+
256
+ def self.select_by_conflict_target_relation(model_class, relation)
257
+ RedisMemo::Tracer.trace(
258
+ 'redis_memo.memoize_query.invalidation',
259
+ "#{__method__}##{model_class.name}",
260
+ ) do
261
+ RedisMemo.without_memo { relation.reload }
262
+ end
263
+ end
90
264
  end
@@ -13,7 +13,7 @@ class RedisMemo::Options
13
13
  )
14
14
  @compress = compress.nil? ? true : compress
15
15
  @compress_threshold = compress_threshold || 1.kilobyte
16
- @redis = redis
16
+ @redis_config = redis
17
17
  @redis_client = nil
18
18
  @redis_error_handler = redis_error_handler
19
19
  @tracer = tracer
@@ -22,20 +22,18 @@ class RedisMemo::Options
22
22
  @expires_in = expires_in
23
23
  end
24
24
 
25
- def redis(&blk)
26
- if blk.nil?
27
- return @redis_client if @redis_client.is_a?(RedisMemo::Redis)
25
+ def redis
26
+ @redis_client ||= RedisMemo::Redis.new(redis_config)
27
+ end
28
28
 
29
- if @redis.respond_to?(:call)
30
- @redis_client = RedisMemo::Redis.new(@redis.call)
31
- elsif @redis
32
- @redis_client = RedisMemo::Redis.new(@redis)
33
- else
34
- @redis_client = RedisMemo::Redis.new
35
- end
36
- else
37
- @redis = blk
38
- end
29
+ def redis_config
30
+ @redis_config || {}
31
+ end
32
+
33
+ def redis=(config)
34
+ @redis_config = config
35
+ @redis_client = nil
36
+ redis
39
37
  end
40
38
 
41
39
  def tracer(&blk)
@@ -74,15 +72,15 @@ class RedisMemo::Options
74
72
  end
75
73
 
76
74
  attr_accessor :async
75
+ attr_accessor :cache_out_of_date_handler
76
+ attr_accessor :cache_validation_sampler
77
77
  attr_accessor :compress
78
78
  attr_accessor :compress_threshold
79
- attr_accessor :redis_error_handler
79
+ attr_accessor :connection_pool
80
80
  attr_accessor :expires_in
81
- attr_accessor :cache_validation_sampler
82
- attr_accessor :cache_out_of_date_handler
81
+ attr_accessor :redis_error_handler
83
82
 
84
83
  attr_writer :global_cache_key_version
85
- attr_writer :redis
86
84
  attr_writer :tracer
87
85
  attr_writer :logger
88
86
  end
@@ -31,7 +31,8 @@ class RedisMemo::Redis < Redis::Distributed
31
31
  end
32
32
 
33
33
  class WithReplicas < ::Redis
34
- def initialize(options)
34
+ def initialize(orig_options)
35
+ options = orig_options.dup
35
36
  primary_option = options.shift
36
37
  @replicas = options.map do |option|
37
38
  option[:logger] ||= RedisMemo::DefaultOptions.logger
@@ -11,13 +11,15 @@ class RedisMemo::Tracer
11
11
  end
12
12
  end
13
13
 
14
- def self.set_tag(cache_hit:)
14
+ def self.set_tag(**tags)
15
15
  tracer = RedisMemo::DefaultOptions.tracer
16
16
  return if tracer.nil? || !tracer.respond_to?(:active_span)
17
17
 
18
18
  active_span = tracer.active_span
19
19
  return if !active_span.respond_to?(:set_tag)
20
20
 
21
- active_span.set_tag('cache_hit', cache_hit)
21
+ tags.each do |name, value|
22
+ active_span.set_tag(name, value)
23
+ end
22
24
  end
23
25
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: redis-memo
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.0.beta.2
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chan Zuckerberg Initiative
@@ -14,44 +14,72 @@ dependencies:
14
14
  name: activesupport
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - "~>"
17
+ - - ">="
18
18
  - !ruby/object:Gem::Version
19
19
  version: '5.2'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - "~>"
24
+ - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: '5.2'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: redis
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - "~>"
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: '4'
33
+ version: 4.0.1
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - "~>"
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: 4.0.1
41
+ - !ruby/object:Gem::Dependency
42
+ name: connection_pool
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: 2.2.3
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
39
53
  - !ruby/object:Gem::Version
40
- version: '4'
54
+ version: 2.2.3
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: activerecord
43
57
  requirement: !ruby/object:Gem::Requirement
44
58
  requirements:
45
- - - "~>"
59
+ - - ">="
46
60
  - !ruby/object:Gem::Version
47
61
  version: '5.2'
48
62
  type: :development
49
63
  prerelease: false
50
64
  version_requirements: !ruby/object:Gem::Requirement
51
65
  requirements:
52
- - - "~>"
66
+ - - ">="
53
67
  - !ruby/object:Gem::Version
54
68
  version: '5.2'
69
+ - !ruby/object:Gem::Dependency
70
+ name: activerecord-import
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
55
83
  - !ruby/object:Gem::Dependency
56
84
  name: codecov
57
85
  requirement: !ruby/object:Gem::Requirement
@@ -147,6 +175,7 @@ files:
147
175
  - lib/redis_memo/after_commit.rb
148
176
  - lib/redis_memo/batch.rb
149
177
  - lib/redis_memo/cache.rb
178
+ - lib/redis_memo/connection_pool.rb
150
179
  - lib/redis_memo/future.rb
151
180
  - lib/redis_memo/memoizable.rb
152
181
  - lib/redis_memo/memoizable/dependency.rb
@@ -179,9 +208,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
179
208
  version: 2.5.0
180
209
  required_rubygems_version: !ruby/object:Gem::Requirement
181
210
  requirements:
182
- - - ">"
211
+ - - ">="
183
212
  - !ruby/object:Gem::Version
184
- version: 1.3.1
213
+ version: '0'
185
214
  requirements: []
186
215
  rubygems_version: 3.0.8
187
216
  signing_key: