redis-memo 0.0.0.beta → 0.0.0.beta.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1316982523363ce87b0ada30b82b8200559a07b2f1a6bea54168b40b2afd397e
4
- data.tar.gz: 1278e1ff6b7cc2275416c7af9b8668a13128c105a4cde74640deccd0a4380373
3
+ metadata.gz: ee7d2004ee718072b93ece80d2e98b038b9f4a21cd73632c1df4a3609f11e9e4
4
+ data.tar.gz: c8a89bb39f5b3e9f5a8b571942045c7b6b147a1d3e925f71ae611944c6aa13c9
5
5
  SHA512:
6
- metadata.gz: 121069c8de985a8977234776ad15fd209c1c0db523c361a5caf9181ed7120533350af16c354d7f76dfc03168230248e37c64b37eb8ce9f6af16cfdc9160448f3
7
- data.tar.gz: 133d18912a95eb599554b9cacb5c5932471772930165249381baaa8aeb403a3a6b21c6e17711a11eda56a1fa6dbf347396c7fe513cec3b53ec6f0c592d5b7d9a
6
+ metadata.gz: 15527040749e75dd0f27c31ecf08867cc63066b7582cdc4d618e269b2e9cb63e35601596d8f03900e55c192a20afd717c8b105426d5511df2c5db1906b4777e5
7
+ data.tar.gz: 1ab368f33e5a17209e28efbe2086a067e9f016443b28d56ece0499f557e15fcb3d801930af535d3d1fe1e1eb125b8f2595e694a71a5476c862201e5886cb826c
data/lib/redis-memo.rb CHANGED
@@ -1 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Automatically require the main file of the redis-memo gem when adding it to
4
+ # the Gemfile
1
5
  require_relative 'redis_memo'
data/lib/redis_memo.rb CHANGED
@@ -1,18 +1,53 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  require 'active_support/all'
3
4
  require 'digest'
4
5
  require 'json'
6
+ require 'securerandom'
5
7
 
6
8
  module RedisMemo
7
9
  require 'redis_memo/memoize_method'
8
- require 'redis_memo/memoize_records'
10
+ require 'redis_memo/memoize_query'
9
11
 
12
+ # A process-level +RedisMemo::Options+ instance that stores the global
13
+ # options. This object can be modified by +RedisMemo.configure+.
14
+ #
15
+ # +memoize_method+ allows users to provide method-level configuration.
16
+ # When no callsite-level configuration specified we will use the values in
17
+ # +DefaultOptions+ as the default value.
10
18
  DefaultOptions = RedisMemo::Options.new
11
19
 
20
+ # @todo Move thread keys to +RedisMemo::ThreadKey+
21
+ THREAD_KEY_WITHOUT_MEMO = :__redis_memo_without_memo__
22
+
23
+ # Configure global-level default options. Those options will be used unless
24
+ # some options specified at +memoize_method+ callsite level. See
25
+ # +RedisMemo::Options+ for all the possible options.
26
+ #
27
+ # @yieldparam [RedisMemo::Options] default_options
28
+ # +RedisMemo::DefaultOptions+
29
+ # @return [void]
12
30
  def self.configure(&blk)
13
31
  blk.call(DefaultOptions)
14
32
  end
15
33
 
34
+ # Batch Redis calls triggered by +memoize_method+ to minimize the round trips
35
+ # to Redis.
36
+ # - Batches cannot be nested
37
+ # - When a batch is still open (while still in the +RedisMemo.batch+ block)
38
+ # the return value of any memoized method is a +RedisMemo::Future+ instead of
39
+ # the actual method value
40
+ # - The actual method values are returned as a list, in the same order as
41
+ # invoking, after exiting the block
42
+ #
43
+ # @example
44
+ # results = RedisMemo.batch do
45
+ # 5.times { |i| memoized_calculation(i) }
46
+ # nil # Not the return value of the block
47
+ # end
48
+ # results.size == 5 # true
49
+ #
50
+ # See +RedisMemo::Batch+ for more information.
16
51
  def self.batch(&blk)
17
52
  RedisMemo::Batch.open
18
53
  blk.call
@@ -21,10 +56,17 @@ module RedisMemo
21
56
  RedisMemo::Batch.close
22
57
  end
23
58
 
59
+ # @todo Move this method out of the top namespace
24
60
  def self.checksum(serialized)
25
61
  Digest::SHA1.base64digest(serialized)
26
62
  end
27
63
 
64
+ # @todo Move this method out of the top namespace
65
+ def self.uuid
66
+ SecureRandom.uuid
67
+ end
68
+
69
+ # @todo Move this method out of the top namespace
28
70
  def self.deep_sort_hash(orig_hash)
29
71
  {}.tap do |new_hash|
30
72
  orig_hash.sort.each do |k, v|
@@ -33,12 +75,17 @@ module RedisMemo
33
75
  end
34
76
  end
35
77
 
36
- THREAD_KEY_WITHOUT_MEMO = :__redis_memo_without_memo__
37
-
78
+ # Whether the current execution context has been configured to skip
79
+ # memoization and use the uncached code path.
80
+ #
81
+ # @return [Boolean]
38
82
  def self.without_memo?
39
83
  Thread.current[THREAD_KEY_WITHOUT_MEMO] == true
40
84
  end
41
85
 
86
+ # Configure the wrapped code in the block to skip memoization.
87
+ #
88
+ # @yield [] no_args The block of code to skip memoization.
42
89
  def self.without_memo
43
90
  prev_value = Thread.current[THREAD_KEY_WITHOUT_MEMO]
44
91
  Thread.current[THREAD_KEY_WITHOUT_MEMO] = true
@@ -47,6 +94,8 @@ module RedisMemo
47
94
  Thread.current[THREAD_KEY_WITHOUT_MEMO] = prev_value
48
95
  end
49
96
 
97
+ # @todo Move errors to a separate file errors.rb
50
98
  class ArgumentError < ::ArgumentError; end
51
99
  class RuntimeError < ::RuntimeError; end
100
+ class WithoutMemoization < Exception; end
52
101
  end
@@ -66,7 +66,12 @@ class RedisMemo::AfterCommit
66
66
  @@pending_memo_versions.each do |key, version|
67
67
  invalidation_queue =
68
68
  RedisMemo::Memoizable::Invalidation.class_variable_get(:@@invalidation_queue)
69
- invalidation_queue << [key, version, @@previous_memo_versions[key]]
69
+
70
+ invalidation_queue << RedisMemo::Memoizable::Invalidation::Task.new(
71
+ key,
72
+ version,
73
+ @@previous_memo_versions[key],
74
+ )
70
75
  end
71
76
 
72
77
  RedisMemo::Memoizable::Invalidation.drain_invalidation_queue
@@ -1,12 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
  require_relative 'options'
3
3
  require_relative 'redis'
4
+ require_relative 'connection_pool'
4
5
 
5
6
  class RedisMemo::Cache < ActiveSupport::Cache::RedisCacheStore
6
7
  class Rescuable < Exception; end
7
8
 
8
- THREAD_KEY_LOCAL_CACHE = :__redis_memo_cache_local_cache__
9
- THREAD_KEY_RAISE_ERROR = :__redis_memo_cache_raise_error__
9
+ THREAD_KEY_LOCAL_CACHE = :__redis_memo_cache_local_cache__
10
+ THREAD_KEY_LOCAL_DEPENDENCY_CACHE = :__redis_memo_local_cache_dependency_cache__
11
+ THREAD_KEY_RAISE_ERROR = :__redis_memo_cache_raise_error__
10
12
 
11
13
  @@redis = nil
12
14
  @@redis_store = nil
@@ -23,7 +25,12 @@ class RedisMemo::Cache < ActiveSupport::Cache::RedisCacheStore
23
25
  end
24
26
 
25
27
  def self.redis
26
- @@redis ||= RedisMemo::DefaultOptions.redis
28
+ @@redis ||=
29
+ if RedisMemo::DefaultOptions.connection_pool
30
+ RedisMemo::ConnectionPool.new(**RedisMemo::DefaultOptions.connection_pool)
31
+ else
32
+ RedisMemo::DefaultOptions.redis
33
+ end
27
34
  end
28
35
 
29
36
  def self.redis_store
@@ -44,16 +51,22 @@ class RedisMemo::Cache < ActiveSupport::Cache::RedisCacheStore
44
51
  Thread.current[THREAD_KEY_LOCAL_CACHE]
45
52
  end
46
53
 
54
+ def self.local_dependency_cache
55
+ Thread.current[THREAD_KEY_LOCAL_DEPENDENCY_CACHE]
56
+ end
57
+
47
58
  class << self
48
59
  def with_local_cache(&blk)
49
60
  Thread.current[THREAD_KEY_LOCAL_CACHE] = {}
61
+ Thread.current[THREAD_KEY_LOCAL_DEPENDENCY_CACHE] = {}
50
62
  blk.call
51
63
  ensure
52
64
  Thread.current[THREAD_KEY_LOCAL_CACHE] = nil
65
+ Thread.current[THREAD_KEY_LOCAL_DEPENDENCY_CACHE] = nil
53
66
  end
54
67
 
55
68
  # RedisCacheStore doesn't read from the local cache before reading from redis
56
- def read_multi(*keys, raise_error: false)
69
+ def read_multi(*keys, raw: false, raise_error: false)
57
70
  return {} if keys.empty?
58
71
 
59
72
  Thread.current[THREAD_KEY_RAISE_ERROR] = raise_error
@@ -64,7 +77,7 @@ class RedisMemo::Cache < ActiveSupport::Cache::RedisCacheStore
64
77
  keys_to_fetch -= local_entries.keys unless local_entries.empty?
65
78
  return local_entries if keys_to_fetch.empty?
66
79
 
67
- remote_entries = redis_store.read_multi(*keys_to_fetch)
80
+ remote_entries = redis_store.read_multi(*keys_to_fetch, raw: raw)
68
81
  local_cache&.merge!(remote_entries)
69
82
 
70
83
  if local_entries.empty?
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+ require 'connection_pool'
3
+ require_relative 'redis'
4
+
5
+ class RedisMemo::ConnectionPool
6
+ def initialize(**options)
7
+ @connection_pool = ::ConnectionPool.new(**options) do
8
+ # Construct a new client every time the block gets called
9
+ RedisMemo::Redis.new(RedisMemo::DefaultOptions.redis_config)
10
+ end
11
+ end
12
+
13
+ # Avoid method_missing when possible for better performance
14
+ %i(get mget mapped_mget set eval).each do |method_name|
15
+ define_method method_name do |*args, &blk|
16
+ @connection_pool.with do |redis|
17
+ redis.send(method_name, *args, &blk)
18
+ end
19
+ end
20
+ end
21
+
22
+ def method_missing(method_name, *args, &blk)
23
+ @connection_pool.with do |redis|
24
+ redis.send(method_name, *args, &blk)
25
+ end
26
+ end
27
+ end
@@ -9,14 +9,14 @@ class RedisMemo::Future
9
9
  ref,
10
10
  method_id,
11
11
  method_args,
12
- depends_on,
12
+ dependent_memos,
13
13
  cache_options,
14
14
  method_name_without_memo
15
15
  )
16
16
  @ref = ref
17
17
  @method_id = method_id
18
18
  @method_args = method_args
19
- @depends_on = depends_on
19
+ @dependent_memos = dependent_memos
20
20
  @cache_options = cache_options
21
21
  @method_name_without_memo = method_name_without_memo
22
22
  @method_cache_key = nil
@@ -28,7 +28,7 @@ class RedisMemo::Future
28
28
  end
29
29
 
30
30
  def context
31
- [@ref, @method_id, @method_args, @depends_on]
31
+ [@method_id, @method_args, @dependent_memos]
32
32
  end
33
33
 
34
34
  def method_cache_key
@@ -1,5 +1,4 @@
1
1
  # frozen_string_literal: true
2
- require 'securerandom'
3
2
 
4
3
  class RedisMemo::Memoizable
5
4
  require_relative 'memoizable/dependency'
@@ -52,7 +51,7 @@ class RedisMemo::Memoizable
52
51
  cache_key = instance.cache_key
53
52
  RedisMemo::Memoizable::Invalidation.bump_version_later(
54
53
  cache_key,
55
- SecureRandom.uuid,
54
+ RedisMemo.uuid,
56
55
  )
57
56
  end
58
57
 
@@ -83,7 +82,11 @@ class RedisMemo::Memoizable
83
82
  if keys_to_fetch.empty?
84
83
  {}
85
84
  else
86
- RedisMemo::Cache.read_multi(*keys_to_fetch, raise_error: true)
85
+ RedisMemo::Cache.read_multi(
86
+ *keys_to_fetch,
87
+ raw: true,
88
+ raise_error: true,
89
+ )
87
90
  end
88
91
  memo_versions.merge!(cached_versions) unless cached_versions.empty?
89
92
 
@@ -96,7 +99,7 @@ class RedisMemo::Memoizable
96
99
  # cached result.
97
100
  need_to_bump_versions = true
98
101
 
99
- new_version = SecureRandom.uuid
102
+ new_version = RedisMemo.uuid
100
103
  RedisMemo::Memoizable::Invalidation.bump_version_later(
101
104
  key,
102
105
  new_version,
@@ -12,25 +12,55 @@ class RedisMemo::Memoizable::Dependency
12
12
  @nodes.values
13
13
  end
14
14
 
15
- def depends_on(memo_or_model, **conditions)
16
- if !memo_or_model.is_a?(RedisMemo::Memoizable)
15
+ def depends_on(dependency, **conditions)
16
+ case dependency
17
+ when self.class
18
+ nodes.merge!(dependency.nodes)
19
+ when RedisMemo::Memoizable
20
+ memo = dependency
21
+ return if nodes.include?(memo.cache_key)
22
+ nodes[memo.cache_key] = memo
23
+
24
+ if memo.depends_on
25
+ # Extract dependencies from the current memoizable and recurse
26
+ instance_exec(&memo.depends_on)
27
+ end
28
+ when ActiveRecord::Relation
29
+ extracted = extract_dependencies_for_relation(dependency)
30
+ nodes.merge!(extracted.nodes)
31
+ when UsingActiveRecord
17
32
  [
18
- memo_or_model.redis_memo_class_memoizable,
19
- RedisMemo::MemoizeRecords.create_memo(memo_or_model, **conditions),
33
+ dependency.redis_memo_class_memoizable,
34
+ RedisMemo::MemoizeQuery.create_memo(dependency, **conditions),
20
35
  ].each do |memo|
21
36
  nodes[memo.cache_key] = memo
22
37
  end
23
-
24
- return
38
+ else
39
+ raise(
40
+ RedisMemo::ArgumentError,
41
+ "Invalid dependency #{dependency}"
42
+ )
25
43
  end
44
+ end
26
45
 
27
- memo = memo_or_model
28
- return if nodes.include?(memo.cache_key)
29
- nodes[memo.cache_key] = memo
46
+ def extract_dependencies_for_relation(relation)
47
+ # Extract the dependent memos of an Arel without calling exec_query to actually execute the query
48
+ RedisMemo::MemoizeQuery::CachedSelect.with_new_query_context do
49
+ connection = ActiveRecord::Base.connection
50
+ query, binds, _ = connection.send(:to_sql_and_binds, relation.arel)
51
+ RedisMemo::MemoizeQuery::CachedSelect.current_query = relation.arel
52
+ is_query_cached = RedisMemo::MemoizeQuery::CachedSelect.extract_bind_params(query)
53
+ raise(
54
+ RedisMemo::WithoutMemoization,
55
+ "Arel query is not cached using RedisMemo."
56
+ ) unless is_query_cached
57
+ extracted_dependency = connection.dependency_of(:exec_query, query, nil, binds)
58
+ end
59
+ end
30
60
 
31
- if memo.depends_on
32
- # Extract dependencies from the current memoizable and recurse
33
- instance_exec(&memo.depends_on)
61
+ class UsingActiveRecord
62
+ def self.===(dependency)
63
+ RedisMemo::MemoizeQuery.using_active_record?(dependency)
34
64
  end
35
65
  end
36
66
  end
@@ -3,12 +3,29 @@ require_relative '../after_commit'
3
3
  require_relative '../cache'
4
4
 
5
5
  module RedisMemo::Memoizable::Invalidation
6
- # This is a thread safe data structure
7
- #
8
- # Handle transient network errors during cache invalidation
9
- # Each item in the queue is a tuple:
10
- #
11
- # [key, version (a UUID), previous_version (a UUID)]
6
+ class Task
7
+ attr_reader :key
8
+ attr_reader :version
9
+ attr_reader :previous_version
10
+
11
+ def initialize(key, version, previous_version)
12
+ @key = key
13
+ @version = version
14
+ @previous_version = previous_version
15
+ @created_at = current_timestamp
16
+ end
17
+
18
+ def current_timestamp
19
+ Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond)
20
+ end
21
+
22
+ def duration
23
+ current_timestamp - @created_at
24
+ end
25
+ end
26
+
27
+ # This is a thread safe data structure to handle transient network errors
28
+ # during cache invalidation
12
29
  #
13
30
  # When an invalidation call arrives at Redis, we only bump to the specified
14
31
  # version (so the cached results using that version will become visible) if
@@ -28,8 +45,6 @@ module RedisMemo::Memoizable::Invalidation
28
45
  @@invalidation_queue = Queue.new
29
46
 
30
47
  def self.bump_version_later(key, version, previous_version: nil)
31
- RedisMemo::DefaultOptions.logger&.info("[received] Bump memo key #{key}")
32
-
33
48
  if RedisMemo::AfterCommit.in_transaction?
34
49
  previous_version ||= RedisMemo::AfterCommit.pending_memo_versions[key]
35
50
  end
@@ -41,7 +56,10 @@ module RedisMemo::Memoizable::Invalidation
41
56
  # Fill an expected previous version so the later calculation results
42
57
  # based on this version can still be rolled out if this version
43
58
  # does not change
44
- previous_version ||= RedisMemo::Cache.read_multi(key)[key]
59
+ previous_version ||= RedisMemo::Cache.read_multi(
60
+ key,
61
+ raw: true,
62
+ )[key]
45
63
  end
46
64
 
47
65
  local_cache&.send(:[]=, key, version)
@@ -52,7 +70,7 @@ module RedisMemo::Memoizable::Invalidation
52
70
  previous_version: previous_version,
53
71
  )
54
72
  else
55
- @@invalidation_queue << [key, version, previous_version]
73
+ @@invalidation_queue << Task.new(key, version, previous_version)
56
74
  end
57
75
  end
58
76
 
@@ -89,31 +107,32 @@ module RedisMemo::Memoizable::Invalidation
89
107
  return redis.call('set', key, new_version, unpack(px))
90
108
  LUA
91
109
 
92
- def self.bump_version(cache_key, version, previous_version:)
93
- RedisMemo::Tracer.trace('redis_memo.memoizable.bump_version', nil) do
110
+ def self.bump_version(task)
111
+ RedisMemo::Tracer.trace('redis_memo.memoizable.bump_version', task.key) do
94
112
  ttl = RedisMemo::DefaultOptions.expires_in
95
113
  ttl = (ttl * 1000.0).to_i if ttl
96
114
  RedisMemo::Cache.redis.eval(
97
115
  LUA_BUMP_VERSION,
98
- keys: [cache_key],
99
- argv: [previous_version, version, SecureRandom.uuid, ttl],
116
+ keys: [task.key],
117
+ argv: [task.previous_version, task.version, RedisMemo.uuid, ttl],
100
118
  )
119
+ RedisMemo::Tracer.set_tag(enqueue_to_finish: task.duration)
101
120
  end
102
- RedisMemo::DefaultOptions.logger&.info("[performed] Bump memo key #{cache_key}")
103
121
  end
104
122
 
105
123
  def self.drain_invalidation_queue_now
106
124
  retry_queue = []
107
125
  until @@invalidation_queue.empty?
108
- tuple = @@invalidation_queue.pop
126
+ task = @@invalidation_queue.pop
109
127
  begin
110
- bump_version(tuple[0], tuple[1], previous_version: tuple[2])
111
- rescue SignalException, Redis::BaseConnectionError
112
- retry_queue << tuple
128
+ bump_version(task)
129
+ rescue SignalException, Redis::BaseConnectionError,
130
+ ::ConnectionPool::TimeoutError
131
+ retry_queue << task
113
132
  end
114
133
  end
115
134
  ensure
116
- retry_queue.each { |t| @@invalidation_queue << t }
135
+ retry_queue.each { |task| @@invalidation_queue << task }
117
136
  end
118
137
 
119
138
  at_exit do