redis_queued_locks 1.18.0 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.claude/project-overview.md +65 -13
- data/.claude/rules/acquirer.md +29 -11
- data/.claude/rules/arguments.md +3 -2
- data/.claude/rules/git.md +3 -2
- data/.claude/rules/logic.md +4 -3
- data/.claude/rules/swarm.md +12 -7
- data/.claude/rules/tests.md +6 -0
- data/.claude/rules/type-checking.md +2 -1
- data/.claude/rules/visitors.md +1 -1
- data/.claude/skills/rql-code-review/SKILL.md +372 -0
- data/.claude/skills/rql-code-review/templates/report.en.md +31 -0
- data/.claude/skills/rql-code-review/templates/report.ru.md +31 -0
- data/CHANGELOG.md +74 -0
- data/CLAUDE.md +4 -2
- data/README.md +338 -24
- data/lib/redis_queued_locks/acquirer/acquire_lock/try_to_lock.rb +684 -128
- data/lib/redis_queued_locks/acquirer/acquire_lock/yield_expire.rb +3 -1
- data/lib/redis_queued_locks/acquirer/acquire_lock.rb +24 -3
- data/lib/redis_queued_locks/acquirer/clear_dead_requests.rb +145 -18
- data/lib/redis_queued_locks/acquirer/extend_lock_ttl.rb +71 -10
- data/lib/redis_queued_locks/acquirer/keys.rb +142 -7
- data/lib/redis_queued_locks/acquirer/lock_info.rb +10 -4
- data/lib/redis_queued_locks/acquirer/lock_series_poc.rb +26 -7
- data/lib/redis_queued_locks/acquirer/locks.rb +195 -83
- data/lib/redis_queued_locks/acquirer/queue_info.rb +12 -18
- data/lib/redis_queued_locks/acquirer/queues.rb +114 -50
- data/lib/redis_queued_locks/acquirer/reindex_locks.rb +140 -0
- data/lib/redis_queued_locks/acquirer/release_all_locks.rb +147 -58
- data/lib/redis_queued_locks/acquirer/release_lock.rb +52 -19
- data/lib/redis_queued_locks/acquirer/release_locks_of.rb +235 -66
- data/lib/redis_queued_locks/acquirer.rb +1 -0
- data/lib/redis_queued_locks/client.rb +116 -16
- data/lib/redis_queued_locks/config.rb +5 -1
- data/lib/redis_queued_locks/resource.rb +99 -2
- data/lib/redis_queued_locks/swarm/flush_zombies.rb +332 -61
- data/lib/redis_queued_locks/swarm/zombie_info.rb +203 -53
- data/lib/redis_queued_locks/swarm.rb +15 -4
- data/lib/redis_queued_locks/version.rb +2 -2
- data/sig/redis_queued_locks/acquirer/acquire_lock/try_to_lock.rbs +107 -1
- data/sig/redis_queued_locks/acquirer/acquire_lock/yield_expire.rbs +1 -1
- data/sig/redis_queued_locks/acquirer/acquire_lock.rbs +2 -0
- data/sig/redis_queued_locks/acquirer/clear_dead_requests.rbs +10 -1
- data/sig/redis_queued_locks/acquirer/extend_lock_ttl.rbs +5 -2
- data/sig/redis_queued_locks/acquirer/keys.rbs +11 -1
- data/sig/redis_queued_locks/acquirer/locks.rbs +15 -6
- data/sig/redis_queued_locks/acquirer/queues.rbs +9 -2
- data/sig/redis_queued_locks/acquirer/reindex_locks.rbs +31 -0
- data/sig/redis_queued_locks/acquirer/release_all_locks.rbs +8 -1
- data/sig/redis_queued_locks/acquirer/release_lock.rbs +3 -1
- data/sig/redis_queued_locks/acquirer/release_locks_of.rbs +20 -1
- data/sig/redis_queued_locks/client.rbs +9 -0
- data/sig/redis_queued_locks/resource.rbs +6 -0
- data/sig/redis_queued_locks/swarm/flush_zombies.rbs +51 -1
- data/sig/redis_queued_locks/swarm/zombie_info.rbs +58 -7
- data/sig/vendor/redis_client.rbs +1 -0
- metadata +6 -1
|
@@ -2,25 +2,32 @@
|
|
|
2
2
|
|
|
3
3
|
# @api private
|
|
4
4
|
# @since 1.0.0
|
|
5
|
-
# @version 1.
|
|
5
|
+
# @version 1.20.0
|
|
6
6
|
module RedisQueuedLocks::Acquirer::Queues
|
|
7
7
|
class << self
|
|
8
8
|
# @param redis_client [RedisClient]
|
|
9
9
|
# @option scan_size [Integer]
|
|
10
10
|
# @option with_info [Boolean]
|
|
11
|
+
# @option lock_index_enabled [Boolean]
|
|
12
|
+
# Find lock queues via the lock index (`index_queues`) instead of the keyspace SCAN
|
|
13
|
+
# (`scan_queues`).
|
|
11
14
|
# @return [Set<String>,Set<Hash<Symbol,Any>>]
|
|
12
15
|
#
|
|
13
16
|
# @api private
|
|
14
17
|
# @since 1.0.0
|
|
15
|
-
|
|
18
|
+
# @version 1.20.0
|
|
19
|
+
def queues(redis_client, scan_size:, with_info:, lock_index_enabled:)
|
|
16
20
|
redis_client.with do |rconn|
|
|
17
|
-
lock_queues =
|
|
18
|
-
|
|
21
|
+
lock_queues =
|
|
22
|
+
lock_index_enabled ? index_queues(rconn, scan_size) : scan_queues(rconn, scan_size)
|
|
23
|
+
with_info ? extract_queues_info(rconn, lock_queues, scan_size) : lock_queues
|
|
19
24
|
end
|
|
20
25
|
end
|
|
21
26
|
|
|
22
27
|
private
|
|
23
28
|
|
|
29
|
+
# Write and read lock queues in one SCAN pass (the disabled lock index).
|
|
30
|
+
#
|
|
24
31
|
# @param redis_client [RedisClient]
|
|
25
32
|
# @param scan_size [Integer]
|
|
26
33
|
# @return [Set<String>]
|
|
@@ -28,80 +35,137 @@ module RedisQueuedLocks::Acquirer::Queues
|
|
|
28
35
|
#
|
|
29
36
|
# @api private
|
|
30
37
|
# @since 1.0.0
|
|
31
|
-
# @version 1.
|
|
38
|
+
# @version 1.20.0
|
|
32
39
|
def scan_queues(redis_client, scan_size)
|
|
33
40
|
Set.new.tap do |lock_queues|
|
|
34
41
|
# @type var lock_queues: Set[String]
|
|
35
42
|
redis_client.scan(
|
|
36
43
|
'MATCH',
|
|
37
|
-
RedisQueuedLocks::Resource::
|
|
44
|
+
RedisQueuedLocks::Resource::ALL_LOCK_QUEUES_PATTERN,
|
|
38
45
|
count: scan_size
|
|
39
|
-
) do |
|
|
40
|
-
#
|
|
41
|
-
|
|
42
|
-
lock_queues.add(lock_queue
|
|
46
|
+
) do |key|
|
|
47
|
+
# NOTE: the pattern matches other keys too (lock names with `queue:`): lock queues only
|
|
48
|
+
family = RedisQueuedLocks::Resource.lock_key_family(key)
|
|
49
|
+
lock_queues.add(key) if family == :lock_queue || family == :read_lock_queue
|
|
43
50
|
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
44
53
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
+
# Write and read lock queues of the lock index (the enabled lock index): one ZSCAN pass over
|
|
55
|
+
# the lock names (`RedisQueuedLocks::Resource::LOCK_INDEX_KEY`, the keyspace is never scanned)
|
|
56
|
+
# with one round trip per batch of `scan_size` lock names (`EXISTS` of both lock queues of
|
|
57
|
+
# each lock name).
|
|
58
|
+
#
|
|
59
|
+
# @param redis_client [RedisClient]
|
|
60
|
+
# @param scan_size [Integer] ZSCAN count and the number of lock names in one batch.
|
|
61
|
+
# @return [Set<String>]
|
|
62
|
+
# - (RW) queues of read lock requests are included;
|
|
63
|
+
#
|
|
64
|
+
# @api private
|
|
65
|
+
# @since 1.20.0
|
|
66
|
+
def index_queues(redis_client, scan_size)
|
|
67
|
+
Set.new.tap do |lock_queues|
|
|
68
|
+
# @type var lock_queues: Set[String]
|
|
69
|
+
each_lock_name_batch(redis_client, scan_size) do |lock_names|
|
|
70
|
+
# NOTE: (pipeline layout) two replies per lock name in the batch order:
|
|
71
|
+
# EXISTS (write lock queue), EXISTS (read lock queue);
|
|
72
|
+
# @type var queues_existence: Array[Integer]
|
|
73
|
+
queues_existence = redis_client.pipelined do |pipeline|
|
|
74
|
+
lock_names.each do |lock_name|
|
|
75
|
+
pipeline.call('EXISTS', RedisQueuedLocks::Resource.prepare_lock_queue(lock_name))
|
|
76
|
+
pipeline.call(
|
|
77
|
+
'EXISTS', RedisQueuedLocks::Resource.prepare_read_lock_queue(lock_name)
|
|
78
|
+
)
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
lock_names.each_with_index do |lock_name, index|
|
|
83
|
+
if queues_existence[index * 2] == 1
|
|
84
|
+
lock_queues.add(RedisQueuedLocks::Resource.prepare_lock_queue(lock_name))
|
|
85
|
+
end
|
|
86
|
+
if queues_existence[(index * 2) + 1] == 1
|
|
87
|
+
lock_queues.add(RedisQueuedLocks::Resource.prepare_read_lock_queue(lock_name))
|
|
88
|
+
end
|
|
89
|
+
end
|
|
54
90
|
end
|
|
55
91
|
end
|
|
56
92
|
end
|
|
57
93
|
|
|
94
|
+
# Iterates over the lock names of the lock index (one ZSCAN pass) in batches of `scan_size`
|
|
95
|
+
# lock names (batches are collected across ZSCAN pages). Expired lock names (not dropped from
|
|
96
|
+
# the index yet) are skipped: all their keys are expired.
|
|
97
|
+
#
|
|
98
|
+
# NOTE: each operation module iterates over the lock index by itself (duplicated on purpose:
|
|
99
|
+
# operation modules are independent of each other, see `Locks.each_lock_name_batch`);
|
|
100
|
+
#
|
|
101
|
+
# @param redis_client [RedisClient]
|
|
102
|
+
# @param scan_size [Integer] ZSCAN count and the number of lock names in one batch.
|
|
103
|
+
# @yield [lock_names]
|
|
104
|
+
# @yieldparam lock_names [Array<String>] The batch of lock names (reused between batches).
|
|
105
|
+
# @return [void]
|
|
106
|
+
#
|
|
107
|
+
# @api private
|
|
108
|
+
# @since 1.20.0
|
|
109
|
+
def each_lock_name_batch(redis_client, scan_size)
|
|
110
|
+
lock_index_time = RedisQueuedLocks::Resource.lock_index_time
|
|
111
|
+
# @type var lock_names: Array[String]
|
|
112
|
+
lock_names = []
|
|
113
|
+
|
|
114
|
+
redis_client.zscan(
|
|
115
|
+
RedisQueuedLocks::Resource::LOCK_INDEX_KEY, count: scan_size
|
|
116
|
+
) do |lock_name, lock_index_score|
|
|
117
|
+
next if lock_index_score.to_f < lock_index_time
|
|
118
|
+
|
|
119
|
+
lock_names << lock_name
|
|
120
|
+
next if lock_names.size < scan_size
|
|
121
|
+
|
|
122
|
+
yield(lock_names)
|
|
123
|
+
lock_names.clear
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
yield(lock_names) unless lock_names.empty?
|
|
127
|
+
end
|
|
128
|
+
|
|
58
129
|
# @param redis_client [RedisClient]
|
|
59
130
|
# @param lock_queues [Set<String>]
|
|
131
|
+
# @param scan_size [Integer] The number of queues whose requests are read in one round trip.
|
|
60
132
|
# @return [Set<Hash<Symbol,Any>>]
|
|
61
133
|
# - each request: `{ 'acq_id' => String, 'score' => Float, 'rw_mode' => 'read'/'write' }`;
|
|
62
134
|
#
|
|
63
135
|
# @api private
|
|
64
136
|
# @since 1.0.0
|
|
65
|
-
# @version 1.
|
|
66
|
-
def extract_queues_info(redis_client, lock_queues)
|
|
137
|
+
# @version 1.20.0
|
|
138
|
+
def extract_queues_info(redis_client, lock_queues, scan_size)
|
|
67
139
|
# NOTE: the same approach as in `QueueInfo` (duplicated on purpose: public operation modules
|
|
68
140
|
# do not reuse each other, they share logic via `Resource`, `Utilities`);
|
|
69
141
|
Set.new.tap do |seeded_queues|
|
|
70
|
-
# Step X: iterate over each lock queue and extract their info
|
|
71
142
|
# @type var seeded_queues: Set[Hash[Symbol,untyped]]
|
|
72
|
-
lock_queues.
|
|
73
|
-
#
|
|
74
|
-
# -
|
|
75
|
-
#
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
end
|
|
83
|
-
# @type var result: [Integer, Array[[String,Float]]]
|
|
143
|
+
lock_queues.each_slice(scan_size) do |lock_queues_batch|
|
|
144
|
+
# Step 1: extract the requests of the batch of lock queues in one round trip
|
|
145
|
+
# - redis deletes empty sorted sets, so a queue that does not exist anymore gives
|
|
146
|
+
# an empty ZRANGE result (no need for EXISTS checks);
|
|
147
|
+
# - (pipeline layout) one reply per queue in the batch order;
|
|
148
|
+
# @type var queues_requests: Array[Array[[String,Float]]]
|
|
149
|
+
queues_requests = redis_client.pipelined do |pipeline|
|
|
150
|
+
lock_queues_batch.each do |lock_queue|
|
|
151
|
+
pipeline.call('ZRANGE', lock_queue, '0', '-1', 'WITHSCORES')
|
|
152
|
+
end
|
|
153
|
+
end
|
|
84
154
|
|
|
85
|
-
|
|
86
|
-
|
|
155
|
+
lock_queues_batch.each_with_index do |lock_queue, index|
|
|
156
|
+
# NOTE:
|
|
157
|
+
# - (RW) the mode of the requests of the queue (`'read'` or `'write'`);
|
|
158
|
+
# - lock queues are scanned by the lock queue patterns, so the mode is never `nil`;
|
|
159
|
+
rw_mode = RedisQueuedLocks::Resource.lock_queue_rw_mode(lock_queue) #: String
|
|
87
160
|
|
|
88
|
-
|
|
89
|
-
|
|
161
|
+
# Step 2: format the result and push it to the result store
|
|
162
|
+
seeded_queues << {
|
|
163
|
+
queue: lock_queue,
|
|
164
|
+
requests: queues_requests[index].map do |val|
|
|
90
165
|
{ 'acq_id' => val[0], 'score' => val[1], 'rw_mode' => rw_mode }
|
|
91
166
|
end
|
|
92
|
-
|
|
93
|
-
# Step 2.Y: lock queue did not exist during the pipeline invocation
|
|
94
|
-
[] #: Array[Hash[String,String|Float]]
|
|
95
|
-
end
|
|
167
|
+
}
|
|
96
168
|
end
|
|
97
|
-
|
|
98
|
-
# @type var queue_info: Array[Hash[String,String|Float]]
|
|
99
|
-
|
|
100
|
-
# Step 3: push the lock queue info to the result store
|
|
101
|
-
seeded_queues << {
|
|
102
|
-
queue: lock_queue,
|
|
103
|
-
requests: queue_info
|
|
104
|
-
}
|
|
105
169
|
end
|
|
106
170
|
end
|
|
107
171
|
end
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# @api private
|
|
4
|
+
# @since 1.20.0
|
|
5
|
+
module RedisQueuedLocks::Acquirer::ReindexLocks
|
|
6
|
+
class << self
|
|
7
|
+
# Adds the lock keys without the lock index entry (keys of older RQL versions created before
|
|
8
|
+
# the lock index) to the lock index (`RedisQueuedLocks::Resource::LOCK_INDEX_KEY`):
|
|
9
|
+
# - one SCAN pass over the RQL keys (`KEY_PATTERN`) with pipelined batches of `scan_size`
|
|
10
|
+
# keys collected across SCAN pages (two round trips per batch: key TTLs, then the index);
|
|
11
|
+
# - write locks and readers registries are indexed with their remaining TTL (`PTTL`):
|
|
12
|
+
# the score is the client time + the remaining TTL + the grace period;
|
|
13
|
+
# - lock queues without TTL (older RQL versions) get the queue TTL first (`PEXPIRE NX`):
|
|
14
|
+
# the lifetime of every indexed key is bounded;
|
|
15
|
+
# - read lock data is not indexed: it is found via its readers registry;
|
|
16
|
+
# - keys without TTL (unreachable for RQL keys) are indexed forever (`+inf` score);
|
|
17
|
+
# - the index score is never decreased (`ZADD GT`): the operation is idempotent and can be
|
|
18
|
+
# invoked at any time (keys of the current RQL version are indexed already);
|
|
19
|
+
#
|
|
20
|
+
# @param redis_client [RedisClient]
|
|
21
|
+
# @param scan_size [Integer]
|
|
22
|
+
# @param queue_ttl [Integer] TTL of the lock queues without TTL (in seconds).
|
|
23
|
+
# @param lock_index_grace_period [Integer]
|
|
24
|
+
# The lock index score margin over the key lifetime (in milliseconds).
|
|
25
|
+
# @return [Hash<Symbol,Boolean|Hash<Symbol,Integer>>]
|
|
26
|
+
# Format: { ok: true, result: { idx_key_cnt: Integer } } (the number of indexed keys)
|
|
27
|
+
#
|
|
28
|
+
# @api private
|
|
29
|
+
# @since 1.20.0
|
|
30
|
+
def reindex_locks(redis_client, scan_size, queue_ttl, lock_index_grace_period)
|
|
31
|
+
# NOTE: `ceil` - queue ttl can be passed as a Float (in seconds), PEXPIRE requires Integer
|
|
32
|
+
queue_pttl = (queue_ttl * 1_000).ceil
|
|
33
|
+
idx_key_cnt = 0
|
|
34
|
+
|
|
35
|
+
redis_client.with do |rconn|
|
|
36
|
+
# NOTE: write locks and readers registries of the batch
|
|
37
|
+
# @type var lock_keys: Array[String]
|
|
38
|
+
lock_keys = []
|
|
39
|
+
# @type var lock_queues: Array[String]
|
|
40
|
+
lock_queues = []
|
|
41
|
+
|
|
42
|
+
rconn.scan('MATCH', RedisQueuedLocks::Resource::KEY_PATTERN, count: scan_size) do |key|
|
|
43
|
+
case RedisQueuedLocks::Resource.lock_key_family(key)
|
|
44
|
+
when :lock, :lock_readers then lock_keys << key
|
|
45
|
+
when :lock_queue, :read_lock_queue then lock_queues << key
|
|
46
|
+
else next # NOTE: read lock data and keys out of the lock key families
|
|
47
|
+
end
|
|
48
|
+
next if (lock_keys.size + lock_queues.size) < scan_size
|
|
49
|
+
|
|
50
|
+
idx_key_cnt += index_lock_keys(
|
|
51
|
+
rconn, lock_keys, lock_queues, queue_pttl, lock_index_grace_period
|
|
52
|
+
)
|
|
53
|
+
lock_keys.clear
|
|
54
|
+
lock_queues.clear
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
unless lock_keys.empty? && lock_queues.empty?
|
|
58
|
+
idx_key_cnt += index_lock_keys(
|
|
59
|
+
rconn, lock_keys, lock_queues, queue_pttl, lock_index_grace_period
|
|
60
|
+
)
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
{ ok: true, result: { idx_key_cnt: } }
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
# Adds the given batch of lock keys to the lock index (two round trips: key TTLs, then
|
|
70
|
+
# the index entries).
|
|
71
|
+
#
|
|
72
|
+
# @param redis_client [RedisClient]
|
|
73
|
+
# @param lock_keys [Array<String>] Write locks and readers registries.
|
|
74
|
+
# @param lock_queues [Array<String>] Lock queues (write and read).
|
|
75
|
+
# @param queue_pttl [Integer] TTL of the lock queues without TTL (in milliseconds).
|
|
76
|
+
# @param lock_index_grace_period [Integer] In milliseconds.
|
|
77
|
+
# @return [Integer] The number of indexed keys (keys deleted after the SCAN are not indexed).
|
|
78
|
+
#
|
|
79
|
+
# @api private
|
|
80
|
+
# @since 1.20.0
|
|
81
|
+
def index_lock_keys(redis_client, lock_keys, lock_queues, queue_pttl, lock_index_grace_period)
|
|
82
|
+
# NOTE: (pipeline layout) `PEXPIRE NX` + `PTTL` for each lock queue (two replies per queue),
|
|
83
|
+
# then `PTTL` for each lock key, in the batch order;
|
|
84
|
+
# @type var key_ttls: Array[Integer]
|
|
85
|
+
key_ttls = redis_client.pipelined do |pipeline|
|
|
86
|
+
lock_queues.each do |lock_queue|
|
|
87
|
+
pipeline.call('PEXPIRE', lock_queue, queue_pttl, 'NX')
|
|
88
|
+
pipeline.call('PTTL', lock_queue)
|
|
89
|
+
end
|
|
90
|
+
lock_keys.each { |lock_key| pipeline.call('PTTL', lock_key) }
|
|
91
|
+
end
|
|
92
|
+
# NOTE: the client time after the TTLs are received: the score is not less than the key
|
|
93
|
+
# expiration time (+ the grace period for the client clock skew);
|
|
94
|
+
lock_index_time = RedisQueuedLocks::Resource.lock_index_time
|
|
95
|
+
indexed_keys_count = 0
|
|
96
|
+
|
|
97
|
+
redis_client.pipelined do |pipeline|
|
|
98
|
+
lock_queues.each_with_index do |lock_queue, index|
|
|
99
|
+
indexed_keys_count += index_lock_key(
|
|
100
|
+
pipeline, lock_queue, key_ttls[(index * 2) + 1], lock_index_time,
|
|
101
|
+
lock_index_grace_period
|
|
102
|
+
)
|
|
103
|
+
end
|
|
104
|
+
lock_keys.each_with_index do |lock_key, index|
|
|
105
|
+
indexed_keys_count += index_lock_key(
|
|
106
|
+
pipeline, lock_key, key_ttls[(lock_queues.size * 2) + index], lock_index_time,
|
|
107
|
+
lock_index_grace_period
|
|
108
|
+
)
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
indexed_keys_count
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Adds the lock name of the given key to the lock index (`ZADD GT`).
|
|
116
|
+
#
|
|
117
|
+
# @param pipeline [RedisClient::Pipeline]
|
|
118
|
+
# @param key [String] Write lock, readers registry or lock queue.
|
|
119
|
+
# @param key_ttl [Integer] The remaining key TTL (`PTTL`: `-2` - no key, `-1` - no TTL).
|
|
120
|
+
# @param lock_index_time [Integer] Epoch, in milliseconds.
|
|
121
|
+
# @param lock_index_grace_period [Integer] In milliseconds.
|
|
122
|
+
# @return [Integer] `1` - the key is indexed, `0` - the key does not exist anymore.
|
|
123
|
+
#
|
|
124
|
+
# @api private
|
|
125
|
+
# @since 1.20.0
|
|
126
|
+
def index_lock_key(pipeline, key, key_ttl, lock_index_time, lock_index_grace_period)
|
|
127
|
+
return 0 if key_ttl == -2
|
|
128
|
+
|
|
129
|
+
lock_name = RedisQueuedLocks::Resource.lock_name_from_key(
|
|
130
|
+
key, RedisQueuedLocks::Resource.lock_key_family(key)
|
|
131
|
+
)
|
|
132
|
+
lock_index_score =
|
|
133
|
+
(key_ttl == -1) ? '+inf' : (lock_index_time + key_ttl + lock_index_grace_period)
|
|
134
|
+
pipeline.call(
|
|
135
|
+
'ZADD', RedisQueuedLocks::Resource::LOCK_INDEX_KEY, 'GT', lock_index_score, lock_name
|
|
136
|
+
)
|
|
137
|
+
1
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|
|
@@ -2,20 +2,29 @@
|
|
|
2
2
|
|
|
3
3
|
# @api private
|
|
4
4
|
# @since 1.0.0
|
|
5
|
+
# @version 1.20.0
|
|
6
|
+
# rubocop:disable Metrics/ModuleLength
|
|
5
7
|
module RedisQueuedLocks::Acquirer::ReleaseAllLocks
|
|
6
8
|
# @since 1.0.0
|
|
7
9
|
extend RedisQueuedLocks::Utilities
|
|
8
10
|
|
|
11
|
+
# rubocop:disable Metrics/ClassLength
|
|
9
12
|
class << self
|
|
10
|
-
# Release all locks:
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
13
|
+
# Release all locks: delete the keys of all lock families (write locks, lock queues,
|
|
14
|
+
# readers registries, read lock queues and read lock data):
|
|
15
|
+
# - the disabled lock index: the keys found in one SCAN pass, with one `DEL` per batch of
|
|
16
|
+
# keys;
|
|
17
|
+
# - the enabled lock index: the keys of the locks of the lock index (one ZSCAN pass), their
|
|
18
|
+
# lock names are dropped from the lock index, with one transaction (`DEL` + `ZREM`) per
|
|
19
|
+
# batch of lock names;
|
|
14
20
|
#
|
|
15
21
|
# @param redis [RedisClient]
|
|
16
22
|
# Redis connection client.
|
|
17
23
|
# @param batch_size [Integer]
|
|
18
|
-
# The number of lock keys
|
|
24
|
+
# The number of lock keys (the disabled lock index) or locks (lock names, the enabled lock
|
|
25
|
+
# index) that should be released in a time.
|
|
26
|
+
# @param lock_index_enabled [Boolean]
|
|
27
|
+
# Find locks via the lock index instead of the keyspace SCAN.
|
|
19
28
|
# @param logger [::Logger,#debug]
|
|
20
29
|
# - Logger object used from `configuration` layer (see config['logger']);
|
|
21
30
|
# - See RedisQueuedLocks::Logging::VoidLogger for example;
|
|
@@ -72,10 +81,11 @@ module RedisQueuedLocks::Acquirer::ReleaseAllLocks
|
|
|
72
81
|
#
|
|
73
82
|
# @api private
|
|
74
83
|
# @since 1.0.0
|
|
75
|
-
# @version 1.
|
|
84
|
+
# @version 1.20.0
|
|
76
85
|
def release_all_locks(
|
|
77
86
|
redis,
|
|
78
87
|
batch_size,
|
|
88
|
+
lock_index_enabled,
|
|
79
89
|
logger,
|
|
80
90
|
instrumenter,
|
|
81
91
|
instrument,
|
|
@@ -89,7 +99,9 @@ module RedisQueuedLocks::Acquirer::ReleaseAllLocks
|
|
|
89
99
|
instr_sample_this
|
|
90
100
|
)
|
|
91
101
|
rel_start_time = clock_gettime
|
|
92
|
-
|
|
102
|
+
# steep:ignore:start
|
|
103
|
+
fully_release_all_locks(redis, batch_size, lock_index_enabled) => { ok:, result: }
|
|
104
|
+
# steep:ignore:end
|
|
93
105
|
|
|
94
106
|
# @type var ok: bool
|
|
95
107
|
# @type var result: Hash[Symbol,Integer]
|
|
@@ -121,74 +133,151 @@ module RedisQueuedLocks::Acquirer::ReleaseAllLocks
|
|
|
121
133
|
|
|
122
134
|
private
|
|
123
135
|
|
|
124
|
-
# Release all locks:
|
|
136
|
+
# Release all locks:
|
|
137
|
+
# - the disabled lock index: one SCAN pass over the RQL keys (`KEY_PATTERN`); the keys of all
|
|
138
|
+
# lock families are deleted with one `DEL` per batch of `batch_size` keys (the keys are
|
|
139
|
+
# deleted during the scan: no list of all keys is kept in memory);
|
|
140
|
+
# - the enabled lock index: one ZSCAN pass over the lock index (`LOCK_INDEX_KEY`: lock names,
|
|
141
|
+
# the keyspace is never scanned); the keys of all lock families of each batch of
|
|
142
|
+
# `batch_size` lock names are deleted with one `DEL` in one transaction with the `ZREM` of
|
|
143
|
+
# the lock names from the lock index (two round trips per batch: the read lock acquirers,
|
|
144
|
+
# then the transaction):
|
|
145
|
+
# - expired lock names are dropped from the lock index first (all their keys are expired);
|
|
146
|
+
# - lock names of the lock requests queued (or the locks obtained) after the transaction of
|
|
147
|
+
# their batch are indexed again by the lock acquirement;
|
|
125
148
|
#
|
|
126
149
|
# @param redis [RedisClient]
|
|
127
150
|
# @param batch_size [Integer]
|
|
151
|
+
# SCAN count and the number of keys in one `DEL` (the disabled lock index) or ZSCAN count and
|
|
152
|
+
# the number of lock names in one transaction (the enabled lock index).
|
|
153
|
+
# @param lock_index_enabled [Boolean]
|
|
128
154
|
# @return [Hash<Symbol,Boolean|Hash<Symbol,Integer>>]
|
|
129
155
|
# - Exmaple: { ok: true, result: { rel_key_cnt: 12345 } }
|
|
130
156
|
#
|
|
131
157
|
# @api private
|
|
132
158
|
# @since 1.0.0
|
|
133
|
-
# @version 1.
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
result = redis.with do |rconn|
|
|
137
|
-
rconn.pipelined do |pipeline|
|
|
138
|
-
# Step A: release all queus and their related locks
|
|
139
|
-
rconn.scan(
|
|
140
|
-
'MATCH',
|
|
141
|
-
RedisQueuedLocks::Resource::LOCK_QUEUE_PATTERN,
|
|
142
|
-
count: batch_size
|
|
143
|
-
) do |lock_queue|
|
|
144
|
-
# TODO: reduce unnecessary iterations
|
|
145
|
-
pipeline.call('EXPIRE', lock_queue, '0')
|
|
146
|
-
end
|
|
159
|
+
# @version 1.20.0
|
|
160
|
+
def fully_release_all_locks(redis, batch_size, lock_index_enabled)
|
|
161
|
+
rel_key_cnt = 0
|
|
147
162
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
end
|
|
163
|
+
redis.with do |rconn|
|
|
164
|
+
if lock_index_enabled
|
|
165
|
+
# Step 1 (lock index): drop the expired lock names from the lock index
|
|
166
|
+
# (all their keys are expired);
|
|
167
|
+
rconn.call(
|
|
168
|
+
'ZREMRANGEBYSCORE', RedisQueuedLocks::Resource::LOCK_INDEX_KEY,
|
|
169
|
+
'-inf', RedisQueuedLocks::Resource.lock_index_time - 1
|
|
170
|
+
)
|
|
157
171
|
|
|
158
|
-
# Step
|
|
159
|
-
rconn
|
|
160
|
-
|
|
161
|
-
RedisQueuedLocks::Resource::READ_LOCK_QUEUE_PATTERN,
|
|
162
|
-
count: batch_size
|
|
163
|
-
) do |read_lock_queue|
|
|
164
|
-
# TODO: reduce unnecessary iterations
|
|
165
|
-
pipeline.call('EXPIRE', read_lock_queue, '0')
|
|
172
|
+
# Step 2 (lock index): release the locks of the lock index in batches of lock names
|
|
173
|
+
each_lock_name_batch(rconn, batch_size) do |lock_names|
|
|
174
|
+
rel_key_cnt += release_locks(rconn, lock_names)
|
|
166
175
|
end
|
|
176
|
+
else
|
|
177
|
+
# Step 1 (keyspace): delete the RQL keys of the lock families during one SCAN pass
|
|
178
|
+
# @type var lock_keys: Array[String]
|
|
179
|
+
lock_keys = []
|
|
167
180
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
# TODO: reduce unnecessary iterations
|
|
175
|
-
pipeline.call('EXPIRE', lock_readers, '0')
|
|
176
|
-
end
|
|
181
|
+
rconn.scan('MATCH', RedisQueuedLocks::Resource::KEY_PATTERN, count: batch_size) do |key|
|
|
182
|
+
# NOTE: lock families only (KEY_PATTERN matches any `rql:lock*` key)
|
|
183
|
+
next if RedisQueuedLocks::Resource.lock_key_family(key) == nil
|
|
184
|
+
|
|
185
|
+
lock_keys << key
|
|
186
|
+
next if lock_keys.size < batch_size
|
|
177
187
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
'MATCH',
|
|
181
|
-
RedisQueuedLocks::Resource::READ_LOCK_PATTERN,
|
|
182
|
-
count: batch_size
|
|
183
|
-
) do |read_lock|
|
|
184
|
-
# TODO: reduce unnecessary iterations
|
|
185
|
-
pipeline.call('EXPIRE', read_lock, '0')
|
|
188
|
+
rel_key_cnt += rconn.call('DEL', *lock_keys)
|
|
189
|
+
lock_keys.clear
|
|
186
190
|
end
|
|
191
|
+
|
|
192
|
+
rel_key_cnt += rconn.call('DEL', *lock_keys) unless lock_keys.empty?
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
{ ok: true, result: { rel_key_cnt: } }
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# (lock index) Releases the locks of the given batch of lock names: deletes all their keys and
|
|
200
|
+
# drops the lock names from the lock index in one transaction.
|
|
201
|
+
# - (RW) read lock data keys are found via the readers registries (one more round trip
|
|
202
|
+
# before the transaction); the data of a read lock obtained between the registry read and
|
|
203
|
+
# the transaction stays till its ttl (the read lock itself is released with the registry);
|
|
204
|
+
#
|
|
205
|
+
# @param redis_client [RedisClient]
|
|
206
|
+
# @param lock_names [Array<String>]
|
|
207
|
+
# @return [Integer] The number of the deleted keys.
|
|
208
|
+
#
|
|
209
|
+
# @api private
|
|
210
|
+
# @since 1.20.0
|
|
211
|
+
def release_locks(redis_client, lock_names)
|
|
212
|
+
# Step A: (RW) read lock acquirers of the readers registries (their read lock data keys)
|
|
213
|
+
# NOTE: (pipeline layout) one reply per lock name in the batch order
|
|
214
|
+
# @type var read_lock_acquirers: Array[Array[String]]
|
|
215
|
+
read_lock_acquirers = redis_client.pipelined do |pipeline|
|
|
216
|
+
lock_names.each do |lock_name|
|
|
217
|
+
pipeline.call(
|
|
218
|
+
'ZRANGE', RedisQueuedLocks::Resource.prepare_lock_readers(lock_name), '0', '-1'
|
|
219
|
+
)
|
|
187
220
|
end
|
|
188
221
|
end
|
|
189
222
|
|
|
190
|
-
|
|
223
|
+
# @type var lock_keys: Array[String]
|
|
224
|
+
lock_keys = []
|
|
225
|
+
lock_names.each_with_index do |lock_name, index|
|
|
226
|
+
lock_keys << RedisQueuedLocks::Resource.prepare_lock_key(lock_name)
|
|
227
|
+
lock_keys << RedisQueuedLocks::Resource.prepare_lock_queue(lock_name)
|
|
228
|
+
lock_keys << RedisQueuedLocks::Resource.prepare_lock_readers(lock_name)
|
|
229
|
+
lock_keys << RedisQueuedLocks::Resource.prepare_read_lock_queue(lock_name)
|
|
230
|
+
read_lock_acquirers[index].each do |acquirer_id|
|
|
231
|
+
lock_keys << RedisQueuedLocks::Resource.prepare_read_lock_key(lock_name, acquirer_id)
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# Step B: delete the keys and drop the lock names from the lock index in one transaction
|
|
236
|
+
# (a lock obtained after the transaction indexes its lock name again);
|
|
237
|
+
# NOTE: (transaction layout) [0] DEL (the number of the deleted keys), [1] ZREM (not read)
|
|
238
|
+
# @type var released: [Integer, Integer]
|
|
239
|
+
released = redis_client.multi do |transact|
|
|
240
|
+
transact.call('DEL', *lock_keys)
|
|
241
|
+
transact.call('ZREM', RedisQueuedLocks::Resource::LOCK_INDEX_KEY, *lock_names)
|
|
242
|
+
end
|
|
243
|
+
released[0]
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# Iterates over the lock names of the lock index (one ZSCAN pass) in batches of `scan_size`
|
|
247
|
+
# lock names (batches are collected across ZSCAN pages). Expired lock names (not dropped from
|
|
248
|
+
# the index yet) are skipped: all their keys are expired.
|
|
249
|
+
#
|
|
250
|
+
# NOTE: each operation module iterates over the lock index by itself (duplicated on purpose:
|
|
251
|
+
# operation modules are independent of each other, see `Locks.each_lock_name_batch`);
|
|
252
|
+
#
|
|
253
|
+
# @param redis_client [RedisClient]
|
|
254
|
+
# @param scan_size [Integer] ZSCAN count and the number of lock names in one batch.
|
|
255
|
+
# @yield [lock_names]
|
|
256
|
+
# @yieldparam lock_names [Array<String>] The batch of lock names (reused between batches).
|
|
257
|
+
# @return [void]
|
|
258
|
+
#
|
|
259
|
+
# @api private
|
|
260
|
+
# @since 1.20.0
|
|
261
|
+
def each_lock_name_batch(redis_client, scan_size)
|
|
262
|
+
lock_index_time = RedisQueuedLocks::Resource.lock_index_time
|
|
263
|
+
# @type var lock_names: Array[String]
|
|
264
|
+
lock_names = []
|
|
265
|
+
|
|
266
|
+
redis_client.zscan(
|
|
267
|
+
RedisQueuedLocks::Resource::LOCK_INDEX_KEY, count: scan_size
|
|
268
|
+
) do |lock_name, lock_index_score|
|
|
269
|
+
next if lock_index_score.to_f < lock_index_time
|
|
270
|
+
|
|
271
|
+
lock_names << lock_name
|
|
272
|
+
next if lock_names.size < scan_size
|
|
273
|
+
|
|
274
|
+
yield(lock_names)
|
|
275
|
+
lock_names.clear
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
yield(lock_names) unless lock_names.empty?
|
|
191
279
|
end
|
|
192
|
-
# rubocop:enable Metrics/MethodLength
|
|
193
280
|
end
|
|
281
|
+
# rubocop:enable Metrics/ClassLength
|
|
194
282
|
end
|
|
283
|
+
# rubocop:enable Metrics/ModuleLength
|