redis_queued_locks 1.21.0 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.claude/project-overview.md +216 -25
- data/.claude/rules/acquirer.md +6 -5
- data/.claude/rules/arguments.md +2 -2
- data/.claude/rules/changelog.md +27 -0
- data/.claude/rules/git.md +1 -1
- data/.claude/rules/logic.md +10 -12
- data/.claude/rules/swarm.md +3 -4
- data/.claude/rules/tests.md +6 -6
- data/.claude/rules/type-checking.md +13 -14
- data/.claude/rules/visitors.md +5 -5
- data/.claude/skills/rql-code-review/SKILL.md +112 -35
- data/CHANGELOG.md +98 -244
- data/CLAUDE.md +9 -8
- data/README.md +275 -85
- data/Rakefile +11 -0
- data/lib/redis_queued_locks/acquirer/acquire_lock/instr_visitor.rb +28 -12
- data/lib/redis_queued_locks/acquirer/acquire_lock/log_visitor.rb +38 -16
- data/lib/redis_queued_locks/acquirer/acquire_lock/try_to_lock/log_visitor.rb +92 -15
- data/lib/redis_queued_locks/acquirer/acquire_lock/try_to_lock.rb +430 -63
- data/lib/redis_queued_locks/acquirer/acquire_lock.rb +72 -22
- data/lib/redis_queued_locks/acquirer/clear_dead_requests.rb +240 -45
- data/lib/redis_queued_locks/acquirer/extend_lock_ttl.rb +3 -1
- data/lib/redis_queued_locks/acquirer/keys.rb +6 -2
- data/lib/redis_queued_locks/acquirer/lock_series_poc/instr_visitor.rb +10 -5
- data/lib/redis_queued_locks/acquirer/lock_series_poc/log_visitor.rb +18 -9
- data/lib/redis_queued_locks/acquirer/lock_series_poc.rb +52 -23
- data/lib/redis_queued_locks/acquirer/locks.rb +6 -2
- data/lib/redis_queued_locks/acquirer/queue_info.rb +30 -7
- data/lib/redis_queued_locks/acquirer/queues.rb +17 -6
- data/lib/redis_queued_locks/acquirer/release_all_locks.rb +6 -2
- data/lib/redis_queued_locks/acquirer/release_locks_of.rb +6 -2
- data/lib/redis_queued_locks/client.rb +66 -24
- data/lib/redis_queued_locks/config.rb +93 -1
- data/lib/redis_queued_locks/resource.rb +124 -9
- data/lib/redis_queued_locks/swarm/swarm_element/threaded.rb +6 -5
- data/lib/redis_queued_locks/swarm/zombie_info.rb +6 -2
- data/lib/redis_queued_locks/version.rb +2 -2
- data/rbs_collection.lock.yaml +4 -0
- data/rbs_collection.yaml +2 -1
- data/sig/manifest.yml +1 -0
- data/sig/redis_queued_locks/acquirer/acquire_lock/dequeue_from_lock_queue/log_visitor.rbs +2 -2
- data/sig/redis_queued_locks/acquirer/acquire_lock/dequeue_from_lock_queue.rbs +2 -2
- data/sig/redis_queued_locks/acquirer/acquire_lock/instr_visitor.rbs +12 -6
- data/sig/redis_queued_locks/acquirer/acquire_lock/log_visitor.rbs +21 -14
- data/sig/redis_queued_locks/acquirer/acquire_lock/try_to_lock/log_visitor.rbs +61 -40
- data/sig/redis_queued_locks/acquirer/acquire_lock/try_to_lock.rbs +59 -11
- data/sig/redis_queued_locks/acquirer/acquire_lock/{yield_with_expire → yield_expire}/log_visitor.rbs +4 -4
- data/sig/redis_queued_locks/acquirer/acquire_lock/yield_expire.rbs +2 -2
- data/sig/redis_queued_locks/acquirer/acquire_lock.rbs +5 -3
- data/sig/redis_queued_locks/acquirer/clear_dead_requests.rbs +14 -1
- data/sig/redis_queued_locks/acquirer/extend_lock_ttl.rbs +2 -2
- data/sig/redis_queued_locks/acquirer/queues.rbs +1 -1
- data/sig/redis_queued_locks/client.rbs +10 -7
- data/sig/redis_queued_locks/config.rbs +54 -1
- data/sig/redis_queued_locks/instrument/sampler.rbs +3 -1
- data/sig/redis_queued_locks/instrument/void_notifier.rbs +3 -1
- data/sig/redis_queued_locks/logging/sampler.rbs +3 -1
- data/sig/redis_queued_locks/resource.rbs +22 -12
- data/sig/redis_queued_locks/swarm/swarm_element/threaded.rbs +1 -1
- data/sig/redis_queued_locks/utilities.rbs +2 -1
- data/sig/redis_queued_locks.rbs +7 -0
- metadata +4 -3
- /data/sig/redis_queued_locks/{acquier.rbs → acquirer.rbs} +0 -0
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# @api private
|
|
4
4
|
# @since 1.0.0
|
|
5
|
-
# @version 1.
|
|
5
|
+
# @version 1.22.0
|
|
6
6
|
# rubocop:disable Metrics/ModuleLength
|
|
7
7
|
# rubocop:disable Metrics/MethodLength
|
|
8
8
|
# rubocop:disable Metrics/ClassLength
|
|
@@ -126,6 +126,22 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
126
126
|
# (but with checking the limist, retries, timeouts and so on). if lock is
|
|
127
127
|
# free to obtain - it will be obtained;
|
|
128
128
|
# - pre-configured in `config['default_access_strategy']`;
|
|
129
|
+
# @option priority [Integer,Symbol]
|
|
130
|
+
# - The lock request priority: from `RedisQueuedLocks::Resource::MIN_LOCK_PRIORITY` (`-100`)
|
|
131
|
+
# to `RedisQueuedLocks::Resource::MAX_LOCK_PRIORITY` (`100`), `0` by default, or a priority
|
|
132
|
+
# name of `lock_priorities` (resolved to its Integer priority: the lock request queues,
|
|
133
|
+
# the logs and the instrumentation use the Integer priority);
|
|
134
|
+
# - `:queued` access strategy: the lock request is placed in the lock request queue after all
|
|
135
|
+
# requests of the same or higher priority and before all requests of the lower priority
|
|
136
|
+
# (strict priority, FIFO inside the same priority; between modes too: the read and write
|
|
137
|
+
# lock request queues use the same positions);
|
|
138
|
+
# - `:random` access strategy: the lock request does not wait for its turn (it ignores the
|
|
139
|
+
# order of the lock request queues and the priorities of other requests), but it is queued
|
|
140
|
+
# at the position of its priority like a `:queued` one: `:queued` lock requests wait for it;
|
|
141
|
+
# - the obtained lock is never taken away: the priority orders the waiting lock requests only;
|
|
142
|
+
# - pre-configured in `config['default_lock_priority']`;
|
|
143
|
+
# @option lock_priorities [Hash<Symbol,Integer>]
|
|
144
|
+
# - The priority names (`config['lock_priorities']`): a priority name => its Integer priority;
|
|
129
145
|
# @option log_sampling_enabled [Boolean]
|
|
130
146
|
# - enables <log sampling>: only the configured percent of RQL cases will be logged;
|
|
131
147
|
# - disabled by default;
|
|
@@ -177,7 +193,7 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
177
193
|
#
|
|
178
194
|
# @api private
|
|
179
195
|
# @since 1.0.0
|
|
180
|
-
# @version 1.
|
|
196
|
+
# @version 1.22.0
|
|
181
197
|
def acquire_lock(
|
|
182
198
|
redis,
|
|
183
199
|
lock_name,
|
|
@@ -206,6 +222,8 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
206
222
|
conflict_strategy:,
|
|
207
223
|
read_write_mode:,
|
|
208
224
|
access_strategy:,
|
|
225
|
+
priority:,
|
|
226
|
+
lock_priorities:,
|
|
209
227
|
log_sampling_enabled:,
|
|
210
228
|
log_sampling_percent:,
|
|
211
229
|
log_sampler:,
|
|
@@ -228,7 +246,9 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
228
246
|
end
|
|
229
247
|
|
|
230
248
|
# Step 0.1 (RW): prevent :read_write_mode incompatabilities
|
|
231
|
-
|
|
249
|
+
case read_write_mode # NOTE: (steep) case/when narrows `Symbol` to `RQL::readWriteMode` below
|
|
250
|
+
when :read, :write then nil
|
|
251
|
+
else
|
|
232
252
|
raise(
|
|
233
253
|
RedisQueuedLocks::ArgumentError,
|
|
234
254
|
"`:read_write_mode` argument should be `:read` or `:write`, " \
|
|
@@ -292,6 +312,27 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
292
312
|
)
|
|
293
313
|
end
|
|
294
314
|
|
|
315
|
+
# Step 0.1: prevent :priority incompatibilities (the same range as its config key)
|
|
316
|
+
# - a Symbol is a priority name of `config['lock_priorities']`: it is resolved to its
|
|
317
|
+
# Integer priority (the lock request queues, the logs and the instrumentation use the
|
|
318
|
+
# Integer priority; an unknown name stays a Symbol and fails the check);
|
|
319
|
+
# - each priority has its own band of positions in the lock request queues: the range
|
|
320
|
+
# bounds the shift of the positions (float scores lose precision with the shift: the FIFO
|
|
321
|
+
# order inside a priority keeps ~61 µs for -100..100, see `LOCK_PRIORITY_BAND`);
|
|
322
|
+
priority = lock_priorities.fetch(priority, priority) if priority.is_a?(::Symbol)
|
|
323
|
+
if !priority.is_a?(::Integer) ||
|
|
324
|
+
priority < RedisQueuedLocks::Resource::MIN_LOCK_PRIORITY ||
|
|
325
|
+
priority > RedisQueuedLocks::Resource::MAX_LOCK_PRIORITY
|
|
326
|
+
raise(
|
|
327
|
+
RedisQueuedLocks::ArgumentError,
|
|
328
|
+
"`:priority` argument should be an Integer from " \
|
|
329
|
+
"#{RedisQueuedLocks::Resource::MIN_LOCK_PRIORITY} to " \
|
|
330
|
+
"#{RedisQueuedLocks::Resource::MAX_LOCK_PRIORITY} or a priority name of " \
|
|
331
|
+
"`config['lock_priorities']` (#{lock_priorities.keys.map(&:inspect).join(', ')}), " \
|
|
332
|
+
"got #{priority.inspect}."
|
|
333
|
+
)
|
|
334
|
+
end
|
|
335
|
+
|
|
295
336
|
# Step 0.2: prevent :meta incompatabiltiies (structure)
|
|
296
337
|
# - Symbol keys are written to the lock data as Strings (`Symbol#name`, as redis-client
|
|
297
338
|
# does it) and would overwrite the reserved lock data fields too;
|
|
@@ -346,7 +387,9 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
346
387
|
RedisQueuedLocks::Resource.prepare_read_lock_key(lock_name, acquirer_id)
|
|
347
388
|
end
|
|
348
389
|
|
|
349
|
-
|
|
390
|
+
# NOTE: the position in the lock request queue: the enqueue time shifted by the priority band
|
|
391
|
+
# (see `RedisQueuedLocks::Resource::LOCK_PRIORITY_BAND`);
|
|
392
|
+
acquirer_position = RedisQueuedLocks::Resource.calc_initial_acquirer_position(priority)
|
|
350
393
|
|
|
351
394
|
log_sampled = RedisQueuedLocks::Logging.should_log?(
|
|
352
395
|
log_sampling_enabled,
|
|
@@ -393,7 +436,7 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
393
436
|
|
|
394
437
|
LogVisitor.start_lock_obtaining(
|
|
395
438
|
logger, log_sampled, lock_key,
|
|
396
|
-
queue_ttl, acquirer_id, host_id, access_strategy, read_write_mode
|
|
439
|
+
queue_ttl, acquirer_id, host_id, access_strategy, read_write_mode, priority
|
|
397
440
|
)
|
|
398
441
|
|
|
399
442
|
# Step 2: try to lock with timeout
|
|
@@ -414,17 +457,18 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
414
457
|
|
|
415
458
|
LogVisitor.start_try_to_lock_cycle(
|
|
416
459
|
logger, log_sampled, lock_key,
|
|
417
|
-
queue_ttl, acquirer_id, host_id, access_strategy, read_write_mode
|
|
460
|
+
queue_ttl, acquirer_id, host_id, access_strategy, read_write_mode, priority
|
|
418
461
|
)
|
|
419
462
|
|
|
420
463
|
# Step 2.X: check the actual score: is it in queue ttl limit or not?
|
|
421
|
-
if RedisQueuedLocks::Resource.dead_score_reached?(acquirer_position, queue_ttl)
|
|
422
|
-
# Step 2.X.X: dead score reached => re-queue the lock request with the new score
|
|
423
|
-
|
|
464
|
+
if RedisQueuedLocks::Resource.dead_score_reached?(acquirer_position, priority, queue_ttl)
|
|
465
|
+
# Step 2.X.X: dead score reached => re-queue the lock request with the new score
|
|
466
|
+
# (the end of the requests of the same priority);
|
|
467
|
+
acquirer_position = RedisQueuedLocks::Resource.calc_initial_acquirer_position(priority)
|
|
424
468
|
|
|
425
469
|
LogVisitor.dead_score_reached__reset_acquirer_position(
|
|
426
470
|
logger, log_sampled, lock_key,
|
|
427
|
-
queue_ttl, acquirer_id, host_id, access_strategy, read_write_mode
|
|
471
|
+
queue_ttl, acquirer_id, host_id, access_strategy, read_write_mode, priority
|
|
428
472
|
)
|
|
429
473
|
end
|
|
430
474
|
|
|
@@ -444,6 +488,7 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
444
488
|
acquirer_id,
|
|
445
489
|
host_id,
|
|
446
490
|
acquirer_position,
|
|
491
|
+
priority,
|
|
447
492
|
lock_ttl,
|
|
448
493
|
queue_ttl,
|
|
449
494
|
lock_index_enabled,
|
|
@@ -477,10 +522,11 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
477
522
|
# instrumetnation: (reentrant lock with ttl extension)
|
|
478
523
|
LogVisitor.extendable_reentrant_lock_obtained(
|
|
479
524
|
logger, log_sampled, result[:lock_key],
|
|
480
|
-
queue_ttl, acquirer_id, host_id, acq_time, access_strategy, read_write_mode
|
|
525
|
+
queue_ttl, acquirer_id, host_id, acq_time, access_strategy, read_write_mode,
|
|
526
|
+
priority
|
|
481
527
|
)
|
|
482
528
|
InstrVisitor.extendable_reentrant_lock_obtained(
|
|
483
|
-
instrumenter, instr_sampled, result[:lock_key], read_write_mode,
|
|
529
|
+
instrumenter, instr_sampled, result[:lock_key], read_write_mode, priority,
|
|
484
530
|
result[:ttl], result[:acq_id], result[:hst_id], result[:ts], acq_time,
|
|
485
531
|
instrument
|
|
486
532
|
)
|
|
@@ -488,10 +534,11 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
488
534
|
# instrumetnation: (reentrant lock without ttl extension)
|
|
489
535
|
LogVisitor.reentrant_lock_obtained(
|
|
490
536
|
logger, log_sampled, result[:lock_key],
|
|
491
|
-
queue_ttl, acquirer_id, host_id, acq_time, access_strategy, read_write_mode
|
|
537
|
+
queue_ttl, acquirer_id, host_id, acq_time, access_strategy, read_write_mode,
|
|
538
|
+
priority
|
|
492
539
|
)
|
|
493
540
|
InstrVisitor.reentrant_lock_obtained(
|
|
494
|
-
instrumenter, instr_sampled, result[:lock_key], read_write_mode,
|
|
541
|
+
instrumenter, instr_sampled, result[:lock_key], read_write_mode, priority,
|
|
495
542
|
result[:ttl], result[:acq_id], result[:hst_id], result[:ts], acq_time,
|
|
496
543
|
instrument
|
|
497
544
|
)
|
|
@@ -500,10 +547,11 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
500
547
|
# NOTE: classic is: acq_process[:result][:process] == :lock_obtaining
|
|
501
548
|
LogVisitor.lock_obtained(
|
|
502
549
|
logger, log_sampled, result[:lock_key],
|
|
503
|
-
queue_ttl, acquirer_id, host_id, acq_time, access_strategy, read_write_mode
|
|
550
|
+
queue_ttl, acquirer_id, host_id, acq_time, access_strategy, read_write_mode,
|
|
551
|
+
priority
|
|
504
552
|
)
|
|
505
553
|
InstrVisitor.lock_obtained(
|
|
506
|
-
instrumenter, instr_sampled, result[:lock_key], read_write_mode,
|
|
554
|
+
instrumenter, instr_sampled, result[:lock_key], read_write_mode, priority,
|
|
507
555
|
result[:ttl], result[:acq_id], result[:hst_id], result[:ts], acq_time,
|
|
508
556
|
instrument
|
|
509
557
|
)
|
|
@@ -661,6 +709,7 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
661
709
|
instr_sampled,
|
|
662
710
|
acq_process[:lock_info][:lock_key],
|
|
663
711
|
read_write_mode,
|
|
712
|
+
priority,
|
|
664
713
|
acq_process[:lock_info][:ttl],
|
|
665
714
|
acq_process[:lock_info][:acq_id],
|
|
666
715
|
acq_process[:lock_info][:hst_id],
|
|
@@ -676,6 +725,7 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
676
725
|
instr_sampled,
|
|
677
726
|
acq_process[:lock_info][:lock_key],
|
|
678
727
|
read_write_mode,
|
|
728
|
+
priority,
|
|
679
729
|
acq_process[:lock_info][:ttl],
|
|
680
730
|
acq_process[:lock_info][:acq_id],
|
|
681
731
|
acq_process[:lock_info][:hst_id],
|
|
@@ -705,11 +755,11 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
705
755
|
acq_time = ((clock_gettime - acq_start_time) / 1_000.0).ceil(2)
|
|
706
756
|
LogVisitor.lock_obtaining_failed(
|
|
707
757
|
logger, log_sampled, lock_key, queue_ttl, acquirer_id, host_id,
|
|
708
|
-
acq_time, access_strategy, read_write_mode, acq_process[:result]
|
|
758
|
+
acq_time, access_strategy, read_write_mode, priority, acq_process[:result]
|
|
709
759
|
)
|
|
710
760
|
InstrVisitor.lock_obtaining_failed(
|
|
711
|
-
instrumenter, instr_sampled, lock_key, read_write_mode, lock_ttl, acquirer_id,
|
|
712
|
-
Time.now.to_f, acq_time, acq_process[:result], instrument
|
|
761
|
+
instrumenter, instr_sampled, lock_key, read_write_mode, priority, lock_ttl, acquirer_id,
|
|
762
|
+
host_id, Time.now.to_f, acq_time, acq_process[:result], instrument
|
|
713
763
|
)
|
|
714
764
|
{ ok: false, result: acq_process[:result] } #: { ok: bool, result: Symbol }
|
|
715
765
|
end
|
|
@@ -730,11 +780,11 @@ module RedisQueuedLocks::Acquirer::AcquireLock
|
|
|
730
780
|
acq_time = ((clock_gettime - acq_start_time) / 1_000.0).ceil(2)
|
|
731
781
|
LogVisitor.lock_obtaining_failed(
|
|
732
782
|
logger, log_sampled, lock_key, queue_ttl, acquirer_id, host_id,
|
|
733
|
-
acq_time, access_strategy, read_write_mode, acq_process[:result]
|
|
783
|
+
acq_time, access_strategy, read_write_mode, priority, acq_process[:result]
|
|
734
784
|
)
|
|
735
785
|
InstrVisitor.lock_obtaining_failed(
|
|
736
|
-
instrumenter, instr_sampled, lock_key, read_write_mode, lock_ttl, acquirer_id,
|
|
737
|
-
Time.now.to_f, acq_time, acq_process[:result], instrument
|
|
786
|
+
instrumenter, instr_sampled, lock_key, read_write_mode, priority, lock_ttl, acquirer_id,
|
|
787
|
+
host_id, Time.now.to_f, acq_time, acq_process[:result], instrument
|
|
738
788
|
)
|
|
739
789
|
end
|
|
740
790
|
# steep:ignore:end
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# @api private
|
|
4
4
|
# @since 1.0.0
|
|
5
|
-
# @version 1.
|
|
5
|
+
# @version 1.22.0
|
|
6
6
|
# rubocop:disable Metrics/ModuleLength
|
|
7
7
|
module RedisQueuedLocks::Acquirer::ClearDeadRequests
|
|
8
8
|
# @since 1.21.0
|
|
@@ -11,13 +11,19 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
|
|
|
11
11
|
# rubocop:disable Metrics/ClassLength
|
|
12
12
|
class << self
|
|
13
13
|
# Drops dead lock requests (requests older than `dead_ttl`) from the lock queues (write and
|
|
14
|
-
# read):
|
|
15
|
-
# - the disabled lock index: the lock queues found in one SCAN pass,
|
|
16
|
-
#
|
|
14
|
+
# read; the requests of any priority: see `RedisQueuedLocks::Resource::LOCK_PRIORITY_BAND`):
|
|
15
|
+
# - the disabled lock index: the lock queues found in one SCAN pass, processed in batches of
|
|
16
|
+
# `scan_size` lock queues (batches are collected across SCAN pages);
|
|
17
17
|
# - the enabled lock index: the lock queues of the locks of the lock index: one ZSCAN pass
|
|
18
|
-
# over the lock names (the keyspace is never scanned),
|
|
19
|
-
#
|
|
20
|
-
#
|
|
18
|
+
# over the lock names (the keyspace is never scanned), processed in batches of `scan_size`
|
|
19
|
+
# lock names (both lock queues of each lock name); expired lock names are dropped from the
|
|
20
|
+
# lock index first;
|
|
21
|
+
# - the batch round trips (see `drop_dead_requests`): the lock requests of the batch queues
|
|
22
|
+
# are read, then the dead requests of each priority band that has them are dropped (two
|
|
23
|
+
# round trips, one without dead requests); `by_priority`: only the first lock request of
|
|
24
|
+
# each band is read (a walk over the bands of the batch queues: the most bands of a batch
|
|
25
|
+
# queue + 1 round trips, the dead requests are dropped on the way, see
|
|
26
|
+
# `drop_dead_requests_by_priority`);
|
|
21
27
|
# - the operation is reported with one log and one instrumentation event
|
|
22
28
|
# (`redis_queued_locks.clear_dead_requests`) with the time of the whole operation;
|
|
23
29
|
#
|
|
@@ -26,6 +32,10 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
|
|
|
26
32
|
# SCAN count and the number of lock queues in one batch (the disabled lock index) or ZSCAN
|
|
27
33
|
# count and the number of lock names in one batch (the enabled lock index).
|
|
28
34
|
# @param dead_ttl [Integer] In milliseconds
|
|
35
|
+
# @param by_priority [Boolean]
|
|
36
|
+
# Read the first lock request of each priority band of the lock queues instead of all lock
|
|
37
|
+
# requests (more round trips: one per band of the batch queue with the most bands + 1; the
|
|
38
|
+
# transfer does not depend on the number of lock requests).
|
|
29
39
|
# @param lock_index_enabled [Boolean]
|
|
30
40
|
# Find lock queues via the lock index instead of the keyspace SCAN.
|
|
31
41
|
# @param logger [::Logger,#debug]
|
|
@@ -43,12 +53,13 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
|
|
|
43
53
|
#
|
|
44
54
|
# @api private
|
|
45
55
|
# @since 1.0.0
|
|
46
|
-
# @version 1.
|
|
56
|
+
# @version 1.22.0
|
|
47
57
|
# rubocop:disable Metrics/MethodLength
|
|
48
58
|
def clear_dead_requests(
|
|
49
59
|
redis_client,
|
|
50
60
|
scan_size,
|
|
51
61
|
dead_ttl,
|
|
62
|
+
by_priority,
|
|
52
63
|
lock_index_enabled,
|
|
53
64
|
logger,
|
|
54
65
|
instrumenter,
|
|
@@ -76,10 +87,16 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
|
|
|
76
87
|
"`:dead_ttl` argument should be a positive Integer, got #{dead_ttl.inspect}."
|
|
77
88
|
)
|
|
78
89
|
end
|
|
90
|
+
if by_priority != true && by_priority != false
|
|
91
|
+
raise(
|
|
92
|
+
RedisQueuedLocks::ArgumentError,
|
|
93
|
+
"`:by_priority` argument should be a boolean, got #{by_priority.inspect}."
|
|
94
|
+
)
|
|
95
|
+
end
|
|
79
96
|
|
|
80
97
|
rel_start_time = clock_gettime
|
|
81
98
|
processed_queues = fully_clear_dead_requests(
|
|
82
|
-
redis_client, scan_size, dead_ttl, lock_index_enabled
|
|
99
|
+
redis_client, scan_size, dead_ttl, by_priority, lock_index_enabled
|
|
83
100
|
)
|
|
84
101
|
time_at = Time.now.to_f
|
|
85
102
|
rel_end_time = clock_gettime
|
|
@@ -129,12 +146,17 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
|
|
|
129
146
|
# @param redis_client [RedisClient]
|
|
130
147
|
# @param scan_size [Integer]
|
|
131
148
|
# @param dead_ttl [Integer] In milliseconds
|
|
149
|
+
# @param by_priority [Boolean]
|
|
132
150
|
# @param lock_index_enabled [Boolean]
|
|
133
151
|
# @return [Set<String>] The processed lock queues (write and read lock request queues).
|
|
134
152
|
#
|
|
135
153
|
# @api private
|
|
136
154
|
# @since 1.21.0
|
|
137
|
-
|
|
155
|
+
# @version 1.22.0
|
|
156
|
+
# rubocop:disable Metrics/MethodLength
|
|
157
|
+
def fully_clear_dead_requests(
|
|
158
|
+
redis_client, scan_size, dead_ttl, by_priority, lock_index_enabled
|
|
159
|
+
)
|
|
138
160
|
dead_score = RedisQueuedLocks::Resource.acquirer_dead_score(dead_ttl / 1_000.0)
|
|
139
161
|
|
|
140
162
|
Set.new.tap do |processed_queues|
|
|
@@ -150,12 +172,15 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
|
|
|
150
172
|
|
|
151
173
|
# Step 2 (lock index): drop dead requests from the lock queues of the lock index
|
|
152
174
|
each_lock_name_batch(rconn, scan_size) do |lock_names|
|
|
153
|
-
drop_dead_requests_of_locks(
|
|
175
|
+
drop_dead_requests_of_locks(
|
|
176
|
+
rconn, lock_names, dead_score, by_priority, processed_queues
|
|
177
|
+
)
|
|
154
178
|
end
|
|
155
179
|
else
|
|
156
180
|
# Step 1 (keyspace): drop dead requests from the lock queues found in one SCAN pass
|
|
157
|
-
# NOTE: dead requests are dropped
|
|
158
|
-
#
|
|
181
|
+
# NOTE: dead requests are dropped per batch of `scan_size` queues (the round trips of
|
|
182
|
+
# `drop_dead_requests`; batches are collected across SCAN pages: lock queues can be
|
|
183
|
+
# rare among other keys);
|
|
159
184
|
# @type var lock_queues: Array[String]
|
|
160
185
|
lock_queues = []
|
|
161
186
|
|
|
@@ -164,15 +189,18 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
|
|
|
164
189
|
lock_queues << lock_queue
|
|
165
190
|
next if lock_queues.size < scan_size
|
|
166
191
|
|
|
167
|
-
drop_dead_requests(rconn, lock_queues, dead_score)
|
|
192
|
+
drop_dead_requests(rconn, lock_queues, dead_score, by_priority)
|
|
168
193
|
lock_queues.clear
|
|
169
194
|
end
|
|
170
195
|
|
|
171
|
-
|
|
196
|
+
unless lock_queues.empty?
|
|
197
|
+
drop_dead_requests(rconn, lock_queues, dead_score, by_priority)
|
|
198
|
+
end
|
|
172
199
|
end
|
|
173
200
|
end
|
|
174
201
|
end
|
|
175
202
|
end
|
|
203
|
+
# rubocop:enable Metrics/MethodLength
|
|
176
204
|
|
|
177
205
|
# Write and read lock queues in one SCAN pass (the disabled lock index).
|
|
178
206
|
#
|
|
@@ -198,62 +226,226 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
|
|
|
198
226
|
end
|
|
199
227
|
end
|
|
200
228
|
|
|
201
|
-
# Drops dead requests of the given lock queues
|
|
229
|
+
# Drops dead requests of the given lock queues (both lookups; `by_priority`: the first lock
|
|
230
|
+
# request of each priority band is read instead of all lock requests, see
|
|
231
|
+
# `drop_dead_requests_by_priority`):
|
|
232
|
+
# - the first round trip reads the lock requests of the lock queues;
|
|
233
|
+
# - the second round trip drops the dead requests of each priority band that has them: one
|
|
234
|
+
# `ZREMRANGEBYSCORE` per band from the lower edge of the band to its time barrier (see
|
|
235
|
+
# `RedisQueuedLocks::Resource::LOCK_PRIORITY_BAND`), so the number of commands depends on
|
|
236
|
+
# the bands of the queued requests, not on the priority range; no round trip without dead
|
|
237
|
+
# requests (an empty pipeline sends nothing);
|
|
238
|
+
# - the requests of a band are adjacent and ordered by the enqueue time (the bands are
|
|
239
|
+
# disjoint position ranges): the first request of a band decides whether the band has dead
|
|
240
|
+
# requests;
|
|
241
|
+
# - requests queued (or re-queued) between the round trips are never dropped: they are after
|
|
242
|
+
# the time barrier of their band;
|
|
243
|
+
# - (infinite positions) RQL never writes them (manual writes of the queues, see
|
|
244
|
+
# `RedisQueuedLocks::Resource.lock_request_priority`): the requests at `-inf` are dead and
|
|
245
|
+
# are dropped by their position (`-inf` .. `-inf`), the requests at `+inf` are never dead;
|
|
202
246
|
#
|
|
203
247
|
# @param redis_client [RedisClient]
|
|
204
248
|
# @param lock_queues [Array<String>]
|
|
205
|
-
# @param dead_score [Float]
|
|
206
|
-
# @
|
|
249
|
+
# @param dead_score [Float] Requests queued before this time are dead.
|
|
250
|
+
# @param by_priority [Boolean]
|
|
251
|
+
# @return [Array<Array<Array<String,Float>>>]
|
|
252
|
+
# The lock requests (acquirer id, position) of each lock queue before the removal (the first
|
|
253
|
+
# lock request with `by_priority`), in the order of `lock_queues` (an empty list for a missing
|
|
254
|
+
# lock queue).
|
|
207
255
|
#
|
|
208
256
|
# @api private
|
|
209
257
|
# @since 1.20.0
|
|
210
|
-
|
|
211
|
-
|
|
258
|
+
# @version 1.22.0
|
|
259
|
+
# rubocop:disable Metrics/MethodLength
|
|
260
|
+
def drop_dead_requests(redis_client, lock_queues, dead_score, by_priority)
|
|
261
|
+
return drop_dead_requests_by_priority(redis_client, lock_queues, dead_score) if by_priority
|
|
262
|
+
|
|
263
|
+
# NOTE: (pipeline layout) one ZRANGE WITHSCORES per lock queue (the `lock_queues` order)
|
|
264
|
+
# @type var queued_requests: Array[Array[[String, Float]]]
|
|
265
|
+
queued_requests = redis_client.pipelined do |pipeline|
|
|
212
266
|
lock_queues.each do |lock_queue|
|
|
213
|
-
pipeline.call('
|
|
267
|
+
pipeline.call('ZRANGE', lock_queue, '0', '-1', 'WITHSCORES')
|
|
214
268
|
end
|
|
215
269
|
end
|
|
270
|
+
|
|
271
|
+
redis_client.pipelined do |pipeline|
|
|
272
|
+
lock_queues.each_with_index do |lock_queue, index|
|
|
273
|
+
# @type var checked_band: Integer?
|
|
274
|
+
checked_band = nil
|
|
275
|
+
|
|
276
|
+
# NOTE: (infinite positions) the requests at `-inf` (manual writes: out of all bands, see
|
|
277
|
+
# `RedisQueuedLocks::Resource.lock_request_priority`) are dead: they are the first
|
|
278
|
+
# requests of the queue and are dropped by their position (no band removal covers them);
|
|
279
|
+
first_request = queued_requests[index].first
|
|
280
|
+
if first_request && first_request[1] == -Float::INFINITY
|
|
281
|
+
pipeline.call('ZREMRANGEBYSCORE', lock_queue, '-inf', '-inf')
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
queued_requests[index].each do |lock_request|
|
|
285
|
+
position = lock_request[1]
|
|
286
|
+
next if position == -Float::INFINITY
|
|
287
|
+
|
|
288
|
+
priority = RedisQueuedLocks::Resource.lock_request_priority(position)
|
|
289
|
+
next if priority == checked_band
|
|
290
|
+
|
|
291
|
+
# NOTE: the first request of the band: the band has dead requests when it is dead
|
|
292
|
+
checked_band = priority
|
|
293
|
+
band_dead_score =
|
|
294
|
+
RedisQueuedLocks::Resource.lock_priority_band_dead_score(dead_score, priority)
|
|
295
|
+
next if position > band_dead_score
|
|
296
|
+
|
|
297
|
+
pipeline.call(
|
|
298
|
+
'ZREMRANGEBYSCORE', lock_queue,
|
|
299
|
+
RedisQueuedLocks::Resource.lock_priority_band_floor(priority), band_dead_score
|
|
300
|
+
)
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
queued_requests
|
|
216
306
|
end
|
|
307
|
+
# rubocop:enable Metrics/MethodLength
|
|
217
308
|
|
|
218
|
-
# Drops dead requests of
|
|
219
|
-
# (the
|
|
309
|
+
# Drops dead requests of the given lock queues by the first lock request of each priority band
|
|
310
|
+
# (`by_priority`, both lookups): the lock requests of the lock queues are never read at once:
|
|
311
|
+
# - a walk over the bands of each lock queue: each round trip reads the first lock request of
|
|
312
|
+
# the next band of each walked lock queue (`ZRANGE <queue> <bound> +inf BYSCORE LIMIT 0 1
|
|
313
|
+
# WITHSCORES`, the first round trip from `-inf`; the next bound is the lower edge of the
|
|
314
|
+
# next, lower, band: `RedisQueuedLocks::Resource.lock_priority_band_floor(priority - 1)`),
|
|
315
|
+
# so the transfer depends on the number of bands, not on the number of lock requests;
|
|
316
|
+
# - the first request of a band decides whether the band has dead requests (see
|
|
317
|
+
# `drop_dead_requests`, the same removal commands): they are dropped by the next round trip
|
|
318
|
+
# (in the same pipeline after the reads: the removal never touches the next bands);
|
|
319
|
+
# - a lock queue is walked till no more lock requests are found, so the round trips of
|
|
320
|
+
# a batch are the most bands of a batch queue + 1 (the last read finds nothing), with or
|
|
321
|
+
# without dead requests;
|
|
322
|
+
# - requests queued (or re-queued) during the walk are never dropped: they are after the time
|
|
323
|
+
# barrier of their band;
|
|
324
|
+
# - (infinite positions) the requests at `-inf` are dead and are dropped by their position
|
|
325
|
+
# (the walk goes on after them: `(-inf`), the walk of a lock queue stops at a request at
|
|
326
|
+
# `+inf` (never dead, the last requests of the queue);
|
|
327
|
+
# - (no progress) a position out of the float precision of the band edges (manual writes far
|
|
328
|
+
# out of the priority range): the lower edge of the next band rounds to the position itself
|
|
329
|
+
# (Redis parses the bound to the same Float), so the walk goes on after the position (an
|
|
330
|
+
# exclusive bound) instead of reading the same request again;
|
|
331
|
+
#
|
|
332
|
+
# @param redis_client [RedisClient]
|
|
333
|
+
# @param lock_queues [Array<String>]
|
|
334
|
+
# @param dead_score [Float] Requests queued before this time are dead.
|
|
335
|
+
# @return [Array<Array<Array<String,Float>>>]
|
|
336
|
+
# The first lock request (acquirer id, position) of each lock queue before the removal, in the
|
|
337
|
+
# order of `lock_queues` (an empty list for a missing lock queue).
|
|
338
|
+
#
|
|
339
|
+
# @api private
|
|
340
|
+
# @since 1.22.0
|
|
341
|
+
# rubocop:disable Metrics/MethodLength
|
|
342
|
+
def drop_dead_requests_by_priority(redis_client, lock_queues, dead_score)
|
|
343
|
+
# NOTE: the walked lock queues (their indexes in `lock_queues`) and the lower bound of the
|
|
344
|
+
# next read of each lock queue
|
|
345
|
+
walked_queues = Array.new(lock_queues.size) { |index| index }
|
|
346
|
+
# @type var bounds: Array[String | Float]
|
|
347
|
+
bounds = Array.new(lock_queues.size, '-inf')
|
|
348
|
+
# @type var removals: Array[[String, String | Integer, String | Float]]
|
|
349
|
+
removals = []
|
|
350
|
+
# @type var first_requests: Array[Array[[String, Float]]]
|
|
351
|
+
first_requests = []
|
|
352
|
+
|
|
353
|
+
until walked_queues.empty?
|
|
354
|
+
# NOTE: (pipeline layout) one ZRANGE per walked lock queue (the `walked_queues` order),
|
|
355
|
+
# then the removals of the dead requests found by the previous round trip (not read)
|
|
356
|
+
# @type var band_heads: Array[Array[[String, Float]]]
|
|
357
|
+
band_heads = redis_client.pipelined do |pipeline|
|
|
358
|
+
walked_queues.each do |index|
|
|
359
|
+
pipeline.call(
|
|
360
|
+
'ZRANGE', lock_queues[index], bounds[index], '+inf',
|
|
361
|
+
'BYSCORE', 'LIMIT', '0', '1', 'WITHSCORES'
|
|
362
|
+
)
|
|
363
|
+
end
|
|
364
|
+
removals.each do |lock_queue, lowest_position, dead_position|
|
|
365
|
+
pipeline.call('ZREMRANGEBYSCORE', lock_queue, lowest_position, dead_position)
|
|
366
|
+
end
|
|
367
|
+
end
|
|
368
|
+
removals.clear
|
|
369
|
+
# NOTE: the first round trip reads all lock queues (in the `lock_queues` order, no removals)
|
|
370
|
+
first_requests = band_heads if first_requests.empty?
|
|
371
|
+
|
|
372
|
+
# NOTE: a lock queue with a found request is walked further (a removal of its band is sent
|
|
373
|
+
# with the next read of the lock queue: no removal is left after the walk)
|
|
374
|
+
# @type var next_queues: Array[Integer]
|
|
375
|
+
next_queues = []
|
|
376
|
+
walked_queues.each_with_index do |index, read_index|
|
|
377
|
+
band_head = band_heads[read_index].first
|
|
378
|
+
next unless band_head
|
|
379
|
+
|
|
380
|
+
position = band_head[1]
|
|
381
|
+
next if position == Float::INFINITY
|
|
382
|
+
|
|
383
|
+
lock_queue = lock_queues[index]
|
|
384
|
+
if position == -Float::INFINITY
|
|
385
|
+
removals << [lock_queue, '-inf', '-inf']
|
|
386
|
+
bounds[index] = '(-inf'
|
|
387
|
+
else
|
|
388
|
+
priority = RedisQueuedLocks::Resource.lock_request_priority(position)
|
|
389
|
+
band_dead_score =
|
|
390
|
+
RedisQueuedLocks::Resource.lock_priority_band_dead_score(dead_score, priority)
|
|
391
|
+
if position <= band_dead_score
|
|
392
|
+
removals << [
|
|
393
|
+
lock_queue,
|
|
394
|
+
RedisQueuedLocks::Resource.lock_priority_band_floor(priority),
|
|
395
|
+
band_dead_score
|
|
396
|
+
]
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
# NOTE: (no progress) see the method description
|
|
400
|
+
next_band_floor =
|
|
401
|
+
RedisQueuedLocks::Resource.lock_priority_band_floor(priority - 1).to_f
|
|
402
|
+
bounds[index] = (next_band_floor > position) ? next_band_floor : "(#{position}"
|
|
403
|
+
end
|
|
404
|
+
next_queues << index
|
|
405
|
+
end
|
|
406
|
+
walked_queues = next_queues
|
|
407
|
+
end
|
|
408
|
+
|
|
409
|
+
first_requests
|
|
410
|
+
end
|
|
411
|
+
# rubocop:enable Metrics/MethodLength
|
|
412
|
+
|
|
413
|
+
# Drops dead requests of both lock queues of the given lock names (the enabled lock index; the
|
|
414
|
+
# round trips of `drop_dead_requests`).
|
|
220
415
|
#
|
|
221
416
|
# @param redis_client [RedisClient]
|
|
222
417
|
# @param lock_names [Array<String>]
|
|
223
418
|
# @param dead_score [Float]
|
|
419
|
+
# @param by_priority [Boolean]
|
|
224
420
|
# @param processed_queues [Set<String>] Collects the existing (processed) lock queues.
|
|
225
421
|
# @return [void]
|
|
226
422
|
#
|
|
227
423
|
# @api private
|
|
228
424
|
# @since 1.20.0
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
pipeline.call('EXISTS', read_lock_queue)
|
|
240
|
-
pipeline.call('ZREMRANGEBYSCORE', read_lock_queue, '-inf', dead_score)
|
|
241
|
-
end
|
|
425
|
+
# @version 1.22.0
|
|
426
|
+
def drop_dead_requests_of_locks(
|
|
427
|
+
redis_client, lock_names, dead_score, by_priority, processed_queues
|
|
428
|
+
)
|
|
429
|
+
# NOTE: both lock queues of each lock name in the batch order (write, read)
|
|
430
|
+
# @type var lock_queues: Array[String]
|
|
431
|
+
lock_queues = []
|
|
432
|
+
lock_names.each do |lock_name|
|
|
433
|
+
lock_queues << RedisQueuedLocks::Resource.prepare_lock_queue(lock_name)
|
|
434
|
+
lock_queues << RedisQueuedLocks::Resource.prepare_read_lock_queue(lock_name)
|
|
242
435
|
end
|
|
243
436
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
processed_queues << RedisQueuedLocks::Resource.prepare_read_lock_queue(lock_name)
|
|
250
|
-
end
|
|
437
|
+
queued_requests = drop_dead_requests(redis_client, lock_queues, dead_score, by_priority)
|
|
438
|
+
|
|
439
|
+
# NOTE: a lock queue is processed when it exists (Redis never keeps empty sorted sets)
|
|
440
|
+
lock_queues.each_with_index do |lock_queue, index|
|
|
441
|
+
processed_queues << lock_queue unless queued_requests[index].empty?
|
|
251
442
|
end
|
|
252
443
|
end
|
|
253
444
|
|
|
254
445
|
# Iterates over the lock names of the lock index (one ZSCAN pass) in batches of `scan_size`
|
|
255
446
|
# lock names (batches are collected across ZSCAN pages). Expired lock names (not dropped from
|
|
256
|
-
# the index yet) are skipped: all their keys are expired
|
|
447
|
+
# the index yet) are skipped: all their keys are expired (`+inf` names, keys without TTL, never
|
|
448
|
+
# expire: see `RedisQueuedLocks::Resource.lock_index_score_expired?`).
|
|
257
449
|
#
|
|
258
450
|
# NOTE: each operation module iterates over the lock index by itself (duplicated on purpose:
|
|
259
451
|
# operation modules are independent of each other, see `Locks.each_lock_name_batch`);
|
|
@@ -266,6 +458,7 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
|
|
|
266
458
|
#
|
|
267
459
|
# @api private
|
|
268
460
|
# @since 1.20.0
|
|
461
|
+
# @version 1.22.0
|
|
269
462
|
def each_lock_name_batch(redis_client, scan_size)
|
|
270
463
|
lock_index_time = RedisQueuedLocks::Resource.lock_index_time
|
|
271
464
|
# @type var lock_names: Array[String]
|
|
@@ -274,7 +467,9 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
|
|
|
274
467
|
redis_client.zscan(
|
|
275
468
|
RedisQueuedLocks::Resource::LOCK_INDEX_KEY, count: scan_size
|
|
276
469
|
) do |lock_name, lock_index_score|
|
|
277
|
-
next if
|
|
470
|
+
next if RedisQueuedLocks::Resource.lock_index_score_expired?(
|
|
471
|
+
lock_index_score, lock_index_time
|
|
472
|
+
)
|
|
278
473
|
|
|
279
474
|
lock_names << lock_name
|
|
280
475
|
next if lock_names.size < scan_size
|
|
@@ -74,7 +74,9 @@ module RedisQueuedLocks::Acquirer::ExtendLockTTL
|
|
|
74
74
|
instr_sampler,
|
|
75
75
|
instr_sample_this
|
|
76
76
|
)
|
|
77
|
-
|
|
77
|
+
case read_write_mode # NOTE: (steep) case/when narrows `Symbol` to `RQL::readWriteMode` below
|
|
78
|
+
when :read, :write then nil
|
|
79
|
+
else
|
|
78
80
|
raise(
|
|
79
81
|
RedisQueuedLocks::ArgumentError,
|
|
80
82
|
"`:read_write_mode` argument should be `:read` or `:write`, " \
|