redis_queued_locks 1.21.0 → 1.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. checksums.yaml +4 -4
  2. data/.claude/project-overview.md +216 -25
  3. data/.claude/rules/acquirer.md +6 -5
  4. data/.claude/rules/arguments.md +2 -2
  5. data/.claude/rules/changelog.md +27 -0
  6. data/.claude/rules/git.md +1 -1
  7. data/.claude/rules/logic.md +10 -12
  8. data/.claude/rules/swarm.md +3 -4
  9. data/.claude/rules/tests.md +6 -6
  10. data/.claude/rules/type-checking.md +13 -14
  11. data/.claude/rules/visitors.md +5 -5
  12. data/.claude/skills/rql-code-review/SKILL.md +112 -35
  13. data/CHANGELOG.md +98 -244
  14. data/CLAUDE.md +9 -8
  15. data/README.md +275 -85
  16. data/Rakefile +11 -0
  17. data/lib/redis_queued_locks/acquirer/acquire_lock/instr_visitor.rb +28 -12
  18. data/lib/redis_queued_locks/acquirer/acquire_lock/log_visitor.rb +38 -16
  19. data/lib/redis_queued_locks/acquirer/acquire_lock/try_to_lock/log_visitor.rb +92 -15
  20. data/lib/redis_queued_locks/acquirer/acquire_lock/try_to_lock.rb +430 -63
  21. data/lib/redis_queued_locks/acquirer/acquire_lock.rb +72 -22
  22. data/lib/redis_queued_locks/acquirer/clear_dead_requests.rb +240 -45
  23. data/lib/redis_queued_locks/acquirer/extend_lock_ttl.rb +3 -1
  24. data/lib/redis_queued_locks/acquirer/keys.rb +6 -2
  25. data/lib/redis_queued_locks/acquirer/lock_series_poc/instr_visitor.rb +10 -5
  26. data/lib/redis_queued_locks/acquirer/lock_series_poc/log_visitor.rb +18 -9
  27. data/lib/redis_queued_locks/acquirer/lock_series_poc.rb +52 -23
  28. data/lib/redis_queued_locks/acquirer/locks.rb +6 -2
  29. data/lib/redis_queued_locks/acquirer/queue_info.rb +30 -7
  30. data/lib/redis_queued_locks/acquirer/queues.rb +17 -6
  31. data/lib/redis_queued_locks/acquirer/release_all_locks.rb +6 -2
  32. data/lib/redis_queued_locks/acquirer/release_locks_of.rb +6 -2
  33. data/lib/redis_queued_locks/client.rb +66 -24
  34. data/lib/redis_queued_locks/config.rb +93 -1
  35. data/lib/redis_queued_locks/resource.rb +124 -9
  36. data/lib/redis_queued_locks/swarm/swarm_element/threaded.rb +6 -5
  37. data/lib/redis_queued_locks/swarm/zombie_info.rb +6 -2
  38. data/lib/redis_queued_locks/version.rb +2 -2
  39. data/rbs_collection.lock.yaml +4 -0
  40. data/rbs_collection.yaml +2 -1
  41. data/sig/manifest.yml +1 -0
  42. data/sig/redis_queued_locks/acquirer/acquire_lock/dequeue_from_lock_queue/log_visitor.rbs +2 -2
  43. data/sig/redis_queued_locks/acquirer/acquire_lock/dequeue_from_lock_queue.rbs +2 -2
  44. data/sig/redis_queued_locks/acquirer/acquire_lock/instr_visitor.rbs +12 -6
  45. data/sig/redis_queued_locks/acquirer/acquire_lock/log_visitor.rbs +21 -14
  46. data/sig/redis_queued_locks/acquirer/acquire_lock/try_to_lock/log_visitor.rbs +61 -40
  47. data/sig/redis_queued_locks/acquirer/acquire_lock/try_to_lock.rbs +59 -11
  48. data/sig/redis_queued_locks/acquirer/acquire_lock/{yield_with_expire → yield_expire}/log_visitor.rbs +4 -4
  49. data/sig/redis_queued_locks/acquirer/acquire_lock/yield_expire.rbs +2 -2
  50. data/sig/redis_queued_locks/acquirer/acquire_lock.rbs +5 -3
  51. data/sig/redis_queued_locks/acquirer/clear_dead_requests.rbs +14 -1
  52. data/sig/redis_queued_locks/acquirer/extend_lock_ttl.rbs +2 -2
  53. data/sig/redis_queued_locks/acquirer/queues.rbs +1 -1
  54. data/sig/redis_queued_locks/client.rbs +10 -7
  55. data/sig/redis_queued_locks/config.rbs +54 -1
  56. data/sig/redis_queued_locks/instrument/sampler.rbs +3 -1
  57. data/sig/redis_queued_locks/instrument/void_notifier.rbs +3 -1
  58. data/sig/redis_queued_locks/logging/sampler.rbs +3 -1
  59. data/sig/redis_queued_locks/resource.rbs +22 -12
  60. data/sig/redis_queued_locks/swarm/swarm_element/threaded.rbs +1 -1
  61. data/sig/redis_queued_locks/utilities.rbs +2 -1
  62. data/sig/redis_queued_locks.rbs +7 -0
  63. metadata +4 -3
  64. /data/sig/redis_queued_locks/{acquier.rbs → acquirer.rbs} +0 -0
@@ -2,7 +2,7 @@
2
2
 
3
3
  # @api private
4
4
  # @since 1.0.0
5
- # @version 1.20.0
5
+ # @version 1.22.0
6
6
  # rubocop:disable Metrics/ModuleLength
7
7
  # rubocop:disable Metrics/MethodLength
8
8
  # rubocop:disable Metrics/ClassLength
@@ -126,6 +126,22 @@ module RedisQueuedLocks::Acquirer::AcquireLock
126
126
  # (but with checking the limist, retries, timeouts and so on). if lock is
127
127
  # free to obtain - it will be obtained;
128
128
  # - pre-configured in `config['default_access_strategy']`;
129
+ # @option priority [Integer,Symbol]
130
+ # - The lock request priority: from `RedisQueuedLocks::Resource::MIN_LOCK_PRIORITY` (`-100`)
131
+ # to `RedisQueuedLocks::Resource::MAX_LOCK_PRIORITY` (`100`), `0` by default, or a priority
132
+ # name of `lock_priorities` (resolved to its Integer priority: the lock request queues,
133
+ # the logs and the instrumentation use the Integer priority);
134
+ # - `:queued` access strategy: the lock request is placed in the lock request queue after all
135
+ # requests of the same or higher priority and before all requests of the lower priority
136
+ # (strict priority, FIFO inside the same priority; between modes too: the read and write
137
+ # lock request queues use the same positions);
138
+ # - `:random` access strategy: the lock request does not wait for its turn (it ignores the
139
+ # order of the lock request queues and the priorities of other requests), but it is queued
140
+ # at the position of its priority like a `:queued` one: `:queued` lock requests wait for it;
141
+ # - the obtained lock is never taken away: the priority orders the waiting lock requests only;
142
+ # - pre-configured in `config['default_lock_priority']`;
143
+ # @option lock_priorities [Hash<Symbol,Integer>]
144
+ # - The priority names (`config['lock_priorities']`): a priority name => its Integer priority;
129
145
  # @option log_sampling_enabled [Boolean]
130
146
  # - enables <log sampling>: only the configured percent of RQL cases will be logged;
131
147
  # - disabled by default;
@@ -177,7 +193,7 @@ module RedisQueuedLocks::Acquirer::AcquireLock
177
193
  #
178
194
  # @api private
179
195
  # @since 1.0.0
180
- # @version 1.21.0
196
+ # @version 1.22.0
181
197
  def acquire_lock(
182
198
  redis,
183
199
  lock_name,
@@ -206,6 +222,8 @@ module RedisQueuedLocks::Acquirer::AcquireLock
206
222
  conflict_strategy:,
207
223
  read_write_mode:,
208
224
  access_strategy:,
225
+ priority:,
226
+ lock_priorities:,
209
227
  log_sampling_enabled:,
210
228
  log_sampling_percent:,
211
229
  log_sampler:,
@@ -228,7 +246,9 @@ module RedisQueuedLocks::Acquirer::AcquireLock
228
246
  end
229
247
 
230
248
  # Step 0.1 (RW): prevent :read_write_mode incompatabilities
231
- if read_write_mode != :read && read_write_mode != :write
249
+ case read_write_mode # NOTE: (steep) case/when narrows `Symbol` to `RQL::readWriteMode` below
250
+ when :read, :write then nil
251
+ else
232
252
  raise(
233
253
  RedisQueuedLocks::ArgumentError,
234
254
  "`:read_write_mode` argument should be `:read` or `:write`, " \
@@ -292,6 +312,27 @@ module RedisQueuedLocks::Acquirer::AcquireLock
292
312
  )
293
313
  end
294
314
 
315
+ # Step 0.1: prevent :priority incompatibilities (the same range as its config key)
316
+ # - a Symbol is a priority name of `config['lock_priorities']`: it is resolved to its
317
+ # Integer priority (the lock request queues, the logs and the instrumentation use the
318
+ # Integer priority; an unknown name stays a Symbol and fails the check);
319
+ # - each priority has its own band of positions in the lock request queues: the range
320
+ # bounds the shift of the positions (float scores lose precision with the shift: the FIFO
321
+ # order inside a priority keeps ~61 µs for -100..100, see `LOCK_PRIORITY_BAND`);
322
+ priority = lock_priorities.fetch(priority, priority) if priority.is_a?(::Symbol)
323
+ if !priority.is_a?(::Integer) ||
324
+ priority < RedisQueuedLocks::Resource::MIN_LOCK_PRIORITY ||
325
+ priority > RedisQueuedLocks::Resource::MAX_LOCK_PRIORITY
326
+ raise(
327
+ RedisQueuedLocks::ArgumentError,
328
+ "`:priority` argument should be an Integer from " \
329
+ "#{RedisQueuedLocks::Resource::MIN_LOCK_PRIORITY} to " \
330
+ "#{RedisQueuedLocks::Resource::MAX_LOCK_PRIORITY} or a priority name of " \
331
+ "`config['lock_priorities']` (#{lock_priorities.keys.map(&:inspect).join(', ')}), " \
332
+ "got #{priority.inspect}."
333
+ )
334
+ end
335
+
295
336
  # Step 0.2: prevent :meta incompatabiltiies (structure)
296
337
  # - Symbol keys are written to the lock data as Strings (`Symbol#name`, as redis-client
297
338
  # does it) and would overwrite the reserved lock data fields too;
@@ -346,7 +387,9 @@ module RedisQueuedLocks::Acquirer::AcquireLock
346
387
  RedisQueuedLocks::Resource.prepare_read_lock_key(lock_name, acquirer_id)
347
388
  end
348
389
 
349
- acquirer_position = RedisQueuedLocks::Resource.calc_initial_acquirer_position
390
+ # NOTE: the position in the lock request queue: the enqueue time shifted by the priority band
391
+ # (see `RedisQueuedLocks::Resource::LOCK_PRIORITY_BAND`);
392
+ acquirer_position = RedisQueuedLocks::Resource.calc_initial_acquirer_position(priority)
350
393
 
351
394
  log_sampled = RedisQueuedLocks::Logging.should_log?(
352
395
  log_sampling_enabled,
@@ -393,7 +436,7 @@ module RedisQueuedLocks::Acquirer::AcquireLock
393
436
 
394
437
  LogVisitor.start_lock_obtaining(
395
438
  logger, log_sampled, lock_key,
396
- queue_ttl, acquirer_id, host_id, access_strategy, read_write_mode
439
+ queue_ttl, acquirer_id, host_id, access_strategy, read_write_mode, priority
397
440
  )
398
441
 
399
442
  # Step 2: try to lock with timeout
@@ -414,17 +457,18 @@ module RedisQueuedLocks::Acquirer::AcquireLock
414
457
 
415
458
  LogVisitor.start_try_to_lock_cycle(
416
459
  logger, log_sampled, lock_key,
417
- queue_ttl, acquirer_id, host_id, access_strategy, read_write_mode
460
+ queue_ttl, acquirer_id, host_id, access_strategy, read_write_mode, priority
418
461
  )
419
462
 
420
463
  # Step 2.X: check the actual score: is it in queue ttl limit or not?
421
- if RedisQueuedLocks::Resource.dead_score_reached?(acquirer_position, queue_ttl)
422
- # Step 2.X.X: dead score reached => re-queue the lock request with the new score;
423
- acquirer_position = RedisQueuedLocks::Resource.calc_initial_acquirer_position
464
+ if RedisQueuedLocks::Resource.dead_score_reached?(acquirer_position, priority, queue_ttl)
465
+ # Step 2.X.X: dead score reached => re-queue the lock request with the new score
466
+ # (the end of the requests of the same priority);
467
+ acquirer_position = RedisQueuedLocks::Resource.calc_initial_acquirer_position(priority)
424
468
 
425
469
  LogVisitor.dead_score_reached__reset_acquirer_position(
426
470
  logger, log_sampled, lock_key,
427
- queue_ttl, acquirer_id, host_id, access_strategy, read_write_mode
471
+ queue_ttl, acquirer_id, host_id, access_strategy, read_write_mode, priority
428
472
  )
429
473
  end
430
474
 
@@ -444,6 +488,7 @@ module RedisQueuedLocks::Acquirer::AcquireLock
444
488
  acquirer_id,
445
489
  host_id,
446
490
  acquirer_position,
491
+ priority,
447
492
  lock_ttl,
448
493
  queue_ttl,
449
494
  lock_index_enabled,
@@ -477,10 +522,11 @@ module RedisQueuedLocks::Acquirer::AcquireLock
477
522
  # instrumetnation: (reentrant lock with ttl extension)
478
523
  LogVisitor.extendable_reentrant_lock_obtained(
479
524
  logger, log_sampled, result[:lock_key],
480
- queue_ttl, acquirer_id, host_id, acq_time, access_strategy, read_write_mode
525
+ queue_ttl, acquirer_id, host_id, acq_time, access_strategy, read_write_mode,
526
+ priority
481
527
  )
482
528
  InstrVisitor.extendable_reentrant_lock_obtained(
483
- instrumenter, instr_sampled, result[:lock_key], read_write_mode,
529
+ instrumenter, instr_sampled, result[:lock_key], read_write_mode, priority,
484
530
  result[:ttl], result[:acq_id], result[:hst_id], result[:ts], acq_time,
485
531
  instrument
486
532
  )
@@ -488,10 +534,11 @@ module RedisQueuedLocks::Acquirer::AcquireLock
488
534
  # instrumetnation: (reentrant lock without ttl extension)
489
535
  LogVisitor.reentrant_lock_obtained(
490
536
  logger, log_sampled, result[:lock_key],
491
- queue_ttl, acquirer_id, host_id, acq_time, access_strategy, read_write_mode
537
+ queue_ttl, acquirer_id, host_id, acq_time, access_strategy, read_write_mode,
538
+ priority
492
539
  )
493
540
  InstrVisitor.reentrant_lock_obtained(
494
- instrumenter, instr_sampled, result[:lock_key], read_write_mode,
541
+ instrumenter, instr_sampled, result[:lock_key], read_write_mode, priority,
495
542
  result[:ttl], result[:acq_id], result[:hst_id], result[:ts], acq_time,
496
543
  instrument
497
544
  )
@@ -500,10 +547,11 @@ module RedisQueuedLocks::Acquirer::AcquireLock
500
547
  # NOTE: classic is: acq_process[:result][:process] == :lock_obtaining
501
548
  LogVisitor.lock_obtained(
502
549
  logger, log_sampled, result[:lock_key],
503
- queue_ttl, acquirer_id, host_id, acq_time, access_strategy, read_write_mode
550
+ queue_ttl, acquirer_id, host_id, acq_time, access_strategy, read_write_mode,
551
+ priority
504
552
  )
505
553
  InstrVisitor.lock_obtained(
506
- instrumenter, instr_sampled, result[:lock_key], read_write_mode,
554
+ instrumenter, instr_sampled, result[:lock_key], read_write_mode, priority,
507
555
  result[:ttl], result[:acq_id], result[:hst_id], result[:ts], acq_time,
508
556
  instrument
509
557
  )
@@ -661,6 +709,7 @@ module RedisQueuedLocks::Acquirer::AcquireLock
661
709
  instr_sampled,
662
710
  acq_process[:lock_info][:lock_key],
663
711
  read_write_mode,
712
+ priority,
664
713
  acq_process[:lock_info][:ttl],
665
714
  acq_process[:lock_info][:acq_id],
666
715
  acq_process[:lock_info][:hst_id],
@@ -676,6 +725,7 @@ module RedisQueuedLocks::Acquirer::AcquireLock
676
725
  instr_sampled,
677
726
  acq_process[:lock_info][:lock_key],
678
727
  read_write_mode,
728
+ priority,
679
729
  acq_process[:lock_info][:ttl],
680
730
  acq_process[:lock_info][:acq_id],
681
731
  acq_process[:lock_info][:hst_id],
@@ -705,11 +755,11 @@ module RedisQueuedLocks::Acquirer::AcquireLock
705
755
  acq_time = ((clock_gettime - acq_start_time) / 1_000.0).ceil(2)
706
756
  LogVisitor.lock_obtaining_failed(
707
757
  logger, log_sampled, lock_key, queue_ttl, acquirer_id, host_id,
708
- acq_time, access_strategy, read_write_mode, acq_process[:result]
758
+ acq_time, access_strategy, read_write_mode, priority, acq_process[:result]
709
759
  )
710
760
  InstrVisitor.lock_obtaining_failed(
711
- instrumenter, instr_sampled, lock_key, read_write_mode, lock_ttl, acquirer_id, host_id,
712
- Time.now.to_f, acq_time, acq_process[:result], instrument
761
+ instrumenter, instr_sampled, lock_key, read_write_mode, priority, lock_ttl, acquirer_id,
762
+ host_id, Time.now.to_f, acq_time, acq_process[:result], instrument
713
763
  )
714
764
  { ok: false, result: acq_process[:result] } #: { ok: bool, result: Symbol }
715
765
  end
@@ -730,11 +780,11 @@ module RedisQueuedLocks::Acquirer::AcquireLock
730
780
  acq_time = ((clock_gettime - acq_start_time) / 1_000.0).ceil(2)
731
781
  LogVisitor.lock_obtaining_failed(
732
782
  logger, log_sampled, lock_key, queue_ttl, acquirer_id, host_id,
733
- acq_time, access_strategy, read_write_mode, acq_process[:result]
783
+ acq_time, access_strategy, read_write_mode, priority, acq_process[:result]
734
784
  )
735
785
  InstrVisitor.lock_obtaining_failed(
736
- instrumenter, instr_sampled, lock_key, read_write_mode, lock_ttl, acquirer_id, host_id,
737
- Time.now.to_f, acq_time, acq_process[:result], instrument
786
+ instrumenter, instr_sampled, lock_key, read_write_mode, priority, lock_ttl, acquirer_id,
787
+ host_id, Time.now.to_f, acq_time, acq_process[:result], instrument
738
788
  )
739
789
  end
740
790
  # steep:ignore:end
@@ -2,7 +2,7 @@
2
2
 
3
3
  # @api private
4
4
  # @since 1.0.0
5
- # @version 1.21.0
5
+ # @version 1.22.0
6
6
  # rubocop:disable Metrics/ModuleLength
7
7
  module RedisQueuedLocks::Acquirer::ClearDeadRequests
8
8
  # @since 1.21.0
@@ -11,13 +11,19 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
11
11
  # rubocop:disable Metrics/ClassLength
12
12
  class << self
13
13
  # Drops dead lock requests (requests older than `dead_ttl`) from the lock queues (write and
14
- # read):
15
- # - the disabled lock index: the lock queues found in one SCAN pass, one round trip per batch
16
- # of `scan_size` lock queues (batches are collected across SCAN pages);
14
+ # read; the requests of any priority: see `RedisQueuedLocks::Resource::LOCK_PRIORITY_BAND`):
15
+ # - the disabled lock index: the lock queues found in one SCAN pass, processed in batches of
16
+ # `scan_size` lock queues (batches are collected across SCAN pages);
17
17
  # - the enabled lock index: the lock queues of the locks of the lock index: one ZSCAN pass
18
- # over the lock names (the keyspace is never scanned), one round trip per batch of
19
- # `scan_size` lock names (both lock queues of each lock name); expired lock names are
20
- # dropped from the lock index first;
18
+ # over the lock names (the keyspace is never scanned), processed in batches of `scan_size`
19
+ # lock names (both lock queues of each lock name); expired lock names are dropped from the
20
+ # lock index first;
21
+ # - the batch round trips (see `drop_dead_requests`): the lock requests of the batch queues
22
+ # are read, then the dead requests of each priority band that has them are dropped (two
23
+ # round trips, one without dead requests); `by_priority`: only the first lock request of
24
+ # each band is read (a walk over the bands of the batch queues: the most bands of a batch
25
+ # queue + 1 round trips, the dead requests are dropped on the way, see
26
+ # `drop_dead_requests_by_priority`);
21
27
  # - the operation is reported with one log and one instrumentation event
22
28
  # (`redis_queued_locks.clear_dead_requests`) with the time of the whole operation;
23
29
  #
@@ -26,6 +32,10 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
26
32
  # SCAN count and the number of lock queues in one batch (the disabled lock index) or ZSCAN
27
33
  # count and the number of lock names in one batch (the enabled lock index).
28
34
  # @param dead_ttl [Integer] In milliseconds
35
+ # @param by_priority [Boolean]
36
+ # Read the first lock request of each priority band of the lock queues instead of all lock
37
+ # requests (more round trips: one per band of the batch queue with the most bands + 1; the
38
+ # transfer does not depend on the number of lock requests).
29
39
  # @param lock_index_enabled [Boolean]
30
40
  # Find lock queues via the lock index instead of the keyspace SCAN.
31
41
  # @param logger [::Logger,#debug]
@@ -43,12 +53,13 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
43
53
  #
44
54
  # @api private
45
55
  # @since 1.0.0
46
- # @version 1.21.0
56
+ # @version 1.22.0
47
57
  # rubocop:disable Metrics/MethodLength
48
58
  def clear_dead_requests(
49
59
  redis_client,
50
60
  scan_size,
51
61
  dead_ttl,
62
+ by_priority,
52
63
  lock_index_enabled,
53
64
  logger,
54
65
  instrumenter,
@@ -76,10 +87,16 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
76
87
  "`:dead_ttl` argument should be a positive Integer, got #{dead_ttl.inspect}."
77
88
  )
78
89
  end
90
+ if by_priority != true && by_priority != false
91
+ raise(
92
+ RedisQueuedLocks::ArgumentError,
93
+ "`:by_priority` argument should be a boolean, got #{by_priority.inspect}."
94
+ )
95
+ end
79
96
 
80
97
  rel_start_time = clock_gettime
81
98
  processed_queues = fully_clear_dead_requests(
82
- redis_client, scan_size, dead_ttl, lock_index_enabled
99
+ redis_client, scan_size, dead_ttl, by_priority, lock_index_enabled
83
100
  )
84
101
  time_at = Time.now.to_f
85
102
  rel_end_time = clock_gettime
@@ -129,12 +146,17 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
129
146
  # @param redis_client [RedisClient]
130
147
  # @param scan_size [Integer]
131
148
  # @param dead_ttl [Integer] In milliseconds
149
+ # @param by_priority [Boolean]
132
150
  # @param lock_index_enabled [Boolean]
133
151
  # @return [Set<String>] The processed lock queues (write and read lock request queues).
134
152
  #
135
153
  # @api private
136
154
  # @since 1.21.0
137
- def fully_clear_dead_requests(redis_client, scan_size, dead_ttl, lock_index_enabled)
155
+ # @version 1.22.0
156
+ # rubocop:disable Metrics/MethodLength
157
+ def fully_clear_dead_requests(
158
+ redis_client, scan_size, dead_ttl, by_priority, lock_index_enabled
159
+ )
138
160
  dead_score = RedisQueuedLocks::Resource.acquirer_dead_score(dead_ttl / 1_000.0)
139
161
 
140
162
  Set.new.tap do |processed_queues|
@@ -150,12 +172,15 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
150
172
 
151
173
  # Step 2 (lock index): drop dead requests from the lock queues of the lock index
152
174
  each_lock_name_batch(rconn, scan_size) do |lock_names|
153
- drop_dead_requests_of_locks(rconn, lock_names, dead_score, processed_queues)
175
+ drop_dead_requests_of_locks(
176
+ rconn, lock_names, dead_score, by_priority, processed_queues
177
+ )
154
178
  end
155
179
  else
156
180
  # Step 1 (keyspace): drop dead requests from the lock queues found in one SCAN pass
157
- # NOTE: dead requests are dropped with one pipeline per batch of `scan_size` queues
158
- # (batches are collected across SCAN pages: lock queues can be rare among other keys);
181
+ # NOTE: dead requests are dropped per batch of `scan_size` queues (the round trips of
182
+ # `drop_dead_requests`; batches are collected across SCAN pages: lock queues can be
183
+ # rare among other keys);
159
184
  # @type var lock_queues: Array[String]
160
185
  lock_queues = []
161
186
 
@@ -164,15 +189,18 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
164
189
  lock_queues << lock_queue
165
190
  next if lock_queues.size < scan_size
166
191
 
167
- drop_dead_requests(rconn, lock_queues, dead_score)
192
+ drop_dead_requests(rconn, lock_queues, dead_score, by_priority)
168
193
  lock_queues.clear
169
194
  end
170
195
 
171
- drop_dead_requests(rconn, lock_queues, dead_score) unless lock_queues.empty?
196
+ unless lock_queues.empty?
197
+ drop_dead_requests(rconn, lock_queues, dead_score, by_priority)
198
+ end
172
199
  end
173
200
  end
174
201
  end
175
202
  end
203
+ # rubocop:enable Metrics/MethodLength
176
204
 
177
205
  # Write and read lock queues in one SCAN pass (the disabled lock index).
178
206
  #
@@ -198,62 +226,226 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
198
226
  end
199
227
  end
200
228
 
201
- # Drops dead requests of the given lock queues in one round trip (the disabled lock index).
229
+ # Drops dead requests of the given lock queues (both lookups; `by_priority`: the first lock
230
+ # request of each priority band is read instead of all lock requests, see
231
+ # `drop_dead_requests_by_priority`):
232
+ # - the first round trip reads the lock requests of the lock queues;
233
+ # - the second round trip drops the dead requests of each priority band that has them: one
234
+ # `ZREMRANGEBYSCORE` per band from the lower edge of the band to its time barrier (see
235
+ # `RedisQueuedLocks::Resource::LOCK_PRIORITY_BAND`), so the number of commands depends on
236
+ # the bands of the queued requests, not on the priority range; no round trip without dead
237
+ # requests (an empty pipeline sends nothing);
238
+ # - the requests of a band are adjacent and ordered by the enqueue time (the bands are
239
+ # disjoint position ranges): the first request of a band decides whether the band has dead
240
+ # requests;
241
+ # - requests queued (or re-queued) between the round trips are never dropped: they are after
242
+ # the time barrier of their band;
243
+ # - (infinite positions) RQL never writes them (manual writes of the queues, see
244
+ # `RedisQueuedLocks::Resource.lock_request_priority`): the requests at `-inf` are dead and
245
+ # are dropped by their position (`-inf` .. `-inf`), the requests at `+inf` are never dead;
202
246
  #
203
247
  # @param redis_client [RedisClient]
204
248
  # @param lock_queues [Array<String>]
205
- # @param dead_score [Float]
206
- # @return [void]
249
+ # @param dead_score [Float] Requests queued before this time are dead.
250
+ # @param by_priority [Boolean]
251
+ # @return [Array<Array<Array<String,Float>>>]
252
+ # The lock requests (acquirer id, position) of each lock queue before the removal (the first
253
+ # lock request with `by_priority`), in the order of `lock_queues` (an empty list for a missing
254
+ # lock queue).
207
255
  #
208
256
  # @api private
209
257
  # @since 1.20.0
210
- def drop_dead_requests(redis_client, lock_queues, dead_score)
211
- redis_client.pipelined do |pipeline|
258
+ # @version 1.22.0
259
+ # rubocop:disable Metrics/MethodLength
260
+ def drop_dead_requests(redis_client, lock_queues, dead_score, by_priority)
261
+ return drop_dead_requests_by_priority(redis_client, lock_queues, dead_score) if by_priority
262
+
263
+ # NOTE: (pipeline layout) one ZRANGE WITHSCORES per lock queue (the `lock_queues` order)
264
+ # @type var queued_requests: Array[Array[[String, Float]]]
265
+ queued_requests = redis_client.pipelined do |pipeline|
212
266
  lock_queues.each do |lock_queue|
213
- pipeline.call('ZREMRANGEBYSCORE', lock_queue, '-inf', dead_score)
267
+ pipeline.call('ZRANGE', lock_queue, '0', '-1', 'WITHSCORES')
214
268
  end
215
269
  end
270
+
271
+ redis_client.pipelined do |pipeline|
272
+ lock_queues.each_with_index do |lock_queue, index|
273
+ # @type var checked_band: Integer?
274
+ checked_band = nil
275
+
276
+ # NOTE: (infinite positions) the requests at `-inf` (manual writes: out of all bands, see
277
+ # `RedisQueuedLocks::Resource.lock_request_priority`) are dead: they are the first
278
+ # requests of the queue and are dropped by their position (no band removal covers them);
279
+ first_request = queued_requests[index].first
280
+ if first_request && first_request[1] == -Float::INFINITY
281
+ pipeline.call('ZREMRANGEBYSCORE', lock_queue, '-inf', '-inf')
282
+ end
283
+
284
+ queued_requests[index].each do |lock_request|
285
+ position = lock_request[1]
286
+ next if position == -Float::INFINITY
287
+
288
+ priority = RedisQueuedLocks::Resource.lock_request_priority(position)
289
+ next if priority == checked_band
290
+
291
+ # NOTE: the first request of the band: the band has dead requests when it is dead
292
+ checked_band = priority
293
+ band_dead_score =
294
+ RedisQueuedLocks::Resource.lock_priority_band_dead_score(dead_score, priority)
295
+ next if position > band_dead_score
296
+
297
+ pipeline.call(
298
+ 'ZREMRANGEBYSCORE', lock_queue,
299
+ RedisQueuedLocks::Resource.lock_priority_band_floor(priority), band_dead_score
300
+ )
301
+ end
302
+ end
303
+ end
304
+
305
+ queued_requests
216
306
  end
307
+ # rubocop:enable Metrics/MethodLength
217
308
 
218
- # Drops dead requests of both lock queues of the given lock names in one round trip
219
- # (the enabled lock index).
309
+ # Drops dead requests of the given lock queues by the first lock request of each priority band
310
+ # (`by_priority`, both lookups): the lock requests of the lock queues are never read at once:
311
+ # - a walk over the bands of each lock queue: each round trip reads the first lock request of
312
+ # the next band of each walked lock queue (`ZRANGE <queue> <bound> +inf BYSCORE LIMIT 0 1
313
+ # WITHSCORES`, the first round trip from `-inf`; the next bound is the lower edge of the
314
+ # next, lower, band: `RedisQueuedLocks::Resource.lock_priority_band_floor(priority - 1)`),
315
+ # so the transfer depends on the number of bands, not on the number of lock requests;
316
+ # - the first request of a band decides whether the band has dead requests (see
317
+ # `drop_dead_requests`, the same removal commands): they are dropped by the next round trip
318
+ # (in the same pipeline after the reads: the removal never touches the next bands);
319
+ # - a lock queue is walked till no more lock requests are found, so the round trips of
320
+ # a batch are the most bands of a batch queue + 1 (the last read finds nothing), with or
321
+ # without dead requests;
322
+ # - requests queued (or re-queued) during the walk are never dropped: they are after the time
323
+ # barrier of their band;
324
+ # - (infinite positions) the requests at `-inf` are dead and are dropped by their position
325
+ # (the walk goes on after them: `(-inf`), the walk of a lock queue stops at a request at
326
+ # `+inf` (never dead, the last requests of the queue);
327
+ # - (no progress) a position out of the float precision of the band edges (manual writes far
328
+ # out of the priority range): the lower edge of the next band rounds to the position itself
329
+ # (Redis parses the bound to the same Float), so the walk goes on after the position (an
330
+ # exclusive bound) instead of reading the same request again;
331
+ #
332
+ # @param redis_client [RedisClient]
333
+ # @param lock_queues [Array<String>]
334
+ # @param dead_score [Float] Requests queued before this time are dead.
335
+ # @return [Array<Array<Array<String,Float>>>]
336
+ # The first lock request (acquirer id, position) of each lock queue before the removal, in the
337
+ # order of `lock_queues` (an empty list for a missing lock queue).
338
+ #
339
+ # @api private
340
+ # @since 1.22.0
341
+ # rubocop:disable Metrics/MethodLength
342
+ def drop_dead_requests_by_priority(redis_client, lock_queues, dead_score)
343
+ # NOTE: the walked lock queues (their indexes in `lock_queues`) and the lower bound of the
344
+ # next read of each lock queue
345
+ walked_queues = Array.new(lock_queues.size) { |index| index }
346
+ # @type var bounds: Array[String | Float]
347
+ bounds = Array.new(lock_queues.size, '-inf')
348
+ # @type var removals: Array[[String, String | Integer, String | Float]]
349
+ removals = []
350
+ # @type var first_requests: Array[Array[[String, Float]]]
351
+ first_requests = []
352
+
353
+ until walked_queues.empty?
354
+ # NOTE: (pipeline layout) one ZRANGE per walked lock queue (the `walked_queues` order),
355
+ # then the removals of the dead requests found by the previous round trip (not read)
356
+ # @type var band_heads: Array[Array[[String, Float]]]
357
+ band_heads = redis_client.pipelined do |pipeline|
358
+ walked_queues.each do |index|
359
+ pipeline.call(
360
+ 'ZRANGE', lock_queues[index], bounds[index], '+inf',
361
+ 'BYSCORE', 'LIMIT', '0', '1', 'WITHSCORES'
362
+ )
363
+ end
364
+ removals.each do |lock_queue, lowest_position, dead_position|
365
+ pipeline.call('ZREMRANGEBYSCORE', lock_queue, lowest_position, dead_position)
366
+ end
367
+ end
368
+ removals.clear
369
+ # NOTE: the first round trip reads all lock queues (in the `lock_queues` order, no removals)
370
+ first_requests = band_heads if first_requests.empty?
371
+
372
+ # NOTE: a lock queue with a found request is walked further (a removal of its band is sent
373
+ # with the next read of the lock queue: no removal is left after the walk)
374
+ # @type var next_queues: Array[Integer]
375
+ next_queues = []
376
+ walked_queues.each_with_index do |index, read_index|
377
+ band_head = band_heads[read_index].first
378
+ next unless band_head
379
+
380
+ position = band_head[1]
381
+ next if position == Float::INFINITY
382
+
383
+ lock_queue = lock_queues[index]
384
+ if position == -Float::INFINITY
385
+ removals << [lock_queue, '-inf', '-inf']
386
+ bounds[index] = '(-inf'
387
+ else
388
+ priority = RedisQueuedLocks::Resource.lock_request_priority(position)
389
+ band_dead_score =
390
+ RedisQueuedLocks::Resource.lock_priority_band_dead_score(dead_score, priority)
391
+ if position <= band_dead_score
392
+ removals << [
393
+ lock_queue,
394
+ RedisQueuedLocks::Resource.lock_priority_band_floor(priority),
395
+ band_dead_score
396
+ ]
397
+ end
398
+
399
+ # NOTE: (no progress) see the method description
400
+ next_band_floor =
401
+ RedisQueuedLocks::Resource.lock_priority_band_floor(priority - 1).to_f
402
+ bounds[index] = (next_band_floor > position) ? next_band_floor : "(#{position}"
403
+ end
404
+ next_queues << index
405
+ end
406
+ walked_queues = next_queues
407
+ end
408
+
409
+ first_requests
410
+ end
411
+ # rubocop:enable Metrics/MethodLength
412
+
413
+ # Drops dead requests of both lock queues of the given lock names (the enabled lock index; the
414
+ # round trips of `drop_dead_requests`).
220
415
  #
221
416
  # @param redis_client [RedisClient]
222
417
  # @param lock_names [Array<String>]
223
418
  # @param dead_score [Float]
419
+ # @param by_priority [Boolean]
224
420
  # @param processed_queues [Set<String>] Collects the existing (processed) lock queues.
225
421
  # @return [void]
226
422
  #
227
423
  # @api private
228
424
  # @since 1.20.0
229
- def drop_dead_requests_of_locks(redis_client, lock_names, dead_score, processed_queues)
230
- # NOTE: (pipeline layout) two replies per lock queue (two lock queues per lock name) in
231
- # the batch order: EXISTS (the queue is processed when it exists), ZREMRANGEBYSCORE;
232
- # @type var replies: Array[Integer]
233
- replies = redis_client.pipelined do |pipeline|
234
- lock_names.each do |lock_name|
235
- lock_queue = RedisQueuedLocks::Resource.prepare_lock_queue(lock_name)
236
- read_lock_queue = RedisQueuedLocks::Resource.prepare_read_lock_queue(lock_name)
237
- pipeline.call('EXISTS', lock_queue)
238
- pipeline.call('ZREMRANGEBYSCORE', lock_queue, '-inf', dead_score)
239
- pipeline.call('EXISTS', read_lock_queue)
240
- pipeline.call('ZREMRANGEBYSCORE', read_lock_queue, '-inf', dead_score)
241
- end
425
+ # @version 1.22.0
426
+ def drop_dead_requests_of_locks(
427
+ redis_client, lock_names, dead_score, by_priority, processed_queues
428
+ )
429
+ # NOTE: both lock queues of each lock name in the batch order (write, read)
430
+ # @type var lock_queues: Array[String]
431
+ lock_queues = []
432
+ lock_names.each do |lock_name|
433
+ lock_queues << RedisQueuedLocks::Resource.prepare_lock_queue(lock_name)
434
+ lock_queues << RedisQueuedLocks::Resource.prepare_read_lock_queue(lock_name)
242
435
  end
243
436
 
244
- lock_names.each_with_index do |lock_name, index|
245
- if replies[index * 4] == 1
246
- processed_queues << RedisQueuedLocks::Resource.prepare_lock_queue(lock_name)
247
- end
248
- if replies[(index * 4) + 2] == 1
249
- processed_queues << RedisQueuedLocks::Resource.prepare_read_lock_queue(lock_name)
250
- end
437
+ queued_requests = drop_dead_requests(redis_client, lock_queues, dead_score, by_priority)
438
+
439
+ # NOTE: a lock queue is processed when it exists (Redis never keeps empty sorted sets)
440
+ lock_queues.each_with_index do |lock_queue, index|
441
+ processed_queues << lock_queue unless queued_requests[index].empty?
251
442
  end
252
443
  end
253
444
 
254
445
  # Iterates over the lock names of the lock index (one ZSCAN pass) in batches of `scan_size`
255
446
  # lock names (batches are collected across ZSCAN pages). Expired lock names (not dropped from
256
- # the index yet) are skipped: all their keys are expired.
447
+ # the index yet) are skipped: all their keys are expired (`+inf` names, keys without TTL, never
448
+ # expire: see `RedisQueuedLocks::Resource.lock_index_score_expired?`).
257
449
  #
258
450
  # NOTE: each operation module iterates over the lock index by itself (duplicated on purpose:
259
451
  # operation modules are independent of each other, see `Locks.each_lock_name_batch`);
@@ -266,6 +458,7 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
266
458
  #
267
459
  # @api private
268
460
  # @since 1.20.0
461
+ # @version 1.22.0
269
462
  def each_lock_name_batch(redis_client, scan_size)
270
463
  lock_index_time = RedisQueuedLocks::Resource.lock_index_time
271
464
  # @type var lock_names: Array[String]
@@ -274,7 +467,9 @@ module RedisQueuedLocks::Acquirer::ClearDeadRequests
274
467
  redis_client.zscan(
275
468
  RedisQueuedLocks::Resource::LOCK_INDEX_KEY, count: scan_size
276
469
  ) do |lock_name, lock_index_score|
277
- next if lock_index_score.to_f < lock_index_time
470
+ next if RedisQueuedLocks::Resource.lock_index_score_expired?(
471
+ lock_index_score, lock_index_time
472
+ )
278
473
 
279
474
  lock_names << lock_name
280
475
  next if lock_names.size < scan_size
@@ -74,7 +74,9 @@ module RedisQueuedLocks::Acquirer::ExtendLockTTL
74
74
  instr_sampler,
75
75
  instr_sample_this
76
76
  )
77
- if read_write_mode != :read && read_write_mode != :write
77
+ case read_write_mode # NOTE: (steep) case/when narrows `Symbol` to `RQL::readWriteMode` below
78
+ when :read, :write then nil
79
+ else
78
80
  raise(
79
81
  RedisQueuedLocks::ArgumentError,
80
82
  "`:read_write_mode` argument should be `:read` or `:write`, " \