redis_queued_locks 1.22.0 → 1.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.claude/project-overview.md +88 -36
  3. data/.claude/rules/acquirer.md +11 -7
  4. data/.claude/rules/arguments.md +2 -2
  5. data/.claude/rules/logic.md +5 -4
  6. data/.claude/rules/tests.md +4 -4
  7. data/.claude/rules/type-checking.md +2 -3
  8. data/.claude/rules/visitors.md +8 -6
  9. data/.claude/skills/rql-code-review/SKILL.md +25 -20
  10. data/.ruby-version +1 -1
  11. data/CHANGELOG.md +24 -1
  12. data/CLAUDE.md +5 -4
  13. data/README.md +247 -96
  14. data/lib/redis_queued_locks/acquirer/acquire_lock/dequeue_from_lock_queue.rb +4 -4
  15. data/lib/redis_queued_locks/acquirer/acquire_lock/try_to_lock.rb +582 -276
  16. data/lib/redis_queued_locks/acquirer/acquire_lock.rb +26 -26
  17. data/lib/redis_queued_locks/acquirer/extend_lock_ttl.rb +13 -13
  18. data/lib/redis_queued_locks/acquirer/lock_series/instr_visitor.rb +243 -0
  19. data/lib/redis_queued_locks/acquirer/lock_series/log_visitor.rb +472 -0
  20. data/lib/redis_queued_locks/acquirer/lock_series/release_lock_series.rb +181 -0
  21. data/lib/redis_queued_locks/acquirer/lock_series/try_to_lock_series.rb +1051 -0
  22. data/lib/redis_queued_locks/acquirer/lock_series.rb +948 -0
  23. data/lib/redis_queued_locks/acquirer.rb +1 -1
  24. data/lib/redis_queued_locks/client.rb +55 -25
  25. data/lib/redis_queued_locks/version.rb +2 -2
  26. data/redis_queued_locks.gemspec +21 -14
  27. data/sig/redis_queued_locks/acquirer/acquire_lock/dequeue_from_lock_queue.rbs +1 -1
  28. data/sig/redis_queued_locks/acquirer/acquire_lock/try_to_lock.rbs +106 -5
  29. data/sig/redis_queued_locks/acquirer/acquire_lock.rbs +1 -1
  30. data/sig/redis_queued_locks/acquirer/extend_lock_ttl.rbs +2 -2
  31. data/sig/redis_queued_locks/acquirer/lock_series/instr_visitor.rbs +102 -0
  32. data/sig/redis_queued_locks/acquirer/lock_series/log_visitor.rbs +154 -0
  33. data/sig/redis_queued_locks/acquirer/lock_series/release_lock_series.rbs +37 -0
  34. data/sig/redis_queued_locks/acquirer/lock_series/try_to_lock_series.rbs +198 -0
  35. data/sig/redis_queued_locks/acquirer/lock_series.rbs +106 -0
  36. data/sig/redis_queued_locks/client.rbs +70 -3
  37. metadata +32 -11
  38. data/lib/redis_queued_locks/acquirer/lock_series_poc/instr_visitor.rb +0 -61
  39. data/lib/redis_queued_locks/acquirer/lock_series_poc/log_visitor.rb +0 -104
  40. data/lib/redis_queued_locks/acquirer/lock_series_poc.rb +0 -513
@@ -0,0 +1,948 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Lock series: a group of locks that are obtained atomically, held together and released together
4
+ # (the second part of the lock acquisition core next to `RedisQueuedLocks::Acquirer::AcquireLock`:
5
+ # it reuses the mixins of `AcquireLock`):
6
+ # - all or nothing: one transaction obtains all locks of the series with the same ttl at the same
7
+ # moment; the attempt that can not obtain some lock obtains nothing (see `TryToLockSeries`);
8
+ # - nothing is held while the series waits for its locks, so the series can not hold locks
9
+ # longer than its `ttl` and can not block other acquirers with partially obtained locks;
10
+ # - the lock series are deadlock-free for any order of the lock names (one position of the lock
11
+ # request in all queues of the series, see `TryToLockSeries`);
12
+ # - the series limits are the limits of the whole series: `timeout`, `retry_count`, `queue_ttl`;
13
+ # - one `rw_mode`, one `priority`, one `ttl` for all locks of the series;
14
+ # - the locks of the series are classic locks with the classic keys: lock info, `#unlock`,
15
+ # `#extend_lock_ttl`, the swarm and other operations work with each of them separately;
16
+ # - reentrant locks (`conflict_strategy`) are processed per lock: they are not obtained and not
17
+ # released by the series;
18
+ #
19
+ # @api private
20
+ # @since 1.23.0
21
+ # @version 1.23.0
22
+ # rubocop:disable Metrics/ModuleLength
23
+ # rubocop:disable Metrics/ClassLength
24
+ # rubocop:disable Metrics/BlockNesting
25
+ module RedisQueuedLocks::Acquirer::LockSeries
26
+ require_relative 'lock_series/log_visitor'
27
+ require_relative 'lock_series/instr_visitor'
28
+ require_relative 'lock_series/release_lock_series'
29
+ require_relative 'lock_series/try_to_lock_series'
30
+
31
+ # @since 1.23.0
32
+ extend RedisQueuedLocks::Utilities
33
+ # @since 1.23.0
34
+ extend RedisQueuedLocks::Acquirer::AcquireLock::DelayExecution
35
+ # @since 1.23.0
36
+ extend RedisQueuedLocks::Acquirer::AcquireLock::YieldExpire
37
+ # @since 1.23.0
38
+ extend TryToLockSeries
39
+
40
+ class << self
41
+ # @param redis [RedisClient]
42
+ # Redis connection client.
43
+ # @param lock_names [Array<String>]
44
+ # Controlled resources (a duplicated name is the same lock). At least one lock name.
45
+ # @option detailed_result [Boolean]
46
+ # Return the detailed result of the series (the block result, times, lock keys).
47
+ # @option process_id [Integer,String]
48
+ # The process that want to acquire the locks.
49
+ # @option thread_id [Integer,String]
50
+ # The process's thread that want to acquire the locks.
51
+ # @option fiber_id [Integer,String]
52
+ # A current fiber that want to acquire the locks.
53
+ # @option ractor_id [Integer,String]
54
+ # The current ractor that want to acquire the locks.
55
+ # @option ttl [Integer]
56
+ # The time to live of each lock of the series (in milliseconds). A positive Integer.
57
+ # @option queue_ttl [Integer]
58
+ # Lifetime of the lock request of the series (in seconds).
59
+ # @option lock_index_enabled [Boolean]
60
+ # Maintain the lock index (`config['lock_index_enabled']`, the client-wide mode).
61
+ # @option lock_index_grace_period [Integer]
62
+ # The lock index score margin over the lock key lifetime (in milliseconds).
63
+ # @option timeout [Integer,NilClass]
64
+ # Time period whe should try to acquire the whole series (in seconds). Nil means "without
65
+ # timeout" (`0` is the same as `nil`: `Timeout.timeout(0)` sets no timeout).
66
+ # @option timed [Boolean]
67
+ # Limit the invocation time period of the passed block of code by the `ttl`.
68
+ # @option retry_count [Integer,NilClass]
69
+ # How many times we should try to acquire the whole series. Nil means "infinite retries".
70
+ # @option retry_delay [Integer]
71
+ # A time-interval between the each retry (in milliseconds).
72
+ # @option retry_jitter [Integer]
73
+ # Time-shift range for retry-delay (in milliseconds).
74
+ # @option raise_errors [Boolean]
75
+ # Raise errors on exceptional cases.
76
+ # @option instrumenter [#notify]
77
+ # See RedisQueuedLocks::Instrument::ActiveSupport for example.
78
+ # @option identity [String]
79
+ # Unique acquire identifier (see `AcquireLock.acquire_lock`).
80
+ # @option fail_fast [Boolean]
81
+ # Should the required locks to be checked before the try and exit immidetly if any lock is
82
+ # already obtained.
83
+ # @option meta [NilClass,Hash<String|Symbol,Any>]
84
+ # - A custom metadata wich will be passed to the data of each lock of the series;
85
+ # - Metadata can not contain reserved lock data keys;
86
+ # @option detailed_acq_timeout_error [Boolean]
87
+ # Add the state of the lock that blocks the series to the acquirement timeout error.
88
+ # @option logger [::Logger,#debug]
89
+ # Logger object used from the configuration layer (see config['logger']).
90
+ # @option log_lock_try [Boolean]
91
+ # Should be logged the each try of the series acquiring (see `config['log_lock_try']`).
92
+ # @option instrument [NilClass,Any]
93
+ # Custom instrumentation data wich will be passed to the instrumenter's payload.
94
+ # @option conflict_strategy [Symbol]
95
+ # The conflict strategy of each lock of the series (see `AcquireLock.acquire_lock`).
96
+ # @option rw_mode [Symbol]
97
+ # The mode of all locks of the series: `:write` or `:read` (see `AcquireLock.acquire_lock`).
98
+ # @option access_strategy [Symbol]
99
+ # The way in which the locks will be obtained (see `AcquireLock.acquire_lock`).
100
+ # @option priority [Integer,Symbol]
101
+ # The lock request priority of the series (see `AcquireLock.acquire_lock`).
102
+ # @option lock_priorities [Hash<Symbol,Integer>]
103
+ # The priority names (`config['lock_priorities']`): a priority name => its Integer priority.
104
+ # @option log_sampling_enabled [Boolean]
105
+ # See `AcquireLock.acquire_lock`.
106
+ # @option log_sampling_percent [Integer]
107
+ # See `AcquireLock.acquire_lock`.
108
+ # @option log_sampler [#sampling_happened?,Module<RedisQueuedLocks::Logging::Sampler>]
109
+ # See `AcquireLock.acquire_lock`.
110
+ # @option log_sample_this [Boolean]
111
+ # See `AcquireLock.acquire_lock`.
112
+ # @option instr_sampling_enabled [Boolean]
113
+ # See `AcquireLock.acquire_lock`.
114
+ # @option instr_sampling_percent [Integer]
115
+ # See `AcquireLock.acquire_lock`.
116
+ # @option instr_sampler [#sampling_happened?,Module<RedisQueuedLocks::Instrument::Sampler>]
117
+ # See `AcquireLock.acquire_lock`.
118
+ # @option instr_sample_this [Boolean]
119
+ # See `AcquireLock.acquire_lock`.
120
+ # @yield A block of code that should be executed under all obtained locks.
121
+ # @yieldreturn [Any]
122
+ # @return [Hash<Symbol,Any>,yield]
123
+ # - Format: { ok: true/false, result: Any };
124
+ # - If block is given the result of block's yield will be returned;
125
+ # - `detailed_result: true`: the detailed result of the obtained series;
126
+ #
127
+ # @api private
128
+ # @since 1.23.0
129
+ # @version 1.23.0
130
+ # rubocop:disable Metrics/MethodLength
131
+ def lock_series(
132
+ redis,
133
+ lock_names,
134
+ detailed_result:,
135
+ process_id:,
136
+ thread_id:,
137
+ fiber_id:,
138
+ ractor_id:,
139
+ ttl:,
140
+ queue_ttl:,
141
+ lock_index_enabled:,
142
+ lock_index_grace_period:,
143
+ timeout:,
144
+ timed:,
145
+ retry_count:,
146
+ retry_delay:,
147
+ retry_jitter:,
148
+ raise_errors:,
149
+ instrumenter:,
150
+ identity:,
151
+ fail_fast:,
152
+ meta:,
153
+ detailed_acq_timeout_error:,
154
+ instrument:,
155
+ logger:,
156
+ log_lock_try:,
157
+ conflict_strategy:,
158
+ rw_mode:,
159
+ access_strategy:,
160
+ priority:,
161
+ lock_priorities:,
162
+ log_sampling_enabled:,
163
+ log_sampling_percent:,
164
+ log_sampler:,
165
+ log_sample_this:,
166
+ instr_sampling_enabled:,
167
+ instr_sampling_percent:,
168
+ instr_sampler:,
169
+ instr_sample_this:,
170
+ &block
171
+ )
172
+ # Step 0: Prevent argument type incompatabilities (the same checks as in
173
+ # RedisQueuedLocks::Acquirer::AcquireLock, duplicated on purpose)
174
+ # Step 0.1: prevent :lock_names incompatibilities
175
+ if lock_names.empty?
176
+ raise(
177
+ RedisQueuedLocks::ArgumentError,
178
+ "`:lock_names` argument should contain at least one lock name, got #{lock_names.inspect}."
179
+ )
180
+ end
181
+
182
+ # Step 0.1: prevent :meta incompatabiltiies (type)
183
+ case meta # NOTE: do not ask why case/when is used here
184
+ when Hash, NilClass then nil
185
+ else
186
+ raise(
187
+ RedisQueuedLocks::ArgumentError,
188
+ "`:meta` argument should be a type of NilClass or Hash, got #{meta.class}."
189
+ )
190
+ end
191
+
192
+ # Step 0.1 (RW): prevent :rw_mode incompatabilities
193
+ case rw_mode # NOTE: (steep) case/when narrows `Symbol` to `RQL::readWriteMode` below
194
+ when :read, :write then nil
195
+ else
196
+ raise(
197
+ RedisQueuedLocks::ArgumentError,
198
+ "`:rw_mode` argument should be `:read` or `:write`, " \
199
+ "got #{rw_mode.inspect}."
200
+ )
201
+ end
202
+
203
+ # Step 0.1: prevent numeric argument incompatibilities (the same ranges as their config keys)
204
+ if !ttl.is_a?(::Integer) || ttl <= 0
205
+ raise(
206
+ RedisQueuedLocks::ArgumentError,
207
+ "`:ttl` argument should be a positive Integer, got #{ttl.inspect}."
208
+ )
209
+ end
210
+ if !queue_ttl.is_a?(::Integer) || queue_ttl <= 0
211
+ raise(
212
+ RedisQueuedLocks::ArgumentError,
213
+ "`:queue_ttl` argument should be a positive Integer, got #{queue_ttl.inspect}."
214
+ )
215
+ end
216
+ if timeout != nil && (!timeout.is_a?(::Integer) || timeout < 0)
217
+ raise(
218
+ RedisQueuedLocks::ArgumentError,
219
+ "`:timeout` argument should be nil or a non-negative Integer, got #{timeout.inspect}."
220
+ )
221
+ end
222
+ if retry_count != nil && (!retry_count.is_a?(::Integer) || retry_count < 0)
223
+ raise(
224
+ RedisQueuedLocks::ArgumentError,
225
+ "`:retry_count` argument should be nil or a non-negative Integer, " \
226
+ "got #{retry_count.inspect}."
227
+ )
228
+ end
229
+ if !retry_delay.is_a?(::Integer) || retry_delay < 0
230
+ raise(
231
+ RedisQueuedLocks::ArgumentError,
232
+ "`:retry_delay` argument should be a non-negative Integer, got #{retry_delay.inspect}."
233
+ )
234
+ end
235
+ if !retry_jitter.is_a?(::Integer) || retry_jitter < 0
236
+ raise(
237
+ RedisQueuedLocks::ArgumentError,
238
+ "`:retry_jitter` argument should be a non-negative Integer, got #{retry_jitter.inspect}."
239
+ )
240
+ end
241
+ if !lock_index_grace_period.is_a?(::Integer) || lock_index_grace_period < 0
242
+ raise(
243
+ RedisQueuedLocks::ArgumentError,
244
+ "`:lock_index_grace_period` argument should be a non-negative Integer, " \
245
+ "got #{lock_index_grace_period.inspect}."
246
+ )
247
+ end
248
+
249
+ # Step 0.1: prevent :priority incompatibilities (the same range as its config key)
250
+ # - a Symbol is a priority name of `config['lock_priorities']`: it is resolved to its
251
+ # Integer priority (an unknown name stays a Symbol and fails the check);
252
+ priority = lock_priorities.fetch(priority, priority) if priority.is_a?(::Symbol)
253
+ if !priority.is_a?(::Integer) ||
254
+ priority < RedisQueuedLocks::Resource::MIN_LOCK_PRIORITY ||
255
+ priority > RedisQueuedLocks::Resource::MAX_LOCK_PRIORITY
256
+ raise(
257
+ RedisQueuedLocks::ArgumentError,
258
+ "`:priority` argument should be an Integer from " \
259
+ "#{RedisQueuedLocks::Resource::MIN_LOCK_PRIORITY} to " \
260
+ "#{RedisQueuedLocks::Resource::MAX_LOCK_PRIORITY} or a priority name of " \
261
+ "`config['lock_priorities']` (#{lock_priorities.keys.map(&:inspect).join(', ')}), " \
262
+ "got #{priority.inspect}."
263
+ )
264
+ end
265
+
266
+ # Step 0.2: prevent :meta incompatabiltiies (structure)
267
+ # - Symbol keys are written to the lock data as Strings (`Symbol#name`, as redis-client
268
+ # does it) and would overwrite the reserved lock data fields too;
269
+ if meta.is_a?(::Hash) && (meta.any? do |key, _value|
270
+ meta_key = key.is_a?(::Symbol) ? key.name : key
271
+ meta_key == 'acq_id' ||
272
+ meta_key == 'hst_id' ||
273
+ meta_key == 'ts' ||
274
+ meta_key == 'ini_ttl' ||
275
+ meta_key == 'lock_key' ||
276
+ meta_key == 'rem_ttl' ||
277
+ meta_key == 'spc_ext_ttl' ||
278
+ meta_key == 'spc_cnt' ||
279
+ meta_key == 'l_spc_ext_ini_ttl' ||
280
+ meta_key == 'l_spc_ext_ts' ||
281
+ meta_key == 'l_spc_ts' ||
282
+ meta_key == 'rw_mode'
283
+ end)
284
+ raise(
285
+ RedisQueuedLocks::ArgumentError,
286
+ '`:meta` keys can not overlap reserved lock data keys ' \
287
+ '"acq_id", "hst_id", "ts", "ini_ttl", "lock_key", "rem_ttl", "spc_cnt", ' \
288
+ '"spc_ext_ttl", "l_spc_ext_ini_ttl", "l_spc_ext_ts", "l_spc_ts", "rw_mode"'
289
+ )
290
+ end
291
+
292
+ # Step 1: prepare the requirements of the series (lock keys, acquirer, lock request position)
293
+ # NOTE: a duplicated lock name is the same lock (the series is obtained by one transaction
294
+ # and the lock state is checked once per lock);
295
+ lock_names = lock_names.uniq(&:to_s)
296
+ read_mode = (rw_mode == :read)
297
+ acquirer_id = RedisQueuedLocks::Resource.acquirer_identifier(
298
+ process_id,
299
+ thread_id,
300
+ fiber_id,
301
+ ractor_id,
302
+ identity
303
+ )
304
+ host_id = RedisQueuedLocks::Resource.host_identifier(
305
+ process_id,
306
+ thread_id,
307
+ ractor_id,
308
+ identity
309
+ )
310
+ lock_keys = lock_names.map do |lock_name|
311
+ RedisQueuedLocks::Resource.prepare_lock_key(lock_name)
312
+ end
313
+ lock_key_queues = lock_names.map do |lock_name|
314
+ RedisQueuedLocks::Resource.prepare_lock_queue(lock_name)
315
+ end
316
+ read_lock_key_queues = lock_names.map do |lock_name|
317
+ RedisQueuedLocks::Resource.prepare_read_lock_queue(lock_name)
318
+ end
319
+ lock_readers_keys = lock_names.map do |lock_name|
320
+ RedisQueuedLocks::Resource.prepare_lock_readers(lock_name)
321
+ end
322
+ # NOTE: (RW) read lock data keys are required by read lock requests only
323
+ # @type var read_lock_keys: Array[String?]
324
+ read_lock_keys = lock_names.map do |lock_name|
325
+ RedisQueuedLocks::Resource.prepare_read_lock_key(lock_name, acquirer_id) if read_mode
326
+ end
327
+ # NOTE: (RW) the lock state keys: write lock requests watch the readers registries too, read
328
+ # lock requests do not (readers should not invalidate transactions of each other);
329
+ watched_keys = read_mode ? lock_keys : (lock_keys + lock_readers_keys)
330
+
331
+ # NOTE: one position in all lock request queues of the series: the lock requests are ordered
332
+ # in the same way in all queues, so they can not wait for each other in a cycle;
333
+ acquirer_position = RedisQueuedLocks::Resource.calc_initial_acquirer_position(priority)
334
+
335
+ log_sampled = RedisQueuedLocks::Logging.should_log?(
336
+ log_sampling_enabled,
337
+ log_sample_this,
338
+ log_sampling_percent,
339
+ log_sampler
340
+ )
341
+ instr_sampled = RedisQueuedLocks::Instrument.should_instrument?(
342
+ instr_sampling_enabled,
343
+ instr_sample_this,
344
+ instr_sampling_percent,
345
+ instr_sampler
346
+ )
347
+
348
+ # Step X: intermediate result observer
349
+ # @type var acq_process: Hash[Symbol,untyped]
350
+ acq_process = {
351
+ should_try: true,
352
+ tries: 0,
353
+ acquired: false,
354
+ result: nil,
355
+ failed_lock_name: nil,
356
+ ts: nil, # NOTE: see `try_to_lock_series`
357
+ lock_processes: nil, # NOTE: see `try_to_lock_series`
358
+ held_rw_modes: nil, # NOTE: see `try_to_lock_series`
359
+ acq_time: nil, # NOTE: in milliseconds
360
+ acq_end_time: nil,
361
+ rel_ts: nil,
362
+ hold_time: nil # NOTE: in milliseconds
363
+ }
364
+ # @type var reentrant_lock_keys: Array[String]
365
+ reentrant_lock_keys = []
366
+ # NOTE: (observability) the locks with the reported intermediate obtaining (once per lock)
367
+ # @type var reported_ready_locks: Array[bool]
368
+ reported_ready_locks = Array.new(lock_names.size, false)
369
+ # @type var yield_result: untyped
370
+ yield_result = nil
371
+
372
+ LogVisitor.start_lock_series_obtaining(
373
+ logger, log_sampled, lock_keys,
374
+ queue_ttl, acquirer_id, host_id, access_strategy, rw_mode, priority
375
+ )
376
+
377
+ # Step 2: try to obtain the series with timeout
378
+ # NOTE: the acquirement start time is defined out of the timeout block: the failed
379
+ # acquirement reports the acquirement time too (Step 3.b);
380
+ acq_start_time = clock_gettime
381
+
382
+ begin
383
+ begin
384
+ ::Timeout.timeout(timeout, RedisQueuedLocks::LockAcquirementIntermediateTimeoutError) do
385
+ # Step 2.1: cyclically try to obtain the series
386
+ while acq_process[:should_try]
387
+ LogVisitor.start_try_to_lock_series_cycle(
388
+ logger, log_sampled, lock_keys,
389
+ queue_ttl, acquirer_id, host_id, access_strategy, rw_mode, priority
390
+ )
391
+
392
+ # Step 2.X: check the actual score: is it in queue ttl limit or not?
393
+ if RedisQueuedLocks::Resource.dead_score_reached?(
394
+ acquirer_position, priority, queue_ttl
395
+ )
396
+ # Step 2.X.X: dead score reached => re-queue the lock request of the series with
397
+ # the new score in all its queues (one position of the series);
398
+ acquirer_position =
399
+ RedisQueuedLocks::Resource.calc_initial_acquirer_position(priority)
400
+
401
+ LogVisitor.dead_score_reached__reset_series_position(
402
+ logger, log_sampled, lock_keys,
403
+ queue_ttl, acquirer_id, host_id, access_strategy, rw_mode, priority
404
+ )
405
+ end
406
+
407
+ attempt = try_to_lock_series(
408
+ redis,
409
+ logger,
410
+ log_lock_try,
411
+ lock_keys,
412
+ rw_mode,
413
+ lock_key_queues,
414
+ read_lock_key_queues,
415
+ lock_readers_keys,
416
+ read_lock_keys,
417
+ lock_names,
418
+ watched_keys,
419
+ acquirer_id,
420
+ host_id,
421
+ acquirer_position,
422
+ priority,
423
+ ttl,
424
+ queue_ttl,
425
+ lock_index_enabled,
426
+ lock_index_grace_period,
427
+ fail_fast,
428
+ conflict_strategy,
429
+ access_strategy,
430
+ meta,
431
+ acq_process[:tries] > 0, # NOTE: retry attempt (see "pre-check" in the attempt)
432
+ acq_process,
433
+ log_sampled
434
+ )
435
+ acq_end_time = clock_gettime
436
+ acq_time = ((acq_end_time - acq_start_time) / 1_000.0).ceil(2)
437
+
438
+ # Step X: save the intermediate results to the result observer
439
+ acq_process[:result] = attempt[:result]
440
+ acq_process[:failed_lock_name] = attempt[:lock_name] if attempt[:lock_name]
441
+
442
+ # Step 2.1: analyze an acquirement attempt
443
+ if attempt[:ok]
444
+ # Step 2.1.a: successfully acquired
445
+ acq_process[:acq_time] = acq_time
446
+ acq_process[:acq_end_time] = acq_end_time
447
+ acq_process[:should_try] = false
448
+ acq_process[:acquired] = true
449
+ end
450
+
451
+ # Step X: (observability) the intermediate obtaining of the locks of the series: the
452
+ # lock passed all checks of an attempt (nothing is held: the locks are obtained
453
+ # together, by the obtaining transaction of the attempt):
454
+ # - once per lock of the series (never per attempt), up to the lock that blocks
455
+ # the series, out of the WATCH block of the attempt;
456
+ if log_sampled || instr_sampled
457
+ ready_ts = Time.now.to_f
458
+ attempt[:ready_locks].each do |lock_index|
459
+ next if reported_ready_locks[lock_index]
460
+
461
+ reported_ready_locks[lock_index] = true
462
+ LogVisitor.lock_series_intermediate_lock_obtained(
463
+ logger, log_sampled, lock_keys[lock_index], lock_keys, queue_ttl, acquirer_id,
464
+ host_id, acq_time, access_strategy, rw_mode, priority
465
+ )
466
+ InstrVisitor.lock_series_intermediate_lock_obtained(
467
+ instrumenter, instr_sampled, lock_keys[lock_index], lock_keys, rw_mode,
468
+ priority, ttl, acquirer_id, host_id, ready_ts, acq_time, instrument
469
+ )
470
+ end
471
+ end
472
+
473
+ unless attempt[:ok]
474
+ # Step 2.2: failed to acquire. anylize each case and act in accordance
475
+ case attempt[:result]
476
+ when :fail_fast_no_try # Step 2.2.a: fail without try
477
+ acq_process[:should_try] = false
478
+ when :conflict_dead_lock, :conflict_lock_upgrade # Step 2.2.b: fail after conflict
479
+ acq_process[:tries] += 1
480
+ acq_process[:should_try] = false
481
+ else
482
+ acq_process[:tries] += 1 # Step RETRY: possible retry case
483
+
484
+ if fail_fast # Step RETRY.A: fail after try
485
+ acq_process[:should_try] = false
486
+ acq_process[:result] = :fail_fast_after_try
487
+ elsif retry_count != nil && acq_process[:tries] >= retry_count
488
+ # Step RETRY.B: fail cuz the retry count is reached
489
+ acq_process[:should_try] = false
490
+ acq_process[:result] = :retry_limit_reached
491
+ else
492
+ # Step RETRY.X: no significant failures => retry easily :)
493
+ # NOTE:
494
+ # delay the exceution in order to prevent chaotic lock-acquire attempts
495
+ # and to allow other processes and threads to obtain the locks too.
496
+ delay_execution(retry_delay, retry_jitter)
497
+ end
498
+ end
499
+ end
500
+ end
501
+ end
502
+ rescue RedisQueuedLocks::LockAcquirementIntermediateTimeoutError
503
+ # NOTE: the acquirement time limit is reached: it is a failed acquirement (see Step 3.b)
504
+ # (the timeout can come right after the obtaining: `acq_process[:acquired]` decides);
505
+ nil
506
+ end
507
+
508
+ # Step 3.a: acquired successfully => run logic or return the result of acquirement
509
+ if acq_process[:acquired]
510
+ # @type var lock_processes: Array[Symbol?]
511
+ lock_processes = acq_process[:lock_processes]
512
+ # NOTE: (observability) the full obtaining of the locks of the series; reentrant locks are
513
+ # not obtained by the series (the series events report them);
514
+ lock_keys.each_with_index do |lock_key, index|
515
+ if lock_processes[index] == :lock_obtaining
516
+ LogVisitor.lock_series_lock_obtained(
517
+ logger, log_sampled, lock_key, lock_keys, queue_ttl, acquirer_id, host_id,
518
+ acq_process[:acq_time], access_strategy, rw_mode, priority
519
+ )
520
+ InstrVisitor.lock_series_lock_obtained(
521
+ instrumenter, instr_sampled, lock_key, lock_keys, rw_mode, priority, ttl,
522
+ acquirer_id, host_id, acq_process[:ts], acq_process[:acq_time], instrument
523
+ )
524
+ else
525
+ reentrant_lock_keys << lock_key
526
+ end
527
+ end
528
+
529
+ LogVisitor.lock_series_obtained(
530
+ logger, log_sampled, lock_keys,
531
+ queue_ttl, acquirer_id, host_id, acq_process[:acq_time], access_strategy,
532
+ rw_mode, priority, reentrant_lock_keys
533
+ )
534
+ InstrVisitor.lock_series_obtained(
535
+ instrumenter, instr_sampled, lock_keys, rw_mode, priority, ttl, acquirer_id,
536
+ host_id, acq_process[:ts], acq_process[:acq_time], reentrant_lock_keys, instrument
537
+ )
538
+
539
+ ttl_shift = (
540
+ (clock_gettime - acq_process[:acq_end_time]) / 1_000.0 -
541
+ RedisQueuedLocks::Resource::REDIS_TIMESHIFT_ERROR
542
+ ).ceil(2)
543
+
544
+ yield_result = yield_lock_series(
545
+ lock_keys, acquirer_id, host_id, timed, ttl_shift, ttl, meta, &block
546
+ )
547
+ end
548
+ ensure
549
+ if !acq_process[:acquired]
550
+ # Step 3.b.X: the series is not obtained (a failure or an interruption of any kind)
551
+ drop_failed_lock_series(
552
+ redis, logger, lock_keys, rw_mode, lock_key_queues, read_lock_key_queues,
553
+ lock_names, queue_ttl, acquirer_id, host_id, access_strategy, priority, acq_process,
554
+ log_sampled
555
+ )
556
+ elsif block_given?
557
+ # Step 3.a.X: release the locks of the series after the block of code (on any failure too)
558
+ spent_time = ((clock_gettime - acq_process[:acq_end_time]) / 1_000.0).ceil
559
+ decreased_ttl = ttl - spent_time - RedisQueuedLocks::Resource::REDIS_TIMESHIFT_ERROR
560
+
561
+ complete_lock_series_release(
562
+ redis, lock_names, acq_process[:lock_processes], acq_process[:held_rw_modes],
563
+ acquirer_id, decreased_ttl
564
+ )
565
+
566
+ acq_process[:rel_ts] = Time.now.to_f
567
+ acq_process[:hold_time] =
568
+ ((clock_gettime - acq_process[:acq_end_time]) / 1_000.0).ceil(2)
569
+
570
+ LogVisitor.expire_lock_series(
571
+ logger, log_sampled, lock_keys,
572
+ queue_ttl, acquirer_id, host_id, access_strategy, rw_mode, priority
573
+ )
574
+ # NOTE: an extended lock is the reentrant lock that was extended by the attempt
575
+ if log_sampled && decreased_ttl > 0
576
+ extended_lock_keys = lock_keys.select.with_index do |_lock_key, index|
577
+ acq_process[:lock_processes][index] == :extendable_conflict_work_through
578
+ end
579
+ unless extended_lock_keys.empty?
580
+ LogVisitor.decrease_lock_series(
581
+ logger, log_sampled, extended_lock_keys, decreased_ttl,
582
+ queue_ttl, acquirer_id, host_id, access_strategy, rw_mode, priority
583
+ )
584
+ end
585
+ end
586
+ InstrVisitor.lock_series_hold_and_release(
587
+ instrumenter, instr_sampled, lock_keys, rw_mode, priority, ttl, acquirer_id,
588
+ host_id, acq_process[:rel_ts], acq_process[:acq_time], acq_process[:hold_time],
589
+ reentrant_lock_keys, instrument
590
+ )
591
+ end
592
+ end
593
+
594
+ if acq_process[:acquired]
595
+ if detailed_result
596
+ {
597
+ yield_result: yield_result,
598
+ locks_release_strategy: block_given? ? :immediate_release_after_yield : :redis_key_ttl,
599
+ locks_released_at: acq_process[:rel_ts],
600
+ locks_acq_time: acq_process[:acq_time],
601
+ locks_hold_time: acq_process[:hold_time],
602
+ lock_series: lock_names,
603
+ rql_lock_series: lock_keys
604
+ }
605
+ elsif block_given?
606
+ yield_result
607
+ else
608
+ {
609
+ ok: true,
610
+ result: {
611
+ lock_series: lock_names,
612
+ rql_lock_series: lock_keys,
613
+ acq_id: acquirer_id,
614
+ hst_id: host_id,
615
+ ts: acq_process[:ts],
616
+ ttl: ttl
617
+ }
618
+ }
619
+ end
620
+ else
621
+ # Step 3.b: the series is not obtained
622
+ # NOTE: the loop is interrupted by the acquirement timeout only (see the `rescue` above):
623
+ # all other interruptions are raised through the `ensure` part;
624
+ acq_process[:result] = :timeout_reached if acq_process[:should_try]
625
+
626
+ # @type var failure_result: Symbol
627
+ failure_result = acq_process[:result]
628
+ # @type var failed_lock_name: String?
629
+ failed_lock_name = acq_process[:failed_lock_name]
630
+ failed_lock_key =
631
+ failed_lock_name ? RedisQueuedLocks::Resource.prepare_lock_key(failed_lock_name) : nil
632
+ acq_time = ((clock_gettime - acq_start_time) / 1_000.0).ceil(2)
633
+ failed_ts = Time.now.to_f
634
+
635
+ # NOTE: the lock that blocks the series (unknown without an attempt with an obstacle)
636
+ if failed_lock_key
637
+ LogVisitor.lock_series_lock_obtaining_failed(
638
+ logger, log_sampled, failed_lock_key, lock_keys, queue_ttl, acquirer_id, host_id,
639
+ acq_time, access_strategy, rw_mode, priority, failure_result
640
+ )
641
+ InstrVisitor.lock_series_lock_obtaining_failed(
642
+ instrumenter, instr_sampled, failed_lock_key, lock_keys, rw_mode, priority, ttl,
643
+ acquirer_id, host_id, failed_ts, acq_time, failure_result, instrument
644
+ )
645
+ end
646
+ LogVisitor.lock_series_obtaining_failed(
647
+ logger, log_sampled, lock_keys, queue_ttl, acquirer_id, host_id, acq_time,
648
+ access_strategy, rw_mode, priority, failure_result, failed_lock_key
649
+ )
650
+ InstrVisitor.lock_series_obtaining_failed(
651
+ instrumenter, instr_sampled, lock_keys, rw_mode, priority, ttl, acquirer_id,
652
+ host_id, failed_ts, acq_time, failure_result, failed_lock_key, instrument
653
+ )
654
+
655
+ if raise_errors
656
+ raise_lock_series_error(
657
+ redis, failure_result, lock_keys, failed_lock_name, failed_lock_key, acquirer_id,
658
+ conflict_strategy, retry_count, timeout, detailed_acq_timeout_error
659
+ )
660
+ end
661
+
662
+ # @type var detailed_errors: Hash[String, { ok: bool, result: Symbol }]
663
+ detailed_errors = {}
664
+ if failed_lock_name
665
+ detailed_errors[failed_lock_name] = { ok: false, result: failure_result }
666
+ end
667
+ # NOTE: nothing is obtained by the failed series (all or nothing)
668
+ # @type var acquired_locks: Array[String]
669
+ acquired_locks = []
670
+
671
+ {
672
+ ok: false,
673
+ result: {
674
+ error: :failed_to_acquire_lock_series,
675
+ detailed_errors:,
676
+ lock_series: lock_names,
677
+ acquired_locks:,
678
+ missing_locks: lock_names,
679
+ failed_on_lock: failed_lock_name
680
+ }
681
+ }
682
+ end
683
+ end
684
+ # rubocop:enable Metrics/MethodLength
685
+
686
+ private
687
+
688
+ # Runs the block of code under the obtained series (`timed`: limited by the `ttl`).
689
+ #
690
+ # @param lock_keys [Array<String>]
691
+ # @param acquirer_id [String]
692
+ # @param host_id [String]
693
+ # @param timed [Boolean]
694
+ # @param ttl_shift [Numeric] In milliseconds.
695
+ # @param ttl [Integer] In milliseconds.
696
+ # @param meta [NilClass,Hash<String|Symbol,Any>]
697
+ # @yield Custom logic that should be invoked under the obtained locks.
698
+ # @yieldreturn [Any]
699
+ # @return [Any,NilClass] nil is returned when no block parametr is provided.
700
+ #
701
+ # @api private
702
+ # @since 1.23.0
703
+ def yield_lock_series(lock_keys, acquirer_id, host_id, timed, ttl_shift, ttl, meta, &block)
704
+ return unless block_given?
705
+ return yield unless timed
706
+
707
+ # NOTE: (steep) see `YieldExpire#yield_expire`: `::Integer - ::Integer|::Float`
708
+ # steep:ignore:start
709
+ # @type var timeout: Float
710
+ timeout = ((ttl - ttl_shift) / 1_000.0).yield_self do |time|
711
+ # NOTE: time in <seconds> cuz Ruby's Timeout requires <seconds>
712
+ (time < 0) ? 0.0 : time
713
+ end
714
+ # steep:ignore:end
715
+
716
+ yield_with_timeout(timeout, lock_keys.join(', '), ttl, acquirer_id, host_id, meta, &block)
717
+ end
718
+
719
+ # Releases the series: an interruption of the release itself (`Timeout.timeout` of the outer
720
+ # logic, `Thread#kill`, a lost connection) is completed by the second release: an interruption
721
+ # is delivered once and the release is idempotent (it checks the owner and works with own read
722
+ # locks only); the error of the first release is raised as is.
723
+ #
724
+ # @param redis [RedisClient]
725
+ # @param lock_names [Array<String>]
726
+ # @param lock_processes [Array<Symbol,NilClass>] See `release_lock_series`.
727
+ # @param held_rw_modes [Array<Symbol>] See `release_lock_series`.
728
+ # @param acquirer_id [String]
729
+ # @param decreased_ttl [Integer] See `release_lock_series`.
730
+ # @return [void]
731
+ #
732
+ # @api private
733
+ # @since 1.23.0
734
+ def complete_lock_series_release(
735
+ redis,
736
+ lock_names,
737
+ lock_processes,
738
+ held_rw_modes,
739
+ acquirer_id,
740
+ decreased_ttl
741
+ )
742
+ is_series_released = false
743
+
744
+ begin
745
+ release_lock_series(
746
+ redis, lock_names, lock_processes, held_rw_modes, acquirer_id, decreased_ttl
747
+ )
748
+ is_series_released = true
749
+ ensure
750
+ unless is_series_released
751
+ run_non_critical do
752
+ release_lock_series(
753
+ redis, lock_names, lock_processes, held_rw_modes, acquirer_id, decreased_ttl
754
+ )
755
+ end
756
+ end
757
+ end
758
+ end
759
+
760
+ # Drops the traces of the series that is not obtained (a failure or an interruption of any
761
+ # kind: the acquirement timeout, an error, `Timeout.timeout` of the outer logic, `Thread#kill`):
762
+ # - the obtaining transaction of an interrupted attempt can be applied by Redis without
763
+ # the reply: its locks are released (only the locks that are obtained by this transaction
764
+ # and only if they are owned by the current acquirer: nothing is released otherwise; the
765
+ # ttl of the extended reentrant locks is not decreased: the extension is unknown);
766
+ # - the lock requests are dropped from all lock request queues of the series;
767
+ # - errors are not raised here (they would replace the original error of the interrupted
768
+ # acquirement): dead lock requests are dropped by other lock requests, locks are released
769
+ # by their ttl;
770
+ #
771
+ # @param redis [RedisClient]
772
+ # @param logger [::Logger,#debug]
773
+ # @param lock_keys [Array<String>]
774
+ # @param rw_mode [Symbol]
775
+ # @param lock_key_queues [Array<String>]
776
+ # @param read_lock_key_queues [Array<String>]
777
+ # @param lock_names [Array<String>]
778
+ # @param queue_ttl [Integer]
779
+ # @param acquirer_id [String]
780
+ # @param host_id [String]
781
+ # @param access_strategy [Symbol]
782
+ # @param priority [Integer]
783
+ # @param acq_process [Hash<Symbol,Any>]
784
+ # @param log_sampled [Boolean]
785
+ # @return [void]
786
+ #
787
+ # @api private
788
+ # @since 1.23.0
789
+ def drop_failed_lock_series(
790
+ redis,
791
+ logger,
792
+ lock_keys,
793
+ rw_mode,
794
+ lock_key_queues,
795
+ read_lock_key_queues,
796
+ lock_names,
797
+ queue_ttl,
798
+ acquirer_id,
799
+ host_id,
800
+ access_strategy,
801
+ priority,
802
+ acq_process,
803
+ log_sampled
804
+ )
805
+ run_non_critical do
806
+ if acq_process[:lock_processes]
807
+ complete_lock_series_release(
808
+ redis, lock_names, acq_process[:lock_processes], acq_process[:held_rw_modes],
809
+ acquirer_id, 0
810
+ )
811
+ LogVisitor.expire_lock_series(
812
+ logger, log_sampled, lock_keys,
813
+ queue_ttl, acquirer_id, host_id, access_strategy, rw_mode, priority
814
+ )
815
+ end
816
+ end
817
+
818
+ # NOTE: the failed "no try" attempt queues nothing
819
+ return if acq_process[:result] == :fail_fast_no_try
820
+
821
+ run_non_critical do
822
+ request_queues = (rw_mode == :read) ? read_lock_key_queues : lock_key_queues
823
+ redis.pipelined do |pipeline|
824
+ request_queues.each { |request_queue| pipeline.call('ZREM', request_queue, acquirer_id) }
825
+ end
826
+ LogVisitor.dequeue_from_lock_series_queues(
827
+ logger, log_sampled, lock_keys,
828
+ queue_ttl, acquirer_id, host_id, access_strategy, rw_mode, priority
829
+ )
830
+ end
831
+ end
832
+
833
+ # Raises the error of the series that is not obtained (`raise_errors: true`): the same errors
834
+ # as in the single lock acquirement.
835
+ #
836
+ # @param redis [RedisClient]
837
+ # @param failure_result [Symbol]
838
+ # @param lock_keys [Array<String>]
839
+ # @param failed_lock_name [String,NilClass]
840
+ # @param failed_lock_key [String,NilClass]
841
+ # @param acquirer_id [String]
842
+ # @param conflict_strategy [Symbol]
843
+ # @param retry_count [Integer,NilClass]
844
+ # @param timeout [Integer,NilClass]
845
+ # @param detailed_acq_timeout_error [Boolean]
846
+ # @return [void]
847
+ #
848
+ # @raise [RedisQueuedLocks::LockAlreadyObtainedError]
849
+ # @raise [RedisQueuedLocks::ConflictLockObtainError]
850
+ # @raise [RedisQueuedLocks::LockAcquirementRetryLimitError]
851
+ # @raise [RedisQueuedLocks::LockAcquirementTimeoutError]
852
+ #
853
+ # @api private
854
+ # @since 1.23.0
855
+ # rubocop:disable Metrics/MethodLength
856
+ def raise_lock_series_error(
857
+ redis,
858
+ failure_result,
859
+ lock_keys,
860
+ failed_lock_name,
861
+ failed_lock_key,
862
+ acquirer_id,
863
+ conflict_strategy,
864
+ retry_count,
865
+ timeout,
866
+ detailed_acq_timeout_error
867
+ )
868
+ series_description = "the lock series #{lock_keys.inspect}"
869
+ series_description += " (blocked by \"#{failed_lock_key}\")" if failed_lock_key
870
+
871
+ case failure_result
872
+ when :fail_fast_no_try, :fail_fast_after_try
873
+ raise(
874
+ RedisQueuedLocks::LockAlreadyObtainedError,
875
+ "Lock \"#{failed_lock_key || lock_keys.join(', ')}\" is already obtained."
876
+ )
877
+ when :conflict_dead_lock
878
+ raise(
879
+ RedisQueuedLocks::ConflictLockObtainError,
880
+ "Lock Conflict: trying to acquire the lock \"#{failed_lock_key}\" " \
881
+ "that is already acquired by the current acquirer (acq_id: \"#{acquirer_id}\")."
882
+ )
883
+ when :conflict_lock_upgrade
884
+ raise(
885
+ RedisQueuedLocks::ConflictLockObtainError,
886
+ "Lock Conflict: trying to acquire the write lock \"#{failed_lock_key}\" " \
887
+ "while the current acquirer holds its read lock (acq_id: \"#{acquirer_id}\"): " \
888
+ "read-to-write lock upgrade is not supported " \
889
+ "(conflict strategy: :#{conflict_strategy})."
890
+ )
891
+ when :retry_limit_reached
892
+ raise(
893
+ RedisQueuedLocks::LockAcquirementRetryLimitError,
894
+ "Failed to acquire #{series_description} " \
895
+ "for the given retry_count limit (#{retry_count} times)."
896
+ )
897
+ else
898
+ details =
899
+ if detailed_acq_timeout_error && failed_lock_name
900
+ " Details: #{blocking_lock_details(redis, failed_lock_name)}"
901
+ end
902
+ raise(
903
+ RedisQueuedLocks::LockAcquirementTimeoutError,
904
+ "Failed to acquire #{series_description} " \
905
+ "for the given <#{timeout} seconds> timeout.#{details}"
906
+ )
907
+ end
908
+ end
909
+ # rubocop:enable Metrics/MethodLength
910
+
911
+ # The raw state of the lock that blocks the series (the detailed acquirement timeout error).
912
+ #
913
+ # @param redis [RedisClient]
914
+ # @param failed_lock_name [String]
915
+ # @return [String]
916
+ #
917
+ # @api private
918
+ # @since 1.23.0
919
+ def blocking_lock_details(redis, failed_lock_name)
920
+ # NOTE: (pipeline layout) [0] the write lock data, [1] the readers registry,
921
+ # [2] write lock requests, [3] read lock requests;
922
+ # @type var lock_state: Array[untyped]
923
+ lock_state = redis.pipelined do |pipeline|
924
+ pipeline.call('HGETALL', RedisQueuedLocks::Resource.prepare_lock_key(failed_lock_name))
925
+ pipeline.call(
926
+ 'ZRANGE', RedisQueuedLocks::Resource.prepare_lock_readers(failed_lock_name),
927
+ '0', '-1', 'WITHSCORES'
928
+ )
929
+ pipeline.call(
930
+ 'ZRANGE', RedisQueuedLocks::Resource.prepare_lock_queue(failed_lock_name),
931
+ '0', '-1', 'WITHSCORES'
932
+ )
933
+ pipeline.call(
934
+ 'ZRANGE', RedisQueuedLocks::Resource.prepare_read_lock_queue(failed_lock_name),
935
+ '0', '-1', 'WITHSCORES'
936
+ )
937
+ end
938
+
939
+ "<Lock Data> => #{lock_state[0].to_h.inspect}; " \
940
+ "<Readers Data> => #{lock_state[1].inspect}; " \
941
+ "<Queue Data> => #{lock_state[2].inspect}; " \
942
+ "<Read Queue Data> => #{lock_state[3].inspect};"
943
+ end
944
+ end
945
+ end
946
+ # rubocop:enable Metrics/ModuleLength
947
+ # rubocop:enable Metrics/ClassLength
948
+ # rubocop:enable Metrics/BlockNesting