redis_queued_locks 1.8.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -1
- data/CHANGELOG.md +51 -0
- data/README.md +402 -46
- data/lib/redis_queued_locks/acquier/acquire_lock/dequeue_from_lock_queue/log_visitor.rb +4 -0
- data/lib/redis_queued_locks/acquier/acquire_lock/dequeue_from_lock_queue.rb +4 -1
- data/lib/redis_queued_locks/acquier/acquire_lock/instr_visitor.rb +20 -5
- data/lib/redis_queued_locks/acquier/acquire_lock/log_visitor.rb +24 -0
- data/lib/redis_queued_locks/acquier/acquire_lock/try_to_lock/log_visitor.rb +56 -0
- data/lib/redis_queued_locks/acquier/acquire_lock/try_to_lock.rb +37 -30
- data/lib/redis_queued_locks/acquier/acquire_lock/yield_expire/log_visitor.rb +8 -0
- data/lib/redis_queued_locks/acquier/acquire_lock/yield_expire.rb +13 -9
- data/lib/redis_queued_locks/acquier/acquire_lock.rb +44 -20
- data/lib/redis_queued_locks/acquier/clear_dead_requests.rb +5 -1
- data/lib/redis_queued_locks/acquier/extend_lock_ttl.rb +5 -1
- data/lib/redis_queued_locks/acquier/lock_info.rb +4 -3
- data/lib/redis_queued_locks/acquier/locks.rb +2 -2
- data/lib/redis_queued_locks/acquier/queue_info.rb +2 -2
- data/lib/redis_queued_locks/acquier/release_all_locks.rb +12 -2
- data/lib/redis_queued_locks/acquier/release_lock.rb +12 -2
- data/lib/redis_queued_locks/client.rb +284 -34
- data/lib/redis_queued_locks/errors.rb +8 -0
- data/lib/redis_queued_locks/instrument.rb +8 -1
- data/lib/redis_queued_locks/logging.rb +8 -1
- data/lib/redis_queued_locks/resource.rb +59 -1
- data/lib/redis_queued_locks/swarm/acquirers.rb +44 -0
- data/lib/redis_queued_locks/swarm/flush_zombies.rb +133 -0
- data/lib/redis_queued_locks/swarm/probe_hosts.rb +69 -0
- data/lib/redis_queued_locks/swarm/redis_client_builder.rb +67 -0
- data/lib/redis_queued_locks/swarm/supervisor.rb +83 -0
- data/lib/redis_queued_locks/swarm/swarm_element/isolated.rb +287 -0
- data/lib/redis_queued_locks/swarm/swarm_element/threaded.rb +351 -0
- data/lib/redis_queued_locks/swarm/swarm_element.rb +8 -0
- data/lib/redis_queued_locks/swarm/zombie_info.rb +145 -0
- data/lib/redis_queued_locks/swarm.rb +241 -0
- data/lib/redis_queued_locks/utilities/lock.rb +22 -0
- data/lib/redis_queued_locks/utilities.rb +75 -0
- data/lib/redis_queued_locks/version.rb +2 -2
- data/lib/redis_queued_locks.rb +2 -0
- metadata +14 -4
- data/lib/redis_queued_locks/watcher.rb +0 -1
data/README.md
CHANGED
@@ -35,6 +35,22 @@ Provides flexible invocation flow, parametrized limits (lock request ttl, lock t
|
|
35
35
|
- [queues_info](#queues_info---get-list-of-queues-with-their-info)
|
36
36
|
- [clear_dead_requests](#clear_dead_requests)
|
37
37
|
- [current_acquirer_id](#current_acquirer_id)
|
38
|
+
- [current_host_id](#current_host_id)
|
39
|
+
- [possible_host_ids](#possible_host_ids)
|
40
|
+
- [Swarm Mode and Zombie Locks](#swarm-mode-and-zombie-locks)
|
41
|
+
- [work and usage preview (temporary example-based docs)](#work-and-usage-preview-temporary-example-based-docs)
|
42
|
+
- [How to Swarm](#how-to-swarm)
|
43
|
+
- [configuration](#)
|
44
|
+
- [swarm_status](#swarm_status)
|
45
|
+
- [swarm_info](#swarm_info)
|
46
|
+
- [swarmize!](#swarmize!)
|
47
|
+
- [deswarmize!](#deswarmize!)
|
48
|
+
- [probe_hosts](#probe_hosts)
|
49
|
+
- [flush_zobmies](#flush_zombies)
|
50
|
+
- [zombies_info](#zombies_info)
|
51
|
+
- [zombie_locks](#zombie_locks)
|
52
|
+
- [zombie_hosts](#zombie_hosts)
|
53
|
+
- [zombie_acquiers](#zombie_acquiers)
|
38
54
|
- [Lock Access Strategies](#lock-access-strategies)
|
39
55
|
- [queued](#lock-access-strategies)
|
40
56
|
- [random](#lock-access-strategies)
|
@@ -244,35 +260,35 @@ clinet = RedisQueuedLocks::Client.new(redis_client) do |config|
|
|
244
260
|
# - should implement `debug(progname = nil, &block)` (minimal requirement) or be an instance of Ruby's `::Logger` class/subclass;
|
245
261
|
# - supports `SemanticLogger::Logger` (see "semantic_logger" gem)
|
246
262
|
# - at this moment the only debug logs are realised in following cases:
|
247
|
-
# - "[redis_queued_locks.start_lock_obtaining]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
248
|
-
# - "[redis_queued_locks.start_try_to_lock_cycle]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
249
|
-
# - "[redis_queued_locks.dead_score_reached__reset_acquier_position]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
250
|
-
# - "[redis_queued_locks.lock_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "acq_time", "acs_strat");
|
251
|
-
# - "[redis_queued_locks.extendable_reentrant_lock_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "acq_time", "acs_strat");
|
252
|
-
# - "[redis_queued_locks.reentrant_lock_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "acq_time", "acs_strat");
|
253
|
-
# - "[redis_queued_locks.fail_fast_or_limits_reached_or_deadlock__dequeue]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
254
|
-
# - "[redis_queued_locks.expire_lock]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
255
|
-
# - "[redis_queued_locks.decrease_lock]" (logs "lock_key", "decreased_ttl", "queue_ttl", "acq_id", "acs_strat");
|
263
|
+
# - "[redis_queued_locks.start_lock_obtaining]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
264
|
+
# - "[redis_queued_locks.start_try_to_lock_cycle]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
265
|
+
# - "[redis_queued_locks.dead_score_reached__reset_acquier_position]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
266
|
+
# - "[redis_queued_locks.lock_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acq_time", "acs_strat");
|
267
|
+
# - "[redis_queued_locks.extendable_reentrant_lock_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acq_time", "acs_strat");
|
268
|
+
# - "[redis_queued_locks.reentrant_lock_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acq_time", "acs_strat");
|
269
|
+
# - "[redis_queued_locks.fail_fast_or_limits_reached_or_deadlock__dequeue]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
270
|
+
# - "[redis_queued_locks.expire_lock]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
271
|
+
# - "[redis_queued_locks.decrease_lock]" (logs "lock_key", "decreased_ttl", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
256
272
|
# - by default uses VoidLogger that does nothing;
|
257
273
|
config.logger = RedisQueuedLocks::Logging::VoidLogger
|
258
274
|
|
259
275
|
# (default: false)
|
260
276
|
# - adds additional debug logs;
|
261
277
|
# - enables additional logs for each internal try-retry lock acquiring (a lot of logs can be generated depending on your retry configurations);
|
262
|
-
# - it adds following logs in addition to the existing:
|
263
|
-
# - "[redis_queued_locks.try_lock.start]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
264
|
-
# - "[redis_queued_locks.try_lock.rconn_fetched]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
265
|
-
# - "[redis_queued_locks.try_lock.same_process_conflict_detected]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
266
|
-
# - "[redis_queued_locks.try_lock.same_process_conflict_analyzed]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status");
|
267
|
-
# - "[redis_queued_locks.try_lock.reentrant_lock__extend_and_work_through]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status", "last_ext_ttl", "last_ext_ts");
|
268
|
-
# - "[redis_queued_locks.try_lock.reentrant_lock__work_through]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status", last_spc_ts);
|
269
|
-
# - "[redis_queued_locks.try_lock.acq_added_to_queue]" (logs "lock_key", "queue_ttl", "acq_id, "acs_strat")";
|
270
|
-
# - "[redis_queued_locks.try_lock.remove_expired_acqs]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
271
|
-
# - "[redis_queued_locks.try_lock.get_first_from_queue]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "first_acq_id_in_queue");
|
272
|
-
# - "[redis_queued_locks.try_lock.exit__queue_ttl_reached]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
273
|
-
# - "[redis_queued_locks.try_lock.exit__no_first]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "first_acq_id_in_queue", "<current_lock_data>");
|
274
|
-
# - "[redis_queued_locks.try_lock.exit__lock_still_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "first_acq_id_in_queue", "locked_by_acq_id", "<current_lock_data>");
|
275
|
-
# - "[redis_queued_locks.try_lock.obtain__free_to_acquire]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
278
|
+
# - it adds following debug logs in addition to the existing:
|
279
|
+
# - "[redis_queued_locks.try_lock.start]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
280
|
+
# - "[redis_queued_locks.try_lock.rconn_fetched]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
281
|
+
# - "[redis_queued_locks.try_lock.same_process_conflict_detected]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
282
|
+
# - "[redis_queued_locks.try_lock.same_process_conflict_analyzed]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status");
|
283
|
+
# - "[redis_queued_locks.try_lock.reentrant_lock__extend_and_work_through]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status", "last_ext_ttl", "last_ext_ts");
|
284
|
+
# - "[redis_queued_locks.try_lock.reentrant_lock__work_through]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status", last_spc_ts);
|
285
|
+
# - "[redis_queued_locks.try_lock.acq_added_to_queue]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat")";
|
286
|
+
# - "[redis_queued_locks.try_lock.remove_expired_acqs]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
287
|
+
# - "[redis_queued_locks.try_lock.get_first_from_queue]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "first_acq_id_in_queue");
|
288
|
+
# - "[redis_queued_locks.try_lock.exit__queue_ttl_reached]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
289
|
+
# - "[redis_queued_locks.try_lock.exit__no_first]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "first_acq_id_in_queue", "<current_lock_data>");
|
290
|
+
# - "[redis_queued_locks.try_lock.exit__lock_still_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "first_acq_id_in_queue", "locked_by_acq_id", "<current_lock_data>");
|
291
|
+
# - "[redis_queued_locks.try_lock.obtain__free_to_acquire]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
276
292
|
config.log_lock_try = false
|
277
293
|
|
278
294
|
# (default: false)
|
@@ -341,6 +357,8 @@ end
|
|
341
357
|
- [queues_info](#queues_info---get-list-of-queues-with-their-info)
|
342
358
|
- [clear_dead_requests](#clear_dead_requests)
|
343
359
|
- [current_acquirer_id](#current_acquirer_id)
|
360
|
+
- [current_host_id](#current_host_id)
|
361
|
+
- [possible_host_ids](#possible_host_ids)
|
344
362
|
|
345
363
|
---
|
346
364
|
|
@@ -381,9 +399,11 @@ def lock(
|
|
381
399
|
log_sampling_enabled: config[:log_sampling_enabled],
|
382
400
|
log_sampling_percent: config[:log_sampling_percent],
|
383
401
|
log_sampler: config[:log_sampler],
|
402
|
+
log_sample_this: false,
|
384
403
|
instr_sampling_enabled: config[:instr_sampling_enabled],
|
385
404
|
instr_sampling_percent: config[:instr_sampling_percent],
|
386
405
|
instr_sampler: config[:instr_sampler],
|
406
|
+
instr_sample_this: false,
|
387
407
|
&block
|
388
408
|
)
|
389
409
|
```
|
@@ -503,6 +523,10 @@ def lock(
|
|
503
523
|
- you can provide your own log sampler with bettter algorithm that should realize
|
504
524
|
`sampling_happened?(percent) => boolean` interface (see `RedisQueuedLocks::Logging::Sampler` for example);
|
505
525
|
- pre-configured in `config[:log_sampler]`;
|
526
|
+
- `log_sample_this` - (optional) `[Boolean]`
|
527
|
+
- marks the method that everything should be logged despite the enabled log sampling;
|
528
|
+
- makes sense when log sampling is enabled;
|
529
|
+
- `false` by default;
|
506
530
|
- `instr_sampling_enabled` - (optional) `[Boolean]`
|
507
531
|
- enables **instrumentaion sampling**: only the configured percent of RQL cases will be instrumented;
|
508
532
|
- disabled by default;
|
@@ -521,6 +545,10 @@ def lock(
|
|
521
545
|
- you can provide your own log sampler with bettter algorithm that should realize
|
522
546
|
`sampling_happened?(percent) => boolean` interface (see `RedisQueuedLocks::Instrument::Sampler` for example);
|
523
547
|
- pre-configured in `config[:instr_sampler]`;
|
548
|
+
- `instr_sample_this` - (optional) `[Boolean]`
|
549
|
+
- marks the method that everything should be instrumneted despite the enabled instrumentation sampling;
|
550
|
+
- makes sense when instrumentation sampling is enabled;
|
551
|
+
- `false` by default;
|
524
552
|
- `block` - (optional) `[Block]`
|
525
553
|
- A block of code that should be executed after the successfully acquired lock.
|
526
554
|
- If block is **passed** the obtained lock will be released after the block execution or it's ttl (what will happen first);
|
@@ -566,6 +594,7 @@ Return value:
|
|
566
594
|
result: {
|
567
595
|
lock_key: String, # acquierd lock key ("rql:lock:your_lock_name")
|
568
596
|
acq_id: String, # acquier identifier ("process_id/thread_id/fiber_id/ractor_id/identity")
|
597
|
+
hst_id: String, # host identifier ("process_id/thread_id/ractor_id/identity")
|
569
598
|
ts: Float, # time (epoch) when lock was obtained (float, Time#to_f)
|
570
599
|
ttl: Integer, # lock's time to live in milliseconds (integer)
|
571
600
|
process: Symbol # which logical process has acquired the lock (:lock_obtaining, :extendable_conflict_work_through, :conflict_work_through, :conflict_dead_lock)
|
@@ -580,6 +609,7 @@ Return value:
|
|
580
609
|
result: {
|
581
610
|
lock_key: "rql:lock:my_lock",
|
582
611
|
acq_id: "rql:acq:26672/2280/2300/2320/70ea5dbf10ea1056",
|
612
|
+
acq_id: "rql:acq:26672/2280/2320/70ea5dbf10ea1056",
|
583
613
|
ts: 1711909612.653696,
|
584
614
|
ttl: 10000,
|
585
615
|
process: :lock_obtaining # for custom conflict strategies may be: :conflict_dead_lock, :conflict_work_through, :extendable_conflict_work_through
|
@@ -655,6 +685,7 @@ rql.lock_info("my_lock")
|
|
655
685
|
{
|
656
686
|
"lock_key" => "rql:lock:my_lock",
|
657
687
|
"acq_id" => "rql:acq:123/456/567/678/374dd74324",
|
688
|
+
"hst_id" => "rql:acq:123/456/678/374dd74324",
|
658
689
|
"ts" => 123456789,
|
659
690
|
"ini_ttl" => 123456,
|
660
691
|
"rem_ttl" => 123440,
|
@@ -780,9 +811,11 @@ def lock!(
|
|
780
811
|
log_sampling_enabled: config[:log_sampling_enabled],
|
781
812
|
log_sampling_percent: config[:log_sampling_percent],
|
782
813
|
log_sampler: config[:log_sampler],
|
814
|
+
log_sample_this: false,
|
783
815
|
instr_sampling_enabled: config[:instr_sampling_enabled],
|
784
816
|
instr_sampling_percent: config[:instr_sampling_percent],
|
785
817
|
instr_sampler: config[:instr_sampler],
|
818
|
+
instr_sample_this: false,
|
786
819
|
&block
|
787
820
|
)
|
788
821
|
```
|
@@ -800,6 +833,7 @@ See `#lock` method [documentation](#lock---obtain-a-lock).
|
|
800
833
|
- lock data (`Hash<String,String|Integer>`):
|
801
834
|
- `"lock_key"` - `string` - lock key in redis;
|
802
835
|
- `"acq_id"` - `string` - acquier identifier (process_id/thread_id/fiber_id/ractor_id/identity);
|
836
|
+
- `"hst_id"` - `string` - host identifier (process_id/thread_id/ractor_id/identity);
|
803
837
|
- `"ts"` - `numeric`/`epoch` - the time when lock was obtained;
|
804
838
|
- `"init_ttl"` - `integer` - (milliseconds) initial lock key ttl;
|
805
839
|
- `"rem_ttl"` - `integer` - (milliseconds) remaining lock key ttl;
|
@@ -822,6 +856,7 @@ rql.lock_info("your_lock_name")
|
|
822
856
|
{
|
823
857
|
"lock_key" => "rql:lock:your_lock_name",
|
824
858
|
"acq_id" => "rql:acq:123/456/567/678/374dd74324",
|
859
|
+
"hst_id" => "rql:acq:123/456/678/374dd74324",
|
825
860
|
"ts" => 123456789.12345,
|
826
861
|
"ini_ttl" => 5_000,
|
827
862
|
"rem_ttl" => 4_999
|
@@ -837,6 +872,7 @@ rql.lock_info("your_lock_name")
|
|
837
872
|
{
|
838
873
|
"lock_key" => "rql:lock:your_lock_name",
|
839
874
|
"acq_id" => "rql:acq:123/456/567/678/374dd74324",
|
875
|
+
"hst_id" => "rql:acq:123/456/678/374dd74324",
|
840
876
|
"ts" => 123456789.12345,
|
841
877
|
"ini_ttl" => 5_000,
|
842
878
|
"rem_ttl" => 4_999,
|
@@ -858,6 +894,7 @@ rql.lock_info("your_lock_name")
|
|
858
894
|
{
|
859
895
|
"lock_key" => "rql:lock:your_lock_name",
|
860
896
|
"acq_id" => "rql:acq:123/456/567/678/374dd74324",
|
897
|
+
"hst_id" => "rql:acq:123/456/678/374dd74324",
|
861
898
|
"ts" => 123456789.12345,
|
862
899
|
"ini_ttl" => 5_000,
|
863
900
|
"rem_ttl" => 9_444,
|
@@ -943,6 +980,7 @@ rql.queued?("your_lock_name") # => true/false
|
|
943
980
|
|
944
981
|
- release the concrete lock with lock request queue;
|
945
982
|
- queue will be relased first;
|
983
|
+
- has an alias: `#release_lock`;
|
946
984
|
- accepts:
|
947
985
|
- `lock_name` - (required) `[String]` - the lock name that should be released.
|
948
986
|
- `:logger` - (optional) `[::Logger,#debug]`
|
@@ -963,6 +1001,10 @@ rql.queued?("your_lock_name") # => true/false
|
|
963
1001
|
- `:log_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Logging::Sampler>]`
|
964
1002
|
- **log sampling**: percent-based log sampler that decides should be RQL case logged or not;
|
965
1003
|
- pre-configured in `config[:log_sampler]`;
|
1004
|
+
- `log_sample_this` - (optional) `[Boolean]`
|
1005
|
+
- marks the method that everything should be logged despite the enabled log sampling;
|
1006
|
+
- makes sense when log sampling is enabled;
|
1007
|
+
- `false` by default;
|
966
1008
|
- `:instr_sampling_enabled` - (optional) `[Boolean]`
|
967
1009
|
- enables **instrumentaion sampling**;
|
968
1010
|
- pre-configured in `config[:instr_sampling_enabled]`;
|
@@ -972,6 +1014,10 @@ rql.queued?("your_lock_name") # => true/false
|
|
972
1014
|
- `instr_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Instrument::Sampler>]`
|
973
1015
|
- percent-based log sampler that decides should be RQL case instrumented or not;
|
974
1016
|
- pre-configured in `config[:instr_sampler]`;
|
1017
|
+
- `instr_sample_this` - (optional) `[Boolean]`
|
1018
|
+
- marks the method that everything should be instrumneted despite the enabled instrumentation sampling;
|
1019
|
+
- makes sense when instrumentation sampling is enabled;
|
1020
|
+
- `false` by default;
|
975
1021
|
- if you try to unlock non-existent lock you will receive `ok: true` result with operation timings
|
976
1022
|
and `:nothing_to_release` result factor inside;
|
977
1023
|
|
@@ -1010,6 +1056,7 @@ rql.unlock("your_lock_name")
|
|
1010
1056
|
|
1011
1057
|
- release all obtained locks and related lock request queues;
|
1012
1058
|
- queues will be released first;
|
1059
|
+
- has an alias: `#release_locks`;
|
1013
1060
|
- accepts:
|
1014
1061
|
- `:batch_size` - (optional) `[Integer]`
|
1015
1062
|
- the size of batch of locks and lock queus that should be cleared under the one pipelined redis command at once;
|
@@ -1031,6 +1078,10 @@ rql.unlock("your_lock_name")
|
|
1031
1078
|
- `:log_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Logging::Sampler>]`
|
1032
1079
|
- **log sampling**: percent-based log sampler that decides should be RQL case logged or not;
|
1033
1080
|
- pre-configured in `config[:log_sampler]`;
|
1081
|
+
- `log_sample_this` - (optional) `[Boolean]`
|
1082
|
+
- marks the method that everything should be logged despite the enabled log sampling;
|
1083
|
+
- makes sense when log sampling is enabled;
|
1084
|
+
- `false` by default;
|
1034
1085
|
- `:instr_sampling_enabled` - (optional) `[Boolean]`
|
1035
1086
|
- enables **instrumentaion sampling**;
|
1036
1087
|
- pre-configured in `config[:instr_sampling_enabled]`;
|
@@ -1040,6 +1091,10 @@ rql.unlock("your_lock_name")
|
|
1040
1091
|
- `instr_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Instrument::Sampler>]`
|
1041
1092
|
- percent-based log sampler that decides should be RQL case instrumented or not;
|
1042
1093
|
- pre-configured in `config[:instr_sampler]`;
|
1094
|
+
- `instr_sample_this` - (optional) `[Boolean]`
|
1095
|
+
- marks the method that everything should be instrumneted despite the enabled instrumentation sampling;
|
1096
|
+
- makes sense when instrumentation sampling is enabled;
|
1097
|
+
- `false` by default;
|
1043
1098
|
- returns:
|
1044
1099
|
- `[Hash<Symbol,Numeric>]` - Format: `{ ok: true, result: Hash<Symbol,Numeric> }`;
|
1045
1100
|
- result data:
|
@@ -1090,6 +1145,10 @@ rql.clear_locks
|
|
1090
1145
|
- `:log_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Logging::Sampler>]`
|
1091
1146
|
- **log sampling**: percent-based log sampler that decides should be RQL case logged or not;
|
1092
1147
|
- pre-configured in `config[:log_sampler]`;
|
1148
|
+
- `log_sample_this` - (optional) `[Boolean]`
|
1149
|
+
- marks the method that everything should be logged despite the enabled log sampling;
|
1150
|
+
- makes sense when log sampling is enabled;
|
1151
|
+
- `false` by default;
|
1093
1152
|
- `:instr_sampling_enabled` - (optional) `[Boolean]`
|
1094
1153
|
- enables **instrumentaion sampling**;
|
1095
1154
|
- pre-configured in `config[:instr_sampling_enabled]`;
|
@@ -1099,6 +1158,10 @@ rql.clear_locks
|
|
1099
1158
|
- `instr_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Instrument::Sampler>]`
|
1100
1159
|
- percent-based log sampler that decides should be RQL case instrumented or not;
|
1101
1160
|
- pre-configured in `config[:instr_sampler]`;
|
1161
|
+
- `instr_sample_this` - (optional) `[Boolean]`
|
1162
|
+
- marks the method that everything should be instrumneted despite the enabled instrumentation sampling;
|
1163
|
+
- makes sense when instrumentation sampling is enabled;
|
1164
|
+
- `false` by default;
|
1102
1165
|
- returns `{ ok: true, result: :ttl_extended }` when ttl is extended;
|
1103
1166
|
- returns `{ ok: false, result: :async_expire_or_no_lock }` when a lock not found or a lock is already expired during
|
1104
1167
|
some steps of invocation (see **Important** section below);
|
@@ -1253,6 +1316,7 @@ rql.locks_info # or rql.locks_info(scan_size: 123)
|
|
1253
1316
|
:status=>:alive,
|
1254
1317
|
:info=>{
|
1255
1318
|
"acq_id"=>"rql:acq:41478/4320/4340/4360/848818f09d8c3420",
|
1319
|
+
"hst_id"=>"rql:hst:41478/4320/4360/848818f09d8c3420"
|
1256
1320
|
"ts"=>1711607112.670343,
|
1257
1321
|
"ini_ttl"=>15000,
|
1258
1322
|
"rem_ttl"=>13998}},
|
@@ -1336,6 +1400,10 @@ Accepts:
|
|
1336
1400
|
- `:log_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Logging::Sampler>]`
|
1337
1401
|
- **log sampling**: percent-based log sampler that decides should be RQL case logged or not;
|
1338
1402
|
- pre-configured in `config[:log_sampler]`;
|
1403
|
+
- `log_sample_this` - (optional) `[Boolean]`
|
1404
|
+
- marks the method that everything should be logged despite the enabled log sampling;
|
1405
|
+
- makes sense when log sampling is enabled;
|
1406
|
+
- `false` by default;
|
1339
1407
|
- `:instr_sampling_enabled` - (optional) `[Boolean]`
|
1340
1408
|
- enables **instrumentaion sampling**;
|
1341
1409
|
- pre-configured in `config[:instr_sampling_enabled]`;
|
@@ -1345,6 +1413,10 @@ Accepts:
|
|
1345
1413
|
- `instr_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Instrument::Sampler>]`
|
1346
1414
|
- percent-based log sampler that decides should be RQL case instrumented or not;
|
1347
1415
|
- pre-configured in `config[:instr_sampler]`;
|
1416
|
+
- `instr_sample_this` - (optional) `[Boolean]`
|
1417
|
+
- marks the method that everything should be instrumneted despite the enabled instrumentation sampling;
|
1418
|
+
- makes sense when instrumentation sampling is enabled;
|
1419
|
+
- `false` by default;
|
1348
1420
|
|
1349
1421
|
Returns: `{ ok: true, processed_queues: Set<String> }` returns the list of processed lock queues;
|
1350
1422
|
|
@@ -1367,6 +1439,8 @@ rql.clear_dead_requests(dead_ttl: 60 * 60 * 1000) # 1 hour in milliseconds
|
|
1367
1439
|
|
1368
1440
|
#### #current_acquirer_id
|
1369
1441
|
|
1442
|
+
<sup>\[[back to top](#usage)\]</sup>
|
1443
|
+
|
1370
1444
|
- get the current acquirer identifier in RQL notation that you can use for debugging purposes during the lock analyzation;
|
1371
1445
|
- acquirer identifier format:
|
1372
1446
|
```ruby
|
@@ -1400,6 +1474,277 @@ rql.current_acquirer_id
|
|
1400
1474
|
|
1401
1475
|
---
|
1402
1476
|
|
1477
|
+
#### #current_host_id
|
1478
|
+
|
1479
|
+
<sup>\[[back to top](#usage)\]</sup>
|
1480
|
+
|
1481
|
+
- get a current host identifier in RQL notation that you can use for debugging purposes during the lock analyzis;
|
1482
|
+
- the host is a ruby worker (a combination of process/thread/ractor) that is alive and can obtain locks;
|
1483
|
+
- the host is limited to `process`/`thread`/`ractor` (without `fiber`) combination cuz we have no abilities to extract
|
1484
|
+
all fiber objects from the current ruby process when at least one ractor object is defined (**ObjectSpace** loses
|
1485
|
+
abilities to extract `Fiber` and `Thread` objects after the any ractor is created) (`Thread` objects are analyzed
|
1486
|
+
via `Thread.list` API which does not lose their abilites);
|
1487
|
+
- host identifier format:
|
1488
|
+
```ruby
|
1489
|
+
"rql:hst:#{process_id}/#{thread_id}/#{ractor_id}/#{uniq_identity}"
|
1490
|
+
```
|
1491
|
+
- because of the moment that `#lock`/`#lock!` gives you a possibility to customize `process_id`,
|
1492
|
+
`fiber_id`, `thread_id`, `ractor_id` and `unique identity` identifiers the `#current_host_id` method provides this possibility too
|
1493
|
+
(except the `fiber_id` correspondingly);
|
1494
|
+
|
1495
|
+
Accepts:
|
1496
|
+
|
1497
|
+
- `process_id:` - (optional) `[Integer,Any]`
|
1498
|
+
- `::Process.pid` by default;
|
1499
|
+
- `thread_id:` - (optional) `[Integer,Any]`;
|
1500
|
+
- `::Thread.current.object_id` by default;
|
1501
|
+
- `ractor_id:` - (optional) `[Integer,Any]`;
|
1502
|
+
- `::Ractor.current.object_id` by default;
|
1503
|
+
- `identity:` - (optional) `[String]`;
|
1504
|
+
- this value is calculated once during `RedisQueuedLock::Client` instantiation and stored in `@uniq_identity`;
|
1505
|
+
- this value can be accessed from `RedisQueuedLock::Client#uniq_identity`;
|
1506
|
+
- [Configuration](#configuration) documentation: see `config[:uniq_identifier]`;
|
1507
|
+
- [#lock](#lock---obtain-a-lock) method documentation: see `uniq_identifier`;
|
1508
|
+
|
1509
|
+
```ruby
|
1510
|
+
rql.current_host_id
|
1511
|
+
|
1512
|
+
# =>
|
1513
|
+
"rql:acq:38529/4500/4360/66093702f24a3129"
|
1514
|
+
```
|
1515
|
+
|
1516
|
+
---
|
1517
|
+
|
1518
|
+
#### #possible_host_ids
|
1519
|
+
|
1520
|
+
<sup>\[[back to top](#usage)\]</sup>
|
1521
|
+
|
1522
|
+
- return the list (`Array<String>`) of possible host identifiers that can be reached from the current ractor;
|
1523
|
+
- the host is a ruby worker (a combination of process/thread/ractor/identity) that is alive and can obtain locks;
|
1524
|
+
- the host is limited to `process`/`thread`/`ractor` (without `fiber`) combination cuz we have no abilities to extract
|
1525
|
+
all fiber objects from the current ruby process when at least one ractor object is defined (**ObjectSpace** loses
|
1526
|
+
abilities to extract `Fiber` and `Thread` objects after the any ractor is created) (`Thread` objects are analyzed
|
1527
|
+
via `Thread.list` API which does not lose their abilites);
|
1528
|
+
- host identifier format:
|
1529
|
+
```ruby
|
1530
|
+
"rql:hst:#{process_id}/#{thread_id}/#{ractor_id}/#{uniq_identity}"
|
1531
|
+
```
|
1532
|
+
|
1533
|
+
Accepts:
|
1534
|
+
|
1535
|
+
- `identity` - (optional) `[String]`;
|
1536
|
+
- this value is calculated once during `RedisQueuedLock::Client` instantiation and stored in `@uniq_identity`;
|
1537
|
+
- this value can be accessed from `RedisQueuedLock::Client#uniq_identity`;
|
1538
|
+
- [Configuration](#configuration) documentation: see `config[:uniq_identifier]`;
|
1539
|
+
- [#lock](#lock---obtain-a-lock) method documentation: see `uniq_identifier`;
|
1540
|
+
|
1541
|
+
```ruby
|
1542
|
+
rql.possible_host_ids
|
1543
|
+
|
1544
|
+
# =>
|
1545
|
+
[
|
1546
|
+
"rql:hst:18814/2300/2280/5ce0c4582fc59c06", # process id / thread id / ractor id / uniq identity
|
1547
|
+
"rql:hst:18814/2320/2280/5ce0c4582fc59c06", # ...
|
1548
|
+
"rql:hst:18814/2340/2280/5ce0c4582fc59c06", # ...
|
1549
|
+
"rql:hst:18814/2360/2280/5ce0c4582fc59c06", # ...
|
1550
|
+
"rql:hst:18814/2380/2280/5ce0c4582fc59c06", # ...
|
1551
|
+
"rql:hst:18814/2400/2280/5ce0c4582fc59c06"
|
1552
|
+
]
|
1553
|
+
```
|
1554
|
+
---
|
1555
|
+
|
1556
|
+
## Swarm Mode and Zombie Locks
|
1557
|
+
|
1558
|
+
<sup>\[[back to top](#table-of-contents)\]</sup>
|
1559
|
+
|
1560
|
+
> Eliminate zombie locks with a swarm.
|
1561
|
+
|
1562
|
+
**This documentation section is in progress!**
|
1563
|
+
|
1564
|
+
[(work and usage preview (temporary example-based docs))](#work-and-usage-preview-temporary-example-based-docs)
|
1565
|
+
|
1566
|
+
- [How to Swarm](#how-to-swarm)
|
1567
|
+
- [configuration](#)
|
1568
|
+
- [swarm_status](#swarm_status)
|
1569
|
+
- [swarm_info](#swarm_info)
|
1570
|
+
- [swarmize!](#swarmize!)
|
1571
|
+
- [deswarmize!](#deswarmize!)
|
1572
|
+
- [probe_hosts](#probe_hosts)
|
1573
|
+
- [flush_zobmies](#flush_zombies)
|
1574
|
+
- [zombies_info](#zombies_info)
|
1575
|
+
- [zombie_locks](#zombie_locks)
|
1576
|
+
- [zombie_hosts](#zombie_hosts)
|
1577
|
+
- [zombie_acquiers](#zombie_acquiers)
|
1578
|
+
|
1579
|
+
<hr>
|
1580
|
+
|
1581
|
+
#### Work and Usage Preview (temporary example-based docs)
|
1582
|
+
|
1583
|
+
<sup>\[[back to top](#swarm-mode-and-zombie-locks)\]</sup>
|
1584
|
+
|
1585
|
+
<details>
|
1586
|
+
<summary>configuration</summary>
|
1587
|
+
|
1588
|
+
```ruby
|
1589
|
+
redis_client = RedisClient.config.new_pool # NOTE: provide your own RedisClient instance
|
1590
|
+
|
1591
|
+
clinet = RedisQueuedLocks::Client.new(redis_client) do |config|
|
1592
|
+
# NOTE: auto-swarm your RQL client after initalization (run swarm elements and their supervisor)
|
1593
|
+
config.swarm.auto_swarm = false
|
1594
|
+
|
1595
|
+
# supervisor configs
|
1596
|
+
config.swarm.supervisor.liveness_probing_period = 2 # NOTE: in seconds
|
1597
|
+
|
1598
|
+
# (probe_hosts) host probing configuration
|
1599
|
+
config.swarm.probe_hosts.enabled_for_swarm = true # NOTE: run host-probing from or not
|
1600
|
+
config.swarm.probe_hosts.probe_period = 2 # NOTE: (in seconds) the period of time when the probing process is triggered
|
1601
|
+
# (probe_hosts) individual redis config
|
1602
|
+
config.swarm.probe_hosts.redis_config.sentinel = false # NOTE: individual redis config
|
1603
|
+
config.swarm.probe_hosts.redis_config.pooled = false # NOTE: individual redis config
|
1604
|
+
config.swarm.probe_hosts.redis_config.config = {} # NOTE: individual redis config
|
1605
|
+
config.swarm.probe_hosts.redis_config.pool_config = {} # NOTE: individual redis config
|
1606
|
+
|
1607
|
+
# (flush_zombies) zombie flushing configuration
|
1608
|
+
config.swarm.flush_zombies.enabled_for_swarm = true # NOTE: run zombie flushing or not
|
1609
|
+
config.swarm.flush_zombies.zombie_flush_period = 10 # NOTE: (in seconds) period of time when the zombie flusher is triggered
|
1610
|
+
config.swarm.flush_zombies.zombie_ttl = 15_000 # NOTE: (in milliseconds) when the lock/host/acquier is considered a zombie
|
1611
|
+
config.swarm.flush_zombies.zombie_lock_scan_size = 500 # NOTE: scan sizec during zombie flushing
|
1612
|
+
config.swarm.flush_zombies.zombie_queue_scan_size = 500 # NOTE: scan sizec during zombie flushing
|
1613
|
+
# (flush_zombies) individual redis config
|
1614
|
+
config.swarm.flush_zombies.redis_config.sentinel = false # NOTE: individual redis config
|
1615
|
+
config.swarm.flush_zombies.redis_config.pooled = false # NOTE: individual redis config
|
1616
|
+
config.swarm.flush_zombies.redis_config.config = {} # NOTE: individual redis config
|
1617
|
+
config.swarm.flush_zombies.redis_config.pool_config = {} # NOTE: individual redis config
|
1618
|
+
```
|
1619
|
+
</details>
|
1620
|
+
|
1621
|
+
<details>
|
1622
|
+
<summary>seed a zombie</summary>
|
1623
|
+
|
1624
|
+
- obtain some long living lock and kill the host process which will lead the lock becoming a zombie:
|
1625
|
+
|
1626
|
+
```ruby
|
1627
|
+
daiver => ~/Projects/redis_queued_locks master [$]
|
1628
|
+
➜ bin/console
|
1629
|
+
[1] pry(main)> rql = RedisQueuedLocks::Client.new(RedisClient.new);
|
1630
|
+
[2] pry(main)> rql.swarmize!
|
1631
|
+
/Users/daiver/Projects/redis_queued_locks/lib/redis_queued_locks/swarm/flush_zombies.rb:107: warning: Ractor is experimental, and the behavior may change in future versions of Ruby! Also there are many implementation issues.
|
1632
|
+
=> {:ok=>true, :result=>:swarming}
|
1633
|
+
[3] pry(main)> rql.lock('kekpek', ttl: 1111111111)
|
1634
|
+
=> {:ok=>true,
|
1635
|
+
:result=>
|
1636
|
+
{:lock_key=>"rql:lock:kekpek",
|
1637
|
+
:acq_id=>"rql:acq:17580/2260/2380/2280/3f16b93973612580",
|
1638
|
+
:hst_id=>"rql:hst:17580/2260/2280/3f16b93973612580",
|
1639
|
+
:ts=>1720305351.069259,
|
1640
|
+
:ttl=>1111111111,
|
1641
|
+
:process=>:lock_obtaining}}
|
1642
|
+
[4] pry(main)> exit
|
1643
|
+
```
|
1644
|
+
</details>
|
1645
|
+
|
1646
|
+
<details>
|
1647
|
+
<summary>find zombies</summary>
|
1648
|
+
|
1649
|
+
- start another process, fetch the swarm info, see that our last process is a zombie now and their hosted lock is a zombie too:
|
1650
|
+
|
1651
|
+
```ruby
|
1652
|
+
daiver => ~/Projects/redis_queued_locks master [$] took 27.2s
|
1653
|
+
➜ bin/console
|
1654
|
+
[1] pry(main)> rql = RedisQueuedLocks::Client.new(RedisClient.new);
|
1655
|
+
[2] pry(main)> rql.swarm_info
|
1656
|
+
=> {"rql:hst:17580/2260/2280/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 12897/262144 +0300, :last_probe_score=>1720305353.0491982},
|
1657
|
+
"rql:hst:17580/2300/2280/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 211107/4194304 +0300, :last_probe_score=>1720305353.0503318},
|
1658
|
+
"rql:hst:17580/2320/2280/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 106615/2097152 +0300, :last_probe_score=>1720305353.050838},
|
1659
|
+
"rql:hst:17580/2260/2340/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 26239/524288 +0300, :last_probe_score=>1720305353.050047},
|
1660
|
+
"rql:hst:17580/2300/2340/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 106359/2097152 +0300, :last_probe_score=>1720305353.050716},
|
1661
|
+
"rql:hst:17580/2320/2340/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 213633/4194304 +0300, :last_probe_score=>1720305353.050934},
|
1662
|
+
"rql:hst:17580/2360/2280/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 214077/4194304 +0300, :last_probe_score=>1720305353.05104},
|
1663
|
+
"rql:hst:17580/2360/2340/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 214505/4194304 +0300, :last_probe_score=>1720305353.051142},
|
1664
|
+
"rql:hst:17580/2400/2280/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 53729/1048576 +0300, :last_probe_score=>1720305353.05124},
|
1665
|
+
"rql:hst:17580/2400/2340/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 3365/65536 +0300, :last_probe_score=>1720305353.0513458}}
|
1666
|
+
[3] pry(main)> rql.swarm_status
|
1667
|
+
=> {:auto_swarm=>false,
|
1668
|
+
:supervisor=>{:running=>false, :state=>"non_initialized", :observable=>"non_initialized"},
|
1669
|
+
:probe_hosts=>{:enabled=>true, :thread=>{:running=>false, :state=>"non_initialized"}, :main_loop=>{:running=>false, :state=>"non_initialized"}},
|
1670
|
+
:flush_zombies=>{:enabled=>true, :ractor=>{:running=>false, :state=>"non_initialized"}, :main_loop=>{:running=>false, :state=>"non_initialized"}}}
|
1671
|
+
[4] pry(main)> rql.zombies_info
|
1672
|
+
=> {:zombie_hosts=>
|
1673
|
+
#<Set:
|
1674
|
+
{"rql:hst:17580/2260/2280/3f16b93973612580",
|
1675
|
+
"rql:hst:17580/2300/2280/3f16b93973612580",
|
1676
|
+
"rql:hst:17580/2320/2280/3f16b93973612580",
|
1677
|
+
"rql:hst:17580/2260/2340/3f16b93973612580",
|
1678
|
+
"rql:hst:17580/2300/2340/3f16b93973612580",
|
1679
|
+
"rql:hst:17580/2320/2340/3f16b93973612580",
|
1680
|
+
"rql:hst:17580/2360/2280/3f16b93973612580",
|
1681
|
+
"rql:hst:17580/2360/2340/3f16b93973612580",
|
1682
|
+
"rql:hst:17580/2400/2280/3f16b93973612580",
|
1683
|
+
"rql:hst:17580/2400/2340/3f16b93973612580"}>,
|
1684
|
+
:zombie_acquirers=>#<Set: {"rql:acq:17580/2260/2380/2280/3f16b93973612580"}>,
|
1685
|
+
:zombie_locks=>#<Set: {"rql:lock:kekpek"}>}
|
1686
|
+
[5] pry(main)> rql.zombie_locks
|
1687
|
+
=> #<Set: {"rql:lock:kekpek"}>
|
1688
|
+
[6] pry(main)> rql.zombie_acquiers
|
1689
|
+
=> #<Set: {"rql:acq:17580/2260/2380/2280/3f16b93973612580"}>
|
1690
|
+
[7] pry(main)> rql.zombie_hosts
|
1691
|
+
=> #<Set:
|
1692
|
+
{"rql:hst:17580/2260/2280/3f16b93973612580",
|
1693
|
+
"rql:hst:17580/2300/2280/3f16b93973612580",
|
1694
|
+
"rql:hst:17580/2320/2280/3f16b93973612580",
|
1695
|
+
"rql:hst:17580/2260/2340/3f16b93973612580",
|
1696
|
+
"rql:hst:17580/2300/2340/3f16b93973612580",
|
1697
|
+
"rql:hst:17580/2320/2340/3f16b93973612580",
|
1698
|
+
"rql:hst:17580/2360/2280/3f16b93973612580",
|
1699
|
+
"rql:hst:17580/2360/2340/3f16b93973612580",
|
1700
|
+
"rql:hst:17580/2400/2280/3f16b93973612580",
|
1701
|
+
"rql:hst:17580/2400/2340/3f16b93973612580"}>
|
1702
|
+
```
|
1703
|
+
</details>
|
1704
|
+
|
1705
|
+
<details>
|
1706
|
+
<summary>kill zombies in a background</summary>
|
1707
|
+
|
1708
|
+
- swarmize the new current ruby process that should run the flush zombies element that will drop zombie locks, zombie hosts and their lock requests in a background:
|
1709
|
+
|
1710
|
+
```ruby
|
1711
|
+
[8] pry(main)> rql.swarmize!
|
1712
|
+
/Users/daiver/Projects/redis_queued_locks/lib/redis_queued_locks/swarm/flush_zombies.rb:107: warning: Ractor is experimental, and the behavior may change in future versions of Ruby! Also there are many implementation issues.
|
1713
|
+
=> {:ok=>true, :result=>:swarming}
|
1714
|
+
[9] pry(main)> rql.swarm_info
|
1715
|
+
=> {"rql:hst:17752/2260/2280/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 4012577/4194304 +0300, :last_probe_score=>1720305399.956673},
|
1716
|
+
"rql:hst:17752/2300/2280/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 4015233/4194304 +0300, :last_probe_score=>1720305399.9573061},
|
1717
|
+
"rql:hst:17752/2320/2280/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 4016755/4194304 +0300, :last_probe_score=>1720305399.957669},
|
1718
|
+
"rql:hst:17752/2260/2340/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 1003611/1048576 +0300, :last_probe_score=>1720305399.957118},
|
1719
|
+
"rql:hst:17752/2300/2340/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 2008027/2097152 +0300, :last_probe_score=>1720305399.957502},
|
1720
|
+
"rql:hst:17752/2320/2340/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 2008715/2097152 +0300, :last_probe_score=>1720305399.95783},
|
1721
|
+
"rql:hst:17752/2360/2280/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 4018063/4194304 +0300, :last_probe_score=>1720305399.9579809},
|
1722
|
+
"rql:hst:17752/2360/2340/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 1004673/1048576 +0300, :last_probe_score=>1720305399.9581308}}
|
1723
|
+
[10] pry(main)> rql.swarm_status
|
1724
|
+
=> {:auto_swarm=>false,
|
1725
|
+
:supervisor=>{:running=>true, :state=>"sleep", :observable=>"initialized"},
|
1726
|
+
:probe_hosts=>{:enabled=>true, :thread=>{:running=>true, :state=>"sleep"}, :main_loop=>{:running=>true, :state=>"sleep"}},
|
1727
|
+
:flush_zombies=>{:enabled=>true, :ractor=>{:running=>true, :state=>"running"}, :main_loop=>{:running=>true, :state=>"sleep"}}}
|
1728
|
+
[11] pry(main)> rql.zombies_info
|
1729
|
+
=> {:zombie_hosts=>#<Set: {}>, :zombie_acquirers=>#<Set: {}>, :zombie_locks=>#<Set: {}>}
|
1730
|
+
[12] pry(main)> rql.zombie_acquiers
|
1731
|
+
=> #<Set: {}>
|
1732
|
+
[13] pry(main)> rql.zombie_hosts
|
1733
|
+
=> #<Set: {}>
|
1734
|
+
[14] pry(main)>
|
1735
|
+
```
|
1736
|
+
</details>
|
1737
|
+
|
1738
|
+
<details>
|
1739
|
+
<summary>swarm hosts key in Redis</summary>
|
1740
|
+
|
1741
|
+
```ruby
|
1742
|
+
"rql:swarm:hsts"
|
1743
|
+
```
|
1744
|
+
</details>
|
1745
|
+
|
1746
|
+
---
|
1747
|
+
|
1403
1748
|
## Lock Access Strategies
|
1404
1749
|
|
1405
1750
|
<sup>\[[back to top](#table-of-contents)\]</sup>
|
@@ -1443,34 +1788,34 @@ rql.current_acquirer_id
|
|
1443
1788
|
- default logs (raised from `#lock`/`#lock!`):
|
1444
1789
|
|
1445
1790
|
```ruby
|
1446
|
-
"[redis_queued_locks.start_lock_obtaining]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1447
|
-
"[redis_queued_locks.start_try_to_lock_cycle]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1448
|
-
"[redis_queued_locks.dead_score_reached__reset_acquier_position]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1449
|
-
"[redis_queued_locks.lock_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "acq_time");
|
1450
|
-
"[redis_queued_locks.extendable_reentrant_lock_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "acq_time");
|
1451
|
-
"[redis_queued_locks.reentrant_lock_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "acq_time");
|
1452
|
-
"[redis_queued_locks.fail_fast_or_limits_reached_or_deadlock__dequeue]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1453
|
-
"[redis_queued_locks.expire_lock]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1454
|
-
"[redis_queued_locks.decrease_lock]" # (logs "lock_key", "decreased_ttl", "queue_ttl", "acq_id", "acs_strat");
|
1791
|
+
"[redis_queued_locks.start_lock_obtaining]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1792
|
+
"[redis_queued_locks.start_try_to_lock_cycle]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1793
|
+
"[redis_queued_locks.dead_score_reached__reset_acquier_position]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1794
|
+
"[redis_queued_locks.lock_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acq_time");
|
1795
|
+
"[redis_queued_locks.extendable_reentrant_lock_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "acq_time");
|
1796
|
+
"[redis_queued_locks.reentrant_lock_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "acq_time");
|
1797
|
+
"[redis_queued_locks.fail_fast_or_limits_reached_or_deadlock__dequeue]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1798
|
+
"[redis_queued_locks.expire_lock]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1799
|
+
"[redis_queued_locks.decrease_lock]" # (logs "lock_key", "decreased_ttl", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1455
1800
|
```
|
1456
1801
|
|
1457
1802
|
- additional logs (raised from `#lock`/`#lock!` with `confg[:log_lock_try] == true`):
|
1458
1803
|
|
1459
1804
|
```ruby
|
1460
|
-
"[redis_queued_locks.try_lock.start]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1461
|
-
"[redis_queued_locks.try_lock.rconn_fetched]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1462
|
-
"[redis_queued_locks.try_lock.same_process_conflict_detected]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1463
|
-
"[redis_queued_locks.try_lock.same_process_conflict_analyzed]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status");
|
1464
|
-
"[redis_queued_locks.try_lock.reentrant_lock__extend_and_work_through]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status", "last_ext_ttl", "last_ext_ts");
|
1465
|
-
"[redis_queued_locks.try_lock.reentrant_lock__work_through]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status", last_spc_ts);
|
1466
|
-
"[redis_queued_locks.try_lock.single_process_lock_conflict__dead_lock]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status", "last_spc_ts");
|
1467
|
-
"[redis_queued_locks.try_lock.acq_added_to_queue]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1468
|
-
"[redis_queued_locks.try_lock.remove_expired_acqs]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1469
|
-
"[redis_queued_locks.try_lock.get_first_from_queue]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "first_acq_id_in_queue");
|
1470
|
-
"[redis_queued_locks.try_lock.exit__queue_ttl_reached]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1471
|
-
"[redis_queued_locks.try_lock.exit__no_first]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "first_acq_id_in_queue", "<current_lock_data>");
|
1472
|
-
"[redis_queued_locks.try_lock.exit__lock_still_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "first_acq_id_in_queue", "locked_by_acq_id", "<current_lock_data>");
|
1473
|
-
"[redis_queued_locks.try_lock.obtain__free_to_acquire]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1805
|
+
"[redis_queued_locks.try_lock.start]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1806
|
+
"[redis_queued_locks.try_lock.rconn_fetched]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1807
|
+
"[redis_queued_locks.try_lock.same_process_conflict_detected]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1808
|
+
"[redis_queued_locks.try_lock.same_process_conflict_analyzed]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status");
|
1809
|
+
"[redis_queued_locks.try_lock.reentrant_lock__extend_and_work_through]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status", "last_ext_ttl", "last_ext_ts");
|
1810
|
+
"[redis_queued_locks.try_lock.reentrant_lock__work_through]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status", last_spc_ts);
|
1811
|
+
"[redis_queued_locks.try_lock.single_process_lock_conflict__dead_lock]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status", "last_spc_ts");
|
1812
|
+
"[redis_queued_locks.try_lock.acq_added_to_queue]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1813
|
+
"[redis_queued_locks.try_lock.remove_expired_acqs]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1814
|
+
"[redis_queued_locks.try_lock.get_first_from_queue]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "first_acq_id_in_queue");
|
1815
|
+
"[redis_queued_locks.try_lock.exit__queue_ttl_reached]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1816
|
+
"[redis_queued_locks.try_lock.exit__no_first]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "first_acq_id_in_queue", "<current_lock_data>");
|
1817
|
+
"[redis_queued_locks.try_lock.exit__lock_still_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "first_acq_id_in_queue", "locked_by_acq_id", "<current_lock_data>");
|
1818
|
+
"[redis_queued_locks.try_lock.obtain__free_to_acquire]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1474
1819
|
```
|
1475
1820
|
|
1476
1821
|
---
|
@@ -1520,6 +1865,7 @@ Detalized event semantics and payload structure:
|
|
1520
1865
|
- payload:
|
1521
1866
|
- `:ttl` - `integer`/`milliseconds` - lock ttl;
|
1522
1867
|
- `:acq_id` - `string` - lock acquier identifier;
|
1868
|
+
- `:hst_id` - `string` - lock's host identifier;
|
1523
1869
|
- `:lock_key` - `string` - lock name;
|
1524
1870
|
- `:ts` - `numeric`/`epoch` - the time when the lock was obtaiend;
|
1525
1871
|
- `:acq_time` - `float`/`milliseconds` - time spent on lock acquiring;
|
@@ -1532,6 +1878,7 @@ Detalized event semantics and payload structure:
|
|
1532
1878
|
- `:lock_key` - `string` - lock name;
|
1533
1879
|
- `:ttl` - `integer`/`milliseconds` - last lock ttl by reentrant locking;
|
1534
1880
|
- `:acq_id` - `string` - lock acquier identifier;
|
1881
|
+
- `:hst_id` - `string` - lock's host identifier;
|
1535
1882
|
- `:ts` - `numeric`/`epoch` - the time when the lock was obtaiend as extendable reentrant lock;
|
1536
1883
|
- `:acq_time` - `float`/`milliseconds` - time spent on lock acquiring;
|
1537
1884
|
- `:instrument` - `nil`/`Any` - custom data passed to the `#lock`/`#lock!` method as `:instrument` attribute;
|
@@ -1543,6 +1890,7 @@ Detalized event semantics and payload structure:
|
|
1543
1890
|
- `:lock_key` - `string` - lock name;
|
1544
1891
|
- `:ttl` - `integer`/`milliseconds` - last lock ttl by reentrant locking;
|
1545
1892
|
- `:acq_id` - `string` - lock acquier identifier;
|
1893
|
+
- `:hst_id` - `string` - lock's host identifier;
|
1546
1894
|
- `:ts` - `numeric`/`epoch` - the time when the lock was obtaiend as reentrant lock;
|
1547
1895
|
- `:acq_time` - `float`/`milliseconds` - time spent on lock acquiring;
|
1548
1896
|
- `:instrument` - `nil`/`Any` - custom data passed to the `#lock`/`#lock!` method as `:instrument` attribute;
|
@@ -1554,6 +1902,7 @@ Detalized event semantics and payload structure:
|
|
1554
1902
|
- `:hold_time` - `float`/`milliseconds` - lock hold time;
|
1555
1903
|
- `:ttl` - `integer`/`milliseconds` - lock ttl;
|
1556
1904
|
- `:acq_id` - `string` - lock acquier identifier;
|
1905
|
+
- `:hst_id` - `string` - lock's host identifier;
|
1557
1906
|
- `:lock_key` - `string` - lock name;
|
1558
1907
|
- `:ts` - `numeric`/`epoch` - the time when lock was obtained;
|
1559
1908
|
- `:acq_time` - `float`/`milliseconds` - time spent on lock acquiring;
|
@@ -1567,6 +1916,7 @@ Detalized event semantics and payload structure:
|
|
1567
1916
|
- `:hold_time` - `float`/`milliseconds` - lock hold time;
|
1568
1917
|
- `:ttl` - `integer`/`milliseconds` - last lock ttl by reentrant locking;
|
1569
1918
|
- `:acq_id` - `string` - lock acquier identifier;
|
1919
|
+
- `:hst_id` - `string` - lock's host identifier;
|
1570
1920
|
- `:ts` - `numeric`/`epoch` - the time when the lock was obtaiend as reentrant lock;
|
1571
1921
|
- `:lock_key` - `string` - lock name;
|
1572
1922
|
- `:acq_time` - `float`/`milliseconds` - time spent on lock acquiring;
|
@@ -1596,6 +1946,12 @@ Detalized event semantics and payload structure:
|
|
1596
1946
|
<sup>\[[back to top](#table-of-contents)\]</sup>
|
1597
1947
|
|
1598
1948
|
- **Major**:
|
1949
|
+
- Swarm:
|
1950
|
+
- circuit-breaker for long-living failures of your infrastructure inside the swarm elements and supervisor:
|
1951
|
+
the supervisor will stop (for some period of time or while the some factor will return `true`)
|
1952
|
+
trying to ressurect unexpectedly terminated swarm elements, and will notify about this;
|
1953
|
+
- swarm logs (thread/ractor has some limitations so the initial implementation does not include swarm logging);
|
1954
|
+
- swarm instrumentation (thread/ractor has some limitations so the initial implementation does not include swarm instrumentation);
|
1599
1955
|
- lock request prioritization;
|
1600
1956
|
- **strict redlock algorithm support** (support for many `RedisClient` instances);
|
1601
1957
|
- `#lock_series` - acquire a series of locks:
|