redis_queued_locks 1.8.0 → 1.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.ruby-version +1 -1
- data/CHANGELOG.md +51 -0
- data/README.md +402 -46
- data/lib/redis_queued_locks/acquier/acquire_lock/dequeue_from_lock_queue/log_visitor.rb +4 -0
- data/lib/redis_queued_locks/acquier/acquire_lock/dequeue_from_lock_queue.rb +4 -1
- data/lib/redis_queued_locks/acquier/acquire_lock/instr_visitor.rb +20 -5
- data/lib/redis_queued_locks/acquier/acquire_lock/log_visitor.rb +24 -0
- data/lib/redis_queued_locks/acquier/acquire_lock/try_to_lock/log_visitor.rb +56 -0
- data/lib/redis_queued_locks/acquier/acquire_lock/try_to_lock.rb +37 -30
- data/lib/redis_queued_locks/acquier/acquire_lock/yield_expire/log_visitor.rb +8 -0
- data/lib/redis_queued_locks/acquier/acquire_lock/yield_expire.rb +13 -9
- data/lib/redis_queued_locks/acquier/acquire_lock.rb +44 -20
- data/lib/redis_queued_locks/acquier/clear_dead_requests.rb +5 -1
- data/lib/redis_queued_locks/acquier/extend_lock_ttl.rb +5 -1
- data/lib/redis_queued_locks/acquier/lock_info.rb +4 -3
- data/lib/redis_queued_locks/acquier/locks.rb +2 -2
- data/lib/redis_queued_locks/acquier/queue_info.rb +2 -2
- data/lib/redis_queued_locks/acquier/release_all_locks.rb +12 -2
- data/lib/redis_queued_locks/acquier/release_lock.rb +12 -2
- data/lib/redis_queued_locks/client.rb +284 -34
- data/lib/redis_queued_locks/errors.rb +8 -0
- data/lib/redis_queued_locks/instrument.rb +8 -1
- data/lib/redis_queued_locks/logging.rb +8 -1
- data/lib/redis_queued_locks/resource.rb +59 -1
- data/lib/redis_queued_locks/swarm/acquirers.rb +44 -0
- data/lib/redis_queued_locks/swarm/flush_zombies.rb +133 -0
- data/lib/redis_queued_locks/swarm/probe_hosts.rb +69 -0
- data/lib/redis_queued_locks/swarm/redis_client_builder.rb +67 -0
- data/lib/redis_queued_locks/swarm/supervisor.rb +83 -0
- data/lib/redis_queued_locks/swarm/swarm_element/isolated.rb +287 -0
- data/lib/redis_queued_locks/swarm/swarm_element/threaded.rb +351 -0
- data/lib/redis_queued_locks/swarm/swarm_element.rb +8 -0
- data/lib/redis_queued_locks/swarm/zombie_info.rb +145 -0
- data/lib/redis_queued_locks/swarm.rb +241 -0
- data/lib/redis_queued_locks/utilities/lock.rb +22 -0
- data/lib/redis_queued_locks/utilities.rb +75 -0
- data/lib/redis_queued_locks/version.rb +2 -2
- data/lib/redis_queued_locks.rb +2 -0
- metadata +14 -4
- data/lib/redis_queued_locks/watcher.rb +0 -1
data/README.md
CHANGED
@@ -35,6 +35,22 @@ Provides flexible invocation flow, parametrized limits (lock request ttl, lock t
|
|
35
35
|
- [queues_info](#queues_info---get-list-of-queues-with-their-info)
|
36
36
|
- [clear_dead_requests](#clear_dead_requests)
|
37
37
|
- [current_acquirer_id](#current_acquirer_id)
|
38
|
+
- [current_host_id](#current_host_id)
|
39
|
+
- [possible_host_ids](#possible_host_ids)
|
40
|
+
- [Swarm Mode and Zombie Locks](#swarm-mode-and-zombie-locks)
|
41
|
+
- [work and usage preview (temporary example-based docs)](#work-and-usage-preview-temporary-example-based-docs)
|
42
|
+
- [How to Swarm](#how-to-swarm)
|
43
|
+
- [configuration](#)
|
44
|
+
- [swarm_status](#swarm_status)
|
45
|
+
- [swarm_info](#swarm_info)
|
46
|
+
- [swarmize!](#swarmize!)
|
47
|
+
- [deswarmize!](#deswarmize!)
|
48
|
+
- [probe_hosts](#probe_hosts)
|
49
|
+
- [flush_zobmies](#flush_zombies)
|
50
|
+
- [zombies_info](#zombies_info)
|
51
|
+
- [zombie_locks](#zombie_locks)
|
52
|
+
- [zombie_hosts](#zombie_hosts)
|
53
|
+
- [zombie_acquiers](#zombie_acquiers)
|
38
54
|
- [Lock Access Strategies](#lock-access-strategies)
|
39
55
|
- [queued](#lock-access-strategies)
|
40
56
|
- [random](#lock-access-strategies)
|
@@ -244,35 +260,35 @@ clinet = RedisQueuedLocks::Client.new(redis_client) do |config|
|
|
244
260
|
# - should implement `debug(progname = nil, &block)` (minimal requirement) or be an instance of Ruby's `::Logger` class/subclass;
|
245
261
|
# - supports `SemanticLogger::Logger` (see "semantic_logger" gem)
|
246
262
|
# - at this moment the only debug logs are realised in following cases:
|
247
|
-
# - "[redis_queued_locks.start_lock_obtaining]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
248
|
-
# - "[redis_queued_locks.start_try_to_lock_cycle]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
249
|
-
# - "[redis_queued_locks.dead_score_reached__reset_acquier_position]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
250
|
-
# - "[redis_queued_locks.lock_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "acq_time", "acs_strat");
|
251
|
-
# - "[redis_queued_locks.extendable_reentrant_lock_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "acq_time", "acs_strat");
|
252
|
-
# - "[redis_queued_locks.reentrant_lock_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "acq_time", "acs_strat");
|
253
|
-
# - "[redis_queued_locks.fail_fast_or_limits_reached_or_deadlock__dequeue]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
254
|
-
# - "[redis_queued_locks.expire_lock]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
255
|
-
# - "[redis_queued_locks.decrease_lock]" (logs "lock_key", "decreased_ttl", "queue_ttl", "acq_id", "acs_strat");
|
263
|
+
# - "[redis_queued_locks.start_lock_obtaining]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
264
|
+
# - "[redis_queued_locks.start_try_to_lock_cycle]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
265
|
+
# - "[redis_queued_locks.dead_score_reached__reset_acquier_position]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
266
|
+
# - "[redis_queued_locks.lock_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acq_time", "acs_strat");
|
267
|
+
# - "[redis_queued_locks.extendable_reentrant_lock_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acq_time", "acs_strat");
|
268
|
+
# - "[redis_queued_locks.reentrant_lock_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acq_time", "acs_strat");
|
269
|
+
# - "[redis_queued_locks.fail_fast_or_limits_reached_or_deadlock__dequeue]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
270
|
+
# - "[redis_queued_locks.expire_lock]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
271
|
+
# - "[redis_queued_locks.decrease_lock]" (logs "lock_key", "decreased_ttl", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
256
272
|
# - by default uses VoidLogger that does nothing;
|
257
273
|
config.logger = RedisQueuedLocks::Logging::VoidLogger
|
258
274
|
|
259
275
|
# (default: false)
|
260
276
|
# - adds additional debug logs;
|
261
277
|
# - enables additional logs for each internal try-retry lock acquiring (a lot of logs can be generated depending on your retry configurations);
|
262
|
-
# - it adds following logs in addition to the existing:
|
263
|
-
# - "[redis_queued_locks.try_lock.start]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
264
|
-
# - "[redis_queued_locks.try_lock.rconn_fetched]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
265
|
-
# - "[redis_queued_locks.try_lock.same_process_conflict_detected]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
266
|
-
# - "[redis_queued_locks.try_lock.same_process_conflict_analyzed]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status");
|
267
|
-
# - "[redis_queued_locks.try_lock.reentrant_lock__extend_and_work_through]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status", "last_ext_ttl", "last_ext_ts");
|
268
|
-
# - "[redis_queued_locks.try_lock.reentrant_lock__work_through]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status", last_spc_ts);
|
269
|
-
# - "[redis_queued_locks.try_lock.acq_added_to_queue]" (logs "lock_key", "queue_ttl", "acq_id, "acs_strat")";
|
270
|
-
# - "[redis_queued_locks.try_lock.remove_expired_acqs]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
271
|
-
# - "[redis_queued_locks.try_lock.get_first_from_queue]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "first_acq_id_in_queue");
|
272
|
-
# - "[redis_queued_locks.try_lock.exit__queue_ttl_reached]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
273
|
-
# - "[redis_queued_locks.try_lock.exit__no_first]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "first_acq_id_in_queue", "<current_lock_data>");
|
274
|
-
# - "[redis_queued_locks.try_lock.exit__lock_still_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "first_acq_id_in_queue", "locked_by_acq_id", "<current_lock_data>");
|
275
|
-
# - "[redis_queued_locks.try_lock.obtain__free_to_acquire]" (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
278
|
+
# - it adds following debug logs in addition to the existing:
|
279
|
+
# - "[redis_queued_locks.try_lock.start]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
280
|
+
# - "[redis_queued_locks.try_lock.rconn_fetched]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
281
|
+
# - "[redis_queued_locks.try_lock.same_process_conflict_detected]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
282
|
+
# - "[redis_queued_locks.try_lock.same_process_conflict_analyzed]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status");
|
283
|
+
# - "[redis_queued_locks.try_lock.reentrant_lock__extend_and_work_through]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status", "last_ext_ttl", "last_ext_ts");
|
284
|
+
# - "[redis_queued_locks.try_lock.reentrant_lock__work_through]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status", last_spc_ts);
|
285
|
+
# - "[redis_queued_locks.try_lock.acq_added_to_queue]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat")";
|
286
|
+
# - "[redis_queued_locks.try_lock.remove_expired_acqs]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
287
|
+
# - "[redis_queued_locks.try_lock.get_first_from_queue]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "first_acq_id_in_queue");
|
288
|
+
# - "[redis_queued_locks.try_lock.exit__queue_ttl_reached]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
289
|
+
# - "[redis_queued_locks.try_lock.exit__no_first]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "first_acq_id_in_queue", "<current_lock_data>");
|
290
|
+
# - "[redis_queued_locks.try_lock.exit__lock_still_obtained]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "first_acq_id_in_queue", "locked_by_acq_id", "<current_lock_data>");
|
291
|
+
# - "[redis_queued_locks.try_lock.obtain__free_to_acquire]" (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
276
292
|
config.log_lock_try = false
|
277
293
|
|
278
294
|
# (default: false)
|
@@ -341,6 +357,8 @@ end
|
|
341
357
|
- [queues_info](#queues_info---get-list-of-queues-with-their-info)
|
342
358
|
- [clear_dead_requests](#clear_dead_requests)
|
343
359
|
- [current_acquirer_id](#current_acquirer_id)
|
360
|
+
- [current_host_id](#current_host_id)
|
361
|
+
- [possible_host_ids](#possible_host_ids)
|
344
362
|
|
345
363
|
---
|
346
364
|
|
@@ -381,9 +399,11 @@ def lock(
|
|
381
399
|
log_sampling_enabled: config[:log_sampling_enabled],
|
382
400
|
log_sampling_percent: config[:log_sampling_percent],
|
383
401
|
log_sampler: config[:log_sampler],
|
402
|
+
log_sample_this: false,
|
384
403
|
instr_sampling_enabled: config[:instr_sampling_enabled],
|
385
404
|
instr_sampling_percent: config[:instr_sampling_percent],
|
386
405
|
instr_sampler: config[:instr_sampler],
|
406
|
+
instr_sample_this: false,
|
387
407
|
&block
|
388
408
|
)
|
389
409
|
```
|
@@ -503,6 +523,10 @@ def lock(
|
|
503
523
|
- you can provide your own log sampler with bettter algorithm that should realize
|
504
524
|
`sampling_happened?(percent) => boolean` interface (see `RedisQueuedLocks::Logging::Sampler` for example);
|
505
525
|
- pre-configured in `config[:log_sampler]`;
|
526
|
+
- `log_sample_this` - (optional) `[Boolean]`
|
527
|
+
- marks the method that everything should be logged despite the enabled log sampling;
|
528
|
+
- makes sense when log sampling is enabled;
|
529
|
+
- `false` by default;
|
506
530
|
- `instr_sampling_enabled` - (optional) `[Boolean]`
|
507
531
|
- enables **instrumentaion sampling**: only the configured percent of RQL cases will be instrumented;
|
508
532
|
- disabled by default;
|
@@ -521,6 +545,10 @@ def lock(
|
|
521
545
|
- you can provide your own log sampler with bettter algorithm that should realize
|
522
546
|
`sampling_happened?(percent) => boolean` interface (see `RedisQueuedLocks::Instrument::Sampler` for example);
|
523
547
|
- pre-configured in `config[:instr_sampler]`;
|
548
|
+
- `instr_sample_this` - (optional) `[Boolean]`
|
549
|
+
- marks the method that everything should be instrumneted despite the enabled instrumentation sampling;
|
550
|
+
- makes sense when instrumentation sampling is enabled;
|
551
|
+
- `false` by default;
|
524
552
|
- `block` - (optional) `[Block]`
|
525
553
|
- A block of code that should be executed after the successfully acquired lock.
|
526
554
|
- If block is **passed** the obtained lock will be released after the block execution or it's ttl (what will happen first);
|
@@ -566,6 +594,7 @@ Return value:
|
|
566
594
|
result: {
|
567
595
|
lock_key: String, # acquierd lock key ("rql:lock:your_lock_name")
|
568
596
|
acq_id: String, # acquier identifier ("process_id/thread_id/fiber_id/ractor_id/identity")
|
597
|
+
hst_id: String, # host identifier ("process_id/thread_id/ractor_id/identity")
|
569
598
|
ts: Float, # time (epoch) when lock was obtained (float, Time#to_f)
|
570
599
|
ttl: Integer, # lock's time to live in milliseconds (integer)
|
571
600
|
process: Symbol # which logical process has acquired the lock (:lock_obtaining, :extendable_conflict_work_through, :conflict_work_through, :conflict_dead_lock)
|
@@ -580,6 +609,7 @@ Return value:
|
|
580
609
|
result: {
|
581
610
|
lock_key: "rql:lock:my_lock",
|
582
611
|
acq_id: "rql:acq:26672/2280/2300/2320/70ea5dbf10ea1056",
|
612
|
+
acq_id: "rql:acq:26672/2280/2320/70ea5dbf10ea1056",
|
583
613
|
ts: 1711909612.653696,
|
584
614
|
ttl: 10000,
|
585
615
|
process: :lock_obtaining # for custom conflict strategies may be: :conflict_dead_lock, :conflict_work_through, :extendable_conflict_work_through
|
@@ -655,6 +685,7 @@ rql.lock_info("my_lock")
|
|
655
685
|
{
|
656
686
|
"lock_key" => "rql:lock:my_lock",
|
657
687
|
"acq_id" => "rql:acq:123/456/567/678/374dd74324",
|
688
|
+
"hst_id" => "rql:acq:123/456/678/374dd74324",
|
658
689
|
"ts" => 123456789,
|
659
690
|
"ini_ttl" => 123456,
|
660
691
|
"rem_ttl" => 123440,
|
@@ -780,9 +811,11 @@ def lock!(
|
|
780
811
|
log_sampling_enabled: config[:log_sampling_enabled],
|
781
812
|
log_sampling_percent: config[:log_sampling_percent],
|
782
813
|
log_sampler: config[:log_sampler],
|
814
|
+
log_sample_this: false,
|
783
815
|
instr_sampling_enabled: config[:instr_sampling_enabled],
|
784
816
|
instr_sampling_percent: config[:instr_sampling_percent],
|
785
817
|
instr_sampler: config[:instr_sampler],
|
818
|
+
instr_sample_this: false,
|
786
819
|
&block
|
787
820
|
)
|
788
821
|
```
|
@@ -800,6 +833,7 @@ See `#lock` method [documentation](#lock---obtain-a-lock).
|
|
800
833
|
- lock data (`Hash<String,String|Integer>`):
|
801
834
|
- `"lock_key"` - `string` - lock key in redis;
|
802
835
|
- `"acq_id"` - `string` - acquier identifier (process_id/thread_id/fiber_id/ractor_id/identity);
|
836
|
+
- `"hst_id"` - `string` - host identifier (process_id/thread_id/ractor_id/identity);
|
803
837
|
- `"ts"` - `numeric`/`epoch` - the time when lock was obtained;
|
804
838
|
- `"init_ttl"` - `integer` - (milliseconds) initial lock key ttl;
|
805
839
|
- `"rem_ttl"` - `integer` - (milliseconds) remaining lock key ttl;
|
@@ -822,6 +856,7 @@ rql.lock_info("your_lock_name")
|
|
822
856
|
{
|
823
857
|
"lock_key" => "rql:lock:your_lock_name",
|
824
858
|
"acq_id" => "rql:acq:123/456/567/678/374dd74324",
|
859
|
+
"hst_id" => "rql:acq:123/456/678/374dd74324",
|
825
860
|
"ts" => 123456789.12345,
|
826
861
|
"ini_ttl" => 5_000,
|
827
862
|
"rem_ttl" => 4_999
|
@@ -837,6 +872,7 @@ rql.lock_info("your_lock_name")
|
|
837
872
|
{
|
838
873
|
"lock_key" => "rql:lock:your_lock_name",
|
839
874
|
"acq_id" => "rql:acq:123/456/567/678/374dd74324",
|
875
|
+
"hst_id" => "rql:acq:123/456/678/374dd74324",
|
840
876
|
"ts" => 123456789.12345,
|
841
877
|
"ini_ttl" => 5_000,
|
842
878
|
"rem_ttl" => 4_999,
|
@@ -858,6 +894,7 @@ rql.lock_info("your_lock_name")
|
|
858
894
|
{
|
859
895
|
"lock_key" => "rql:lock:your_lock_name",
|
860
896
|
"acq_id" => "rql:acq:123/456/567/678/374dd74324",
|
897
|
+
"hst_id" => "rql:acq:123/456/678/374dd74324",
|
861
898
|
"ts" => 123456789.12345,
|
862
899
|
"ini_ttl" => 5_000,
|
863
900
|
"rem_ttl" => 9_444,
|
@@ -943,6 +980,7 @@ rql.queued?("your_lock_name") # => true/false
|
|
943
980
|
|
944
981
|
- release the concrete lock with lock request queue;
|
945
982
|
- queue will be relased first;
|
983
|
+
- has an alias: `#release_lock`;
|
946
984
|
- accepts:
|
947
985
|
- `lock_name` - (required) `[String]` - the lock name that should be released.
|
948
986
|
- `:logger` - (optional) `[::Logger,#debug]`
|
@@ -963,6 +1001,10 @@ rql.queued?("your_lock_name") # => true/false
|
|
963
1001
|
- `:log_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Logging::Sampler>]`
|
964
1002
|
- **log sampling**: percent-based log sampler that decides should be RQL case logged or not;
|
965
1003
|
- pre-configured in `config[:log_sampler]`;
|
1004
|
+
- `log_sample_this` - (optional) `[Boolean]`
|
1005
|
+
- marks the method that everything should be logged despite the enabled log sampling;
|
1006
|
+
- makes sense when log sampling is enabled;
|
1007
|
+
- `false` by default;
|
966
1008
|
- `:instr_sampling_enabled` - (optional) `[Boolean]`
|
967
1009
|
- enables **instrumentaion sampling**;
|
968
1010
|
- pre-configured in `config[:instr_sampling_enabled]`;
|
@@ -972,6 +1014,10 @@ rql.queued?("your_lock_name") # => true/false
|
|
972
1014
|
- `instr_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Instrument::Sampler>]`
|
973
1015
|
- percent-based log sampler that decides should be RQL case instrumented or not;
|
974
1016
|
- pre-configured in `config[:instr_sampler]`;
|
1017
|
+
- `instr_sample_this` - (optional) `[Boolean]`
|
1018
|
+
- marks the method that everything should be instrumneted despite the enabled instrumentation sampling;
|
1019
|
+
- makes sense when instrumentation sampling is enabled;
|
1020
|
+
- `false` by default;
|
975
1021
|
- if you try to unlock non-existent lock you will receive `ok: true` result with operation timings
|
976
1022
|
and `:nothing_to_release` result factor inside;
|
977
1023
|
|
@@ -1010,6 +1056,7 @@ rql.unlock("your_lock_name")
|
|
1010
1056
|
|
1011
1057
|
- release all obtained locks and related lock request queues;
|
1012
1058
|
- queues will be released first;
|
1059
|
+
- has an alias: `#release_locks`;
|
1013
1060
|
- accepts:
|
1014
1061
|
- `:batch_size` - (optional) `[Integer]`
|
1015
1062
|
- the size of batch of locks and lock queus that should be cleared under the one pipelined redis command at once;
|
@@ -1031,6 +1078,10 @@ rql.unlock("your_lock_name")
|
|
1031
1078
|
- `:log_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Logging::Sampler>]`
|
1032
1079
|
- **log sampling**: percent-based log sampler that decides should be RQL case logged or not;
|
1033
1080
|
- pre-configured in `config[:log_sampler]`;
|
1081
|
+
- `log_sample_this` - (optional) `[Boolean]`
|
1082
|
+
- marks the method that everything should be logged despite the enabled log sampling;
|
1083
|
+
- makes sense when log sampling is enabled;
|
1084
|
+
- `false` by default;
|
1034
1085
|
- `:instr_sampling_enabled` - (optional) `[Boolean]`
|
1035
1086
|
- enables **instrumentaion sampling**;
|
1036
1087
|
- pre-configured in `config[:instr_sampling_enabled]`;
|
@@ -1040,6 +1091,10 @@ rql.unlock("your_lock_name")
|
|
1040
1091
|
- `instr_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Instrument::Sampler>]`
|
1041
1092
|
- percent-based log sampler that decides should be RQL case instrumented or not;
|
1042
1093
|
- pre-configured in `config[:instr_sampler]`;
|
1094
|
+
- `instr_sample_this` - (optional) `[Boolean]`
|
1095
|
+
- marks the method that everything should be instrumneted despite the enabled instrumentation sampling;
|
1096
|
+
- makes sense when instrumentation sampling is enabled;
|
1097
|
+
- `false` by default;
|
1043
1098
|
- returns:
|
1044
1099
|
- `[Hash<Symbol,Numeric>]` - Format: `{ ok: true, result: Hash<Symbol,Numeric> }`;
|
1045
1100
|
- result data:
|
@@ -1090,6 +1145,10 @@ rql.clear_locks
|
|
1090
1145
|
- `:log_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Logging::Sampler>]`
|
1091
1146
|
- **log sampling**: percent-based log sampler that decides should be RQL case logged or not;
|
1092
1147
|
- pre-configured in `config[:log_sampler]`;
|
1148
|
+
- `log_sample_this` - (optional) `[Boolean]`
|
1149
|
+
- marks the method that everything should be logged despite the enabled log sampling;
|
1150
|
+
- makes sense when log sampling is enabled;
|
1151
|
+
- `false` by default;
|
1093
1152
|
- `:instr_sampling_enabled` - (optional) `[Boolean]`
|
1094
1153
|
- enables **instrumentaion sampling**;
|
1095
1154
|
- pre-configured in `config[:instr_sampling_enabled]`;
|
@@ -1099,6 +1158,10 @@ rql.clear_locks
|
|
1099
1158
|
- `instr_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Instrument::Sampler>]`
|
1100
1159
|
- percent-based log sampler that decides should be RQL case instrumented or not;
|
1101
1160
|
- pre-configured in `config[:instr_sampler]`;
|
1161
|
+
- `instr_sample_this` - (optional) `[Boolean]`
|
1162
|
+
- marks the method that everything should be instrumneted despite the enabled instrumentation sampling;
|
1163
|
+
- makes sense when instrumentation sampling is enabled;
|
1164
|
+
- `false` by default;
|
1102
1165
|
- returns `{ ok: true, result: :ttl_extended }` when ttl is extended;
|
1103
1166
|
- returns `{ ok: false, result: :async_expire_or_no_lock }` when a lock not found or a lock is already expired during
|
1104
1167
|
some steps of invocation (see **Important** section below);
|
@@ -1253,6 +1316,7 @@ rql.locks_info # or rql.locks_info(scan_size: 123)
|
|
1253
1316
|
:status=>:alive,
|
1254
1317
|
:info=>{
|
1255
1318
|
"acq_id"=>"rql:acq:41478/4320/4340/4360/848818f09d8c3420",
|
1319
|
+
"hst_id"=>"rql:hst:41478/4320/4360/848818f09d8c3420"
|
1256
1320
|
"ts"=>1711607112.670343,
|
1257
1321
|
"ini_ttl"=>15000,
|
1258
1322
|
"rem_ttl"=>13998}},
|
@@ -1336,6 +1400,10 @@ Accepts:
|
|
1336
1400
|
- `:log_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Logging::Sampler>]`
|
1337
1401
|
- **log sampling**: percent-based log sampler that decides should be RQL case logged or not;
|
1338
1402
|
- pre-configured in `config[:log_sampler]`;
|
1403
|
+
- `log_sample_this` - (optional) `[Boolean]`
|
1404
|
+
- marks the method that everything should be logged despite the enabled log sampling;
|
1405
|
+
- makes sense when log sampling is enabled;
|
1406
|
+
- `false` by default;
|
1339
1407
|
- `:instr_sampling_enabled` - (optional) `[Boolean]`
|
1340
1408
|
- enables **instrumentaion sampling**;
|
1341
1409
|
- pre-configured in `config[:instr_sampling_enabled]`;
|
@@ -1345,6 +1413,10 @@ Accepts:
|
|
1345
1413
|
- `instr_sampler` - (optional) `[#sampling_happened?,Module<RedisQueuedLocks::Instrument::Sampler>]`
|
1346
1414
|
- percent-based log sampler that decides should be RQL case instrumented or not;
|
1347
1415
|
- pre-configured in `config[:instr_sampler]`;
|
1416
|
+
- `instr_sample_this` - (optional) `[Boolean]`
|
1417
|
+
- marks the method that everything should be instrumneted despite the enabled instrumentation sampling;
|
1418
|
+
- makes sense when instrumentation sampling is enabled;
|
1419
|
+
- `false` by default;
|
1348
1420
|
|
1349
1421
|
Returns: `{ ok: true, processed_queues: Set<String> }` returns the list of processed lock queues;
|
1350
1422
|
|
@@ -1367,6 +1439,8 @@ rql.clear_dead_requests(dead_ttl: 60 * 60 * 1000) # 1 hour in milliseconds
|
|
1367
1439
|
|
1368
1440
|
#### #current_acquirer_id
|
1369
1441
|
|
1442
|
+
<sup>\[[back to top](#usage)\]</sup>
|
1443
|
+
|
1370
1444
|
- get the current acquirer identifier in RQL notation that you can use for debugging purposes during the lock analyzation;
|
1371
1445
|
- acquirer identifier format:
|
1372
1446
|
```ruby
|
@@ -1400,6 +1474,277 @@ rql.current_acquirer_id
|
|
1400
1474
|
|
1401
1475
|
---
|
1402
1476
|
|
1477
|
+
#### #current_host_id
|
1478
|
+
|
1479
|
+
<sup>\[[back to top](#usage)\]</sup>
|
1480
|
+
|
1481
|
+
- get a current host identifier in RQL notation that you can use for debugging purposes during the lock analyzis;
|
1482
|
+
- the host is a ruby worker (a combination of process/thread/ractor) that is alive and can obtain locks;
|
1483
|
+
- the host is limited to `process`/`thread`/`ractor` (without `fiber`) combination cuz we have no abilities to extract
|
1484
|
+
all fiber objects from the current ruby process when at least one ractor object is defined (**ObjectSpace** loses
|
1485
|
+
abilities to extract `Fiber` and `Thread` objects after the any ractor is created) (`Thread` objects are analyzed
|
1486
|
+
via `Thread.list` API which does not lose their abilites);
|
1487
|
+
- host identifier format:
|
1488
|
+
```ruby
|
1489
|
+
"rql:hst:#{process_id}/#{thread_id}/#{ractor_id}/#{uniq_identity}"
|
1490
|
+
```
|
1491
|
+
- because of the moment that `#lock`/`#lock!` gives you a possibility to customize `process_id`,
|
1492
|
+
`fiber_id`, `thread_id`, `ractor_id` and `unique identity` identifiers the `#current_host_id` method provides this possibility too
|
1493
|
+
(except the `fiber_id` correspondingly);
|
1494
|
+
|
1495
|
+
Accepts:
|
1496
|
+
|
1497
|
+
- `process_id:` - (optional) `[Integer,Any]`
|
1498
|
+
- `::Process.pid` by default;
|
1499
|
+
- `thread_id:` - (optional) `[Integer,Any]`;
|
1500
|
+
- `::Thread.current.object_id` by default;
|
1501
|
+
- `ractor_id:` - (optional) `[Integer,Any]`;
|
1502
|
+
- `::Ractor.current.object_id` by default;
|
1503
|
+
- `identity:` - (optional) `[String]`;
|
1504
|
+
- this value is calculated once during `RedisQueuedLock::Client` instantiation and stored in `@uniq_identity`;
|
1505
|
+
- this value can be accessed from `RedisQueuedLock::Client#uniq_identity`;
|
1506
|
+
- [Configuration](#configuration) documentation: see `config[:uniq_identifier]`;
|
1507
|
+
- [#lock](#lock---obtain-a-lock) method documentation: see `uniq_identifier`;
|
1508
|
+
|
1509
|
+
```ruby
|
1510
|
+
rql.current_host_id
|
1511
|
+
|
1512
|
+
# =>
|
1513
|
+
"rql:acq:38529/4500/4360/66093702f24a3129"
|
1514
|
+
```
|
1515
|
+
|
1516
|
+
---
|
1517
|
+
|
1518
|
+
#### #possible_host_ids
|
1519
|
+
|
1520
|
+
<sup>\[[back to top](#usage)\]</sup>
|
1521
|
+
|
1522
|
+
- return the list (`Array<String>`) of possible host identifiers that can be reached from the current ractor;
|
1523
|
+
- the host is a ruby worker (a combination of process/thread/ractor/identity) that is alive and can obtain locks;
|
1524
|
+
- the host is limited to `process`/`thread`/`ractor` (without `fiber`) combination cuz we have no abilities to extract
|
1525
|
+
all fiber objects from the current ruby process when at least one ractor object is defined (**ObjectSpace** loses
|
1526
|
+
abilities to extract `Fiber` and `Thread` objects after the any ractor is created) (`Thread` objects are analyzed
|
1527
|
+
via `Thread.list` API which does not lose their abilites);
|
1528
|
+
- host identifier format:
|
1529
|
+
```ruby
|
1530
|
+
"rql:hst:#{process_id}/#{thread_id}/#{ractor_id}/#{uniq_identity}"
|
1531
|
+
```
|
1532
|
+
|
1533
|
+
Accepts:
|
1534
|
+
|
1535
|
+
- `identity` - (optional) `[String]`;
|
1536
|
+
- this value is calculated once during `RedisQueuedLock::Client` instantiation and stored in `@uniq_identity`;
|
1537
|
+
- this value can be accessed from `RedisQueuedLock::Client#uniq_identity`;
|
1538
|
+
- [Configuration](#configuration) documentation: see `config[:uniq_identifier]`;
|
1539
|
+
- [#lock](#lock---obtain-a-lock) method documentation: see `uniq_identifier`;
|
1540
|
+
|
1541
|
+
```ruby
|
1542
|
+
rql.possible_host_ids
|
1543
|
+
|
1544
|
+
# =>
|
1545
|
+
[
|
1546
|
+
"rql:hst:18814/2300/2280/5ce0c4582fc59c06", # process id / thread id / ractor id / uniq identity
|
1547
|
+
"rql:hst:18814/2320/2280/5ce0c4582fc59c06", # ...
|
1548
|
+
"rql:hst:18814/2340/2280/5ce0c4582fc59c06", # ...
|
1549
|
+
"rql:hst:18814/2360/2280/5ce0c4582fc59c06", # ...
|
1550
|
+
"rql:hst:18814/2380/2280/5ce0c4582fc59c06", # ...
|
1551
|
+
"rql:hst:18814/2400/2280/5ce0c4582fc59c06"
|
1552
|
+
]
|
1553
|
+
```
|
1554
|
+
---
|
1555
|
+
|
1556
|
+
## Swarm Mode and Zombie Locks
|
1557
|
+
|
1558
|
+
<sup>\[[back to top](#table-of-contents)\]</sup>
|
1559
|
+
|
1560
|
+
> Eliminate zombie locks with a swarm.
|
1561
|
+
|
1562
|
+
**This documentation section is in progress!**
|
1563
|
+
|
1564
|
+
[(work and usage preview (temporary example-based docs))](#work-and-usage-preview-temporary-example-based-docs)
|
1565
|
+
|
1566
|
+
- [How to Swarm](#how-to-swarm)
|
1567
|
+
- [configuration](#)
|
1568
|
+
- [swarm_status](#swarm_status)
|
1569
|
+
- [swarm_info](#swarm_info)
|
1570
|
+
- [swarmize!](#swarmize!)
|
1571
|
+
- [deswarmize!](#deswarmize!)
|
1572
|
+
- [probe_hosts](#probe_hosts)
|
1573
|
+
- [flush_zobmies](#flush_zombies)
|
1574
|
+
- [zombies_info](#zombies_info)
|
1575
|
+
- [zombie_locks](#zombie_locks)
|
1576
|
+
- [zombie_hosts](#zombie_hosts)
|
1577
|
+
- [zombie_acquiers](#zombie_acquiers)
|
1578
|
+
|
1579
|
+
<hr>
|
1580
|
+
|
1581
|
+
#### Work and Usage Preview (temporary example-based docs)
|
1582
|
+
|
1583
|
+
<sup>\[[back to top](#swarm-mode-and-zombie-locks)\]</sup>
|
1584
|
+
|
1585
|
+
<details>
|
1586
|
+
<summary>configuration</summary>
|
1587
|
+
|
1588
|
+
```ruby
|
1589
|
+
redis_client = RedisClient.config.new_pool # NOTE: provide your own RedisClient instance
|
1590
|
+
|
1591
|
+
clinet = RedisQueuedLocks::Client.new(redis_client) do |config|
|
1592
|
+
# NOTE: auto-swarm your RQL client after initalization (run swarm elements and their supervisor)
|
1593
|
+
config.swarm.auto_swarm = false
|
1594
|
+
|
1595
|
+
# supervisor configs
|
1596
|
+
config.swarm.supervisor.liveness_probing_period = 2 # NOTE: in seconds
|
1597
|
+
|
1598
|
+
# (probe_hosts) host probing configuration
|
1599
|
+
config.swarm.probe_hosts.enabled_for_swarm = true # NOTE: run host-probing from or not
|
1600
|
+
config.swarm.probe_hosts.probe_period = 2 # NOTE: (in seconds) the period of time when the probing process is triggered
|
1601
|
+
# (probe_hosts) individual redis config
|
1602
|
+
config.swarm.probe_hosts.redis_config.sentinel = false # NOTE: individual redis config
|
1603
|
+
config.swarm.probe_hosts.redis_config.pooled = false # NOTE: individual redis config
|
1604
|
+
config.swarm.probe_hosts.redis_config.config = {} # NOTE: individual redis config
|
1605
|
+
config.swarm.probe_hosts.redis_config.pool_config = {} # NOTE: individual redis config
|
1606
|
+
|
1607
|
+
# (flush_zombies) zombie flushing configuration
|
1608
|
+
config.swarm.flush_zombies.enabled_for_swarm = true # NOTE: run zombie flushing or not
|
1609
|
+
config.swarm.flush_zombies.zombie_flush_period = 10 # NOTE: (in seconds) period of time when the zombie flusher is triggered
|
1610
|
+
config.swarm.flush_zombies.zombie_ttl = 15_000 # NOTE: (in milliseconds) when the lock/host/acquier is considered a zombie
|
1611
|
+
config.swarm.flush_zombies.zombie_lock_scan_size = 500 # NOTE: scan sizec during zombie flushing
|
1612
|
+
config.swarm.flush_zombies.zombie_queue_scan_size = 500 # NOTE: scan sizec during zombie flushing
|
1613
|
+
# (flush_zombies) individual redis config
|
1614
|
+
config.swarm.flush_zombies.redis_config.sentinel = false # NOTE: individual redis config
|
1615
|
+
config.swarm.flush_zombies.redis_config.pooled = false # NOTE: individual redis config
|
1616
|
+
config.swarm.flush_zombies.redis_config.config = {} # NOTE: individual redis config
|
1617
|
+
config.swarm.flush_zombies.redis_config.pool_config = {} # NOTE: individual redis config
|
1618
|
+
```
|
1619
|
+
</details>
|
1620
|
+
|
1621
|
+
<details>
|
1622
|
+
<summary>seed a zombie</summary>
|
1623
|
+
|
1624
|
+
- obtain some long living lock and kill the host process which will lead the lock becoming a zombie:
|
1625
|
+
|
1626
|
+
```ruby
|
1627
|
+
daiver => ~/Projects/redis_queued_locks master [$]
|
1628
|
+
➜ bin/console
|
1629
|
+
[1] pry(main)> rql = RedisQueuedLocks::Client.new(RedisClient.new);
|
1630
|
+
[2] pry(main)> rql.swarmize!
|
1631
|
+
/Users/daiver/Projects/redis_queued_locks/lib/redis_queued_locks/swarm/flush_zombies.rb:107: warning: Ractor is experimental, and the behavior may change in future versions of Ruby! Also there are many implementation issues.
|
1632
|
+
=> {:ok=>true, :result=>:swarming}
|
1633
|
+
[3] pry(main)> rql.lock('kekpek', ttl: 1111111111)
|
1634
|
+
=> {:ok=>true,
|
1635
|
+
:result=>
|
1636
|
+
{:lock_key=>"rql:lock:kekpek",
|
1637
|
+
:acq_id=>"rql:acq:17580/2260/2380/2280/3f16b93973612580",
|
1638
|
+
:hst_id=>"rql:hst:17580/2260/2280/3f16b93973612580",
|
1639
|
+
:ts=>1720305351.069259,
|
1640
|
+
:ttl=>1111111111,
|
1641
|
+
:process=>:lock_obtaining}}
|
1642
|
+
[4] pry(main)> exit
|
1643
|
+
```
|
1644
|
+
</details>
|
1645
|
+
|
1646
|
+
<details>
|
1647
|
+
<summary>find zombies</summary>
|
1648
|
+
|
1649
|
+
- start another process, fetch the swarm info, see that our last process is a zombie now and their hosted lock is a zombie too:
|
1650
|
+
|
1651
|
+
```ruby
|
1652
|
+
daiver => ~/Projects/redis_queued_locks master [$] took 27.2s
|
1653
|
+
➜ bin/console
|
1654
|
+
[1] pry(main)> rql = RedisQueuedLocks::Client.new(RedisClient.new);
|
1655
|
+
[2] pry(main)> rql.swarm_info
|
1656
|
+
=> {"rql:hst:17580/2260/2280/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 12897/262144 +0300, :last_probe_score=>1720305353.0491982},
|
1657
|
+
"rql:hst:17580/2300/2280/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 211107/4194304 +0300, :last_probe_score=>1720305353.0503318},
|
1658
|
+
"rql:hst:17580/2320/2280/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 106615/2097152 +0300, :last_probe_score=>1720305353.050838},
|
1659
|
+
"rql:hst:17580/2260/2340/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 26239/524288 +0300, :last_probe_score=>1720305353.050047},
|
1660
|
+
"rql:hst:17580/2300/2340/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 106359/2097152 +0300, :last_probe_score=>1720305353.050716},
|
1661
|
+
"rql:hst:17580/2320/2340/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 213633/4194304 +0300, :last_probe_score=>1720305353.050934},
|
1662
|
+
"rql:hst:17580/2360/2280/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 214077/4194304 +0300, :last_probe_score=>1720305353.05104},
|
1663
|
+
"rql:hst:17580/2360/2340/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 214505/4194304 +0300, :last_probe_score=>1720305353.051142},
|
1664
|
+
"rql:hst:17580/2400/2280/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 53729/1048576 +0300, :last_probe_score=>1720305353.05124},
|
1665
|
+
"rql:hst:17580/2400/2340/3f16b93973612580"=>{:zombie=>true, :last_probe_time=>2024-07-07 01:35:53 3365/65536 +0300, :last_probe_score=>1720305353.0513458}}
|
1666
|
+
[3] pry(main)> rql.swarm_status
|
1667
|
+
=> {:auto_swarm=>false,
|
1668
|
+
:supervisor=>{:running=>false, :state=>"non_initialized", :observable=>"non_initialized"},
|
1669
|
+
:probe_hosts=>{:enabled=>true, :thread=>{:running=>false, :state=>"non_initialized"}, :main_loop=>{:running=>false, :state=>"non_initialized"}},
|
1670
|
+
:flush_zombies=>{:enabled=>true, :ractor=>{:running=>false, :state=>"non_initialized"}, :main_loop=>{:running=>false, :state=>"non_initialized"}}}
|
1671
|
+
[4] pry(main)> rql.zombies_info
|
1672
|
+
=> {:zombie_hosts=>
|
1673
|
+
#<Set:
|
1674
|
+
{"rql:hst:17580/2260/2280/3f16b93973612580",
|
1675
|
+
"rql:hst:17580/2300/2280/3f16b93973612580",
|
1676
|
+
"rql:hst:17580/2320/2280/3f16b93973612580",
|
1677
|
+
"rql:hst:17580/2260/2340/3f16b93973612580",
|
1678
|
+
"rql:hst:17580/2300/2340/3f16b93973612580",
|
1679
|
+
"rql:hst:17580/2320/2340/3f16b93973612580",
|
1680
|
+
"rql:hst:17580/2360/2280/3f16b93973612580",
|
1681
|
+
"rql:hst:17580/2360/2340/3f16b93973612580",
|
1682
|
+
"rql:hst:17580/2400/2280/3f16b93973612580",
|
1683
|
+
"rql:hst:17580/2400/2340/3f16b93973612580"}>,
|
1684
|
+
:zombie_acquirers=>#<Set: {"rql:acq:17580/2260/2380/2280/3f16b93973612580"}>,
|
1685
|
+
:zombie_locks=>#<Set: {"rql:lock:kekpek"}>}
|
1686
|
+
[5] pry(main)> rql.zombie_locks
|
1687
|
+
=> #<Set: {"rql:lock:kekpek"}>
|
1688
|
+
[6] pry(main)> rql.zombie_acquiers
|
1689
|
+
=> #<Set: {"rql:acq:17580/2260/2380/2280/3f16b93973612580"}>
|
1690
|
+
[7] pry(main)> rql.zombie_hosts
|
1691
|
+
=> #<Set:
|
1692
|
+
{"rql:hst:17580/2260/2280/3f16b93973612580",
|
1693
|
+
"rql:hst:17580/2300/2280/3f16b93973612580",
|
1694
|
+
"rql:hst:17580/2320/2280/3f16b93973612580",
|
1695
|
+
"rql:hst:17580/2260/2340/3f16b93973612580",
|
1696
|
+
"rql:hst:17580/2300/2340/3f16b93973612580",
|
1697
|
+
"rql:hst:17580/2320/2340/3f16b93973612580",
|
1698
|
+
"rql:hst:17580/2360/2280/3f16b93973612580",
|
1699
|
+
"rql:hst:17580/2360/2340/3f16b93973612580",
|
1700
|
+
"rql:hst:17580/2400/2280/3f16b93973612580",
|
1701
|
+
"rql:hst:17580/2400/2340/3f16b93973612580"}>
|
1702
|
+
```
|
1703
|
+
</details>
|
1704
|
+
|
1705
|
+
<details>
|
1706
|
+
<summary>kill zombies in a background</summary>
|
1707
|
+
|
1708
|
+
- swarmize the new current ruby process that should run the flush zombies element that will drop zombie locks, zombie hosts and their lock requests in a background:
|
1709
|
+
|
1710
|
+
```ruby
|
1711
|
+
[8] pry(main)> rql.swarmize!
|
1712
|
+
/Users/daiver/Projects/redis_queued_locks/lib/redis_queued_locks/swarm/flush_zombies.rb:107: warning: Ractor is experimental, and the behavior may change in future versions of Ruby! Also there are many implementation issues.
|
1713
|
+
=> {:ok=>true, :result=>:swarming}
|
1714
|
+
[9] pry(main)> rql.swarm_info
|
1715
|
+
=> {"rql:hst:17752/2260/2280/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 4012577/4194304 +0300, :last_probe_score=>1720305399.956673},
|
1716
|
+
"rql:hst:17752/2300/2280/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 4015233/4194304 +0300, :last_probe_score=>1720305399.9573061},
|
1717
|
+
"rql:hst:17752/2320/2280/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 4016755/4194304 +0300, :last_probe_score=>1720305399.957669},
|
1718
|
+
"rql:hst:17752/2260/2340/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 1003611/1048576 +0300, :last_probe_score=>1720305399.957118},
|
1719
|
+
"rql:hst:17752/2300/2340/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 2008027/2097152 +0300, :last_probe_score=>1720305399.957502},
|
1720
|
+
"rql:hst:17752/2320/2340/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 2008715/2097152 +0300, :last_probe_score=>1720305399.95783},
|
1721
|
+
"rql:hst:17752/2360/2280/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 4018063/4194304 +0300, :last_probe_score=>1720305399.9579809},
|
1722
|
+
"rql:hst:17752/2360/2340/89beef198021f16d"=>{:zombie=>false, :last_probe_time=>2024-07-07 01:36:39 1004673/1048576 +0300, :last_probe_score=>1720305399.9581308}}
|
1723
|
+
[10] pry(main)> rql.swarm_status
|
1724
|
+
=> {:auto_swarm=>false,
|
1725
|
+
:supervisor=>{:running=>true, :state=>"sleep", :observable=>"initialized"},
|
1726
|
+
:probe_hosts=>{:enabled=>true, :thread=>{:running=>true, :state=>"sleep"}, :main_loop=>{:running=>true, :state=>"sleep"}},
|
1727
|
+
:flush_zombies=>{:enabled=>true, :ractor=>{:running=>true, :state=>"running"}, :main_loop=>{:running=>true, :state=>"sleep"}}}
|
1728
|
+
[11] pry(main)> rql.zombies_info
|
1729
|
+
=> {:zombie_hosts=>#<Set: {}>, :zombie_acquirers=>#<Set: {}>, :zombie_locks=>#<Set: {}>}
|
1730
|
+
[12] pry(main)> rql.zombie_acquiers
|
1731
|
+
=> #<Set: {}>
|
1732
|
+
[13] pry(main)> rql.zombie_hosts
|
1733
|
+
=> #<Set: {}>
|
1734
|
+
[14] pry(main)>
|
1735
|
+
```
|
1736
|
+
</details>
|
1737
|
+
|
1738
|
+
<details>
|
1739
|
+
<summary>swarm hosts key in Redis</summary>
|
1740
|
+
|
1741
|
+
```ruby
|
1742
|
+
"rql:swarm:hsts"
|
1743
|
+
```
|
1744
|
+
</details>
|
1745
|
+
|
1746
|
+
---
|
1747
|
+
|
1403
1748
|
## Lock Access Strategies
|
1404
1749
|
|
1405
1750
|
<sup>\[[back to top](#table-of-contents)\]</sup>
|
@@ -1443,34 +1788,34 @@ rql.current_acquirer_id
|
|
1443
1788
|
- default logs (raised from `#lock`/`#lock!`):
|
1444
1789
|
|
1445
1790
|
```ruby
|
1446
|
-
"[redis_queued_locks.start_lock_obtaining]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1447
|
-
"[redis_queued_locks.start_try_to_lock_cycle]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1448
|
-
"[redis_queued_locks.dead_score_reached__reset_acquier_position]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1449
|
-
"[redis_queued_locks.lock_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "acq_time");
|
1450
|
-
"[redis_queued_locks.extendable_reentrant_lock_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "acq_time");
|
1451
|
-
"[redis_queued_locks.reentrant_lock_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "acq_time");
|
1452
|
-
"[redis_queued_locks.fail_fast_or_limits_reached_or_deadlock__dequeue]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1453
|
-
"[redis_queued_locks.expire_lock]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1454
|
-
"[redis_queued_locks.decrease_lock]" # (logs "lock_key", "decreased_ttl", "queue_ttl", "acq_id", "acs_strat");
|
1791
|
+
"[redis_queued_locks.start_lock_obtaining]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1792
|
+
"[redis_queued_locks.start_try_to_lock_cycle]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1793
|
+
"[redis_queued_locks.dead_score_reached__reset_acquier_position]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1794
|
+
"[redis_queued_locks.lock_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acq_time");
|
1795
|
+
"[redis_queued_locks.extendable_reentrant_lock_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "acq_time");
|
1796
|
+
"[redis_queued_locks.reentrant_lock_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "acq_time");
|
1797
|
+
"[redis_queued_locks.fail_fast_or_limits_reached_or_deadlock__dequeue]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1798
|
+
"[redis_queued_locks.expire_lock]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1799
|
+
"[redis_queued_locks.decrease_lock]" # (logs "lock_key", "decreased_ttl", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1455
1800
|
```
|
1456
1801
|
|
1457
1802
|
- additional logs (raised from `#lock`/`#lock!` with `confg[:log_lock_try] == true`):
|
1458
1803
|
|
1459
1804
|
```ruby
|
1460
|
-
"[redis_queued_locks.try_lock.start]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1461
|
-
"[redis_queued_locks.try_lock.rconn_fetched]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1462
|
-
"[redis_queued_locks.try_lock.same_process_conflict_detected]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1463
|
-
"[redis_queued_locks.try_lock.same_process_conflict_analyzed]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status");
|
1464
|
-
"[redis_queued_locks.try_lock.reentrant_lock__extend_and_work_through]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status", "last_ext_ttl", "last_ext_ts");
|
1465
|
-
"[redis_queued_locks.try_lock.reentrant_lock__work_through]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status", last_spc_ts);
|
1466
|
-
"[redis_queued_locks.try_lock.single_process_lock_conflict__dead_lock]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "spc_status", "last_spc_ts");
|
1467
|
-
"[redis_queued_locks.try_lock.acq_added_to_queue]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1468
|
-
"[redis_queued_locks.try_lock.remove_expired_acqs]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1469
|
-
"[redis_queued_locks.try_lock.get_first_from_queue]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "first_acq_id_in_queue");
|
1470
|
-
"[redis_queued_locks.try_lock.exit__queue_ttl_reached]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1471
|
-
"[redis_queued_locks.try_lock.exit__no_first]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "first_acq_id_in_queue", "<current_lock_data>");
|
1472
|
-
"[redis_queued_locks.try_lock.exit__lock_still_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat", "first_acq_id_in_queue", "locked_by_acq_id", "<current_lock_data>");
|
1473
|
-
"[redis_queued_locks.try_lock.obtain__free_to_acquire]" # (logs "lock_key", "queue_ttl", "acq_id", "acs_strat");
|
1805
|
+
"[redis_queued_locks.try_lock.start]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1806
|
+
"[redis_queued_locks.try_lock.rconn_fetched]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1807
|
+
"[redis_queued_locks.try_lock.same_process_conflict_detected]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1808
|
+
"[redis_queued_locks.try_lock.same_process_conflict_analyzed]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status");
|
1809
|
+
"[redis_queued_locks.try_lock.reentrant_lock__extend_and_work_through]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status", "last_ext_ttl", "last_ext_ts");
|
1810
|
+
"[redis_queued_locks.try_lock.reentrant_lock__work_through]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status", last_spc_ts);
|
1811
|
+
"[redis_queued_locks.try_lock.single_process_lock_conflict__dead_lock]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "spc_status", "last_spc_ts");
|
1812
|
+
"[redis_queued_locks.try_lock.acq_added_to_queue]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1813
|
+
"[redis_queued_locks.try_lock.remove_expired_acqs]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1814
|
+
"[redis_queued_locks.try_lock.get_first_from_queue]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "first_acq_id_in_queue");
|
1815
|
+
"[redis_queued_locks.try_lock.exit__queue_ttl_reached]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1816
|
+
"[redis_queued_locks.try_lock.exit__no_first]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "first_acq_id_in_queue", "<current_lock_data>");
|
1817
|
+
"[redis_queued_locks.try_lock.exit__lock_still_obtained]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat", "first_acq_id_in_queue", "locked_by_acq_id", "<current_lock_data>");
|
1818
|
+
"[redis_queued_locks.try_lock.obtain__free_to_acquire]" # (logs "lock_key", "queue_ttl", "acq_id", "hst_id", "acs_strat");
|
1474
1819
|
```
|
1475
1820
|
|
1476
1821
|
---
|
@@ -1520,6 +1865,7 @@ Detalized event semantics and payload structure:
|
|
1520
1865
|
- payload:
|
1521
1866
|
- `:ttl` - `integer`/`milliseconds` - lock ttl;
|
1522
1867
|
- `:acq_id` - `string` - lock acquier identifier;
|
1868
|
+
- `:hst_id` - `string` - lock's host identifier;
|
1523
1869
|
- `:lock_key` - `string` - lock name;
|
1524
1870
|
- `:ts` - `numeric`/`epoch` - the time when the lock was obtaiend;
|
1525
1871
|
- `:acq_time` - `float`/`milliseconds` - time spent on lock acquiring;
|
@@ -1532,6 +1878,7 @@ Detalized event semantics and payload structure:
|
|
1532
1878
|
- `:lock_key` - `string` - lock name;
|
1533
1879
|
- `:ttl` - `integer`/`milliseconds` - last lock ttl by reentrant locking;
|
1534
1880
|
- `:acq_id` - `string` - lock acquier identifier;
|
1881
|
+
- `:hst_id` - `string` - lock's host identifier;
|
1535
1882
|
- `:ts` - `numeric`/`epoch` - the time when the lock was obtaiend as extendable reentrant lock;
|
1536
1883
|
- `:acq_time` - `float`/`milliseconds` - time spent on lock acquiring;
|
1537
1884
|
- `:instrument` - `nil`/`Any` - custom data passed to the `#lock`/`#lock!` method as `:instrument` attribute;
|
@@ -1543,6 +1890,7 @@ Detalized event semantics and payload structure:
|
|
1543
1890
|
- `:lock_key` - `string` - lock name;
|
1544
1891
|
- `:ttl` - `integer`/`milliseconds` - last lock ttl by reentrant locking;
|
1545
1892
|
- `:acq_id` - `string` - lock acquier identifier;
|
1893
|
+
- `:hst_id` - `string` - lock's host identifier;
|
1546
1894
|
- `:ts` - `numeric`/`epoch` - the time when the lock was obtaiend as reentrant lock;
|
1547
1895
|
- `:acq_time` - `float`/`milliseconds` - time spent on lock acquiring;
|
1548
1896
|
- `:instrument` - `nil`/`Any` - custom data passed to the `#lock`/`#lock!` method as `:instrument` attribute;
|
@@ -1554,6 +1902,7 @@ Detalized event semantics and payload structure:
|
|
1554
1902
|
- `:hold_time` - `float`/`milliseconds` - lock hold time;
|
1555
1903
|
- `:ttl` - `integer`/`milliseconds` - lock ttl;
|
1556
1904
|
- `:acq_id` - `string` - lock acquier identifier;
|
1905
|
+
- `:hst_id` - `string` - lock's host identifier;
|
1557
1906
|
- `:lock_key` - `string` - lock name;
|
1558
1907
|
- `:ts` - `numeric`/`epoch` - the time when lock was obtained;
|
1559
1908
|
- `:acq_time` - `float`/`milliseconds` - time spent on lock acquiring;
|
@@ -1567,6 +1916,7 @@ Detalized event semantics and payload structure:
|
|
1567
1916
|
- `:hold_time` - `float`/`milliseconds` - lock hold time;
|
1568
1917
|
- `:ttl` - `integer`/`milliseconds` - last lock ttl by reentrant locking;
|
1569
1918
|
- `:acq_id` - `string` - lock acquier identifier;
|
1919
|
+
- `:hst_id` - `string` - lock's host identifier;
|
1570
1920
|
- `:ts` - `numeric`/`epoch` - the time when the lock was obtaiend as reentrant lock;
|
1571
1921
|
- `:lock_key` - `string` - lock name;
|
1572
1922
|
- `:acq_time` - `float`/`milliseconds` - time spent on lock acquiring;
|
@@ -1596,6 +1946,12 @@ Detalized event semantics and payload structure:
|
|
1596
1946
|
<sup>\[[back to top](#table-of-contents)\]</sup>
|
1597
1947
|
|
1598
1948
|
- **Major**:
|
1949
|
+
- Swarm:
|
1950
|
+
- circuit-breaker for long-living failures of your infrastructure inside the swarm elements and supervisor:
|
1951
|
+
the supervisor will stop (for some period of time or while the some factor will return `true`)
|
1952
|
+
trying to ressurect unexpectedly terminated swarm elements, and will notify about this;
|
1953
|
+
- swarm logs (thread/ractor has some limitations so the initial implementation does not include swarm logging);
|
1954
|
+
- swarm instrumentation (thread/ractor has some limitations so the initial implementation does not include swarm instrumentation);
|
1599
1955
|
- lock request prioritization;
|
1600
1956
|
- **strict redlock algorithm support** (support for many `RedisClient` instances);
|
1601
1957
|
- `#lock_series` - acquire a series of locks:
|