redis_queued_locks 1.8.0 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/.ruby-version +1 -1
  3. data/CHANGELOG.md +51 -0
  4. data/README.md +402 -46
  5. data/lib/redis_queued_locks/acquier/acquire_lock/dequeue_from_lock_queue/log_visitor.rb +4 -0
  6. data/lib/redis_queued_locks/acquier/acquire_lock/dequeue_from_lock_queue.rb +4 -1
  7. data/lib/redis_queued_locks/acquier/acquire_lock/instr_visitor.rb +20 -5
  8. data/lib/redis_queued_locks/acquier/acquire_lock/log_visitor.rb +24 -0
  9. data/lib/redis_queued_locks/acquier/acquire_lock/try_to_lock/log_visitor.rb +56 -0
  10. data/lib/redis_queued_locks/acquier/acquire_lock/try_to_lock.rb +37 -30
  11. data/lib/redis_queued_locks/acquier/acquire_lock/yield_expire/log_visitor.rb +8 -0
  12. data/lib/redis_queued_locks/acquier/acquire_lock/yield_expire.rb +13 -9
  13. data/lib/redis_queued_locks/acquier/acquire_lock.rb +44 -20
  14. data/lib/redis_queued_locks/acquier/clear_dead_requests.rb +5 -1
  15. data/lib/redis_queued_locks/acquier/extend_lock_ttl.rb +5 -1
  16. data/lib/redis_queued_locks/acquier/lock_info.rb +4 -3
  17. data/lib/redis_queued_locks/acquier/locks.rb +2 -2
  18. data/lib/redis_queued_locks/acquier/queue_info.rb +2 -2
  19. data/lib/redis_queued_locks/acquier/release_all_locks.rb +12 -2
  20. data/lib/redis_queued_locks/acquier/release_lock.rb +12 -2
  21. data/lib/redis_queued_locks/client.rb +284 -34
  22. data/lib/redis_queued_locks/errors.rb +8 -0
  23. data/lib/redis_queued_locks/instrument.rb +8 -1
  24. data/lib/redis_queued_locks/logging.rb +8 -1
  25. data/lib/redis_queued_locks/resource.rb +59 -1
  26. data/lib/redis_queued_locks/swarm/acquirers.rb +44 -0
  27. data/lib/redis_queued_locks/swarm/flush_zombies.rb +133 -0
  28. data/lib/redis_queued_locks/swarm/probe_hosts.rb +69 -0
  29. data/lib/redis_queued_locks/swarm/redis_client_builder.rb +67 -0
  30. data/lib/redis_queued_locks/swarm/supervisor.rb +83 -0
  31. data/lib/redis_queued_locks/swarm/swarm_element/isolated.rb +287 -0
  32. data/lib/redis_queued_locks/swarm/swarm_element/threaded.rb +351 -0
  33. data/lib/redis_queued_locks/swarm/swarm_element.rb +8 -0
  34. data/lib/redis_queued_locks/swarm/zombie_info.rb +145 -0
  35. data/lib/redis_queued_locks/swarm.rb +241 -0
  36. data/lib/redis_queued_locks/utilities/lock.rb +22 -0
  37. data/lib/redis_queued_locks/utilities.rb +75 -0
  38. data/lib/redis_queued_locks/version.rb +2 -2
  39. data/lib/redis_queued_locks.rb +2 -0
  40. metadata +14 -4
  41. data/lib/redis_queued_locks/watcher.rb +0 -1
@@ -0,0 +1,351 @@
1
+ # frozen_string_literal: true
2
+
3
+ # @api private
4
+ # @since 1.9.0
5
+ # rubocop:disable Metrics/ClassLength
6
+ class RedisQueuedLocks::Swarm::SwarmElement::Threaded
7
+ # @since 1.9.0
8
+ include RedisQueuedLocks::Utilities
9
+
10
+ # @return [RedisQueuedLocks::Client]
11
+ #
12
+ # @api private
13
+ # @since 1.9.0
14
+ attr_reader :rql_client
15
+
16
+ # @return [Thread,NilClass]
17
+ #
18
+ # @api private
19
+ # @since 1.9.0
20
+ attr_reader :swarm_element
21
+
22
+ # The main loop reference is only used inside the swarm element thread and
23
+ # swarm element termination. It should not be used everywhere else!
24
+ # This is strongly technical variable refenrece.
25
+ #
26
+ # @return [Thread,NilClass]
27
+ #
28
+ # @api private
29
+ # @since 1.9.0
30
+ attr_reader :main_loop
31
+
32
+ # @return [Thread::SizedQueue,NilClass]
33
+ #
34
+ # @api private
35
+ # @since 1.9.0
36
+ attr_reader :swarm_element_commands
37
+
38
+ # @return [Thread::SizedQueue]
39
+ #
40
+ # @api private
41
+ # @since 1.9.0
42
+ attr_reader :swarm_element_results
43
+
44
+ # @return [RedisQueuedLocks::Utilities::Lock]
45
+ #
46
+ # @api private
47
+ # @since 1.9.0
48
+ attr_reader :sync
49
+
50
+ # @param rql_client [RedisQueuedLocks::Client]
51
+ # @return [void]
52
+ #
53
+ # @api private
54
+ # @since 1.9.0
55
+ def initialize(rql_client)
56
+ @rql_client = rql_client
57
+ @swarm_element = nil
58
+ @main_loop = nil # NOTE: strongly technical variable refenrece (see attr_reader docs)
59
+ @swarm_element_commands = nil
60
+ @swarm_element_results = nil
61
+ @sync = RedisQueuedLocks::Utilities::Lock.new
62
+ end
63
+
64
+ # @return [void]
65
+ #
66
+ # @api private
67
+ # @since 1.9.0
68
+ def try_swarm!
69
+ return unless enabled?
70
+
71
+ sync.synchronize do
72
+ swarm_element__termiante
73
+ swarm!
74
+ swarm_loop__start
75
+ end
76
+ end
77
+
78
+ # @return [void]
79
+ #
80
+ # @api private
81
+ # @since 19.0.0
82
+ def reswarm_if_dead!
83
+ return unless enabled?
84
+
85
+ sync.synchronize do
86
+ if swarmed__stopped?
87
+ swarm_loop__start
88
+ elsif swarmed__dead? || idle?
89
+ swarm!
90
+ swarm_loop__start
91
+ end
92
+ end
93
+ end
94
+
95
+ # @return [void]
96
+ #
97
+ # @api private
98
+ # @since 1.9.0
99
+ def try_kill!
100
+ sync.synchronize do
101
+ swarm_element__termiante
102
+ end
103
+ end
104
+
105
+ # @return [Boolean]
106
+ #
107
+ # @api private
108
+ # @since 1.9.0
109
+ def enabled?
110
+ # NOTE: provide an <is enabled> logic here by analyzing the redis queued locks config.
111
+ end
112
+
113
+ # @return [Hash<Symbol,Boolean|Hash<Symbol,String|Boolean>>]
114
+ # Format: {
115
+ # enabled: <Boolean>,
116
+ # thread: {
117
+ # running: <Boolean>,
118
+ # state: <String>,
119
+ # },
120
+ # main_loop: {
121
+ # running: <Boolean>,
122
+ # state: <String>
123
+ # }
124
+ # }
125
+ #
126
+ # @api private
127
+ # @since 1.9.0
128
+ # rubocop:disable Style/RedundantBegin
129
+ def status
130
+ sync.synchronize do
131
+ thread_running = swarmed__alive?
132
+ thread_state = swarmed? ? thread_state(swarm_element) : 'non_initialized'
133
+
134
+ main_loop_running = swarmed__running?
135
+ main_loop_state = begin
136
+ if main_loop_running
137
+ swarm_loop__status[:result][:main_loop][:state]
138
+ else
139
+ 'non_initialized'
140
+ end
141
+ end
142
+
143
+ {
144
+ enabled: enabled?,
145
+ thread: {
146
+ running: thread_running,
147
+ state: thread_state
148
+ },
149
+ main_loop: {
150
+ running: main_loop_running,
151
+ state: main_loop_state
152
+ }
153
+ }
154
+ end
155
+ end
156
+ # rubocop:enable Style/RedundantBegin
157
+
158
+ private
159
+
160
+ # Swarm element lifecycle have the following flow:
161
+ # => 1) init (swarm!): create a thread, main loop is not started;
162
+ # => 2) start (swarm_loop__start): run the main lopp inside the created thread;
163
+ # => 3) stop (swarm_loop__stop): stop the main loop inside the created thread;
164
+ # => 4) terminate (swarm_element__termiante): kill the created thread;
165
+ # In addition you should implement `#spawn_main_loop!` method that will be the main logic
166
+ # of the your swarm element.
167
+ #
168
+ # @return [void]
169
+ #
170
+ # @api private
171
+ # @since 1.9.0
172
+ # rubocop:disable Metrics/MethodLength
173
+ def swarm!
174
+ # NOTE: kill the main loop at start to prevent any async-thread-race-based memory leaks;
175
+ main_loop&.kill
176
+
177
+ @swarm_element_commands = Thread::SizedQueue.new(1)
178
+ @swarm_element_results = Thread::SizedQueue.new(1)
179
+
180
+ @swarm_element = Thread.new do
181
+ loop do
182
+ command = swarm_element_commands.pop
183
+
184
+ case command
185
+ when :status
186
+ main_loop_alive = main_loop != nil && main_loop.alive?
187
+ main_loop_state = (main_loop == nil) ? 'non_initialized' : thread_state(main_loop)
188
+ swarm_element_results.push({
189
+ ok: true,
190
+ result: { main_loop: { alive: main_loop_alive, state: main_loop_state } }
191
+ })
192
+ when :is_active
193
+ is_active = main_loop != nil && main_loop.alive?
194
+ swarm_element_results.push({ ok: true, result: { is_active: } })
195
+ when :start
196
+ main_loop&.kill
197
+ @main_loop = spawn_main_loop!.tap { |thread| thread.abort_on_exception = false }
198
+ swarm_element_results.push({ ok: true, result: nil })
199
+ when :stop
200
+ main_loop&.kill
201
+ swarm_element_results.push({ ok: true, result: nil })
202
+ end
203
+ end
204
+ end
205
+ end
206
+ # rubocop:enable Metrics/MethodLength
207
+
208
+ # @return [Thread] Thread with #abort_onexception == false that wraps loop'ed logic;
209
+ #
210
+ # @api private
211
+ # @since 1.9.0
212
+ def spawn_main_loop!
213
+ # NOTE:
214
+ # - provide the swarm element looped logic here wrapped into the thread;
215
+ # - created thread will be reconfigured inside the swarm_element logic with a
216
+ # `abort_on_exception = false` (cuz the stauts of the thread is
217
+ # totally controlled by the @swarm_element's logic);
218
+ end
219
+
220
+ # @return [Boolean]
221
+ #
222
+ # @api private
223
+ # @since 1.9.0
224
+ def idle?
225
+ swarm_element == nil
226
+ end
227
+
228
+ # @return [Boolean]
229
+ #
230
+ # @api private
231
+ # @since 1.9.0
232
+ def swarmed?
233
+ swarm_element != nil
234
+ end
235
+
236
+ # @return [Boolean]
237
+ #
238
+ # @api private
239
+ # @since 1.9.0
240
+ def swarmed__alive?
241
+ swarmed? && swarm_element.alive? && !terminating?
242
+ end
243
+
244
+ # @return [Boolean]
245
+ #
246
+ # @api private
247
+ # @since 1.9.0
248
+ def swarmed__dead?
249
+ swarmed? && (!swarm_element.alive? || terminating?)
250
+ end
251
+
252
+ # @return [Boolean]
253
+ #
254
+ # @api private
255
+ # @since 1.9.0
256
+ def swarmed__running?
257
+ swarmed__alive? && !terminating? && (swarm_loop__is_active.yield_self do |result|
258
+ result && result[:ok] && result[:result][:is_active]
259
+ end)
260
+ end
261
+
262
+ # @return [Boolean,NilClass]
263
+ #
264
+ # @api private
265
+ # @since 1.9.0
266
+ # rubocop:disable Layout/MultilineOperationIndentation
267
+ def terminating?
268
+ swarm_element_commands == nil ||
269
+ swarm_element_commands&.closed? ||
270
+ swarm_element_results == nil ||
271
+ swarm_element_results&.closed?
272
+ end
273
+ # rubocop:enable Layout/MultilineOperationIndentation
274
+
275
+ # @return [Boolean]
276
+ #
277
+ # @api private
278
+ # @since 1.9.0
279
+ def swarmed__stopped?
280
+ swarmed__alive? && (terminating? || !(swarm_loop__is_active.yield_self do |result|
281
+ result && result[:ok] && result[:result][:is_active]
282
+ end))
283
+ end
284
+
285
+ # @return [Boolean,NilClass]
286
+ #
287
+ # @api private
288
+ # @since 1.9.0
289
+ def swarm_loop__is_active
290
+ return if idle? || swarmed__dead? || terminating?
291
+ sync.synchronize do
292
+ swarm_element_commands.push(:is_active)
293
+ swarm_element_results.pop
294
+ end
295
+ end
296
+
297
+ # @return [Hash,NilClass]
298
+ #
299
+ # @api private
300
+ # @since 1.9.0
301
+ def swarm_loop__status
302
+ return if idle? || swarmed__dead? || terminating?
303
+ sync.synchronize do
304
+ swarm_element_commands.push(:status)
305
+ swarm_element_results.pop
306
+ end
307
+ end
308
+
309
+ # @return [void]
310
+ #
311
+ # @api private
312
+ # @since 1.9.0
313
+ def swarm_loop__start
314
+ return if idle? || swarmed__dead? || terminating?
315
+ sync.synchronize do
316
+ swarm_element_commands.push(:start)
317
+ swarm_element_results.pop
318
+ end
319
+ end
320
+
321
+ # @return [void]
322
+ #
323
+ # @api private
324
+ # @since 1.9.0
325
+ def swarm_loop__stop
326
+ return if idle? || swarmed__dead? || terminating?
327
+ sync.synchronize do
328
+ swarm_element_commands.push(:stop)
329
+ swarm_element_results.pop
330
+ end
331
+ end
332
+
333
+ # @return [void]
334
+ #
335
+ # @api private
336
+ # @since 1.9.0
337
+ def swarm_element__termiante
338
+ return if idle? || swarmed__dead?
339
+ sync.synchronize do
340
+ main_loop&.kill
341
+ swarm_element.kill
342
+ swarm_element_commands.close
343
+ swarm_element_results.close
344
+ swarm_element_commands.clear
345
+ swarm_element_results.clear
346
+ @swarm_element_commands = nil
347
+ @swarm_element_results = nil
348
+ end
349
+ end
350
+ end
351
+ # rubocop:enable Metrics/ClassLength
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ # @api private
4
+ # @since 1.9.0
5
+ module RedisQueuedLocks::Swarm::SwarmElement
6
+ require_relative 'swarm_element/isolated'
7
+ require_relative 'swarm_element/threaded'
8
+ end
@@ -0,0 +1,145 @@
1
+ # frozen_string_literal: true
2
+
3
+ # @api private
4
+ # @since 1.9.0
5
+ module RedisQueuedLocks::Swarm::ZombieInfo
6
+ class << self
7
+ # @param redis_client [RedisClient]
8
+ # @param zombie_ttl [Integer]
9
+ # @param lock_scan_size [Integer]
10
+ # @return [Hash<Symbol,Set<String>>]
11
+ # Format: {
12
+ # zombie_hosts: <Set<String>>,
13
+ # zombie_acquirers: <Set<String>>,
14
+ # zombie_locks: <Set<String>>
15
+ # }
16
+ #
17
+ # @api private
18
+ # @since 1.9.0
19
+ def zombies_info(redis_client, zombie_ttl, lock_scan_size)
20
+ redis_client.with do |rconn|
21
+ extract_all(rconn, zombie_ttl, lock_scan_size)
22
+ end
23
+ end
24
+
25
+ # @param redis_client [RedisClient]
26
+ # @param zombie_ttl [Integer]
27
+ # @param lock_scan_size [Integer]
28
+ # @return [Set<String>]
29
+ #
30
+ # @api private
31
+ # @since 1.9.0
32
+ def zombie_locks(redis_client, zombie_ttl, lock_scan_size)
33
+ redis_client.with do |rconn|
34
+ extract_zombie_locks(rconn, zombie_ttl, lock_scan_size)
35
+ end
36
+ end
37
+
38
+ # @param redis_client [RedisClient]
39
+ # @param zombie_ttl [Integer]
40
+ # @return [Set<String>]
41
+ #
42
+ # @api private
43
+ # @since 1.9.0
44
+ def zombie_hosts(redis_client, zombie_ttl)
45
+ redis_client.with do |rconn|
46
+ extract_zombie_hosts(rconn, zombie_ttl)
47
+ end
48
+ end
49
+
50
+ # @param redis_client [RedisClient]
51
+ # @param zombie_ttl [Integer]
52
+ # @param lock_scan_size [Integer]
53
+ # @return [Set<String>]
54
+ #
55
+ # @api private
56
+ # @since 1.9.0
57
+ def zombie_acquiers(redis_client, zombie_ttl, lock_scan_size)
58
+ redis_client.with do |rconn|
59
+ extract_zombie_acquiers(rconn, zombie_ttl, lock_scan_size)
60
+ end
61
+ end
62
+
63
+ private
64
+
65
+ # @param rconn [RedisClient] redis connection obtained via `#with` from RedisClient instance;
66
+ # @param zombie_ttl [Integer]
67
+ # @return [Set<String>]
68
+ #
69
+ # @api private
70
+ # @since 1.9.0
71
+ def extract_zombie_hosts(rconn, zombie_ttl)
72
+ zombie_score = RedisQueuedLocks::Resource.calc_zombie_score(zombie_ttl / 1_000)
73
+ swarmed_hosts = rconn.call('HGETALL', RedisQueuedLocks::Resource::SWARM_KEY)
74
+ swarmed_hosts.each_with_object(Set.new) do |(hst_id, ts), zombies|
75
+ (zombies << hst_id) if (zombie_score > ts.to_f)
76
+ end
77
+ end
78
+
79
+ # @param rconn [RedisClient] redis connection obtained via `#with` from RedisClient instance;
80
+ # @param zombie_ttl [Integer]
81
+ # @param lock_scan_size [Integer]
82
+ # @return [Set<String>]
83
+ #
84
+ # @api private
85
+ # @since 1.9.0
86
+ def extract_zombie_locks(rconn, zombie_ttl, lock_scan_size)
87
+ zombie_hosts = extract_zombie_hosts(rconn, zombie_ttl)
88
+ zombie_locks = Set.new
89
+ rconn.scan(
90
+ 'MATCH', RedisQueuedLocks::Resource::LOCK_PATTERN, count: lock_scan_size
91
+ ) do |lock_key|
92
+ _acquier_id, host_id = rconn.call('HMGET', lock_key, 'acq_id', 'hst_id')
93
+ zombie_locks << lock_key if zombie_hosts.include?(host_id)
94
+ end
95
+ zombie_locks
96
+ end
97
+
98
+ # @param rconn [RedisClient] redis connection obtained via `#with` from RedisClient instance;
99
+ # @param zombie_ttl [Integer]
100
+ # @param lock_scan_size [Integer]
101
+ # @return [Set<String>]
102
+ #
103
+ # @api private
104
+ # @since 1.9.0
105
+ def extract_zombie_acquiers(rconn, zombie_ttl, lock_scan_size)
106
+ zombie_hosts = extract_zombie_hosts(rconn, zombie_ttl)
107
+ zombie_acquirers = Set.new
108
+ rconn.scan(
109
+ 'MATCH', RedisQueuedLocks::Resource::LOCK_PATTERN, count: lock_scan_size
110
+ ) do |lock_key|
111
+ acquier_id, host_id = rconn.call('HMGET', lock_key, 'acq_id', 'hst_id')
112
+ zombie_acquirers << acquier_id if zombie_hosts.include?(host_id)
113
+ end
114
+ zombie_acquirers
115
+ end
116
+
117
+ # @param rconn [RedisClient] redis connection obtained via `#with` from RedisClient instance;
118
+ # @param zombie_ttl [Integer]
119
+ # @param lock_scan_size [Integer]
120
+ # @return [Hash<Symbol,<Set<String>>]
121
+ # Format: {
122
+ # zombie_hosts: <Set<String>>,
123
+ # zombie_acquirers: <Set<String>>,
124
+ # zombie_locks: <Set<String>>
125
+ # }
126
+ #
127
+ # @api private
128
+ # @since 1.9.0
129
+ def extract_all(rconn, zombie_ttl, lock_scan_size)
130
+ zombie_hosts = extract_zombie_hosts(rconn, zombie_ttl)
131
+ zombie_locks = Set.new
132
+ zombie_acquirers = Set.new
133
+ rconn.scan(
134
+ 'MATCH', RedisQueuedLocks::Resource::LOCK_PATTERN, count: lock_scan_size
135
+ ) do |lock_key|
136
+ acquier_id, host_id = rconn.call('HMGET', lock_key, 'acq_id', 'hst_id')
137
+ if zombie_hosts.include?(host_id)
138
+ zombie_acquirers << acquier_id
139
+ zombie_locks << lock_key
140
+ end
141
+ end
142
+ { zombie_hosts:, zombie_acquirers:, zombie_locks: }
143
+ end
144
+ end
145
+ end