legion-crypt 1.5.42 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,841 +1,15 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'legion/logging/helper'
4
3
  require 'singleton'
5
- require 'timeout'
6
- require 'legion/crypt/vault/operation'
4
+ require 'legion/crypt/vault/leases'
7
5
 
8
6
  module Legion
9
7
  module Crypt
10
- class LeaseManager
8
+ # The process-wide lease object. legion-settings (lease:// resolution) and legion-data (consumer
9
+ # reconnects) call Legion::Crypt::LeaseManager.instance as a stable cross-gem contract, and it
10
+ # must answer whether or not Vault is enabled, so it exists from load rather than per session.
11
+ class LeaseManager < Legion::Crypt::Vault::Leases
11
12
  include Singleton
12
- include Legion::Logging::Helper
13
-
14
- RENEWAL_CHECK_INTERVAL = 5
15
- MINIMUM_SLEEP = 0.01
16
- MAX_RETRY_DELAY = 60
17
-
18
- def initialize
19
- @lease_cache = {}
20
- @active_leases = {}
21
- @owned_leases = {}
22
- @pending = {}
23
- @refs = {}
24
- @ref_bindings = {}
25
- @installations = {}.compare_by_identity
26
- @lease_locks = {}
27
- @state_mutex = Mutex.new
28
- @wake = ConditionVariable.new
29
- @running = false
30
- @renewal_thread = nil
31
- @generation = 0
32
- @stopped = false
33
- @renewal_generation = nil
34
- end
35
-
36
- def start(definitions, vault_client: nil)
37
- client = vault_client || Legion::Crypt.vault_client
38
- generation = @state_mutex.synchronize do
39
- return if Legion::Crypt.respond_to?(:stopping) && Legion::Crypt.stopping
40
-
41
- @stopped = false
42
- @vault_client = client
43
- @generation
44
- end
45
- register_at_exit_hook
46
- (definitions || {}).each do |name, opts|
47
- break unless lifecycle_current?(generation: generation)
48
-
49
- name = name.to_sym
50
- path = opts['path'] || opts[:path]
51
- next unless path
52
-
53
- pending = @state_mutex.synchronize { @pending[name] }
54
- next if lease_valid?(name, path: path, client: client)
55
- next if pending && pending[:path] == path && pending[:client].equal?(client) && pending[:next_attempt_at] > monotonic_now
56
-
57
- acquire_lease(name: name, path: path, client: client, generation: generation)
58
- end
59
- end
60
-
61
- def fetched_count
62
- @state_mutex.synchronize { @active_leases.count { |name, _lease| cached_lease_data(name: name) } }
63
- end
64
-
65
- def fetch(name, key)
66
- data = lease_data(name)
67
- return unless data
68
-
69
- data.key?(key.to_sym) ? data[key.to_sym] : data[key.to_s]
70
- end
71
-
72
- def lease_data(name)
73
- @state_mutex.synchronize { cached_lease_data(name: name.to_sym) }
74
- end
75
-
76
- attr_reader :active_leases
77
-
78
- def register_ref(name, key, path, generation: @generation)
79
- return false unless path.is_a?(Array) && !path.empty?
80
-
81
- name = name.to_sym
82
- key = key.to_sym
83
- with_current_state(generation: generation) do
84
- @refs.delete_if do |registered_name, keys|
85
- keys.delete_if do |registered_key, destinations|
86
- @ref_bindings.delete(path) if !(registered_name == name && registered_key == key) && destinations.delete(path)
87
- destinations.empty?
88
- end
89
- keys.empty?
90
- end
91
- @refs[name] ||= {}
92
- destinations = (@refs[name][key] ||= [])
93
- destinations << path.dup.freeze unless destinations.include?(path)
94
- end
95
- end
96
-
97
- # Preparation and publication must only perform in-memory work, without calling manager methods.
98
- def update_settings_refs(settings:, publish: nil)
99
- raise ArgumentError, 'Settings reference preparation requires a block' unless block_given?
100
-
101
- @state_mutex.synchronize do
102
- raise 'Cannot publish settings from an in-flight lease reconnect' if @installations.value?(Thread.current)
103
-
104
- generation = @generation
105
- @wake.wait(@state_mutex) while !@installations.empty? && lifecycle_current?(generation: generation)
106
- raise 'Cannot update settings references in an inactive lifecycle' unless lifecycle_current?(generation: generation)
107
-
108
- reader = lambda do |name:, key:|
109
- data = cached_lease_data(name: name.to_sym)
110
- data && (data.key?(key.to_sym) ? data[key.to_sym] : data[key.to_s])
111
- end
112
- proposal = yield reader
113
- apply_settings_proposal(settings: settings, proposal: proposal, publish: publish)
114
- proposal
115
- end
116
- end
117
-
118
- def push_to_settings(name, generation: @generation, active: nil)
119
- count = with_current_state(generation: generation, active: active) do
120
- refs = @refs[name.to_sym]
121
- data = cached_lease_data(name: name.to_sym)
122
- next 0 if refs.nil? || refs.empty? || data.nil?
123
-
124
- refs.sum do |key, paths|
125
- next 0 unless data.key?(key) || data.key?(key.to_s)
126
-
127
- value = data.key?(key) ? data[key] : data[key.to_s]
128
- paths.count { |path| write_setting(path, value) }
129
- end
130
- end
131
- return unless count
132
-
133
- log.info("Lease '#{name}' rotated — updated #{count} settings reference(s)")
134
- count
135
- end
136
-
137
- def vault_logical
138
- client = @state_mutex.synchronize { @vault_client } || Legion::Crypt.vault_client
139
- client.logical
140
- end
141
-
142
- def vault_sys
143
- client = @state_mutex.synchronize { @vault_client } || Legion::Crypt.vault_client
144
- client.sys
145
- end
146
-
147
- def reissue_all(active: nil)
148
- names, generation = @state_mutex.synchronize { [@active_leases.keys, @generation] }
149
- names.each do |name|
150
- break unless lifecycle_current?(generation: generation, active: active)
151
-
152
- reissue_lease(name, generation: generation, active: active)
153
- end
154
- end
155
-
156
- def register_dynamic_lease(name:, path:, response:, settings_refs:)
157
- name = name.to_sym
158
- generation, client = @state_mutex.synchronize { [@generation, @vault_client] }
159
- client ||= Legion::Crypt.vault_client
160
- register_at_exit_hook
161
- ownership = track_ownership(name: name, client: client, lease_id: response.lease_id)
162
- install_started = false
163
- with_lease_operation(name: name) do |deadline|
164
- settings_refs.each { |ref| register_ref(name, ref[:key], ref[:path], generation: generation) }
165
- install_started = true
166
- install_lease(name: name, path: path, response: response, client: client, generation: generation,
167
- deadline: deadline, reconnect: true, ownership: ownership)
168
- end
169
- rescue StandardError => e
170
- handle_exception(e, level: :warn, operation: 'crypt.lease_manager.register_dynamic_lease', lease_name: name)
171
- unless install_started
172
- @state_mutex.synchronize do
173
- entry = @owned_leases[ownership]
174
- entry[:retired] = true if entry && @active_leases[name]&.dig(:ownership) != ownership
175
- end
176
- end
177
- schedule_retry(name: name, path: path, client: client, generation: generation)
178
- false
179
- end
180
-
181
- def reissue_lease(name, generation: @generation, active: nil, deadline: nil)
182
- name = name.to_sym
183
- lease = @state_mutex.synchronize { @active_leases[name]&.dup }
184
- return false unless lease && lease[:path] && lifecycle_current?(generation: generation, active: active)
185
-
186
- acquire_lease(name: name, path: lease[:path], client: lease[:client], generation: generation,
187
- active: active, deadline: deadline, lease: lease)
188
- end
189
-
190
- def start_renewal_thread
191
- previous = @state_mutex.synchronize { @renewal_thread if @renewal_generation != @generation }
192
- Legion::Crypt::Vault::Operation.with_timeout { previous.join } if previous&.alive?
193
- @state_mutex.synchronize do
194
- return if @renewal_thread&.alive? || @stopped
195
- return if Legion::Crypt.respond_to?(:stopping) && Legion::Crypt.stopping
196
-
197
- @running = true
198
- generation = @generation
199
- @renewal_generation = generation
200
- @renewal_thread = Thread.new { renewal_loop(generation: generation) }
201
- end
202
- log.info 'LeaseManager renewal thread started'
203
- end
204
-
205
- def renewal_thread_alive?
206
- @state_mutex.synchronize { @renewal_thread&.alive? || false }
207
- end
208
-
209
- def request_stop
210
- @state_mutex.synchronize do
211
- @stopped = true
212
- @running = false
213
- @generation += 1
214
- @lease_locks.clear
215
- @wake.broadcast
216
- @renewal_thread
217
- end
218
- end
219
-
220
- def shutdown(deadline: nil)
221
- deadline ||= monotonic_now + Legion::Crypt::Vault::Operation::SHUTDOWN_TIMEOUT
222
- request_stop
223
- @state_mutex.synchronize do
224
- @active_leases.clear
225
- @lease_cache.clear
226
- @refs.clear
227
- @ref_bindings.clear
228
- @pending.clear
229
- @vault_client = nil
230
- @owned_leases.each_value { |entry| entry[:retired] = true }
231
- end
232
- stop_renewal_thread(deadline: deadline)
233
- retire_unused_leases(deadline: deadline, force: true)
234
- log.info 'LeaseManager shutdown complete'
235
- end
236
-
237
- def reset!
238
- stop_renewal_thread
239
- @state_mutex.synchronize do
240
- @lease_cache.clear
241
- @active_leases.clear
242
- @owned_leases.clear
243
- @pending.clear
244
- @refs.clear
245
- @ref_bindings.clear
246
- @vault_client = nil
247
- @stopped = false
248
- end
249
- end
250
-
251
- private
252
-
253
- def apply_settings_proposal(settings:, proposal:, publish:)
254
- validate_settings_proposal!(settings: settings, proposal: proposal)
255
- references, bindings = settings_reference_registry(settings: settings, proposal: proposal)
256
- assignments = proposal[:assignments].map do |assignment|
257
- [assignment[:container], assignment[:key], assignment[:container][assignment[:key]], assignment[:value]]
258
- end
259
- binding_values = proposal[:references].map { |reference| [reference[:binding], reference[:binding][:value]] }
260
- previous_refs = @refs
261
- previous_bindings = @ref_bindings
262
- begin
263
- assignments.each { |container, key, _previous, value| container[key] = value }
264
- @refs = references
265
- @ref_bindings = bindings
266
- publish_settings_proposal(settings: settings, proposal: proposal, publish: publish)
267
- rescue StandardError
268
- @refs = previous_refs
269
- @ref_bindings = previous_bindings
270
- assignments.reverse_each { |container, key, previous, _value| container[key] = previous }
271
- binding_values.each { |binding, previous| binding[:value] = previous }
272
- raise
273
- end
274
- end
275
-
276
- def settings_reference_registry(settings:, proposal:)
277
- paths = proposal[:paths]
278
- references = @refs.each_with_object({}) do |(name, keys), result|
279
- remaining = keys.transform_values { |destinations| destinations.reject { |path| paths.include?(path) } }
280
- remaining.reject! { |_key, destinations| destinations.empty? }
281
- result[name] = remaining unless remaining.empty?
282
- end
283
- bindings = @ref_bindings.except(*paths)
284
- proposal[:references].each do |reference|
285
- name = reference[:name].to_sym
286
- key = reference[:key].to_sym
287
- path = reference[:path].dup.freeze
288
- references[name] ||= {}
289
- (references[name][key] ||= []) << path
290
- bindings[path] = { settings: settings, binding: reference[:binding] }
291
- end
292
- [references, bindings]
293
- end
294
-
295
- def publish_settings_proposal(settings:, proposal:, publish:)
296
- proposal[:references].each do |reference|
297
- target, key = setting_target(path: reference[:path], settings: settings)
298
- raise ArgumentError, 'Settings assignment removed a reference destination' unless target
299
-
300
- reference[:binding][:value] = target[key]
301
- end
302
- publish&.call(proposal)
303
- return unless defined?(Legion::Settings) && Legion::Settings.respond_to?(:loader) && Legion::Settings.loader&.settings.equal?(settings)
304
-
305
- proposal[:references].each { |reference| @ref_bindings[reference[:path]][:settings] = nil }
306
- end
307
-
308
- def validate_settings_proposal!(settings:, proposal:)
309
- raise ArgumentError, 'Settings root must be a Hash or Array' unless settings.is_a?(Hash) || settings.is_a?(Array)
310
- unless proposal.is_a?(Hash) && %i[paths assignments references].all? { |key| proposal[key].is_a?(Array) }
311
- raise ArgumentError, 'Invalid settings reference proposal'
312
- end
313
-
314
- proposal[:paths].each do |path|
315
- unless path.is_a?(Array) && !path.empty? && path.all? { |part| part.is_a?(Symbol) || (part.is_a?(Integer) && part >= 0) }
316
- raise ArgumentError, 'Invalid typed settings path'
317
- end
318
- end
319
- validate_settings_assignments!(assignments: proposal[:assignments])
320
- validate_settings_references!(settings: settings, paths: proposal[:paths], references: proposal[:references])
321
- end
322
-
323
- def validate_settings_assignments!(assignments:)
324
- destinations = {}
325
- assignments.each do |assignment|
326
- raise ArgumentError, 'Invalid settings assignment' unless assignment.is_a?(Hash) && assignment.key?(:value)
327
-
328
- container = assignment[:container]
329
- key = assignment[:key]
330
- raise ArgumentError, 'Unwritable settings assignment' unless valid_setting_key?(container: container, key: key) && !container.frozen?
331
- raise ArgumentError, 'Duplicate settings assignment' if destinations[[container.object_id, key]]
332
-
333
- destinations[[container.object_id, key]] = true
334
- end
335
- end
336
-
337
- def validate_settings_references!(settings:, paths:, references:)
338
- destinations = {}
339
- binding_ids = {}.compare_by_identity
340
- references.each do |reference|
341
- unless reference.is_a?(Hash) && paths.include?(reference[:path]) &&
342
- [reference[:name], reference[:key]].all? { |value| value.is_a?(String) || value.is_a?(Symbol) }
343
- raise ArgumentError, 'Invalid settings reference'
344
- end
345
-
346
- target, key = setting_target(path: reference[:path], settings: settings)
347
- raise ArgumentError, 'Unwritable settings reference destination' unless target
348
- raise ArgumentError, 'Ambiguous settings reference destination' if destinations[[target.object_id, key]]
349
-
350
- binding = reference[:binding]
351
- unless binding.is_a?(Hash) && !binding.frozen? && binding.key?(:value) && !binding_ids[binding]
352
- raise ArgumentError, 'Settings references require distinct mutable binding records'
353
- end
354
-
355
- destinations[[target.object_id, key]] = true
356
- binding_ids[binding] = true
357
- end
358
- end
359
-
360
- def register_at_exit_hook
361
- return if @at_exit_registered
362
-
363
- at_exit do
364
- next if @state_mutex.synchronize { @owned_leases.empty? }
365
-
366
- shutdown
367
- rescue StandardError => e
368
- handle_exception(e, level: :warn, operation: 'crypt.lease_manager.at_exit')
369
- end
370
- @at_exit_registered = true
371
- end
372
-
373
- def monotonic_now
374
- ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
375
- end
376
-
377
- def with_lease_operation(name:, deadline: nil, **)
378
- lock = @state_mutex.synchronize { @lease_locks[name] ||= Mutex.new }
379
- Legion::Crypt::Vault::Operation.with_timeout(deadline: deadline) do |operation_deadline|
380
- lock.synchronize { yield operation_deadline }
381
- end
382
- end
383
-
384
- def lease_operation_current?(name:, pending: nil, lease: nil, **)
385
- @state_mutex.synchronize do
386
- next false if pending && !@pending[name].equal?(pending)
387
-
388
- lease.nil? || @active_leases.dig(name, :ownership) == lease[:ownership]
389
- end
390
- end
391
-
392
- def acquire_lease(name:, path:, client:, generation:, **options)
393
- active = options[:active]
394
- deadline = options[:deadline]
395
- with_lease_operation(name: name, deadline: deadline) do |operation_deadline|
396
- return false unless lifecycle_current?(generation: generation, active: active)
397
- return false unless lease_operation_current?(name: name, pending: options[:pending], lease: options[:lease])
398
- raise ArgumentError, 'Lease acquisition requires a path' unless path
399
-
400
- response = client.logical.read(path: path)
401
- raise "Vault returned no lease for #{name}" unless response
402
-
403
- install_lease(name: name, path: path, response: response, client: client, generation: generation,
404
- active: active, deadline: operation_deadline)
405
- end
406
- rescue StandardError => e
407
- handle_exception(e, level: :warn, operation: 'crypt.lease_manager.acquire_lease', lease_name: name, path: path)
408
- schedule_retry(name: name, path: path, client: client, generation: generation, active: active,
409
- pending: options[:pending], lease: options[:lease])
410
- false
411
- end
412
-
413
- def track_ownership(name:, client:, lease_id:, **)
414
- return if lease_id.nil? || lease_id.empty?
415
-
416
- key = [client, lease_id]
417
- @state_mutex.synchronize do
418
- @owned_leases[key] ||= { name: name, client: client, lease_id: lease_id, retired: false,
419
- revoking: false, attempts: 0, next_attempt_at: 0 }
420
- end
421
- key
422
- end
423
-
424
- def install_lease(name:, path:, response:, client:, **context)
425
- lease_id = context.fetch(:lease_id) { response.lease_id }
426
- ownership = context.fetch(:ownership) { track_ownership(name: name, client: client, lease_id: lease_id) }
427
- handoff = { name: name, ownership: ownership, context: context }
428
- now = Time.now
429
- ttl = response.lease_duration || 0
430
- candidate = { lease_id: lease_id, lease_duration: ttl, renewable: response.renewable?,
431
- expires_at: ttl.positive? ? now + ttl : nil, fetched_at: now, path: path,
432
- client: client, ownership: ownership }
433
- handoff[:candidate] = candidate
434
- handoff[:data] = context.fetch(:data) { response.data }
435
- return false unless stage_install(handoff: handoff)
436
-
437
- generation = context.fetch(:generation)
438
- active = context[:active]
439
- handoff[:attempted_reconnect] = handoff[:reconnect]
440
- raise "Lease consumer reconnect failed for #{name}" if handoff[:reconnect] && !trigger_reconnect(name, generation: generation, active: active)
441
-
442
- committed = with_current_state(generation: generation, active: active) do
443
- next false unless @active_leases[name].equal?(candidate)
444
-
445
- @pending.delete(name)
446
- @owned_leases.each do |key, entry|
447
- entry[:retired] = true if entry[:name] == name && key != ownership
448
- end
449
- @wake.broadcast
450
- true
451
- end
452
- committed
453
- ensure
454
- begin
455
- abort_install(handoff: handoff) if handoff && !committed
456
- ensure
457
- @state_mutex.synchronize do
458
- @installations.delete(candidate)
459
- @wake.broadcast
460
- end
461
- end
462
- retire_unused_leases(name: name, deadline: context[:deadline])
463
- end
464
-
465
- def stage_install(handoff:, **)
466
- name = handoff[:name]
467
- data = handoff[:data]
468
- context = handoff[:context]
469
-
470
- with_current_state(generation: context.fetch(:generation), active: context[:active]) do
471
- validate_lease_data!(name: name, data: data, require_credentials: context[:reconnect] || @refs.dig(name, :password))
472
- owner = @owned_leases[handoff[:ownership]]
473
- raise 'Lease candidate is being revoked or has already been retired' if handoff[:ownership] && (!owner || owner[:revoking])
474
-
475
- owner[:retired] = false if owner
476
-
477
- previous = @active_leases[name]
478
- snapshots = setting_snapshots(name: name, data: data)
479
- transaction = { previous: previous, data: @lease_cache[name], settings: snapshots, candidate: handoff[:candidate] }
480
- handoff[:transaction] = transaction
481
- @installations[handoff[:candidate]] = Thread.current
482
- snapshots.each do |target, key, old, value, binding|
483
- target[key] = value unless old == value
484
- binding[:value] = value if binding
485
- end
486
- @lease_cache[name] = data
487
- @active_leases[name] = handoff[:candidate]
488
- handoff[:reconnect] = context[:reconnect] || (previous && transaction[:data] != data) ||
489
- snapshots.any? { |_target, _key, old, value| old != value }
490
- true
491
- end
492
- end
493
-
494
- def validate_lease_data!(name:, data:, require_credentials:, **)
495
- raise "Vault returned invalid lease data for #{name}" unless data.is_a?(Hash)
496
- return unless name == :rabbitmq && require_credentials
497
-
498
- username = data[:username] || data['username']
499
- password = data[:password] || data['password']
500
- raise 'RabbitMQ lease requires username and password' unless username && password
501
- end
502
-
503
- def abort_install(handoff:, **)
504
- name = handoff[:name]
505
- context = handoff[:context]
506
- generation = context.fetch(:generation)
507
- active = context[:active]
508
- ownership = handoff[:ownership]
509
- restored = rollback_install(name: name, transaction: handoff[:transaction], generation: generation, active: active)
510
- safe_to_retire = !handoff[:attempted_reconnect] || !lifecycle_current?(generation: generation, active: active)
511
- if handoff[:attempted_reconnect] && restored
512
- begin
513
- safe_to_retire = Legion::Crypt::Vault::Operation.with_timeout(deadline: context[:deadline]) do
514
- trigger_reconnect(name, generation: generation, active: active)
515
- end
516
- rescue StandardError => e
517
- handle_exception(e, level: :warn, operation: 'crypt.lease_manager.rollback_reconnect', lease_name: name)
518
- end
519
- end
520
- @state_mutex.synchronize do
521
- entry = @owned_leases[ownership]
522
- entry[:retired] = true if entry && safe_to_retire && @active_leases[name]&.dig(:ownership) != ownership
523
- end
524
- end
525
-
526
- def setting_snapshots(name:, data:, **)
527
- (@refs[name] || {}).flat_map do |key, paths|
528
- raise "Lease #{name} omitted referenced key #{key}" unless data.key?(key) || data.key?(key.to_s)
529
-
530
- value = data.key?(key) ? data[key] : data[key.to_s]
531
- paths.filter_map do |path|
532
- record = @ref_bindings[path]
533
- target, segment = setting_target(path: path, settings: record&.dig(:settings))
534
- raise "Lease #{name} has an unwritable settings destination #{path.inspect}" unless target
535
-
536
- binding = record&.dig(:binding)
537
- raise "Lease #{name} has an immutable settings binding" if binding&.frozen?
538
- next if target[segment] == value && (!binding || binding[:value] == value)
539
-
540
- [target, segment, target[segment], value, binding, binding&.dig(:value)]
541
- end
542
- end
543
- end
544
-
545
- def rollback_install(name:, transaction:, generation:, active:, **)
546
- return false unless transaction
547
-
548
- with_current_state(generation: generation, active: active) do
549
- current = @active_leases[name]
550
- next false unless current.equal?(transaction[:candidate]) || current.equal?(transaction[:previous])
551
-
552
- transaction[:settings].each do |snapshot|
553
- target, key, previous, candidate, binding, previous_binding = snapshot
554
- target[key] = previous if target[key] == candidate
555
- binding[:value] = previous_binding if binding && binding[:value] == candidate
556
- end
557
- if transaction[:previous]
558
- @active_leases[name] = transaction[:previous]
559
- @lease_cache[name] = transaction[:data]
560
- else
561
- @active_leases.delete(name)
562
- @lease_cache.delete(name)
563
- end
564
- true
565
- end
566
- rescue StandardError => e
567
- handle_exception(e, level: :error, operation: 'crypt.lease_manager.rollback_install', lease_name: name)
568
- false
569
- end
570
-
571
- def schedule_retry(name:, path:, client:, generation:, **options)
572
- with_current_state(generation: generation, active: options[:active]) do
573
- previous = @pending[name]
574
- next false if options[:pending] && !previous.equal?(options[:pending])
575
- next false if options[:lease] && @active_leases.dig(name, :ownership) != options[:lease][:ownership]
576
-
577
- attempts = previous && previous[:path] == path && previous[:client].equal?(client) ? previous[:attempts] : 0
578
- attempts = [attempts + 1, 7].min
579
- @pending[name] = { path: path, client: client, operation: options.fetch(:operation, :acquire), attempts: attempts,
580
- next_attempt_at: monotonic_now + [2**(attempts - 1), MAX_RETRY_DELAY].min }
581
- @wake.broadcast
582
- end
583
- end
584
-
585
- def retire_unused_leases(name: nil, deadline: nil, force: false, **)
586
- keys = @state_mutex.synchronize do
587
- @owned_leases.filter_map do |key, entry|
588
- key if entry[:retired] && (!name || entry[:name] == name) && (force || entry[:next_attempt_at] <= monotonic_now)
589
- end
590
- end
591
- keys.each { |key| revoke_owned_lease(key: key, deadline: deadline) }
592
- end
593
-
594
- def revoke_owned_lease(key:, deadline:, **)
595
- entry = @state_mutex.synchronize do
596
- item = @owned_leases[key]
597
- next unless item && item[:retired] && !item[:revoking]
598
- next if @active_leases.any? { |_name, lease| lease[:ownership] == key }
599
-
600
- item[:revoking] = true
601
- item
602
- end
603
- return unless entry
604
-
605
- Legion::Crypt::Vault::Operation.with_timeout(deadline: deadline) { entry[:client].sys.revoke(lease_id: entry[:lease_id]) }
606
- @state_mutex.synchronize { @owned_leases.delete(key) if @owned_leases[key].equal?(entry) }
607
- rescue StandardError => e
608
- handle_exception(e, level: :warn, operation: 'crypt.lease_manager.revoke_owned_lease', lease_name: entry&.dig(:name))
609
- @state_mutex.synchronize do
610
- if entry && @owned_leases[key].equal?(entry)
611
- entry[:attempts] = [entry[:attempts] + 1, 7].min
612
- entry[:next_attempt_at] = monotonic_now + [2**(entry[:attempts] - 1), MAX_RETRY_DELAY].min
613
- end
614
- end
615
- ensure
616
- @state_mutex.synchronize { entry[:revoking] = false if entry }
617
- end
618
-
619
- def stop_renewal_thread(deadline: nil)
620
- thread = request_stop
621
- return unless thread
622
-
623
- Legion::Crypt::Vault::Operation.with_timeout(deadline: deadline) { thread.join }
624
- @state_mutex.synchronize { @renewal_thread = nil if @renewal_thread.equal?(thread) }
625
- rescue StandardError => e
626
- handle_exception(e, level: :warn, operation: 'crypt.lease_manager.stop_renewal_thread')
627
- end
628
-
629
- def renewal_loop(generation:)
630
- while running?(generation: generation)
631
- renew_approaching_leases(generation: generation)
632
- interruptible_sleep(generation: generation)
633
- end
634
- rescue StandardError => e
635
- handle_exception(e, level: :error, operation: 'crypt.lease_manager.renewal_loop')
636
- interruptible_sleep(RENEWAL_CHECK_INTERVAL, generation: generation)
637
- retry if running?(generation: generation)
638
- end
639
-
640
- # Called while holding @state_mutex, immediately before the condition wait.
641
- def next_maintenance_delay
642
- delays = @pending.values.map { |entry| entry[:next_attempt_at] - monotonic_now }
643
- @active_leases.each do |name, lease|
644
- delays << renewal_delay(lease: lease) unless @pending.key?(name)
645
- end
646
- @owned_leases.each do |key, entry|
647
- next unless entry[:retired] && !entry[:revoking]
648
- next if @active_leases.each_value.any? { |lease| lease[:ownership] == key }
649
-
650
- delays << (entry[:next_attempt_at] - monotonic_now)
651
- end
652
- (delays.min || RENEWAL_CHECK_INTERVAL).clamp(MINIMUM_SLEEP, RENEWAL_CHECK_INTERVAL)
653
- end
654
-
655
- def renew_approaching_leases(generation: @generation)
656
- leases = @state_mutex.synchronize { @active_leases.to_a.sort_by { |_name, lease| renewal_delay(lease: lease) } }
657
- attempted = []
658
- leases.each do |name, lease|
659
- break unless lifecycle_current?(generation: generation)
660
- next if @state_mutex.synchronize { @pending.key?(name) }
661
- next if renewal_delay(lease: lease).positive?
662
-
663
- attempted << name
664
- if lease_renewable?(lease: lease)
665
- renew_lease(name, lease.dup, generation: generation)
666
- else
667
- acquire_lease(name: name, path: lease[:path], client: lease[:client], generation: generation, lease: lease)
668
- end
669
- end
670
- pending = @state_mutex.synchronize { @pending.to_a }
671
- pending.each do |name, entry|
672
- break unless lifecycle_current?(generation: generation)
673
- next if attempted.include?(name) || entry[:next_attempt_at] > monotonic_now
674
-
675
- lease = @state_mutex.synchronize { @active_leases[name]&.dup }
676
- if entry[:operation] == :renew && lease_renewable?(lease: lease)
677
- renew_lease(name, lease, generation: generation, pending: entry)
678
- else
679
- acquire_lease(name: name, path: entry[:path], client: entry[:client], generation: generation, pending: entry, lease: lease)
680
- end
681
- end
682
- retire_unused_leases if lifecycle_current?(generation: generation)
683
- end
684
-
685
- def renew_lease(name, lease, generation: @generation, pending: nil)
686
- name = name.to_sym
687
- operation_deadline = nil
688
- with_lease_operation(name: name) do |deadline|
689
- operation_deadline = deadline
690
- return false unless lifecycle_current?(generation: generation)
691
- return false unless lease_operation_current?(name: name, pending: pending, lease: lease)
692
-
693
- raise 'Lease cannot be renewed without an ID' if lease[:lease_id].to_s.empty?
694
-
695
- response = lease[:client].sys.renew(lease_id: lease[:lease_id])
696
- raise 'Lease renewal did not return a positive TTL' unless response&.lease_duration&.positive?
697
-
698
- data = response.data
699
- data = @state_mutex.synchronize { @lease_cache[name] } if data.nil? || data.empty?
700
- lease_id = response.lease_id.to_s.empty? ? lease[:lease_id] : response.lease_id
701
- ownership = if lease_id == lease[:lease_id]
702
- lease[:ownership]
703
- else
704
- track_ownership(name: name, client: lease[:client], lease_id: lease_id)
705
- end
706
- install_lease(name: name, path: lease[:path], response: response, client: lease[:client], generation: generation,
707
- deadline: deadline, data: data, lease_id: lease_id, ownership: ownership)
708
- end
709
- rescue StandardError => e
710
- handle_exception(e, level: :warn, operation: 'crypt.lease_manager.renew_lease', lease_name: name)
711
- return reissue_lease(name, generation: generation, deadline: operation_deadline) if operation_deadline && !e.is_a?(Timeout::Error) && lease[:path]
712
-
713
- schedule_retry(name: name, path: lease[:path], client: lease[:client], generation: generation, operation: :renew, pending: pending)
714
- false
715
- end
716
-
717
- def lease_valid?(name, path:, client:)
718
- lease = @state_mutex.synchronize { @active_leases[name.to_sym] }
719
- lease && lease[:path] == path && lease[:client].equal?(client) && lease_unexpired?(lease: lease)
720
- end
721
-
722
- # Called while holding @state_mutex; expired credentials are never exposed to readers.
723
- def cached_lease_data(name:, **)
724
- lease = @active_leases[name]
725
- return @lease_cache[name] if lease && lease_unexpired?(lease: lease)
726
-
727
- @lease_cache.delete(name)
728
- nil
729
- end
730
-
731
- def lease_unexpired?(lease:, **)
732
- lease[:expires_at].nil? || lease[:expires_at] > Time.now
733
- end
734
-
735
- def lease_renewable?(lease:, **)
736
- lease && lease[:renewable] && lease_unexpired?(lease: lease)
737
- end
738
-
739
- def renewal_delay(lease:, **)
740
- ttl = lease[:lease_duration] || 0
741
- return lease[:fetched_at] + RENEWAL_CHECK_INTERVAL - Time.now unless ttl.positive?
742
- return 0 unless lease[:expires_at]
743
-
744
- lease[:expires_at] - Time.now - (ttl * 0.5)
745
- end
746
-
747
- def setting_target(path:, settings: nil, **)
748
- return unless path.is_a?(Array) && !path.empty?
749
-
750
- target = settings
751
- target ||= Legion::Settings.loader&.settings if defined?(Legion::Settings) && Legion::Settings.respond_to?(:loader)
752
- path.each_with_index do |segment, index|
753
- return nil unless valid_setting_key?(container: target, key: segment)
754
- return target.frozen? ? nil : [target, segment] if index == path.length - 1
755
-
756
- target = target[segment]
757
- end
758
- end
759
-
760
- def valid_setting_key?(container:, key:)
761
- case container
762
- when Hash then key.is_a?(Symbol) && container.key?(key)
763
- when Array then key.is_a?(Integer) && key >= 0 && key < container.length
764
- end
765
- end
766
-
767
- def write_setting(path, value)
768
- record = @ref_bindings[path]
769
- target, key = setting_target(path: path, settings: record&.dig(:settings))
770
- binding = record&.dig(:binding)
771
- return false unless target && !binding&.frozen?
772
-
773
- target[key] = value
774
- binding[:value] = value if binding
775
- true
776
- rescue StandardError => e
777
- handle_exception(e, level: :warn, operation: 'crypt.lease_manager.write_setting', path: path.inspect)
778
- false
779
- end
780
-
781
- def trigger_reconnect(name, generation: @generation, active: nil)
782
- return false unless lifecycle_current?(generation: generation, active: active)
783
-
784
- case name.to_sym
785
- when :rabbitmq
786
- return true unless defined?(Legion::Transport::Connection)
787
-
788
- connection = Legion::Transport::Connection
789
- connection.force_reconnect == true && connection.session_open?
790
- when :postgresql
791
- return true unless defined?(Legion::Data::Connection)
792
-
793
- connection = Legion::Data::Connection
794
- connection.respond_to?(:reconnect_with_fresh_creds) && connection.reconnect_with_fresh_creds == true
795
- when :redis
796
- return true unless defined?(Legion::Cache)
797
-
798
- cache = Legion::Cache
799
- result = if cache.respond_to?(:restart)
800
- cache.restart
801
- elsif cache.respond_to?(:reconnect)
802
- cache.reconnect
803
- end
804
- result == true && (!cache.respond_to?(:connected?) || cache.connected?)
805
- else
806
- true
807
- end
808
- rescue StandardError => e
809
- handle_exception(e, level: :error, operation: 'crypt.lease_manager.trigger_reconnect', lease_name: name)
810
- false
811
- end
812
-
813
- def with_current_state(generation:, active: nil, **)
814
- @state_mutex.synchronize do
815
- next false unless lifecycle_current?(generation: generation)
816
-
817
- next yield unless active
818
-
819
- result = false
820
- active.call { result = yield }
821
- result
822
- end
823
- end
824
-
825
- def lifecycle_current?(generation:, active: nil, **)
826
- !@stopped && generation == @generation &&
827
- !(Legion::Crypt.respond_to?(:stopping) && Legion::Crypt.stopping) && (!active || active.call)
828
- end
829
-
830
- def running?(generation: @generation)
831
- @state_mutex.synchronize { @running && lifecycle_current?(generation: generation) }
832
- end
833
-
834
- def interruptible_sleep(seconds = nil, generation: @generation)
835
- @state_mutex.synchronize do
836
- @wake.wait(@state_mutex, seconds || next_maintenance_delay) if @running && lifecycle_current?(generation: generation)
837
- end
838
- end
839
13
  end
840
14
  end
841
15
  end