legion-crypt 1.5.42 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,871 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'faraday'
4
+ require 'time'
5
+ require 'timeout'
6
+ require 'legion/logging/helper'
7
+ require 'legion/crypt/vault/client'
8
+ require 'legion/crypt/vault/operation'
9
+
10
+ module Legion
11
+ module Crypt
12
+ module Vault
13
+ # Maintains dynamic-secret leases (PostgreSQL, RabbitMQ, Redis, ...): acquisition, renew-at-50%
14
+ # on its own thread, reissue under a replaced token, settings publication, consumer reconnect,
15
+ # and revocation. It holds lease:// settings references with or without Vault, so it is built
16
+ # without a client; the default cluster's session supplies one when it has a token.
17
+ class Leases
18
+ include Legion::Logging::Helper
19
+
20
+ RENEWAL_CHECK_INTERVAL = 5
21
+ MINIMUM_SLEEP = 0.01
22
+ MAX_RETRY_DELAY = 60
23
+
24
+ def initialize(**)
25
+ @vault_client = nil
26
+ @lease_cache = {}
27
+ @active_leases = {}
28
+ @owned_leases = {}
29
+ @pending = {}
30
+ @refs = {}
31
+ @ref_bindings = {}
32
+ @installations = {}.compare_by_identity
33
+ @lease_locks = {}
34
+ @state_mutex = Mutex.new
35
+ @wake = ConditionVariable.new
36
+ @running = false
37
+ @renewal_thread = nil
38
+ @generation = 0
39
+ @stopped = false
40
+ @renewal_generation = nil
41
+ end
42
+
43
+ def start(definitions:, client:, **)
44
+ generation = @state_mutex.synchronize do
45
+ return if Legion::Crypt.respond_to?(:stopping) && Legion::Crypt.stopping
46
+
47
+ @stopped = false
48
+ @vault_client = client
49
+ @generation
50
+ end
51
+ register_at_exit_hook
52
+ (definitions || {}).each do |name, opts|
53
+ break unless lifecycle_current?(generation: generation)
54
+
55
+ name = name.to_sym
56
+ path = opts['path'] || opts[:path]
57
+ next unless path
58
+
59
+ pending = @state_mutex.synchronize { @pending[name] }
60
+ next if lease_valid?(name, path: path, client: @vault_client)
61
+ next if pending && pending[:path] == path && pending[:client].equal?(@vault_client) && pending[:next_attempt_at] > monotonic_now
62
+
63
+ acquire_lease(name: name, path: path, client: @vault_client, generation: generation)
64
+ end
65
+ end
66
+
67
+ def fetched_count
68
+ @state_mutex.synchronize { @active_leases.count { |name, _lease| cached_lease_data(name: name) } }
69
+ end
70
+
71
+ def fetch(name, key)
72
+ data = lease_data(name)
73
+ return unless data
74
+
75
+ data.key?(key.to_sym) ? data[key.to_sym] : data[key.to_s]
76
+ end
77
+
78
+ def lease_data(name)
79
+ @state_mutex.synchronize { cached_lease_data(name: name.to_sym) }
80
+ end
81
+
82
+ attr_reader :active_leases
83
+
84
+ def register_ref(name, key, path, generation: @generation)
85
+ return false unless path.is_a?(Array) && !path.empty?
86
+
87
+ name = name.to_sym
88
+ key = key.to_sym
89
+ with_current_state(generation: generation) do
90
+ @refs.delete_if do |registered_name, keys|
91
+ keys.delete_if do |registered_key, destinations|
92
+ @ref_bindings.delete(path) if !(registered_name == name && registered_key == key) && destinations.delete(path)
93
+ destinations.empty?
94
+ end
95
+ keys.empty?
96
+ end
97
+ @refs[name] ||= {}
98
+ destinations = (@refs[name][key] ||= [])
99
+ destinations << path.dup.freeze unless destinations.include?(path)
100
+ end
101
+ end
102
+
103
+ # Preparation and publication must only perform in-memory work, without calling manager methods.
104
+ def update_settings_refs(settings:, publish: nil)
105
+ raise ArgumentError, 'Settings reference preparation requires a block' unless block_given?
106
+
107
+ @state_mutex.synchronize do
108
+ raise 'Cannot publish settings from an in-flight lease reconnect' if @installations.value?(Thread.current)
109
+
110
+ generation = @generation
111
+ @wake.wait(@state_mutex) while !@installations.empty? && lifecycle_current?(generation: generation)
112
+ raise 'Cannot update settings references in an inactive lifecycle' unless lifecycle_current?(generation: generation)
113
+
114
+ reader = lambda do |name:, key:|
115
+ data = cached_lease_data(name: name.to_sym)
116
+ data && (data.key?(key.to_sym) ? data[key.to_sym] : data[key.to_s])
117
+ end
118
+ proposal = yield reader
119
+ apply_settings_proposal(settings: settings, proposal: proposal, publish: publish)
120
+ proposal
121
+ end
122
+ end
123
+
124
+ def push_to_settings(name, generation: @generation, active: nil)
125
+ count = with_current_state(generation: generation, active: active) do
126
+ refs = @refs[name.to_sym]
127
+ data = cached_lease_data(name: name.to_sym)
128
+ next 0 if refs.nil? || refs.empty? || data.nil?
129
+
130
+ refs.sum do |key, paths|
131
+ next 0 unless data.key?(key) || data.key?(key.to_s)
132
+
133
+ value = data.key?(key) ? data[key] : data[key.to_s]
134
+ paths.count { |path| write_setting(path, value) }
135
+ end
136
+ end
137
+ return unless count
138
+
139
+ log.info("Lease '#{name}' rotated — updated #{count} settings reference(s)")
140
+ count
141
+ end
142
+
143
+ def vault_logical
144
+ vault_client.logical
145
+ end
146
+
147
+ def vault_sys
148
+ vault_client.sys
149
+ end
150
+
151
+ # Reacquires every active lease issued through the client; returns whether all of them were replaced.
152
+ def reissue_all(client:, active: nil, **)
153
+ names, generation = @state_mutex.synchronize do
154
+ [@active_leases.select { |_name, lease| lease[:client].equal?(client) }.keys, @generation]
155
+ end
156
+ names.map do |name|
157
+ lifecycle_current?(generation: generation, active: active) && reissue_lease(name, generation: generation, active: active)
158
+ end.all?
159
+ end
160
+
161
+ def register_dynamic_lease(name:, path:, response:, settings_refs:)
162
+ name = name.to_sym
163
+ generation = @state_mutex.synchronize { @generation }
164
+ client = vault_client
165
+ ownership = track_ownership(name: name, client: client, lease_id: response.lease_id)
166
+ register_at_exit_hook
167
+ install_started = false
168
+ with_lease_operation(name: name) do |deadline|
169
+ settings_refs.each { |ref| register_ref(name, ref[:key], ref[:path], generation: generation) }
170
+ install_started = true
171
+ install_lease(name: name, path: path, response: response, client: client, generation: generation,
172
+ deadline: deadline, reconnect: true, ownership: ownership)
173
+ end
174
+ rescue StandardError => e
175
+ handle_exception(e, level: :warn, operation: 'crypt.vault_leases.register_dynamic_lease', lease_name: name)
176
+ unless install_started
177
+ @state_mutex.synchronize do
178
+ entry = @owned_leases[ownership]
179
+ entry[:retired] = true if entry && @active_leases[name]&.dig(:ownership) != ownership
180
+ end
181
+ end
182
+ schedule_retry(name: name, path: path, client: client, generation: generation)
183
+ false
184
+ end
185
+
186
+ def reissue_lease(name, generation: @generation, active: nil, deadline: nil)
187
+ name = name.to_sym
188
+ lease = @state_mutex.synchronize { @active_leases[name]&.dup }
189
+ return false unless lease && lease[:path] && lifecycle_current?(generation: generation, active: active)
190
+
191
+ acquire_lease(name: name, path: lease[:path], client: lease[:client], generation: generation,
192
+ active: active, deadline: deadline, lease: lease)
193
+ end
194
+
195
+ def start_renewal_thread
196
+ previous = @state_mutex.synchronize { @renewal_thread if @renewal_generation != @generation }
197
+ Legion::Crypt::Vault::Operation.with_timeout { previous.join } if previous&.alive?
198
+ @state_mutex.synchronize do
199
+ return if @renewal_thread&.alive? || @stopped
200
+ return if Legion::Crypt.respond_to?(:stopping) && Legion::Crypt.stopping
201
+
202
+ @running = true
203
+ generation = @generation
204
+ @renewal_generation = generation
205
+ @renewal_thread = Thread.new { renewal_loop(generation: generation) }
206
+ end
207
+ log.info 'Vault lease renewal thread started'
208
+ end
209
+
210
+ def renewal_thread_alive?
211
+ @state_mutex.synchronize { @renewal_thread&.alive? || false }
212
+ end
213
+
214
+ def request_stop
215
+ @state_mutex.synchronize do
216
+ @stopped = true
217
+ @running = false
218
+ @generation += 1
219
+ @lease_locks.clear
220
+ @wake.broadcast
221
+ @renewal_thread
222
+ end
223
+ end
224
+
225
+ # Revokes every owned lease, skipping clients whose token Vault already rejected; returns whether none remain.
226
+ def shutdown(deadline: nil, denied_clients: {}.compare_by_identity, **)
227
+ deadline ||= monotonic_now + Legion::Crypt::Vault::Operation::SHUTDOWN_TIMEOUT
228
+ request_stop
229
+ @state_mutex.synchronize do
230
+ @active_leases.clear
231
+ @lease_cache.clear
232
+ @refs.clear
233
+ @ref_bindings.clear
234
+ @pending.clear
235
+ @vault_client = nil
236
+ @owned_leases.each_value { |entry| entry[:retired] = true }
237
+ end
238
+ stop_renewal_thread(deadline: deadline)
239
+ keys = @state_mutex.synchronize { @owned_leases.select { |_key, entry| entry[:retired] }.keys }
240
+ keys.each do |key|
241
+ revoke_owned_lease(key: key, deadline: deadline, denied_clients: denied_clients) unless denied_clients.key?(key.first)
242
+ end
243
+ remaining = @state_mutex.synchronize { @owned_leases.size }
244
+ if remaining.zero?
245
+ log.info 'Vault lease shutdown complete'
246
+ else
247
+ log.error "Vault lease shutdown incomplete: #{remaining} owned lease(s) not revoked"
248
+ end
249
+ remaining.zero?
250
+ end
251
+
252
+ def reset!
253
+ stop_renewal_thread
254
+ @state_mutex.synchronize do
255
+ @lease_cache.clear
256
+ @active_leases.clear
257
+ @owned_leases.clear
258
+ @pending.clear
259
+ @refs.clear
260
+ @ref_bindings.clear
261
+ @vault_client = nil
262
+ @stopped = false
263
+ end
264
+ end
265
+
266
+ private
267
+
268
+ # The session's client once it has a token; before that, the default cluster's client.
269
+ def vault_client
270
+ @state_mutex.synchronize { @vault_client } || Legion::Crypt.vault_client
271
+ end
272
+
273
+ def apply_settings_proposal(settings:, proposal:, publish:)
274
+ validate_settings_proposal!(settings: settings, proposal: proposal)
275
+ references, bindings = settings_reference_registry(settings: settings, proposal: proposal)
276
+ assignments = proposal[:assignments].map do |assignment|
277
+ [assignment[:container], assignment[:key], assignment[:container][assignment[:key]], assignment[:value]]
278
+ end
279
+ binding_values = proposal[:references].map { |reference| [reference[:binding], reference[:binding][:value]] }
280
+ previous_refs = @refs
281
+ previous_bindings = @ref_bindings
282
+ begin
283
+ assignments.each { |container, key, _previous, value| container[key] = value }
284
+ @refs = references
285
+ @ref_bindings = bindings
286
+ publish_settings_proposal(settings: settings, proposal: proposal, publish: publish)
287
+ rescue StandardError
288
+ @refs = previous_refs
289
+ @ref_bindings = previous_bindings
290
+ assignments.reverse_each { |container, key, previous, _value| container[key] = previous }
291
+ binding_values.each { |binding, previous| binding[:value] = previous }
292
+ raise
293
+ end
294
+ end
295
+
296
+ def settings_reference_registry(settings:, proposal:)
297
+ paths = proposal[:paths]
298
+ references = @refs.each_with_object({}) do |(name, keys), result|
299
+ remaining = keys.transform_values { |destinations| destinations.reject { |path| paths.include?(path) } }
300
+ remaining.reject! { |_key, destinations| destinations.empty? }
301
+ result[name] = remaining unless remaining.empty?
302
+ end
303
+ bindings = @ref_bindings.except(*paths)
304
+ proposal[:references].each do |reference|
305
+ name = reference[:name].to_sym
306
+ key = reference[:key].to_sym
307
+ path = reference[:path].dup.freeze
308
+ references[name] ||= {}
309
+ (references[name][key] ||= []) << path
310
+ bindings[path] = { settings: settings, binding: reference[:binding] }
311
+ end
312
+ [references, bindings]
313
+ end
314
+
315
+ def publish_settings_proposal(settings:, proposal:, publish:)
316
+ proposal[:references].each do |reference|
317
+ target, key = setting_target(path: reference[:path], settings: settings)
318
+ raise ArgumentError, 'Settings assignment removed a reference destination' unless target
319
+
320
+ reference[:binding][:value] = target[key]
321
+ end
322
+ publish&.call(proposal)
323
+ return unless defined?(Legion::Settings) && Legion::Settings.respond_to?(:loader) && Legion::Settings.loader&.settings.equal?(settings)
324
+
325
+ proposal[:references].each { |reference| @ref_bindings[reference[:path]][:settings] = nil }
326
+ end
327
+
328
+ def validate_settings_proposal!(settings:, proposal:)
329
+ raise ArgumentError, 'Settings root must be a Hash or Array' unless settings.is_a?(Hash) || settings.is_a?(Array)
330
+ unless proposal.is_a?(Hash) && %i[paths assignments references].all? { |key| proposal[key].is_a?(Array) }
331
+ raise ArgumentError, 'Invalid settings reference proposal'
332
+ end
333
+
334
+ proposal[:paths].each do |path|
335
+ unless path.is_a?(Array) && !path.empty? && path.all? { |part| part.is_a?(Symbol) || (part.is_a?(Integer) && part >= 0) }
336
+ raise ArgumentError, 'Invalid typed settings path'
337
+ end
338
+ end
339
+ validate_settings_assignments!(assignments: proposal[:assignments])
340
+ validate_settings_references!(settings: settings, paths: proposal[:paths], references: proposal[:references])
341
+ end
342
+
343
+ def validate_settings_assignments!(assignments:)
344
+ destinations = {}
345
+ assignments.each do |assignment|
346
+ raise ArgumentError, 'Invalid settings assignment' unless assignment.is_a?(Hash) && assignment.key?(:value)
347
+
348
+ container = assignment[:container]
349
+ key = assignment[:key]
350
+ raise ArgumentError, 'Unwritable settings assignment' unless valid_setting_key?(container: container, key: key) && !container.frozen?
351
+ raise ArgumentError, 'Duplicate settings assignment' if destinations[[container.object_id, key]]
352
+
353
+ destinations[[container.object_id, key]] = true
354
+ end
355
+ end
356
+
357
+ def validate_settings_references!(settings:, paths:, references:)
358
+ destinations = {}
359
+ binding_ids = {}.compare_by_identity
360
+ references.each do |reference|
361
+ unless reference.is_a?(Hash) && paths.include?(reference[:path]) &&
362
+ [reference[:name], reference[:key]].all? { |value| value.is_a?(String) || value.is_a?(Symbol) }
363
+ raise ArgumentError, 'Invalid settings reference'
364
+ end
365
+
366
+ target, key = setting_target(path: reference[:path], settings: settings)
367
+ raise ArgumentError, 'Unwritable settings reference destination' unless target
368
+ raise ArgumentError, 'Ambiguous settings reference destination' if destinations[[target.object_id, key]]
369
+
370
+ binding = reference[:binding]
371
+ unless binding.is_a?(Hash) && !binding.frozen? && binding.key?(:value) && !binding_ids[binding]
372
+ raise ArgumentError, 'Settings references require distinct mutable binding records'
373
+ end
374
+
375
+ destinations[[target.object_id, key]] = true
376
+ binding_ids[binding] = true
377
+ end
378
+ end
379
+
380
+ def register_at_exit_hook
381
+ return if @at_exit_registered
382
+
383
+ at_exit do
384
+ next if @state_mutex.synchronize { @owned_leases.empty? }
385
+
386
+ shutdown
387
+ rescue StandardError => e
388
+ handle_exception(e, level: :warn, operation: 'crypt.vault_leases.at_exit')
389
+ end
390
+ @at_exit_registered = true
391
+ end
392
+
393
+ def monotonic_now
394
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
395
+ end
396
+
397
+ def with_lease_operation(name:, deadline: nil, **)
398
+ lock = @state_mutex.synchronize { @lease_locks[name] ||= Mutex.new }
399
+ Legion::Crypt::Vault::Operation.with_timeout(deadline: deadline) do |operation_deadline|
400
+ lock.synchronize { yield operation_deadline }
401
+ end
402
+ end
403
+
404
+ def lease_operation_current?(name:, pending: nil, lease: nil, **)
405
+ @state_mutex.synchronize do
406
+ next false if pending && !@pending[name].equal?(pending)
407
+
408
+ lease.nil? || @active_leases.dig(name, :ownership) == lease[:ownership]
409
+ end
410
+ end
411
+
412
+ def acquire_lease(name:, path:, client:, generation:, **options)
413
+ active = options[:active]
414
+ deadline = options[:deadline]
415
+ with_lease_operation(name: name, deadline: deadline) do |operation_deadline|
416
+ return false unless lifecycle_current?(generation: generation, active: active)
417
+ return false unless lease_operation_current?(name: name, pending: options[:pending], lease: options[:lease])
418
+ raise ArgumentError, 'Lease acquisition requires a path' unless path
419
+
420
+ response = client.logical.read(path: path)
421
+ raise "Vault returned no lease for #{name}" unless response
422
+
423
+ install_lease(name: name, path: path, response: response, client: client, generation: generation,
424
+ active: active, deadline: operation_deadline)
425
+ end
426
+ rescue StandardError => e
427
+ handle_exception(e, level: :warn, operation: 'crypt.vault_leases.acquire_lease', lease_name: name, path: path)
428
+ schedule_retry(name: name, path: path, client: client, generation: generation, active: active,
429
+ pending: options[:pending], lease: options[:lease])
430
+ false
431
+ end
432
+
433
+ def track_ownership(name:, client:, lease_id:, **)
434
+ return if lease_id.nil? || lease_id.empty?
435
+
436
+ key = [client, lease_id]
437
+ @state_mutex.synchronize do
438
+ @owned_leases[key] ||= { name: name, client: client, lease_id: lease_id, retired: false,
439
+ revoking: false, attempts: 0, next_attempt_at: 0 }
440
+ end
441
+ key
442
+ end
443
+
444
+ def install_lease(name:, path:, response:, client:, **context)
445
+ lease_id = context.fetch(:lease_id) { response.lease_id }
446
+ ownership = context.fetch(:ownership) { track_ownership(name: name, client: client, lease_id: lease_id) }
447
+ handoff = { name: name, ownership: ownership, context: context }
448
+ now = Time.now
449
+ ttl = response.lease_duration || 0
450
+ candidate = { lease_id: lease_id, lease_duration: ttl, renewable: response.renewable?,
451
+ expires_at: ttl.positive? ? now + ttl : nil, fetched_at: now, path: path,
452
+ client: client, ownership: ownership }
453
+ handoff[:candidate] = candidate
454
+ handoff[:data] = context.fetch(:data) { response.data }
455
+ return false unless stage_install(handoff: handoff)
456
+
457
+ generation = context.fetch(:generation)
458
+ active = context[:active]
459
+ handoff[:attempted_reconnect] = handoff[:reconnect]
460
+ raise "Lease consumer reconnect failed for #{name}" if handoff[:reconnect] && !trigger_reconnect(name, generation: generation, active: active)
461
+
462
+ committed = with_current_state(generation: generation, active: active) do
463
+ next false unless @active_leases[name].equal?(candidate)
464
+
465
+ @pending.delete(name)
466
+ @owned_leases.each do |key, entry|
467
+ entry[:retired] = true if entry[:name] == name && key != ownership
468
+ end
469
+ @wake.broadcast
470
+ true
471
+ end
472
+ committed
473
+ ensure
474
+ begin
475
+ abort_install(handoff: handoff) if handoff && !committed
476
+ ensure
477
+ @state_mutex.synchronize do
478
+ @installations.delete(candidate)
479
+ @wake.broadcast
480
+ end
481
+ end
482
+ retire_unused_leases(name: name, deadline: context[:deadline])
483
+ end
484
+
485
+ def stage_install(handoff:, **)
486
+ name = handoff[:name]
487
+ data = handoff[:data]
488
+ context = handoff[:context]
489
+
490
+ with_current_state(generation: context.fetch(:generation), active: context[:active]) do
491
+ validate_lease_data!(name: name, data: data, require_credentials: context[:reconnect] || @refs.dig(name, :password))
492
+ owner = @owned_leases[handoff[:ownership]]
493
+ raise 'Lease candidate is being revoked or has already been retired' if handoff[:ownership] && (!owner || owner[:revoking])
494
+
495
+ owner[:retired] = false if owner
496
+
497
+ previous = @active_leases[name]
498
+ snapshots = setting_snapshots(name: name, data: data)
499
+ transaction = { previous: previous, data: @lease_cache[name], settings: snapshots, candidate: handoff[:candidate] }
500
+ handoff[:transaction] = transaction
501
+ @installations[handoff[:candidate]] = Thread.current
502
+ snapshots.each do |target, key, old, value, binding|
503
+ target[key] = value unless old == value
504
+ binding[:value] = value if binding
505
+ end
506
+ @lease_cache[name] = data
507
+ @active_leases[name] = handoff[:candidate]
508
+ handoff[:reconnect] = context[:reconnect] || (previous && transaction[:data] != data) ||
509
+ snapshots.any? { |_target, _key, old, value| old != value }
510
+ true
511
+ end
512
+ end
513
+
514
+ def validate_lease_data!(name:, data:, require_credentials:, **)
515
+ raise "Vault returned invalid lease data for #{name}" unless data.is_a?(Hash)
516
+ return unless name == :rabbitmq && require_credentials
517
+
518
+ username = data[:username] || data['username']
519
+ password = data[:password] || data['password']
520
+ raise 'RabbitMQ lease requires username and password' unless username && password
521
+ end
522
+
523
+ def abort_install(handoff:, **)
524
+ name = handoff[:name]
525
+ context = handoff[:context]
526
+ generation = context.fetch(:generation)
527
+ active = context[:active]
528
+ ownership = handoff[:ownership]
529
+ restored = rollback_install(name: name, transaction: handoff[:transaction], generation: generation, active: active)
530
+ safe_to_retire = !handoff[:attempted_reconnect] || !lifecycle_current?(generation: generation, active: active)
531
+ if handoff[:attempted_reconnect] && restored
532
+ begin
533
+ safe_to_retire = Legion::Crypt::Vault::Operation.with_timeout(deadline: context[:deadline]) do
534
+ trigger_reconnect(name, generation: generation, active: active)
535
+ end
536
+ rescue StandardError => e
537
+ handle_exception(e, level: :warn, operation: 'crypt.vault_leases.rollback_reconnect')
538
+ end
539
+ end
540
+ @state_mutex.synchronize do
541
+ entry = @owned_leases[ownership]
542
+ entry[:retired] = true if entry && safe_to_retire && @active_leases[name]&.dig(:ownership) != ownership
543
+ end
544
+ end
545
+
546
+ def setting_snapshots(name:, data:, **)
547
+ (@refs[name] || {}).flat_map do |key, paths|
548
+ raise "Lease #{name} omitted referenced key #{key}" unless data.key?(key) || data.key?(key.to_s)
549
+
550
+ value = data.key?(key) ? data[key] : data[key.to_s]
551
+ paths.filter_map do |path|
552
+ record = @ref_bindings[path]
553
+ target, segment = setting_target(path: path, settings: record&.dig(:settings))
554
+ raise "Lease #{name} has an unwritable settings destination #{path.inspect}" unless target
555
+
556
+ binding = record&.dig(:binding)
557
+ raise "Lease #{name} has an immutable settings binding" if binding&.frozen?
558
+ next if target[segment] == value && (!binding || binding[:value] == value)
559
+
560
+ [target, segment, target[segment], value, binding, binding&.dig(:value)]
561
+ end
562
+ end
563
+ end
564
+
565
+ def rollback_install(name:, transaction:, generation:, active:)
566
+ return false unless transaction
567
+
568
+ with_current_state(generation: generation, active: active) do
569
+ current = @active_leases[name]
570
+ next false unless current.equal?(transaction[:candidate]) || current.equal?(transaction[:previous])
571
+
572
+ transaction[:settings].each do |snapshot|
573
+ target, key, previous, candidate, binding, previous_binding = snapshot
574
+ target[key] = previous if target[key] == candidate
575
+ binding[:value] = previous_binding if binding && binding[:value] == candidate
576
+ end
577
+ if transaction[:previous]
578
+ @active_leases[name] = transaction[:previous]
579
+ @lease_cache[name] = transaction[:data]
580
+ else
581
+ @active_leases.delete(name)
582
+ @lease_cache.delete(name)
583
+ end
584
+ true
585
+ end
586
+ rescue StandardError => e
587
+ handle_exception(e, level: :error, operation: 'crypt.vault_leases.rollback_install', lease_name: name)
588
+ false
589
+ end
590
+
591
+ def schedule_retry(name:, path:, client:, generation:, **options)
592
+ with_current_state(generation: generation, active: options[:active]) do
593
+ previous = @pending[name]
594
+ next false if options[:pending] && !previous.equal?(options[:pending])
595
+ next false if options[:lease] && @active_leases.dig(name, :ownership) != options[:lease][:ownership]
596
+
597
+ attempts = previous && previous[:path] == path && previous[:client].equal?(client) ? previous[:attempts] : 0
598
+ attempts = [attempts + 1, 7].min
599
+ @pending[name] = { path: path, client: client, operation: options.fetch(:operation, :acquire), attempts: attempts,
600
+ next_attempt_at: monotonic_now + [2**(attempts - 1), MAX_RETRY_DELAY].min }
601
+ @wake.broadcast
602
+ end
603
+ end
604
+
605
+ def retire_unused_leases(name: nil, deadline: nil, **)
606
+ keys = @state_mutex.synchronize do
607
+ @owned_leases.filter_map do |key, entry|
608
+ key if entry[:retired] && (!name || entry[:name] == name) && entry[:next_attempt_at] <= monotonic_now
609
+ end
610
+ end
611
+ keys.each { |key| revoke_owned_lease(key: key, deadline: deadline) }
612
+ end
613
+
614
+ def revoke_owned_lease(key:, deadline: nil, denied_clients: nil, **)
615
+ entry = @state_mutex.synchronize do
616
+ item = @owned_leases[key]
617
+ next unless item && item[:retired] && !item[:revoking]
618
+ next if @active_leases.any? { |_name, lease| lease[:ownership] == key }
619
+
620
+ item[:revoking] = true
621
+ item
622
+ end
623
+ return unless entry
624
+
625
+ Legion::Crypt::Vault::Operation.with_timeout(deadline: deadline) { entry[:client].sys.revoke(lease_id: entry[:lease_id]) }
626
+ @state_mutex.synchronize { @owned_leases.delete(key) if @owned_leases[key].equal?(entry) }
627
+ rescue StandardError => e
628
+ note_denied_client(denied_clients, entry, e)
629
+ handle_exception(e, level: :warn, operation: 'crypt.vault_leases.revoke_owned_lease', lease_name: entry&.dig(:name))
630
+ @state_mutex.synchronize do
631
+ if entry && @owned_leases[key].equal?(entry)
632
+ entry[:attempts] = [entry[:attempts] + 1, 7].min
633
+ entry[:next_attempt_at] = monotonic_now + [2**(entry[:attempts] - 1), MAX_RETRY_DELAY].min
634
+ end
635
+ end
636
+ ensure
637
+ @state_mutex.synchronize { entry[:revoking] = false if entry }
638
+ end
639
+
640
+ # Records a client whose token Vault rejected (403) so shared shutdown cleanup skips it.
641
+ def note_denied_client(denied_clients, entry, error)
642
+ return if denied_clients.nil? || entry.nil?
643
+ return unless error.is_a?(Legion::Crypt::Vault::HTTPClientError) && error.code == 403
644
+
645
+ denied_clients[entry[:client]] = true
646
+ end
647
+
648
+ def stop_renewal_thread(deadline: nil)
649
+ thread = request_stop
650
+ return unless thread
651
+
652
+ Legion::Crypt::Vault::Operation.with_timeout(deadline: deadline) { thread.join }
653
+ @state_mutex.synchronize { @renewal_thread = nil if @renewal_thread.equal?(thread) }
654
+ rescue StandardError => e
655
+ handle_exception(e, level: :warn, operation: 'crypt.vault_leases.stop_renewal_thread')
656
+ end
657
+
658
+ def renewal_loop(generation:)
659
+ while running?(generation: generation)
660
+ renew_approaching_leases(generation: generation)
661
+ interruptible_sleep(generation: generation)
662
+ end
663
+ rescue StandardError => e
664
+ handle_exception(e, level: :error, operation: 'crypt.vault_leases.renewal_loop')
665
+ interruptible_sleep(RENEWAL_CHECK_INTERVAL, generation: generation)
666
+ retry if running?(generation: generation)
667
+ end
668
+
669
+ # Called while holding @state_mutex, immediately before the condition wait.
670
+ def next_maintenance_delay
671
+ delays = @pending.values.map { |entry| entry[:next_attempt_at] - monotonic_now }
672
+ @active_leases.each do |name, lease|
673
+ delays << renewal_delay(lease: lease) unless @pending.key?(name)
674
+ end
675
+ @owned_leases.each do |key, entry|
676
+ next unless entry[:retired] && !entry[:revoking]
677
+ next if @active_leases.each_value.any? { |lease| lease[:ownership] == key }
678
+
679
+ delays << (entry[:next_attempt_at] - monotonic_now)
680
+ end
681
+ (delays.min || RENEWAL_CHECK_INTERVAL).clamp(MINIMUM_SLEEP, RENEWAL_CHECK_INTERVAL)
682
+ end
683
+
684
+ def renew_approaching_leases(generation: @generation)
685
+ leases = @state_mutex.synchronize { @active_leases.to_a.sort_by { |_name, lease| renewal_delay(lease: lease) } }
686
+ attempted = []
687
+ leases.each do |name, lease|
688
+ break unless lifecycle_current?(generation: generation)
689
+ next if @state_mutex.synchronize { @pending.key?(name) }
690
+ next if renewal_delay(lease: lease).positive?
691
+
692
+ attempted << name
693
+ if lease_renewable?(lease: lease)
694
+ renew_lease(name, lease.dup, generation: generation)
695
+ else
696
+ acquire_lease(name: name, path: lease[:path], client: lease[:client], generation: generation, lease: lease)
697
+ end
698
+ end
699
+ pending = @state_mutex.synchronize { @pending.to_a }
700
+ pending.each do |name, entry|
701
+ break unless lifecycle_current?(generation: generation)
702
+ next if attempted.include?(name) || entry[:next_attempt_at] > monotonic_now
703
+
704
+ lease = @state_mutex.synchronize { @active_leases[name]&.dup }
705
+ if entry[:operation] == :renew && lease_renewable?(lease: lease)
706
+ renew_lease(name, lease, generation: generation, pending: entry)
707
+ else
708
+ acquire_lease(name: name, path: entry[:path], client: entry[:client], generation: generation, pending: entry, lease: lease)
709
+ end
710
+ end
711
+ retire_unused_leases if lifecycle_current?(generation: generation)
712
+ end
713
+
714
+ def renew_lease(name, lease, generation: @generation, pending: nil)
715
+ name = name.to_sym
716
+ operation_deadline = nil
717
+ with_lease_operation(name: name) do |deadline|
718
+ operation_deadline = deadline
719
+ return false unless lifecycle_current?(generation: generation)
720
+ return false unless lease_operation_current?(name: name, pending: pending, lease: lease)
721
+
722
+ raise 'Lease cannot be renewed without an ID' if lease[:lease_id].to_s.empty?
723
+
724
+ response = lease[:client].sys.renew(lease_id: lease[:lease_id])
725
+ raise 'Lease renewal did not return a positive TTL' unless response&.lease_duration&.positive?
726
+
727
+ data = response.data
728
+ data = @state_mutex.synchronize { @lease_cache[name] } if data.nil? || data.empty?
729
+ lease_id = response.lease_id.to_s.empty? ? lease[:lease_id] : response.lease_id
730
+ ownership = if lease_id == lease[:lease_id]
731
+ lease[:ownership]
732
+ else
733
+ track_ownership(name: name, client: lease[:client], lease_id: lease_id)
734
+ end
735
+ install_lease(name: name, path: lease[:path], response: response, client: lease[:client], generation: generation,
736
+ deadline: deadline, data: data, lease_id: lease_id, ownership: ownership)
737
+ end
738
+ rescue StandardError => e
739
+ handle_exception(e, level: :warn, operation: 'crypt.vault_leases.renew_lease', lease_name: name)
740
+ return reissue_lease(name, generation: generation, deadline: operation_deadline) if operation_deadline && !e.is_a?(Timeout::Error) && lease[:path]
741
+
742
+ schedule_retry(name: name, path: lease[:path], client: lease[:client], generation: generation, operation: :renew, pending: pending)
743
+ false
744
+ end
745
+
746
+ def lease_valid?(name, path:, client:)
747
+ lease = @state_mutex.synchronize { @active_leases[name.to_sym] }
748
+ lease && lease[:path] == path && lease[:client].equal?(client) && lease_unexpired?(lease: lease)
749
+ end
750
+
751
+ # Called while holding @state_mutex; expired credentials are never exposed to readers.
752
+ def cached_lease_data(name:, **)
753
+ lease = @active_leases[name]
754
+ return @lease_cache[name] if lease && lease_unexpired?(lease: lease)
755
+
756
+ @lease_cache.delete(name)
757
+ nil
758
+ end
759
+
760
+ def lease_unexpired?(lease:, **)
761
+ lease[:expires_at].nil? || lease[:expires_at] > Time.now
762
+ end
763
+
764
+ def lease_renewable?(lease:, **)
765
+ lease && lease[:renewable] && lease_unexpired?(lease: lease)
766
+ end
767
+
768
+ def renewal_delay(lease:, **)
769
+ ttl = lease[:lease_duration] || 0
770
+ return lease[:fetched_at] + RENEWAL_CHECK_INTERVAL - Time.now unless ttl.positive?
771
+ return 0 unless lease[:expires_at]
772
+
773
+ lease[:expires_at] - Time.now - (ttl * 0.5)
774
+ end
775
+
776
+ def setting_target(path:, settings: nil)
777
+ return unless path.is_a?(Array) && !path.empty?
778
+
779
+ target = settings
780
+ target ||= Legion::Settings.loader&.settings if defined?(Legion::Settings) && Legion::Settings.respond_to?(:loader)
781
+ path.each_with_index do |segment, index|
782
+ return nil unless valid_setting_key?(container: target, key: segment)
783
+ return target.frozen? ? nil : [target, segment] if index == path.length - 1
784
+
785
+ target = target[segment]
786
+ end
787
+ end
788
+
789
+ def valid_setting_key?(container:, key:)
790
+ case container
791
+ when Hash then key.is_a?(Symbol) && container.key?(key)
792
+ when Array then key.is_a?(Integer) && key >= 0 && key < container.length
793
+ end
794
+ end
795
+
796
+ def write_setting(path, value)
797
+ record = @ref_bindings[path]
798
+ target, key = setting_target(path: path, settings: record&.dig(:settings))
799
+ binding = record&.dig(:binding)
800
+ return false unless target && !binding&.frozen?
801
+
802
+ target[key] = value
803
+ binding[:value] = value if binding
804
+ true
805
+ rescue StandardError => e
806
+ handle_exception(e, level: :warn, operation: 'crypt.vault_leases.write_setting', path: path.inspect)
807
+ false
808
+ end
809
+
810
+ def trigger_reconnect(name, generation: @generation, active: nil)
811
+ return false unless lifecycle_current?(generation: generation, active: active)
812
+
813
+ case name.to_sym
814
+ when :rabbitmq
815
+ return true unless defined?(Legion::Transport::Connection)
816
+
817
+ connection = Legion::Transport::Connection
818
+ connection.force_reconnect == true && connection.session_open?
819
+ when :postgresql
820
+ return true unless defined?(Legion::Data::Connection)
821
+
822
+ connection = Legion::Data::Connection
823
+ connection.respond_to?(:reconnect_with_fresh_creds) && connection.reconnect_with_fresh_creds == true
824
+ when :redis
825
+ return true unless defined?(Legion::Cache)
826
+
827
+ cache = Legion::Cache
828
+ result = if cache.respond_to?(:restart)
829
+ cache.restart
830
+ elsif cache.respond_to?(:reconnect)
831
+ cache.reconnect
832
+ end
833
+ result == true && (!cache.respond_to?(:connected?) || cache.connected?)
834
+ else
835
+ true
836
+ end
837
+ rescue StandardError => e
838
+ handle_exception(e, level: :error, operation: 'crypt.vault_leases.trigger_reconnect', lease_name: name)
839
+ false
840
+ end
841
+
842
+ def with_current_state(generation:, active: nil, **)
843
+ @state_mutex.synchronize do
844
+ next false unless lifecycle_current?(generation: generation)
845
+
846
+ next yield unless active
847
+
848
+ result = false
849
+ active.call { result = yield }
850
+ result
851
+ end
852
+ end
853
+
854
+ def lifecycle_current?(generation:, active: nil, **)
855
+ !@stopped && generation == @generation &&
856
+ !(Legion::Crypt.respond_to?(:stopping) && Legion::Crypt.stopping) && (!active || active.call)
857
+ end
858
+
859
+ def running?(generation: @generation)
860
+ @state_mutex.synchronize { @running && lifecycle_current?(generation: generation) }
861
+ end
862
+
863
+ def interruptible_sleep(seconds = nil, generation: @generation)
864
+ @state_mutex.synchronize do
865
+ @wake.wait(@state_mutex, seconds || next_maintenance_delay) if @running && lifecycle_current?(generation: generation)
866
+ end
867
+ end
868
+ end
869
+ end
870
+ end
871
+ end