legion-data 1.10.9 → 1.10.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,9 +2,11 @@
2
2
 
3
3
  require 'legion/logging/helper'
4
4
  require 'legion/data/auth_failure_handler'
5
+ require 'legion/data/connection_pool'
5
6
 
6
7
  require 'fileutils'
7
8
  require 'sequel'
9
+ require 'uri'
8
10
 
9
11
  module Legion
10
12
  module Data
@@ -142,6 +144,9 @@ module Legion
142
144
  @closed = true
143
145
  @file.close unless @file.closed?
144
146
  end
147
+ rescue IOError, SystemCallError => e
148
+ handle_exception(e, level: :warn, handled: true, operation: :query_file_close, path: @path)
149
+ nil
145
150
  end
146
151
 
147
152
  private
@@ -152,9 +157,7 @@ module Legion
152
157
 
153
158
  @file.puts "[#{Time.now.strftime('%Y-%m-%d %H:%M:%S.%L')}] #{level} #{message}"
154
159
  end
155
- rescue IOError => e
156
- return nil if @closed || @file.closed?
157
-
160
+ rescue IOError, SystemCallError => e
158
161
  handle_exception(e, level: :warn, handled: true, operation: :query_file_write, path: @path)
159
162
  nil
160
163
  end
@@ -163,43 +166,61 @@ module Legion
163
166
  class << self
164
167
  include Legion::Logging::Helper
165
168
 
166
- attr_accessor :sequel
169
+ attr_reader :sequel
170
+
171
+ def sequel=(database)
172
+ if database
173
+ pool = database.pool
174
+ pool.extend(Legion::Data::ConnectionPool) unless pool.is_a?(Legion::Data::ConnectionPool)
175
+ end
176
+ Legion::Data.reset_privileges!
177
+ @sequel = database
178
+ end
167
179
 
168
180
  def adapter
169
181
  @adapter ||= Legion::Settings[:data][:adapter]&.to_sym || :sqlite
170
182
  end
171
183
 
172
184
  def setup
185
+ Legion::Settings[:data][:connected] = false
186
+ @sequel&.disconnect
187
+ @sequel = nil
173
188
  @adapter = Legion::Settings[:data][:adapter]&.to_sym || :sqlite
174
189
  opts = sequel_opts
175
190
  log.info("Legion::Data::Connection setup adapter=#{adapter}")
176
191
  @fallback_active = false
177
- @sequel = if adapter == :sqlite
178
- ::Sequel.connect(opts.merge(adapter: :sqlite, database: sqlite_path))
179
- else
180
- attempted_adapter = adapter
181
- begin
182
- ::Sequel.connect(connection_opts_for(adapter: attempted_adapter, opts: opts))
183
- rescue UnresolvedCredentialError => e
184
- handle_exception(e, level: :fatal, handled: false, operation: :data_connect)
185
- raise
186
- rescue StandardError => e
187
- raise unless dev_fallback?
188
-
189
- log.error("Legion::Data FALLING BACK TO SQLITE — #{attempted_adapter} network DB connection failed: #{e.message}")
190
- log.error("Legion::Data WARNING: Data written to SQLite will NOT be visible when #{attempted_adapter} reconnects. " \
191
- 'Apollo knowledge, audit logs, and other DB-backed services will use a local-only store.')
192
- handle_exception(e, level: :error, handled: true, operation: :shared_connect, fallback: :sqlite)
193
- @adapter = :sqlite
194
- @fallback_active = true
195
- sqlite_opts = sequel_opts
196
- ::Sequel.connect(sqlite_opts.merge(adapter: :sqlite, database: sqlite_path))
197
- end
198
- end
199
- Legion::Settings[:data][:connected] = true
192
+ @replica_servers = []
193
+ self.sequel = if adapter == :sqlite
194
+ ::Sequel.connect(opts.merge(adapter: :sqlite, database: sqlite_path))
195
+ else
196
+ attempted_adapter = adapter
197
+ begin
198
+ ::Sequel.connect(connection_opts_for(adapter: attempted_adapter, opts: opts))
199
+ rescue UnresolvedCredentialError => e
200
+ handle_exception(e, level: :fatal, handled: false, operation: :data_connect)
201
+ raise
202
+ rescue ::Sequel::DatabaseConnectionError => e
203
+ raise unless dev_fallback?
204
+
205
+ log.error("Legion::Data FALLING BACK TO SQLITE — #{attempted_adapter} network DB connection failed: #{e.message}")
206
+ log.error("Legion::Data WARNING: Data written to SQLite will NOT be visible when #{attempted_adapter} reconnects. " \
207
+ 'Apollo knowledge, audit logs, and other DB-backed services will use a local-only store.')
208
+ handle_exception(e, level: :error, handled: true, operation: :shared_connect, fallback: :sqlite)
209
+ @adapter = :sqlite
210
+ @fallback_active = true
211
+ sqlite_opts = sequel_opts
212
+ ::Sequel.connect(sqlite_opts.merge(adapter: :sqlite, database: sqlite_path))
213
+ end
214
+ end
215
+ @replica_servers = @sequel.opts.fetch(:servers, {}).keys
200
216
  log_connection_info
201
217
  configure_extensions
202
- connect_with_replicas
218
+ Legion::Settings[:data][:connected] = true
219
+ rescue StandardError, LoadError => e
220
+ shutdown
221
+ @sequel = nil
222
+ handle_exception(e, level: :fatal, handled: false, operation: :connection_setup)
223
+ raise
203
224
  end
204
225
 
205
226
  # Returns connection metadata for health checks and diagnostics.
@@ -210,6 +231,7 @@ module Legion
210
231
  connected: Legion::Settings[:data][:connected],
211
232
  fallback_active: @fallback_active || false,
212
233
  configured_adapter: Legion::Settings[:data][:adapter]&.to_sym || :sqlite,
234
+ shutdown: @sequel&.pool&.shutdown_status,
213
235
  sequel_alive: (begin
214
236
  !@sequel&.test_connection.nil?
215
237
  rescue StandardError => e
@@ -276,15 +298,26 @@ module Legion
276
298
  end
277
299
 
278
300
  def shutdown
279
- if @sequel
280
- timeout = Legion::Settings[:data][:shutdown_timeout]
281
- force_disconnect_pool(timeout: timeout)
301
+ database = @sequel
302
+ logger = @query_file_logger
303
+ on_closed = proc do
304
+ logger&.close
305
+ if @sequel.equal?(database)
306
+ @query_file_logger = nil
307
+ @fallback_active = false
308
+ @replica_servers = []
309
+ Legion::Settings[:data][:connected] = false
310
+ end
311
+ log.info 'Legion::Data connection closed'
282
312
  end
283
- @query_file_logger&.close
284
- @query_file_logger = nil
285
- @fallback_active = false
286
- Legion::Settings[:data][:connected] = false
287
- log.info 'Legion::Data connection closed'
313
+ unless database
314
+ on_closed.call
315
+ return true
316
+ end
317
+
318
+ closed = database.pool.shutdown(timeout: Legion::Settings[:data][:shutdown_timeout], &on_closed)
319
+ log.warn("Legion::Data shutdown incomplete: #{database.pool.shutdown_status}") unless closed
320
+ closed
288
321
  end
289
322
 
290
323
  def reconnect_with_fresh_creds
@@ -305,6 +338,7 @@ module Legion
305
338
  old_user = @sequel.opts[:user]
306
339
  @sequel.opts[:user] = new_user
307
340
  @sequel.opts[:password] = new_pass
341
+ Legion::Data.reset_privileges!
308
342
 
309
343
  @sequel.disconnect
310
344
  expire_all_pooled_connections
@@ -328,28 +362,6 @@ module Legion
328
362
  handle_exception(e, level: :warn, handled: true, operation: :expire_all_pooled_connections)
329
363
  end
330
364
 
331
- def connect_with_replicas
332
- return unless adapter == :postgres
333
-
334
- replica_url = Legion::Settings[:data][:read_replica_url]
335
- replica_list = Array(Legion::Settings[:data][:replicas]).dup
336
-
337
- replica_list.prepend(replica_url) if replica_url && !replica_url.empty?
338
- replica_list.uniq!
339
- replica_list.compact!
340
-
341
- return if replica_list.empty?
342
-
343
- @sequel.extension(:server_block)
344
-
345
- replica_list.each_with_index do |url, idx|
346
- @sequel.add_servers("read_#{idx}": url)
347
- end
348
-
349
- @replica_servers = replica_list.each_with_index.map { |_, idx| :"read_#{idx}" }
350
- log.debug "Registered #{@replica_servers.size} read replica(s)"
351
- end
352
-
353
365
  def read_server
354
366
  return :default if @replica_servers.nil? || @replica_servers.empty?
355
367
 
@@ -386,6 +398,20 @@ module Legion
386
398
  creds
387
399
  end
388
400
 
401
+ def configure_sqlite_busy_handler(connection:, timeout:)
402
+ timeout_seconds = timeout.to_f / 1000
403
+ deadline = nil
404
+ connection.busy_handler do |attempt|
405
+ now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
406
+ deadline = now + timeout_seconds if attempt.zero?
407
+ remaining = deadline - now
408
+ next false unless remaining.positive?
409
+
410
+ sleep([0.001, remaining].min)
411
+ true
412
+ end
413
+ end
414
+
389
415
  def creds_builder(final_creds = {})
390
416
  final_creds.merge! Legion::Data::Settings.creds(adapter)
391
417
  final_creds.merge! Legion::Settings[:data][:creds] if Legion::Settings[:data][:creds].is_a? Hash
@@ -437,28 +463,6 @@ module Legion
437
463
  data_settings[:dev_mode] == true && data_settings[:dev_fallback] != false
438
464
  end
439
465
 
440
- def force_disconnect_pool(timeout:)
441
- pool = @sequel.pool
442
- in_use = pool.size - (pool.respond_to?(:available_connections) ? pool.available_connections.size : 0)
443
-
444
- if in_use.positive?
445
- log.warn("Legion::Data shutdown: #{in_use} connection(s) still checked out, waiting up to #{timeout}s")
446
- deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout
447
- while ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) < deadline
448
- break if pool.size <= (pool.respond_to?(:available_connections) ? pool.available_connections.size : 0)
449
-
450
- sleep 0.1
451
- end
452
- remaining = pool.size - (pool.respond_to?(:available_connections) ? pool.available_connections.size : 0)
453
- log.warn("Legion::Data shutdown: force-disconnecting #{remaining} wedged connection(s)") if remaining.positive?
454
- end
455
-
456
- @sequel.disconnect
457
- rescue StandardError => e
458
- handle_exception(e, level: :warn, handled: true, operation: :force_disconnect_pool)
459
- @sequel&.disconnect rescue nil # rubocop:disable Style/RescueModifier
460
- end
461
-
462
466
  def sqlite_path
463
467
  path = Legion::Settings[:data][:creds][:database] || 'legionio.db'
464
468
  return path if File.absolute_path?(path)
@@ -472,9 +476,25 @@ module Legion
472
476
  connection_opts = opts.merge(adapter: adapter, **creds_builder)
473
477
  validate_no_unresolved_uris!(connection_opts)
474
478
  connection_opts[:preconnect] = false if adapter != :sqlite && dev_fallback?
479
+ if adapter == :postgres
480
+ servers = postgres_replica_servers
481
+ connection_opts[:servers] = servers unless servers.empty?
482
+ end
475
483
  connection_opts
476
484
  end
477
485
 
486
+ def postgres_replica_servers
487
+ data = Legion::Settings[:data]
488
+ urls = [data[:read_replica_url], *Array(data[:replicas])].compact.reject(&:empty?).uniq
489
+ urls.each_with_index.to_h do |url, index|
490
+ uri = URI.parse(url)
491
+ raise ArgumentError, 'Read replicas require PostgreSQL URLs' unless %w[postgres postgresql].include?(uri.scheme)
492
+
493
+ options = ::Sequel::Database.send(:options_from_uri, uri).compact
494
+ [:"read_#{index}", options]
495
+ end
496
+ end
497
+
478
498
  def validate_no_unresolved_uris!(connection_opts)
479
499
  %i[user username password].each do |key|
480
500
  value = connection_opts[key]
@@ -518,10 +538,18 @@ module Legion
518
538
  val = data.key?(key) && !data[key].nil? ? data[key] : defaults[key]
519
539
  opts[key] = val unless val.nil?
520
540
  end
541
+ opts[:after_connect] = proc { |connection| configure_sqlite_busy_handler(connection: connection, timeout: opts[:timeout]) } if adapter == :sqlite
542
+ opts[:connect_sqls] = postgres_connect_sqls(data) if adapter == :postgres
521
543
 
522
544
  opts
523
545
  end
524
546
 
547
+ def postgres_connect_sqls(data)
548
+ %i[statement_timeout lock_timeout].filter_map do |setting|
549
+ "SET #{setting} = '#{data[setting].to_i}ms'" if data[setting]
550
+ end
551
+ end
552
+
525
553
  def tuning_stats(data)
526
554
  tuning = {}
527
555
 
@@ -655,7 +683,7 @@ module Legion
655
683
  if adapter == :postgres
656
684
  Sequel.extension(:pg_array)
657
685
  @sequel.extension(:pg_array)
658
- install_postgres_session_timeouts
686
+ @sequel.extension(:server_block) unless @replica_servers.empty?
659
687
  end
660
688
 
661
689
  if data[:connection_validation] != false
@@ -670,28 +698,15 @@ module Legion
670
698
 
671
699
  install_auth_failure_hook
672
700
  rescue StandardError => e
673
- handle_exception(e, level: :warn, handled: true, operation: :configure_extensions, adapter: adapter)
674
- end
675
-
676
- def install_postgres_session_timeouts
677
- data = Legion::Settings[:data]
678
- stmt_timeout = data[:statement_timeout]
679
- lck_timeout = data[:lock_timeout]
680
-
681
- @sequel.pool.after_connect = proc do |conn|
682
- conn.exec("SET statement_timeout = '#{stmt_timeout.to_i}ms'") if stmt_timeout
683
- conn.exec("SET lock_timeout = '#{lck_timeout.to_i}ms'") if lck_timeout
684
- end
685
-
686
- log.info("Postgres session timeouts: statement_timeout=#{stmt_timeout}ms, lock_timeout=#{lck_timeout}ms")
687
- rescue StandardError => e
688
- handle_exception(e, level: :warn, handled: true, operation: :install_postgres_session_timeouts)
701
+ handle_exception(e, level: :error, handled: false, operation: :configure_extensions, adapter: adapter)
702
+ raise
689
703
  end
690
704
 
691
705
  def install_auth_failure_hook
692
706
  Legion::Data::AuthFailureHandler.install(@sequel)
693
707
  rescue StandardError => e
694
- handle_exception(e, level: :warn, handled: true, operation: :install_auth_failure_hook)
708
+ handle_exception(e, level: :error, handled: false, operation: :install_auth_failure_hook)
709
+ raise
695
710
  end
696
711
 
697
712
  def build_data_logger
@@ -0,0 +1,83 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'legion/logging/helper'
4
+
5
+ module Legion
6
+ module Data
7
+ module ConnectionPool
8
+ include Legion::Logging::Helper
9
+
10
+ def self.extended(pool)
11
+ pool.instance_variable_set(:@shutdown_mutex, Mutex.new)
12
+ pool.instance_variable_set(:@shutdown_condition, ConditionVariable.new)
13
+ pool.instance_variable_set(:@shutdown_owners, {}.compare_by_identity)
14
+ pool.instance_variable_set(:@shutdown_state, :open)
15
+ end
16
+
17
+ def hold(server = :default)
18
+ owner = Sequel.current
19
+ @shutdown_mutex.synchronize do
20
+ raise Sequel::Error, 'Database pool is shutting down or closed' unless @shutdown_state == :open || @shutdown_owners.key?(owner)
21
+
22
+ @shutdown_owners[owner] = @shutdown_owners.fetch(owner, 0) + 1
23
+ end
24
+ begin
25
+ super
26
+ ensure
27
+ finish = @shutdown_mutex.synchronize do
28
+ @shutdown_owners[owner] -= 1
29
+ @shutdown_owners.delete(owner) if @shutdown_owners[owner].zero?
30
+ if @shutdown_state == :draining && @shutdown_owners.empty?
31
+ @shutdown_state = :closing
32
+ true
33
+ end
34
+ end
35
+ finish_shutdown if finish
36
+ end
37
+ end
38
+
39
+ def shutdown(timeout:, &on_closed)
40
+ deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout
41
+ finish = @shutdown_mutex.synchronize do
42
+ @shutdown_callback ||= on_closed
43
+ @shutdown_state = :draining if %i[open failed].include?(@shutdown_state)
44
+ if @shutdown_state == :draining && @shutdown_owners.empty?
45
+ @shutdown_state = :closing
46
+ true
47
+ end
48
+ end
49
+ finish_shutdown if finish
50
+ @shutdown_mutex.synchronize do
51
+ until @shutdown_state == :closed
52
+ remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
53
+ return false if remaining <= 0 || @shutdown_state == :failed || @shutdown_owners.key?(Sequel.current)
54
+
55
+ @shutdown_condition.wait(@shutdown_mutex, remaining)
56
+ end
57
+ end
58
+ true
59
+ end
60
+
61
+ def shutdown_status
62
+ @shutdown_mutex.synchronize { { state: @shutdown_state, active_requests: @shutdown_owners.size } }
63
+ end
64
+
65
+ private
66
+
67
+ def finish_shutdown
68
+ disconnect
69
+ @shutdown_callback.call
70
+ @shutdown_mutex.synchronize do
71
+ @shutdown_state = :closed
72
+ @shutdown_condition.broadcast
73
+ end
74
+ rescue StandardError => e
75
+ @shutdown_mutex.synchronize do
76
+ @shutdown_state = :failed
77
+ @shutdown_condition.broadcast
78
+ end
79
+ handle_exception(e, level: :error, handled: true, operation: :pool_shutdown)
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Legion
4
+ module Data
5
+ module DatabaseLock
6
+ def self.with_claim(database:, table:, key:)
7
+ options = { savepoint: true }
8
+ options[:mode] = :immediate if database.database_type == :sqlite
9
+ database.transaction(**options) do
10
+ claim = database[table].where(key)
11
+ # InnoDB needs exclusive duplicate-key locks to avoid shared-lock upgrade deadlocks.
12
+ claim = claim.on_duplicate_key_update(*key.keys) if database.database_type == :mysql
13
+ claim.insert(key)
14
+ result = yield
15
+ # The unique-key lock lasts until commit; historical keys need no retained rows.
16
+ claim.delete
17
+ result
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -15,8 +15,7 @@ module Legion
15
15
  end
16
16
 
17
17
  def key_for(tenant_id: nil)
18
- cache_key = tenant_id || '__default__'
19
- @key_cache[cache_key] ||= derive_key(tenant_id)
18
+ @key_cache[tenant_id] ||= derive_key(tenant_id)
20
19
  end
21
20
 
22
21
  def clear_cache!
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'legion/logging/helper'
4
4
  require 'digest'
5
+ require 'legion/data/database_lock'
5
6
 
6
7
  module Legion
7
8
  module Data
@@ -21,11 +22,11 @@ module Legion
21
22
  return { error: 'db unavailable' } unless db_ready?
22
23
 
23
24
  conn = Legion::Data.connection
24
- conn.transaction do
25
- last = conn[:governance_events]
26
- .where(stream_id: stream)
27
- .order(Sequel.desc(:sequence_number))
28
- .first
25
+ Legion::Data::DatabaseLock.with_claim(database: conn, table: :append_claims,
26
+ key: { kind: 'event', stream_id: stream }) do
27
+ records = conn[:governance_events].where(stream_id: stream).order(Sequel.desc(:sequence_number))
28
+ records = records.for_update unless conn.database_type == :sqlite
29
+ last = records.first
29
30
 
30
31
  seq = (last&.[](:sequence_number) || 0) + 1
31
32
  prev_hash = last&.[](:event_hash) || ('0' * 64)
@@ -32,6 +32,11 @@ module Legion
32
32
  val = data.key?(key) && !data[key].nil? ? data[key] : sqlite_defaults[key]
33
33
  opts[key] = val unless val.nil?
34
34
  end
35
+ opts[:timeout] = data[:timeout] || 30_000
36
+ opts[:after_connect] = proc do |connection|
37
+ Legion::Data::Connection.configure_sqlite_busy_handler(connection: connection, timeout: opts[:timeout])
38
+ connection.execute_batch('PRAGMA synchronous=NORMAL; PRAGMA cache_size=-20000; PRAGMA mmap_size=268435456')
39
+ end
35
40
 
36
41
  if local_settings[:query_log]
37
42
  log_path = File.join(Legion::Data::Connection::QUERY_LOG_DIR, 'data-local-query.log')
@@ -46,14 +51,11 @@ module Legion
46
51
 
47
52
  @connection = ::Sequel.connect(opts)
48
53
  @connection.run('PRAGMA journal_mode=WAL')
49
- @connection.run('PRAGMA busy_timeout=30000')
50
- @connection.run('PRAGMA synchronous=NORMAL')
51
- @connection.run('PRAGMA cache_size=-20000')
52
- @connection.run('PRAGMA mmap_size=268435456')
53
- @connected = true
54
54
  run_migrations
55
- log.info "Legion::Data::Local connected to #{db_file} (WAL mode, 30s busy_timeout, 20MB cache, 256MB mmap)"
56
- rescue StandardError => e
55
+ @connected = true
56
+ log.info "Legion::Data::Local connected to #{db_file} (WAL mode, #{opts[:timeout]}ms busy timeout, 20MB cache, 256MB mmap)"
57
+ rescue StandardError, LoadError => e
58
+ shutdown
57
59
  handle_exception(e, level: :error, handled: false, operation: :local_setup, database: db_file)
58
60
  raise
59
61
  end
@@ -74,6 +76,10 @@ module Legion
74
76
  @registered_migrations ||= {}
75
77
  @registered_migrations[name] = path
76
78
  run_single_migration(name, path) if connected?
79
+ rescue StandardError, LoadError => e
80
+ shutdown
81
+ handle_exception(e, level: :error, handled: false, operation: :register_migrations, name: name, path: path)
82
+ raise
77
83
  end
78
84
 
79
85
  def registered_migrations
@@ -142,8 +148,9 @@ module Legion
142
148
 
143
149
  table = :"schema_migrations_#{name}"
144
150
  ::Sequel::TimestampMigrator.new(@connection, path, table: table).run
145
- rescue StandardError => e
146
- handle_exception(e, level: :warn, handled: true, operation: :local_migration, name: name, path: path)
151
+ rescue StandardError, LoadError => e
152
+ handle_exception(e, level: :error, handled: false, operation: :local_migration, name: name, path: path)
153
+ raise
147
154
  end
148
155
 
149
156
  def local_settings
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ Sequel.migration do
4
+ up do
5
+ alter_table(:tasks_archive) do
6
+ add_column :function_id, Integer
7
+ add_column :parent_id, Integer
8
+ add_column :master_id, Integer
9
+ add_column :payload, String, text: true
10
+ add_column :worker_id, String, size: 36
11
+ add_column :tenant_id, String, size: 64
12
+ add_column :depth, Integer
13
+ add_column :cancelled_at, DateTime
14
+ add_column :task_delay, Integer
15
+ add_column :created_at, DateTime
16
+ add_column :idempotency_key, String, size: 64
17
+ add_column :idempotency_expires_at, DateTime
18
+ end
19
+ end
20
+
21
+ down do
22
+ alter_table(:tasks_archive) do
23
+ drop_column :function_id
24
+ drop_column :parent_id
25
+ drop_column :master_id
26
+ drop_column :payload
27
+ drop_column :worker_id
28
+ drop_column :tenant_id
29
+ drop_column :depth
30
+ drop_column :cancelled_at
31
+ drop_column :task_delay
32
+ drop_column :created_at
33
+ drop_column :idempotency_key
34
+ drop_column :idempotency_expires_at
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ Sequel.migration do
4
+ change do
5
+ create_table(:append_claims) do
6
+ String :kind, size: 16, null: false
7
+ String :stream_id, size: 255, null: false
8
+ primary_key %i[kind stream_id]
9
+ end
10
+ end
11
+ end
@@ -30,7 +30,10 @@ module Legion
30
30
  def load
31
31
  log.info 'Loading Legion::Data::Models'
32
32
  @loaded_models ||= []
33
+ database = Legion::Data::Connection.sequel
34
+ Sequel::Model.db = database
33
35
  require_sequel_models(models)
36
+ bind_database(namespace: Legion::Data::Model, database: database)
34
37
  Legion::Settings[:data][:models][:loaded] = true
35
38
  end
36
39
 
@@ -49,6 +52,22 @@ module Legion
49
52
  handle_exception(e, level: :fatal, operation: :load_sequel_model, model: model)
50
53
  raise e unless Legion::Settings[:data][:models][:continue_on_load_fail]
51
54
  end
55
+
56
+ private
57
+
58
+ def bind_database(namespace:, database:)
59
+ namespace.constants(false).each do |name|
60
+ value = namespace.const_get(name, false)
61
+ if value.is_a?(Class) && value < Sequel::Model
62
+ next if value.db.equal?(database)
63
+
64
+ value.set_dataset(database[value.table_name])
65
+ value.association_reflections.each_value { |reflection| reflection[:cache]&.clear }
66
+ elsif value.is_a?(Module)
67
+ bind_database(namespace: value, database: database)
68
+ end
69
+ end
70
+ end
52
71
  end
53
72
  end
54
73
  end
@@ -8,9 +8,11 @@ module Legion
8
8
  module Model
9
9
  class AuditLog < Sequel::Model(:audit_log)
10
10
  include Legion::Logging::Helper
11
+ extend Legion::Logging::Helper
11
12
 
12
13
  VALID_EVENT_TYPES = %w[runner_execution lifecycle_transition].freeze
13
14
  VALID_STATUSES = %w[success failure denied].freeze
15
+ HASH_CHAIN_COLUMNS = [:id, :record_hash, *Legion::Data::AuditLogHashChain::CANONICAL_FIELDS].freeze
14
16
 
15
17
  def validate
16
18
  super
@@ -39,9 +41,39 @@ module Legion
39
41
  Legion::Data::AuditLogHashChain.compute_hash(record)
40
42
  end
41
43
 
42
- def self.verify_chain(records = order(:created_at, :id).all)
43
- Legion::Data::AuditLogHashChain.verify(records)
44
+ def self.verify_chain(records = nil)
45
+ return Legion::Data::AuditLogHashChain.verify(records) unless records.nil?
46
+
47
+ hot = dataset.naked.select(*HASH_CHAIN_COLUMNS)
48
+ unless audit_archive_exists?
49
+ result = Legion::Data::AuditLogHashChain.verify(hot.order(:created_at, :id))
50
+ # Retention may create the archive and move rows after the first presence check.
51
+ return result unless audit_archive_exists?
52
+ end
53
+
54
+ warm = db[:audit_log_archive].select(*HASH_CHAIN_COLUMNS)
55
+ Legion::Data::AuditLogHashChain.verify(hot.union(warm, all: true).order(:created_at, :id))
56
+ end
57
+
58
+ def self.audit_archive_exists?
59
+ db.transaction(savepoint: :only) { db[:audit_log_archive].get(Sequel::NULL) }
60
+ true
61
+ rescue Sequel::DatabaseError => e
62
+ cause = e.wrapped_exception
63
+ missing = case db.database_type
64
+ when :postgres
65
+ cause.is_a?(::PG::UndefinedTable)
66
+ when :mysql
67
+ cause.respond_to?(:error_number) && cause.error_number == 1146
68
+ when :sqlite
69
+ cause.is_a?(::SQLite3::SQLException) && cause.message.start_with?('no such table: audit_log_archive')
70
+ end
71
+ raise unless missing
72
+
73
+ handle_exception(e, level: :debug, handled: true, operation: :audit_archive_presence)
74
+ false
44
75
  end
76
+ private_class_method :audit_archive_exists?
45
77
  end
46
78
  end
47
79
  end