legion-data 1.10.17 → 1.11.7

data/lib/legion/data/migration.rb

@@ -16,6 +16,30 @@ module Legion
             return
           end
 
+          if connection.database_type == :postgres
+            role_state = connection.fetch(<<~SQL).first
+              SELECT session_user,
+                     current_user,
+                     current_setting('role') AS active_role,
+                     to_regrole('legionio_migrator')::text AS migrator_role,
+                     CASE
+                       WHEN to_regrole('legionio_migrator') IS NULL THEN NULL
+                       ELSE pg_has_role(session_user, 'legionio_migrator', 'MEMBER')
+                     END AS migrator_member,
+                     CASE
+                       WHEN to_regrole('legionio_migrator') IS NULL THEN NULL
+                       ELSE pg_has_role(session_user, 'legionio_migrator', 'SET')
+                     END AS migrator_set,
+                     has_schema_privilege(session_user, 'public', 'CREATE') AS session_create,
+                     has_schema_privilege(current_user, 'public', 'CREATE') AS current_create,
+                     CASE
+                       WHEN to_regrole('legionio_migrator') IS NULL THEN NULL
+                       ELSE has_schema_privilege('legionio_migrator', 'public', 'CREATE')
+                     END AS migrator_create
+            SQL
+            Legion::Logging.unknown "migration role state=#{role_state.inspect}"
+          end
+
           Legion::Settings[:data][:migrations][:version] = Sequel::Migrator.run(connection, path, **)
           log.info("Legion::Data::Migration ran successfully to version #{Legion::Settings[:data][:migrations][:version]}")
           Legion::Settings[:data][:migrations][:ran] = true

data/lib/legion/data/migrations/137_preserve_task_archive_fields.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:tasks_archive) do
+      add_column :function_id, Integer
+      add_column :parent_id, Integer
+      add_column :master_id, Integer
+      add_column :payload, String, text: true
+      add_column :worker_id, String, size: 36
+      add_column :tenant_id, String, size: 64
+      add_column :depth, Integer
+      add_column :cancelled_at, DateTime
+      add_column :task_delay, Integer
+      add_column :created_at, DateTime
+      add_column :idempotency_key, String, size: 64
+      add_column :idempotency_expires_at, DateTime
+    end
+  end
+
+  down do
+    alter_table(:tasks_archive) do
+      drop_column :function_id
+      drop_column :parent_id
+      drop_column :master_id
+      drop_column :payload
+      drop_column :worker_id
+      drop_column :tenant_id
+      drop_column :depth
+      drop_column :cancelled_at
+      drop_column :task_delay
+      drop_column :created_at
+      drop_column :idempotency_key
+      drop_column :idempotency_expires_at
+    end
+  end
+end

data/lib/legion/data/migrations/138_add_append_claims.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:append_claims) do
+      String :kind, size: 16, null: false
+      String :stream_id, size: 255, null: false
+      primary_key %i[kind stream_id]
+    end
+  end
+end

data/lib/legion/data/migrations/139_create_task_logs.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:task_logs) do
+      primary_key :id
+      String :uuid, size: 36, null: false, unique: true
+      String :access_scope, size: 20, null: false, default: 'global', index: true
+      foreign_key :identity_principal_id, :identity_principals, null: true, on_delete: :set_null, on_update: :cascade
+      foreign_key :identity_id, :identities, null: true, on_delete: :set_null, on_update: :cascade
+      String :identity_canonical_name, size: 255, null: true, index: true
+      foreign_key :task_id, :tasks, null: true, on_delete: :set_null, on_update: :cascade
+      foreign_key :node_id, :nodes, null: true, on_delete: :set_null, on_update: :cascade
+      foreign_key :function_id, :functions, null: true, on_delete: :set_null, on_update: :cascade
+      String :entry, text: true, null: false
+      DateTime :created, null: false, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :inserted_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :updated_at, null: true
+      index :identity_principal_id
+      index :task_id
+      index :node_id
+      index :function_id
+      index :created
+    end
+  end
+end

data/lib/legion/data/model.rb

@@ -11,9 +11,9 @@ module Legion
         attr_reader :loaded_models
 
         def models
-          %w[extension function relationship chain task runner node setting digital_worker
+          %w[extension function relationship chain task task_log runner node setting digital_worker
              apollo_entry apollo_relation apollo_expertise apollo_access_log audit_log
-             audit_record extract_step_timing
+             audit_record governance_event append_claim data_archive extract_step_timing
              identity/identity identity/principal identity/providers identity/group
              identity/group_memberships identity/audit_log
              apollo/entries apollo/relation apollo/access_log apollo/expertise

data/lib/legion/data/models/append_claim.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Model
+      class AppendClaim < Sequel::Model(:append_claims)
+      end
+    end
+  end
+end

data/lib/legion/data/models/audit_log.rb

@@ -8,9 +8,11 @@ module Legion
     module Model
       class AuditLog < Sequel::Model(:audit_log)
         include Legion::Logging::Helper
+        extend Legion::Logging::Helper
 
         VALID_EVENT_TYPES = %w[runner_execution lifecycle_transition].freeze
         VALID_STATUSES    = %w[success failure denied].freeze
+        HASH_CHAIN_COLUMNS = [:id, :record_hash, *Legion::Data::AuditLogHashChain::CANONICAL_FIELDS].freeze
 
         def validate
           super
@@ -39,9 +41,39 @@ module Legion
           Legion::Data::AuditLogHashChain.compute_hash(record)
         end
 
-        def self.verify_chain(records = order(:created_at, :id).all)
-          Legion::Data::AuditLogHashChain.verify(records)
+        def self.verify_chain(records = nil)
+          return Legion::Data::AuditLogHashChain.verify(records) unless records.nil?
+
+          hot = dataset.naked.select(*HASH_CHAIN_COLUMNS)
+          unless audit_archive_exists?
+            result = Legion::Data::AuditLogHashChain.verify(hot.order(:created_at, :id))
+            # Retention may create the archive and move rows after the first presence check.
+            return result unless audit_archive_exists?
+          end
+
+          warm = db[:audit_log_archive].select(*HASH_CHAIN_COLUMNS)
+          Legion::Data::AuditLogHashChain.verify(hot.union(warm, all: true).order(:created_at, :id))
+        end
+
+        def self.audit_archive_exists?
+          db.transaction(savepoint: :only) { db[:audit_log_archive].get(Sequel::NULL) }
+          true
+        rescue Sequel::DatabaseError => e
+          cause = e.wrapped_exception
+          missing = case db.database_type
+                    when :postgres
+                      cause.is_a?(::PG::UndefinedTable)
+                    when :mysql
+                      cause.respond_to?(:error_number) && cause.error_number == 1146
+                    when :sqlite
+                      cause.is_a?(::SQLite3::SQLException) && cause.message.start_with?('no such table: audit_log_archive')
+                    end
+          raise unless missing
+
+          handle_exception(e, level: :debug, handled: true, operation: :audit_archive_presence)
+          false
         end
+        private_class_method :audit_archive_exists?
       end
     end
   end

data/lib/legion/data/models/data_archive.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Model
+      class DataArchive < Sequel::Model(:data_archive)
+      end
+    end
+  end
+end

data/lib/legion/data/models/governance_event.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Model
+      class GovernanceEvent < Sequel::Model(:governance_events)
+      end
+    end
+  end
+end

data/lib/legion/data/models/task.rb

@@ -2,15 +2,15 @@
 
 require 'digest'
 require 'legion/json'
+require 'legion/data/task/status'
 require 'time'
 
 module Legion
   module Data
     module Model
       class Task < Sequel::Model
-        TERMINAL_STATUSES = %w[
-          completed complete failed error cancelled canceled timeout timed_out
-        ].freeze
+        TERMINAL_STATUSES = (Legion::Data::Task::Status::TERMINAL_STATUSES +
+                             Legion::Data::Task::Status::LEGACY_TERMINAL_STATUSES).freeze
 
         many_to_one :function
         many_to_one :relationship
@@ -60,9 +60,9 @@ module Legion
           when Array
             value.map { |item| canonical_payload(item) }
           when Time
-            value.utc.iso8601(6)
+            value.getutc.iso8601(6)
           when DateTime
-            value.to_time.utc.iso8601(6)
+            value.to_time.getutc.iso8601(6)
           else
             value
           end

data/lib/legion/data/models/task_log.rb

@@ -1,11 +1,27 @@
 # frozen_string_literal: true
 
+require 'securerandom'
+
 module Legion
   module Data
     module Model
       class TaskLog < Sequel::Model
         many_to_one :task
         many_to_one :node
+        many_to_one :function
+
+        def self.insert(values)
+          create(values).id
+        end
+
+        def before_create
+          self[:uuid] ||= SecureRandom.uuid
+          self[:created] ||= Time.now
+          self[:created] = self[:created].getlocal if self[:created].is_a?(Time)
+          self[:created_at] ||= self[:created]
+          self[:created_at] = self[:created_at].getlocal if self[:created_at].is_a?(Time)
+          super
+        end
       end
     end
   end

data/lib/legion/data/retention.rb

@@ -12,24 +12,36 @@ module Legion
       class << self
         include Legion::Logging::Helper
 
-        def archive_old_records(table:, date_column: nil, archive_after_days: DEFAULT_ARCHIVE_AFTER_DAYS)
+        def archive_old_records(table:, date_column: nil, archive_after_days: DEFAULT_ARCHIVE_AFTER_DAYS, batch_size: 1000)
           db = Legion::Data.connection
           return { archived: 0, table: table } unless db
+          raise ArgumentError, 'batch_size must be a positive Integer' unless batch_size.is_a?(Integer) && batch_size.positive?
 
           date_column = resolve_date_column(table, date_column)
           cutoff = Time.now - (archive_after_days * 86_400)
           archive_table = archive_table_name(table)
+          primary_keys = db.schema(table).filter_map { |name, info| name if info[:primary_key] }
+          raise Sequel::Error, "#{table} needs a primary key for bounded retention" if primary_keys.empty?
 
           ensure_archive_table!(db, table, archive_table)
 
           count = 0
-          db.transaction do
-            records = db[table].where(Sequel.identifier(date_column) < cutoff)
-            count = records.count
-            if count.positive?
-              db[archive_table].multi_insert(records.all)
-              records.delete
+          loop do
+            archived = db.transaction(savepoint: true, **(db.database_type == :sqlite ? { mode: :immediate } : {})) do
+              records = db[table].where(Sequel.identifier(date_column) < cutoff).order(*primary_keys).limit(batch_size)
+              records = records.for_update unless db.database_type == :sqlite
+              rows = records.all
+              next 0 if rows.empty?
+
+              db[archive_table].multi_insert(rows)
+              deleted = delete_archived_rows(db: db, table: table, rows: rows, primary_keys: primary_keys)
+              raise Sequel::Error, "Retention copied #{rows.length} rows but deleted #{deleted} from #{table}" unless deleted == rows.length
+
+              deleted
             end
+            break if archived.zero?
+
+            count += archived
           end
 
           log.info "Archived #{count} row(s) from #{table}" if count.positive?
@@ -113,18 +125,37 @@ module Legion
           end
         end
 
+        def delete_archived_rows(db:, table:, rows:, primary_keys:)
+          if primary_keys.one?
+            key = primary_keys.first
+            return db[table].where(key => rows.map { |row| row.fetch(key) }).delete
+          end
+
+          # SQLite expands composite-key predicates into OR expressions with a depth limit.
+          identity_batch_size = db.database_type == :sqlite ? 100 : rows.length
+          rows.each_slice(identity_batch_size).sum do |batch|
+            db[table].where(primary_keys => batch.map { |row| row.values_at(*primary_keys) }).delete
+          end
+        end
+
         def ensure_archive_table!(db, source_table, archive_table)
           return if db.table_exists?(archive_table)
 
           source_schema = db.schema(source_table).to_h
 
           log.info "Creating archive table #{archive_table} from #{source_table}"
-          db.create_table(archive_table) do
-            source_schema.each do |col_name, col_info|
-              column col_name, col_info[:db_type]
+          db.transaction(savepoint: true) do
+            db.create_table?(archive_table) do
+              source_schema.each do |col_name, col_info|
+                column col_name, col_info[:db_type]
+              end
+              DateTime :archived_at, default: Sequel::CURRENT_TIMESTAMP
             end
-            DateTime :archived_at, default: Sequel::CURRENT_TIMESTAMP
           end
+        rescue Sequel::UniqueConstraintViolation => e
+          raise unless db.table_exists?(archive_table)
+
+          handle_exception(e, level: :debug, handled: true, operation: :concurrent_archive_creation, table: archive_table)
         end
       end
     end

data/lib/legion/data/settings.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'legion/logging/helper'
+require 'legion/settings'
 
 module Legion
   module Data
@@ -76,10 +77,6 @@ module Legion
           keepalives_count:              3,
           tcp_user_timeout:              15_000,
 
-          # Shutdown: max seconds to wait for checked-out connections to return before
-          # force-disconnecting them.
-          shutdown_timeout:              5,
-
           # Adapter-specific (nil = use adapter built-in default)
           connect_timeout:               nil,
           read_timeout:                  nil,
@@ -95,6 +92,7 @@ module Legion
 
           # Grouped settings
           creds:                         creds,
+          tls:                           { enabled: false },
           cache:                         cache,
           migrations:                    migrations,
           models:                        models,
@@ -141,7 +139,10 @@ module Legion
       def self.archival
         {
           retention_days:  90,
+          warm_after_days: 7,
+          cold_after_days: 90,
           batch_size:      1000,
+          tables:          %w[tasks metering_records],
           storage_backend: nil
         }
       end
@@ -158,8 +159,4 @@ module Legion
   end
 end
 
-begin
-  Legion::Settings.merge_settings('data', Legion::Data::Settings.default) if Legion.const_defined?('Settings', false)
-rescue StandardError => e
-  Legion::Data::Settings.handle_exception(e, level: :fatal, operation: :merge_settings)
-end
+Legion::Settings.register_library(:data, Legion::Data::Settings.default)

data/lib/legion/data/spool.rb

@@ -61,8 +61,10 @@ module Legion
 
         def read(sub_namespace)
           sorted_files(sub_namespace).each_with_object([]) do |path, events|
-            event = load_event_file(path, sub_namespace)
-            events << event if event
+            with_locked_event(path) do |file|
+              event = load_event_file(file, path, sub_namespace)
+              events << event if event
+            end
           end
         rescue StandardError => e
           handle_exception(e, level: :error, handled: false, operation: :spool_read, sub_namespace: sub_namespace)
@@ -72,13 +74,16 @@ module Legion
         def flush(sub_namespace)
           count = 0
           path = nil
-          sorted_files(sub_namespace).each do |path|
-            event = load_event_file(path, sub_namespace)
-            next unless event
-
-            yield event
-            File.delete(path)
-            count += 1
+          sorted_files(sub_namespace).each do |entry_path|
+            path = entry_path
+            with_locked_event(path) do |file|
+              event = load_event_file(file, path, sub_namespace)
+              next unless event
+
+              yield event
+              File.delete(path)
+              count += 1
+            end
           end
           log.info "Spool drained #{count} item(s) from #{sub_namespace}" if count.positive?
           count
@@ -95,8 +100,10 @@ module Legion
           dir = sub_dir(sub_namespace)
           return unless Dir.exist?(dir)
 
-          Dir[File.join(dir, '*.json')].each { |f| File.delete(f) }
-          log.info "Spool cleared #{sub_namespace}"
+          sorted_files(sub_namespace).each do |path|
+            with_locked_event(path) { |_file| File.delete(path) }
+          end
+          log.info "Spool cleared unclaimed events from #{sub_namespace}"
         rescue StandardError => e
           handle_exception(e, level: :error, handled: false, operation: :spool_clear, sub_namespace: sub_namespace)
           raise
@@ -115,11 +122,24 @@ module Legion
           Dir.glob(File.join(dir, '*.json'), sort: true)
         end
 
-        def load_event_file(path, sub_namespace)
-          ::JSON.parse(File.binread(path), symbolize_names: true)
-        rescue Errno::ENOENT => e
-          log.debug("spool event file not found: #{path}: #{e.message}")
-          nil
+        def with_locked_event(path)
+          file = begin
+            File.open(path, 'rb')
+          rescue Errno::ENOENT => e
+            log.debug("spool event file not found: #{path}: #{e.message}")
+            nil
+          end
+          return unless file
+          return unless file.flock(File::LOCK_EX | File::LOCK_NB)
+          return unless File.identical?(file, path)
+
+          yield file
+        ensure
+          file&.close
+        end
+
+        def load_event_file(file, path, sub_namespace)
+          ::JSON.parse(file.read, symbolize_names: true)
         rescue ::JSON::ParserError, EOFError, ArgumentError => e
           quarantine_corrupt_file(path, sub_namespace, e)
           nil

data/lib/legion/data/storage_tiers.rb

@@ -10,56 +10,62 @@ module Legion
       class << self
         include Legion::Logging::Helper
 
-        def archive_to_warm(table:, age_days: 90, batch_size: 1000)
-          return { archived: 0, reason: 'no_connection' } unless Legion::Data.connection
-          return { archived: 0, reason: 'no_archive_table' } unless Legion::Data.connection.table_exists?(:data_archive)
+        def archive_to_warm(model:, age_days: 90, batch_size: 1000)
+          return { archived: 0, reason: 'no_connection' } unless Legion::Data.connected?
+
+          table = model.table_name
+          db = model.db
+          raise ArgumentError, 'source model must use the DataArchive database' unless db.equal?(Legion::Data::Model::DataArchive.db)
 
           cutoff = Time.now - (age_days * 86_400)
-          records = Legion::Data.connection[table].where { created_at < cutoff }.limit(batch_size).all
-          return { archived: 0 } if records.empty?
+          archived = db.transaction(savepoint: true, **(db.database_type == :sqlite ? { mode: :immediate } : {})) do
+            candidates = model.where { created_at < cutoff }.order(:id).limit(batch_size)
+            candidates = candidates.for_update unless db.database_type == :sqlite
+            records = candidates.all
+            next 0 if records.empty?
 
-          Legion::Data.connection.transaction do
             records.each do |record|
-              Legion::Data.connection[:data_archive].insert(
+              Legion::Data::Model::DataArchive.create(
                 source_table: table.to_s, source_id: record[:id],
-                data:         Legion::JSON.dump(record),
+                data:         Legion::JSON.dump(record.values),
                 tier:         TIERS[:warm],
                 archived_at:  Time.now.utc
               )
             end
 
             ids = records.map { |r| r[:id] }
-            Legion::Data.connection[table].where(id: ids).delete
+            model.where(id: ids).delete
           end
+          return { archived: 0 } if archived.zero?
 
-          log.info "Archived #{records.size} row(s) from #{table} to warm tier"
-          { archived: records.size, table: table.to_s }
+          log.info "Archived #{archived} row(s) from #{table} to warm tier"
+          { archived: archived, table: table.to_s }
         rescue StandardError => e
           handle_exception(e, level: :error, handled: false, operation: :archive_to_warm, table: table, age_days: age_days, batch_size: batch_size)
           raise
         end
 
         def export_to_cold(age_days: 365, batch_size: 5000)
-          return { exported: 0 } unless Legion::Data.connection&.table_exists?(:data_archive)
+          return { exported: 0 } unless Legion::Data.connected?
 
           cutoff = Time.now - (age_days * 86_400)
-          records = Legion::Data.connection[:data_archive]
-                                .where(tier: TIERS[:warm])
-                                .where { archived_at < cutoff }
-                                .limit(batch_size).all
+          records = Legion::Data::Model::DataArchive
+                    .where(tier: TIERS[:warm])
+                    .where { archived_at < cutoff }
+                    .limit(batch_size).all
           return { exported: 0 } if records.empty?
 
           ids = records.map { |r| r[:id] }
-          Legion::Data.connection[:data_archive].where(id: ids).update(tier: TIERS[:cold])
+          Legion::Data::Model::DataArchive.where(id: ids).update(tier: TIERS[:cold])
           log.info "Exported #{records.size} row(s) to cold tier"
-          { exported: records.size, data: records }
+          { exported: records.size, data: records.map(&:values) }
         rescue StandardError => e
           handle_exception(e, level: :error, handled: false, operation: :export_to_cold, age_days: age_days, batch_size: batch_size)
           raise
         end
 
         def stats
-          return {} unless Legion::Data.connection&.table_exists?(:data_archive)
+          return {} unless Legion::Data.connected?
 
           { warm: count_tier(:warm), cold: count_tier(:cold) }
         rescue StandardError => e
@@ -70,7 +76,7 @@ module Legion
         private
 
         def count_tier(tier)
-          Legion::Data.connection[:data_archive].where(tier: TIERS[tier]).count
+          Legion::Data::Model::DataArchive.where(tier: TIERS[tier]).count
         rescue StandardError => e
           handle_exception(e, level: :warn, handled: true, operation: :storage_tiers_count, tier: tier)
           0

data/lib/legion/data/task/status.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Task
+      module Status
+        STATUSES = {
+          scheduled: 'task.scheduled', queued: 'task.queued', running: 'task.running', completed: 'task.completed',
+          exception: 'task.exception', failed: 'task.failed', skipped: 'task.skipped',
+          cancelled: 'task.cancelled', expired: 'task.expired', delayed: 'task.delayed',
+          dispatching: 'task.dispatching', multiplied: 'task.multiplied', retry_pending: 'task.retry_pending',
+          unknown: 'task.unknown', conditioner_queued: 'conditioner.queued', conditioner_failed: 'conditioner.failed',
+          conditioner_exception: 'conditioner.exception', transformer_queued: 'transformer.queued',
+          transformer_succeeded: 'transformer.succeeded', transformer_validation_failed: 'transformer.validation_failed',
+          transformer_failed: 'transformer.failed', transformer_exception: 'transformer.exception', blocked: 'task.blocked'
+        }.freeze
+        TERMINAL_STATUSES = STATUSES.values_at(:completed, :failed, :exception, :skipped, :cancelled, :expired, :multiplied,
+                                               :conditioner_failed, :conditioner_exception, :transformer_validation_failed,
+                                               :transformer_failed, :transformer_exception, :unknown, :blocked).freeze
+        LEGACY_TERMINAL_STATUSES = %w[completed complete failed error cancelled canceled timeout timed_out exception skipped expired multiplied].freeze
+
+        module Helper
+          def task_status(name:)
+            raise ArgumentError, 'Task status name must be a String or Symbol' unless name.is_a?(String) || name.is_a?(Symbol)
+
+            STATUSES.fetch(name.to_sym) { raise ArgumentError, "Unknown task status name #{name.inspect}" }
+          end
+
+          def task_terminal_status?(status:)
+            TERMINAL_STATUSES.include?(status) || LEGACY_TERMINAL_STATUSES.include?(status)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.10.17'
+    VERSION = '1.11.7'
   end
 end