legion-data 1.10.28 → 1.11.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec41889e66777b7940c00db2ed4f19ec704145ea27b149604ba82a168fed9e0b
-  data.tar.gz: a1d0eec9f3cd9e84d63a4b1eb4e145330b4cdd6354d742ca6f003ea78558ca4c
+  metadata.gz: a3113304f2f0357f9ea74b932ffc50231616e8eca856e69f5f8c85cb62adad99
+  data.tar.gz: 3387584a5f01c29edcedccc27f1d4ef8a8b3d3147ab4272943a4902cef50e7c9
 SHA512:
-  metadata.gz: b62c13c70c6330f24c17c92c645887f7d506ac83b7978cd33226ca198b0294d49823c0143d778cb3cd03e73e508b87b4b90c1ed229f3fd558e7c5f4b7c9a8710
-  data.tar.gz: 3def1ace7b8172f1b8bd3b9ebd513fae7a65bc10cab304a5f6f7a593f6b44ef3431e93484c12af547b34961107fe02901d0930aefa6a8c8405501c95f7bf3de5
+  metadata.gz: 044e75084811c9b1d71b8364e81b01a7351d6cccc25e93b1866f8e13a55ec5ea50e053ebe63c35efc284acbf962cc6746d599dc8c3d858090e57510c0cc5d533
+  data.tar.gz: f1e3f19d5e29ef237ffb3a2a548d67e7f8b3932ddb9ecf6c7cb6119b59fa44a807c1f299109b82aec4510e260516dfcaf862c26c5c1700aa10d9cbe45bdc5d72

data/.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
   ci:
     permissions:
       contents: read
-    uses: LegionIO/.github/.github/workflows/ci.yml@main
+    uses: LegionIO/.github/.github/workflows/ci.yml@e0fc7364b73f446d4cb9bdc162a66ed34bf0daa7
 
   ci-postgres:
     name: "RSpec (PostgreSQL)"
@@ -61,7 +61,7 @@ jobs:
     permissions:
       contents: read
       security-events: write
-    uses: LegionIO/.github/.github/workflows/security-scan.yml@main
+    uses: LegionIO/.github/.github/workflows/security-scan.yml@e0fc7364b73f446d4cb9bdc162a66ed34bf0daa7
 
   version-changelog:
     permissions:

data/AGENTS.md

@@ -8,13 +8,44 @@ Always run a full `bundle exec rspec` and `bundle exec rubocop -A` and fix all e
 
 HTTP APIs, runtime orchestration, extension behavior, and UI concerns belong in their owning repos. This repo should expose clean model contracts that those layers can call.
 
+## Implementation Rules
+
+- Follow the existing Legion framework patterns. Reuse the established implementation instead of inventing a replacement, generic abstraction, or convenience layer.
+- Keep method-returned settings defaults directly in the existing methods. Do not extract the default/connection settings tree into a constant such as `CONNECTION_DEFAULTS`, a class variable, or a memoized snapshot. Register defaults at boot and read live values through `Legion::Settings`.
+- Application-table reads and writes must use real, named Sequel models. Do not substitute a raw-database facade or move `connection[table]` behind a differently named helper.
+- When a Data-owned table lacks a required model, add a normal `class Name < Sequel::Model(:table_name)` in the existing model namespace and register it in `Legion::Data::Models.models`.
+- Do not introduce `Class.new`, anonymous/dynamic models, fake models, or model-loading workarounds. Use the named model classes in tests too; bind them to the isolated test database and restore their original datasets afterward.
+- Verify table ownership from its migration before adding a model. An existing reference is not proof that legion-data owns or creates the table. Check adapter restrictions as well as the owning repository.
+- Respect startup order: connect, migrate, load models, mark Data connected, then cache and local setup. Do not require model declarations before the database and schema are ready or hide a load-order problem with fallback classes or guards.
+- Use fully qualified model names across module boundaries. Return plain Hashes rather than leaking Sequel model instances across those boundaries.
+- Preserve existing behavior and public contracts unless the authorized change requires updating them. When an API changes, migrate its callers and documentation; do not add compatibility shims.
+- The async persistence contract assumes prefetch 1: one persistence message performs one model insert or update. Do not reintroduce batching, attempt ownership, leases, or transactional outbox coordination for that path.
+- Raw database access is appropriate for migrations and database infrastructure, not a substitute for models in application-table CRUD. Do not expand a model conversion into excluded migration or connection-management work.
+
+## Scope and Interaction
+
+- Work only in the repository and files the user authorizes. Do not sweep adjacent files, convert additional helpers, or redesign unrelated behavior.
+- For per-file conversion work, finish the selected file before moving on: implement with the required models and caller/spec updates, verify, apply the requested patch-version and changelog bump, commit, and push to the existing branch. Then stop and wait for the next file.
+- Necessary supporting changes are not permission for unrelated conversions. If the requested change needs work outside the authorized scope, identify the concrete dependency and obtain direction rather than improvising around it.
+- When the user asks a question during work, answer that question first. Do not treat the question as permission for more edits or replace the answer with an action.
+- Use direct Ruby, Bundler, and shell commands. Do not wrap ordinary project commands in Python or add orchestration scripts for simple edits, tests, commits, or pushes.
+- Work in the existing development checkout and branch. Do not create another branch, worktree, clone, or PR without explicit direction. Never force-push.
+- Preserve unrelated user changes. Stage explicit filenames only; never use `git add .` or `git add -A`.
+- Do not claim completion before the scoped behavior is verified and the requested commit/push is finished. Report only checks actually run.
+
 ## Required Commands
 
 Run from the repo root:
 
+Use an isolated temporary `HOME` for Data tests: `spec/spec_helper.rb` deletes its test database under `~/.legionio/data/`. Never run it against the developer's real HOME. Set `LEGION_DNS_BOOTSTRAP=false` and preserve the real `RBENV_ROOT` when using rbenv. Apply those variables to the test command, not the parent environment.
+
 ```bash
+mkdir -p tmp
+test_home="$(mktemp -d)"
 bundle exec rubocop -A
-bundle exec rspec --format json --out tmp/rspec_results.json --format progress --out tmp/rspec_progress.txt
+env HOME="$test_home" RBENV_ROOT="${RBENV_ROOT:-$HOME/.rbenv}" LEGION_DNS_BOOTSTRAP=false \
+  bundle exec rspec --format json --out tmp/rspec_results.json --format progress --out tmp/rspec_progress.txt \
+  > tmp/rspec_console.log 2>&1
 ```
 
 If RSpec fails, extract failures with:
@@ -25,6 +56,8 @@ jq '[.examples[] | select(.status != "passed") | {file_path, line_number, full_d
 
 Do not run partial RSpec or partial RuboCop for release validation.
 
+For behavior changes, run the changed path against real models and an isolated database as a smoke check; a green suite alone is not proof of the changed path. Do not create mock-echo or default-pinning tests merely to restate the implementation.
+
 ## Migration Rules
 
 - Never edit published migrations. Add a new migration instead.
@@ -60,7 +93,7 @@ Rules:
 
 ## Current Schema Notes
 
-- Migrations currently run through `096`.
+- Use `lib/legion/data/migrations/` as the authoritative migration inventory.
 - `074`-`076` are mainline Apollo/task/extract migrations.
 - `077`-`090` define the LLM lifecycle ledger.
 - `091`-`096` define portable identity companion tables.

data/CHANGELOG.md

@@ -1,5 +1,55 @@
 # Legion::Data Changelog
 
+## [1.11.7] - 2026-11-06
+
+### Changed
+- Simplify `Connection.shutdown` to a direct disconnect: it closes the shared connection and query log and returns `true`. Pool drain ownership, `shutdown_status` reporting, and the `shutdown_timeout` setting were removed along with the `ConnectionPool` module. `Connection.connection_info` no longer reports the `shutdown` key.
+- `Connection.setup` dev-mode fallback now rescues any connection failure (not only `Sequel::DatabaseConnectionError`) before falling back to SQLite.
+- `Models.load` no longer rebinds loaded model datasets after connection setup.
+
+## [1.11.6] - 2026-11-05
+
+### Fixed
+- Reject storage-tier source models connected to a different database before moving rows, preserving the single-database transaction boundary.
+
+## [1.11.5] - 2026-11-05
+
+### Changed
+- Register the `DataArchive` Sequel model and use it for warm/cold storage operations. `StorageTiers.archive_to_warm` now accepts the source `model:` instead of `table:`; source selection and deletion use that model, and exported rows remain plain Hashes.
+
+## [1.11.4] - 2026-11-05
+
+### Changed
+- Register the `AppendClaim` Sequel model and use it for database append locks instead of direct table access. Audit and event callers now supply only the claim key; retain existing transaction and adapter-specific locking behavior.
+
+## [1.11.3] - 2026-11-05
+
+### Changed
+- Use the existing `AuditRecord` Sequel model for audit appends, chain verification, and queries instead of direct connection/table access. Preserve append locking, signatures, metadata, and hash-chain results.
+
+## [1.11.2] - 2026-11-05
+
+### Changed
+- Register the `GovernanceEvent` Sequel model and use it for event-store appends, reads, and chain verification instead of direct table access. Preserve existing stream locking, sequence numbering, and hash-chain behavior.
+
+## [1.11.1] - 2026-11-05
+
+### Changed
+- Persist extraction step timings through the existing `ExtractStepTiming` Sequel model instead of direct connection queries and bulk inserts.
+
+## [1.11.0] - 2026-11-03
+
+### Added
+- Migration 139 provisions `task_logs` and registers its model. Both existing `TaskLog.insert` call shapes initialize UUIDs and event timestamps while preserving the integer return value; deleting parent records retains log evidence. Unsupported manually created tables fail without data loss and require operator reconciliation.
+
+### Changed
+- Simplify task persistence to normal per-message model inserts or updates, assuming RabbitMQ prefetch of one and runtime acknowledgement after a successful write. Remove the added execution facade, attempt/dispatch models, leases, outbox coordination, execution-policy settings, and unmerged coordination schema migrations. Renumber the unmerged task-log migration to 139; migrations already on main remain unchanged.
+- `Legion::Data::Task::Status` retains immutable persisted identities independently of Transport and Core, including scheduled tasks and terminal transformer exceptions.
+- Preserve existing Task idempotency helpers: positional arguments, Sequel model returns, historical key lookup, optional TTL, and key reuse after terminal completion or expiry. Recognize canonical namespaced terminal statuses as well as legacy statuses, so completed runtime tasks do not retain active keys. Frozen timestamp payloads are canonicalized without mutation.
+- Retain ordinary task archival eligibility without execution-specific exclusions.
+- Read runtime configuration from the live `Legion::Settings[:data]` tree, including TLS, local migration permission, archival policy, and cache TTL. Register defaults once at library boot without resetting live connection state during reads.
+- Define connection defaults directly in `Settings.default`, following the framework's method-returned settings Hash pattern; remove `CONNECTION_DEFAULTS`.
+
 ## [1.10.28] - 2026-10-30
 
 ### Changed

data/CLAUDE.md

@@ -1,5 +1,3 @@
-Always run a full `bundle exec rspec` and `bundle exec rubocop -A` and fix all errors before committing.
-
 # legion-data
 
 Persistent storage gem for LegionIO. Owns Sequel database connections, numbered migrations, models, local SQLite state, extract timing persistence, audit/governance storage, identity/RBAC storage, Apollo storage, and the LLM lifecycle ledger.
@@ -7,8 +5,6 @@ Persistent storage gem for LegionIO. Owns Sequel database connections, numbered
 ## Commands
 
 ```bash
-bundle install
-bundle exec rubocop -A
 bundle exec rspec --format json --out tmp/rspec_results.json --format progress --out tmp/rspec_progress.txt
 ```
 
@@ -23,7 +19,7 @@ jq '[.examples[] | select(.status != "passed") | {file_path, line_number, full_d
 - `lib/legion/data/connection.rb`: Sequel connection setup, diagnostics, fallback, query logging.
 - `lib/legion/data/migration.rb`: numbered Sequel migrations.
 - `lib/legion/data/model.rb`: shared model loader.
-- `lib/legion/data/models/`: flat and namespaced Sequel model classes.
+- `lib/legion/data/models/`: flat and namespaced Sequel model classes (`Identity::*`, `Apollo::*`, `RBAC::*`, `LLM::*`).
 - `lib/legion/data/local.rb`: local SQLite database for on-node state.
 - `lib/legion/data/extract.rb`: text extraction and persisted extract step timings.
 - `lib/legion/data/spool.rb`: filesystem write buffer when DB writes are unavailable.
@@ -95,20 +91,6 @@ Sequel.migration do
 end
 ```
 
-### Column Option Reference
-
-| Option | Purpose |
-|--------|---------|
-| `:null` | `false` = NOT NULL, `true` = nullable |
-| `:default` | Default value (use `Sequel::CURRENT_TIMESTAMP` for timestamps) |
-| `:index` | `true` creates an index on this column; pass a Hash for index options |
-| `:unique` | `true` adds a UNIQUE constraint |
-| `:on_delete` | FK behavior: `:cascade`, `:set_null`, `:restrict`, `:no_action` |
-| `:on_update` | FK behavior: `:cascade`, `:set_null`, `:restrict`, `:no_action` |
-| `:key` | For FKs — the referenced column (unnecessary if referencing primary key) |
-| `:size` | Column width for String/Decimal |
-| `:text` | `true` for TEXT columns (unlimited length) |
-
 ### Foreign Key Conventions
 
 ```ruby
@@ -191,17 +173,6 @@ All new tables in legion-data should follow this column convention. Required fie
 - The denormalized string field is always `identity_canonical_name` — not `canonical_name`, `actor`, `agent_id`, or `identity_name`.
 - Existing columns (`agent_id`, `source_agent`, `submitted_by`, `actor`, etc.) on pre-existing tables are **not renamed or removed** — they are historical record and intentionally left as-is. New identity columns are purely additive.
 
-## Current Schema Landmarks
-
-- `074`-`076`: Apollo field width, task idempotency, extract step timings.
-- `077`-`090`: LLM lifecycle ledger.
-- `091`-`096`: portable identity companion tables.
-- `097`: LLM dispatch fields (operation, correlation_id, provider_instance, dispatch_path).
-- `098`-`099`: Legacy identity table drop + rename (portable_identity_* → identity_*).
-- `100`-`102`: Apollo identity columns + access_scope + indexes.
-- `103`-`114`: LLM table identity standardization (access_scope, identity_principal_id, identity_id, identity_canonical_name).
-- Namespaced models: `Identity::*`, `Apollo::*`, `RBAC::*`, `LLM::*`.
-
 ## Boundaries
 
 - REST APIs belong in LegionIO, not this gem.

data/Gemfile

@@ -4,6 +4,9 @@ source 'https://rubygems.org'
 
 gemspec
 
+settings_path = File.expand_path('../legion-settings', __dir__)
+gem 'legion-settings', path: settings_path if File.directory?(settings_path)
+
 group :test do
   gem 'rake'
   gem 'rspec'

data/README.md

@@ -1,8 +1,8 @@
 # legion-data
 
-Persistent database storage for the [LegionIO](https://github.com/LegionIO/LegionIO) async job engine and AI coding assistant platform. Provides database connectivity via the [Sequel ORM](https://sequel.jeremyevans.net/), automatic schema migrations (97 numbered migrations), Sequel models for the full LegionIO control plane, and a parallel local SQLite database for on-node agentic cognitive state.
+Persistent database storage for the [LegionIO](https://github.com/LegionIO/LegionIO) async job engine and AI coding assistant platform. Provides database connectivity via the [Sequel ORM](https://sequel.jeremyevans.net/), numbered schema migrations, Sequel models for the full LegionIO control plane, and a parallel local SQLite database for on-node agentic cognitive state.
 
-**Version**: 1.10.18 | **Ruby**: >= 3.4 | **License**: Apache-2.0
+**Version**: 1.11.6 | **Ruby**: >= 3.4 | **License**: Apache-2.0
 
 ---
 
@@ -25,6 +25,30 @@ The schema is portable by default across SQLite, MySQL, and PostgreSQL. PostgreS
 
 `Legion::Data.setup` raises on fatal connection configuration, migration, or model-loading failures rather than announcing successful startup. Failed setup closes its database connections and clears readiness; fix the underlying error before retrying. Local migrations registered after startup also propagate failures and leave the local database disconnected until setup succeeds.
 
+## Framework Conventions
+
+### Startup and Table Ownership
+
+The shared database starts in this order: **connect → run migrations → load models → mark Data connected**. Cache setup and local storage setup follow. Application code uses the models registered by that lifecycle; model declarations must not be loaded before their tables are migrated.
+
+A table belongs to the component that creates and maintains its schema. A reference in an existing helper does not establish ownership. Before adding a model, identify the owning migration and its supported adapters. A PostgreSQL-only migration does not create its table on SQLite or MySQL.
+
+### Model-backed Table Access
+
+- Read and write application tables through named Sequel models, such as `Legion::Data::Model::Task`, rather than `connection[:tasks]` or `db[:tasks]`.
+- If a Data-owned table is used but lacks a model, add a normal named class under `lib/legion/data/models/` and register it in `Legion::Data::Models.models`.
+- Use the existing model namespace and association conventions. Do not replace named models with `Class.new`, anonymous models, or a facade around raw table access.
+- Shared operations can accept an explicit source model rather than infer a class from a table name. Keep transactional source and destination models on the same database.
+- Keep model instances within database operations. Return plain Hashes across module boundaries, using `.values` or an explicit result Hash.
+
+Migrations and database infrastructure still need direct Sequel database operations for schema changes, connection setup, and adapter/session management. Those operations are distinct from application-table reads and writes.
+
+### Settings
+
+Keep method-returned settings defaults in the existing methods, including `Legion::Data::Settings.default`. Do not extract that settings tree into a constant such as `CONNECTION_DEFAULTS`, a class variable, or a memoized snapshot.
+
+Register defaults through Legion Settings at library startup. Runtime code reads the current `Legion::Settings[:data]` values so configured overrides and settings reloads remain authoritative. Follow the framework's existing loading and access patterns rather than adding a second settings mechanism.
+
 ## Supported Databases
 
 | Database | Adapter | Gem | Default |
@@ -73,9 +97,9 @@ Legion::Data (singleton module)
 │   ├── .connection_info  # Adapter, liveness, and fallback diagnostics
 │   ├── .fallback_active? # True when dev fallback moved a network DB to SQLite
 │   ├── .stats          # Pool metrics, tuning snapshot, adapter-specific DB stats
-│   └── .shutdown       # Drain accepted work; return true only after cleanup
+│   └── .shutdown       # Disconnect shared connection; close query log
 │
-├── Migration           # Auto-migration system (97 numbered Sequel DSL migrations)
+├── Migration           # Numbered Sequel DSL migrations
 │
 ├── Model               # Sequel model autoloader
 │   └── Models: Extension, Function, Runner, Node, Task, TaskLog, Setting,
@@ -151,13 +175,9 @@ Legion::Data::Connection.connection_info
 Legion::Data.shutdown
 ```
 
-### Graceful Database Shutdown
-
-`Legion::Data.shutdown` returns `true` when the shared pool has drained and its connections and query logger are closed. An idle pool closes immediately. Once shutdown starts, new checkouts are rejected; work already accepted by the pool may finish, including nested checkouts in the same execution context.
+### Database Shutdown
 
-If `shutdown_timeout` expires, shutdown returns `false` rather than claiming active connections were forcibly closed. The last accepted checkout performs deferred cleanup. `Connection.connection_info[:shutdown]` reports `state` (`open`, `draining`, `closing`, `closed`, or `failed`) and `active_requests`, which counts accepted execution contexts, including pool waiters—not physical connections. A cleanup failure is logged and can be retried by calling shutdown again.
-
-Shutdown does not cancel SQL, kill worker threads, or close borrowed connections. Configure database statement timeouts separately to bound query execution. To resume work after completed shutdown, call `Legion::Data.setup`; it creates a new pool and rebinds loaded models.
+`Legion::Data.shutdown` disconnects the shared connection, closes the query log, and marks Data as disconnected. It does not cancel in-flight SQL — configure database statement timeouts separately to bound query execution. To resume work, call `Legion::Data.setup`; it creates a new connection.
 
 ### Model Associations
 
@@ -215,21 +235,37 @@ Supported formats: `.txt`, `.md`, `.csv`, `.json`, `.jsonl`, `.html`, `.xlsx`, `
 When migration 076 is present, Extract also persists the same per-step timing rows to `extract_step_timings`
 under the returned `extract_id`.
 
-### Task Idempotency
+### Task Persistence
 
-`Task.idempotency_key_for` computes a stable SHA-256 key from canonical JSON payloads. `Task.create_idempotent`
-returns an existing non-terminal task for the same key inside the optional TTL window, or creates a new task
-with `idempotency_key` and `idempotency_expires_at` populated:
+`Legion::Data::Model::Task` is the normal Sequel model for task persistence.
+The message-processing contract assumes RabbitMQ prefetch of one: each persistence message
+performs one model insert or update, and the runtime acknowledges the message only after
+that operation succeeds. A failed write must not be acknowledged as successful.
+Data does not add message batching, attempt ownership, leases, or outbox coordination.
 
-```ruby
-task = Legion::Data::Model::Task.create_idempotent(
-  { status: 'pending', payload: Legion::JSON.dump(payload) },
-  payload: payload,
-  ttl: 300
-)
-```
+Task statuses retain their existing persisted identities in `Legion::Data::Task::Status`.
+Task rows follow the ordinary archival age, status, and dependency rules.
+
+The existing model idempotency API is unchanged:
+
+- `Legion::Data::Model::Task.idempotency_key_for(payload)` computes a stable SHA-256 key from canonical JSON.
+- `create_idempotent(values, payload: nil, idempotency_key: nil, ttl: nil)` accepts a positional
+  values Hash and returns a Sequel model. Without `ttl`, it does not add an expiry.
+- `find_active_by_idempotency_key(key, now: Time.now)` reads the original `idempotency_key`
+  column and returns a Sequel model or nil. Terminal tasks and expired keys allow a new occurrence.
+
+These helpers do not promise concurrent deduplication or permanent execution identity.
+Frozen timestamp payloads are canonicalized without mutation.
 
-Terminal tasks and expired TTLs allow a replacement. This database helper uses a lookup followed by an insert; concurrent callers are not guaranteed to receive the same task. It does not provide RabbitMQ delivery or execution deduplication.
+Migration 139 supplies the previously unprovisioned `task_logs` table and the normal model
+loader registers `Legion::Data::Model::TaskLog`. Both `TaskLog.create` and `TaskLog.insert`
+generate its public UUID and retain supplied event timestamps. `insert` supports the existing
+positional Hash and keyword-Hash call shapes and returns the inserted integer ID; `create`
+returns a model. Entries are text and may reference a task, node, or function. Removing those
+referenced records nulls the reference rather than deleting the
+log evidence. An unsupported manually created `task_logs` table requires operator
+reconciliation before upgrading: migration 139 fails safely if it already exists and never
+overwrites, truncates, or automatically adopts existing logs.
 
 ### Serialized Audit and Governance Appends
 
@@ -237,6 +273,16 @@ After migration 138, `AuditRecord.append` serializes each chain and `EventStore.
 
 Audit tails, verification, and walks use append IDs rather than timestamps, preserving chain order across writer clock skew. Governance events retain their sequential stream numbering. PostgreSQL lock/statement timeouts and native adapter lock limits bound contention; timeout and unrelated database errors propagate rather than being retried indefinitely. MySQL claims use exclusive duplicate-key locking to avoid InnoDB shared-lock upgrade deadlocks.
 
+### Model-backed Storage Tiers
+
+```ruby
+Legion::Data::StorageTiers.archive_to_warm(
+  model: Legion::Data::Model::Task, age_days: 90, batch_size: 1000
+)
+```
+
+Pass the source Sequel model using `model:`, not a table name. The source must have `id` and `created_at` columns and use the same database as `Legion::Data::Model::DataArchive`. Selected rows are copied as JSON and deleted in the existing transaction. `export_to_cold` returns plain archived-row hashes; `stats` reports warm/cold counts.
+
 ### Bounded Retention
 
 ```ruby
@@ -361,6 +407,8 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
 
 ### Full Configuration Reference
 
+Defaults are registered when the library loads; runtime consumers read the current `Legion::Settings[:data]` tree. User configuration and subsequent settings reloads take precedence, including explicit `false` migration/TLS flags and empty archival table lists. Connection options apply on the next connection setup, while local migration registration and `Archival.archive!` (without an explicit policy) read the current configuration on each call. A full `Legion::Settings.reset!` requires explicit library-default registration again before using Data.
+
 ```json
 {
   "data": {
@@ -390,6 +438,15 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
     "replicas": [],
 
     "creds": { "database": "legionio.db" },
+    "tls": { "enabled": false },
+    "archival": {
+      "retention_days": 90,
+      "warm_after_days": 7,
+      "cold_after_days": 90,
+      "batch_size": 1000,
+      "tables": ["tasks", "metering_records"],
+      "storage_backend": null
+    },
 
     "migrations": {
       "continue_on_fail": false,
@@ -582,7 +639,7 @@ The `Legion::Data::Model::Identity::*`, `Apollo::*`, and `RBAC::*` namespaces pr
 | `csv` (>= 3.2) | CSV extraction handler |
 | `legion-json` | JSON serialization via Legion::JSON |
 | `legion-logging` (>= 1.5.0) | Structured logging |
-| `legion-settings` (>= 1.3.26) | Configuration management |
+| `legion-settings` (>= 1.4.5) | Configuration management |
 | `mysql2` (>= 0.5.5) | MySQL adapter (optional) |
 | `pg` (>= 1.5) | PostgreSQL adapter (optional) |
 
@@ -590,7 +647,7 @@ The `Legion::Data::Model::Identity::*`, `Apollo::*`, and `RBAC::*` namespaces pr
 
 ## Migrations
 
-97 numbered Sequel DSL migrations run automatically on startup (`auto_migrate: true`). Key milestones:
+Numbered Sequel DSL migrations run automatically on startup (`auto_migrate: true`). The files in `lib/legion/data/migrations/` are the authoritative inventory. Key milestones:
 
 | Range | What was added |
 |-------|---------------|
@@ -666,10 +723,20 @@ bundle exec legionio_migrate
 git clone https://github.com/LegionIO/legion-data
 cd legion-data
 bundle install
-bundle exec rspec        # all tests must pass
+bundle exec rspec --format json --out tmp/rspec_results.json --format progress --out tmp/rspec_progress.txt
 bundle exec rubocop -A   # zero offenses expected
 ```
 
+Task-execution lifecycle and concurrency regressions use temporary SQLite storage by default.
+To exercise the same lifecycle suite on PostgreSQL, set `LEGION_DATA_ADAPTER=postgres` and
+`LEGION_DATA_HOST`, `LEGION_DATA_PORT`, `LEGION_DATA_USER`, `LEGION_DATA_PASSWORD`, and
+`LEGION_DATA_DATABASE` for a **disposable test database**, then run the full RSpec command
+above. These examples create and remove a random isolated schema, applying the actual task
+migrations with minimal foreign-key fixtures. This verifies PostgreSQL ownership locks,
+savepoint rollback, unique identities, and reconnect behavior; it does not claim coverage of
+the unrelated historical `pgvector` migration chain. Other suites retain their own database
+fixtures.
+
 This repo also includes a pre-commit configuration:
 
 ```bash

data/legion-data.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'csv', '>= 3.2'
   spec.add_dependency 'legion-json'
   spec.add_dependency 'legion-logging', '>= 1.5.0'
-  spec.add_dependency 'legion-settings', '>= 1.3.26'
+  spec.add_dependency 'legion-settings', '>= 1.4.5'
   spec.add_dependency 'sequel', '>= 5.70'
   spec.add_dependency 'sqlite3', '>= 2.0'
 end

data/lib/legion/data/archival/policy.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'legion/logging/helper'
-
 module Legion
   module Data
     module Archival
@@ -39,21 +37,8 @@ module Legion
           Time.now - (cold_after_days * 86_400)
         end
 
-        class << self
-          include Legion::Logging::Helper
-        end
-
         def self.from_settings
-          return new unless defined?(Legion::Settings)
-
-          data_settings = Legion::Settings[:data]
-          archival = data_settings.is_a?(Hash) ? data_settings[:archival] : nil
-          return new unless archival.is_a?(Hash)
-
-          new(**archival.slice(:warm_after_days, :cold_after_days, :batch_size, :tables))
-        rescue StandardError => e
-          handle_exception(e, level: :warn, handled: true, operation: :policy_from_settings)
-          new
+          new(**Legion::Settings[:data][:archival].slice(:warm_after_days, :cold_after_days, :batch_size, :tables))
         end
       end
     end

data/lib/legion/data/archival.rb

@@ -19,7 +19,7 @@ module Legion
       class << self
         include Legion::Logging::Helper
 
-        def archive!(policy: Policy.new, dry_run: false)
+        def archive!(policy: Policy.from_settings, dry_run: false)
           log.info "Archival run started dry_run=#{dry_run} tables=#{policy.tables.size}"
           results = {}
           policy.tables.each do |table_name|

data/lib/legion/data/archiver.rb

@@ -115,7 +115,8 @@ module Legion
         end
 
         def archive_batch(conn:, table:, cutoff:, batch_size:, now:, storage_backend:)
-          rows = conn[table].where { created_at < cutoff }.limit(batch_size).all
+          candidates = conn[table].where { created_at < cutoff }.limit(batch_size)
+          rows = candidates.all
           return if rows.empty?
 
           batch_id = SecureRandom.uuid

data/lib/legion/data/audit_record.rb

@@ -21,18 +21,16 @@ module Legion
         # @param metadata     [Hash]    optional structured context (serialised to JSON)
         # @param sign         [Boolean] when true, attempt signing via legion-crypt
         def append(chain_id:, content_type:, content_hash:, metadata: {}, sign: false)
-          return { error: 'db unavailable' } unless db_ready?
+          return { error: 'db unavailable' } unless Legion::Data.connected?
 
-          conn = Legion::Data.connection
-          Legion::Data::DatabaseLock.with_claim(database: conn, table: :append_claims,
-                                                key: { kind: 'audit', stream_id: chain_id }) do
-            parent_hash = latest_chain_hash(conn, chain_id)
+          Legion::Data::DatabaseLock.with_claim(key: { kind: 'audit', stream_id: chain_id }) do
+            parent_hash = latest_chain_hash(chain_id: chain_id)
             ts          = truncate_to_us(Time.now)
             ch          = compute_chain_hash(parent_hash, content_hash, ts, content_type)
             sig         = sign ? sign_record(ch) : nil
             meta_json   = metadata.empty? ? nil : Legion::JSON.dump(metadata)
 
-            id = conn[:audit_records].insert(
+            record = Legion::Data::Model::AuditRecord.create(
               chain_id:     chain_id,
               content_type: content_type,
               content_hash: content_hash,
@@ -43,8 +41,8 @@ module Legion
               created_at:   ts
             )
 
-            log.debug "AuditRecord append: chain=#{chain_id} type=#{content_type} id=#{id}"
-            { id: id, chain_id: chain_id, chain_hash: ch, parent_hash: parent_hash }
+            log.debug "AuditRecord append: chain=#{chain_id} type=#{content_type} id=#{record.id}"
+            { id: record.id, chain_id: chain_id, chain_hash: ch, parent_hash: parent_hash }
           end
         end
 
@@ -54,12 +52,12 @@ module Legion
         # @param chain_id [String]
         # @return [Hash] { valid: Boolean, length: Integer, broken_at: Integer? }
         def verify(chain_id:)
-          return { valid: false, error: 'db unavailable' } unless db_ready?
+          return { valid: false, error: 'db unavailable' } unless Legion::Data.connected?
 
-          records = Legion::Data.connection[:audit_records]
-                                .where(chain_id: chain_id)
-                                .order(:id)
-                                .all
+          records = Legion::Data::Model::AuditRecord
+                    .where(chain_id: chain_id)
+                    .order(:id)
+                    .all
 
           prev_hash = GENESIS_HASH
           records.each do |r|
@@ -86,9 +84,9 @@ module Legion
         # @param since    [Time, nil] optional lower bound on created_at
         # @param limit    [Integer]
         def walk(chain_id:, since: nil, limit: 1000)
-          return [] unless db_ready?
+          return [] unless Legion::Data.connected?
 
-          ds = Legion::Data.connection[:audit_records].where(chain_id: chain_id)
+          ds = Legion::Data::Model::AuditRecord.where(chain_id: chain_id)
           ds = ds.where { created_at >= since } if since
           ds.order(:id).limit(limit).all.map { |r| deserialize(r) }
         end
@@ -99,9 +97,9 @@ module Legion
         # @param since        [Time, nil]
         # @param limit        [Integer]
         def query_by_type(content_type:, since: nil, limit: 100)
-          return [] unless db_ready?
+          return [] unless Legion::Data.connected?
 
-          ds = Legion::Data.connection[:audit_records].where(content_type: content_type)
+          ds = Legion::Data::Model::AuditRecord.where(content_type: content_type)
           ds = ds.where { created_at >= since } if since
           ds.order(Sequel.desc(:created_at)).limit(limit).all.map { |r| deserialize(r) }
         end
@@ -137,12 +135,12 @@ module Legion
           ::Time.at(Rational(us, 1_000_000))
         end
 
-        def latest_chain_hash(conn, chain_id)
-          records = conn[:audit_records]
+        def latest_chain_hash(chain_id:)
+          records = Legion::Data::Model::AuditRecord
                     .select(:chain_hash)
                     .where(chain_id: chain_id)
                     .order(Sequel.desc(:id))
-          records = records.for_update unless conn.database_type == :sqlite
+          records = records.for_update unless Legion::Data::Model::AuditRecord.db.database_type == :sqlite
           last = records.first
           last ? last[:chain_hash] : GENESIS_HASH
         end
@@ -169,13 +167,6 @@ module Legion
             created_at:   row[:created_at]
           }
         end
-
-        def db_ready?
-          defined?(Legion::Data) && Legion::Data.connection&.table_exists?(:audit_records)
-        rescue StandardError => e
-          handle_exception(e, level: :warn, handled: true, operation: :audit_record_db_ready?)
-          false
-        end
       end
     end
   end