@zq-silk/yui 0.12.0 → 0.12.2

package/dist/storage/sqliteSchema.js

@@ -37,6 +37,10 @@ export const TELEMETRY_RUN_CAP = 50_000;
  * `synchronous`/`foreign_keys`/`busy_timeout` are per-connection PRAGMAs set by
  * the store; `journal_mode=WAL` is a persistent database property set on open.
  * The migration itself only contains schema objects.
+ *
+ * This released SQL string, including its comments, is checksummed persistent
+ * history and must remain byte-for-byte unchanged. The `operator-notification`
+ * projection kind it creates is retained only for old layout-7 Homes.
  */
 const MIGRATION_1_SQL = `
 -- Global catalog and coordination (§4.1) -------------------------------------
@@ -327,8 +331,7 @@ CREATE TABLE IF NOT EXISTS events (
 );
 CREATE INDEX IF NOT EXISTS idx_events_type_time ON events(task_id, type, occurred_at);
 
--- Per-task scheduler projections. operator-notification is a retired legacy
--- kind retained only so old layout-7 Homes can be read by the upgrade path.
+-- Per-task scheduler projections (leaderFailure, operatorNotification).
 CREATE TABLE IF NOT EXISTS task_projections (
   task_id    TEXT NOT NULL,
   kind       TEXT NOT NULL CHECK (kind IN ('leader-failure','operator-notification')),
@@ -931,6 +934,12 @@ CREATE TABLE IF NOT EXISTS context_snapshots (
 CREATE UNIQUE INDEX IF NOT EXISTS idx_context_snapshots_scope_sequence
   ON context_snapshots(task_id, scope, COALESCE(scope_ref, ''), sequence);
 `;
+/**
+ * The single forward migration history for current SQLite Homes. New durable
+ * layout, aggregate, and record changes belong here and must be expressible as
+ * one atomic in-place transaction. The separate logical migration registry is
+ * only the bridge from valid pre-SQLite Homes and is allowed to rebuild them.
+ */
 const MIGRATIONS = [
     { version: 1, axis: "layout", sql: MIGRATION_1_SQL },
     { version: 2, axis: "record", recordKind: "durableJob+capability-grant+release-workflow", sql: MIGRATION_2_SQL },
@@ -1137,41 +1146,68 @@ function validatePendingProjectionBackfillSources(db) {
         throw new SqliteSchemaMigrationError(`pending Turn completion source is invalid for ${mismatch.task_id}/${mismatch.role_name}`, "backfill");
     }
 }
+/**
+ * Inspect a database's checksummed migration prefix without changing it.
+ * Pending versions are a supported upgrade state; malformed, gapped, changed,
+ * or future ledger entries remain explicit integrity failures.
+ */
+export function inspectSqliteSchemaMigrations(db) {
+    const ledgerWasCreated = ensureMigrationLedger(db, "validate");
+    const applied = validateAppliedMigrations(db, ledgerWasCreated);
+    const pendingVersions = MIGRATIONS
+        .filter((migration) => !applied.has(migration.version))
+        .map((migration) => migration.version);
+    if (pendingVersions.length === 0)
+        validateSchemaObjects(db);
+    const current = MIGRATIONS[applied.size - 1];
+    const target = MIGRATIONS.at(-1);
+    if (current === undefined) {
+        throw new SqliteSchemaMigrationError("schema_migrations ledger has no current head");
+    }
+    return {
+        currentVersion: applied.size,
+        currentChecksum: checksum(current.sql),
+        targetVersion: SQLITE_SCHEMA_VERSION,
+        targetChecksum: checksum(target.sql),
+        pendingVersions
+    };
+}
 /**
  * Apply or validate schema migrations without letting an ordinary open mutate
  * an existing authoritative database.
  *
- * In `apply` mode each migration runs in its own transaction: the DDL and the
- * `schema_migrations` bookkeeping commit atomically, so a crash mid-migration
- * rolls back and the disposable staged database can be rebuilt cleanly.
- * `validate` mode rejects a pending version before executing any migration.
+ * In `apply` mode every pending DDL/data step and every ledger row runs in one
+ * outer transaction. The database therefore advances to the release version
+ * as one commit or remains entirely at its previous version. `validate` mode
+ * rejects a pending version before executing any migration.
  */
 export function migrateSqliteSchema(db, options) {
-    const ledgerWasCreated = ensureMigrationLedger(db, options.mode);
-    // Validate the complete ledger before touching any pending migration.  This
-    // prevents a manually altered or partially recorded ledger from silently
-    // skipping the partial-index/projection migrations added after a valid Home.
-    const applied = validateAppliedMigrations(db, ledgerWasCreated);
-    const pending = MIGRATIONS.filter((migration) => !applied.has(migration.version));
-    if (options.mode === "validate" && pending.length > 0) {
-        throw new SqliteSchemaMigrationError(`pending SQLite schema migration ${pending[0].version}; `
-            + "run the offline staged upgrade before opening this database", "admission");
-    }
-    const newlyApplied = [];
-    for (const migration of MIGRATIONS) {
-        if (applied.has(migration.version))
-            continue;
-        const apply = db.transaction(() => {
+    const migrate = () => {
+        const ledgerWasCreated = ensureMigrationLedger(db, options.mode);
+        // Validate the complete ledger before touching any pending migration. This
+        // prevents a manually altered or partially recorded ledger from silently
+        // skipping a later schema/data step.
+        const applied = validateAppliedMigrations(db, ledgerWasCreated);
+        const pending = MIGRATIONS.filter((migration) => !applied.has(migration.version));
+        if (options.mode === "validate" && pending.length > 0) {
+            throw new SqliteSchemaMigrationError(`pending SQLite schema migration ${pending[0].version}; `
+                + "run the explicit storage upgrade before opening this database", "admission");
+        }
+        const newlyApplied = [];
+        for (const migration of pending) {
             if (migration.version === 13)
                 validatePendingProjectionBackfillSources(db);
             db.exec(migration.sql);
             db.prepare(`INSERT INTO schema_migrations (version, axis, record_kind, applied_at, checksum)
          VALUES (?, ?, ?, ?, ?)`).run(migration.version, migration.axis, migration.recordKind ?? null, new Date().toISOString(), checksum(migration.sql));
-        });
-        apply();
-        newlyApplied.push(migration.version);
-    }
-    validateSchemaObjects(db);
+            newlyApplied.push(migration.version);
+        }
+        validateSchemaObjects(db);
+        return newlyApplied;
+    };
+    const newlyApplied = options.mode === "apply" && !db.inTransaction
+        ? db.transaction(migrate)()
+        : migrate();
     return { applied: newlyApplied, version: SQLITE_SCHEMA_VERSION };
 }
 /** The names of every table the schema creates (for tests/introspection). */

package/dist/storage/sqliteStore.js

@@ -55,7 +55,7 @@ import { assertHomeWritable } from "./upgradeFence.js";
 import { CURRENT_CONFIG_SCHEMA_VERSION, CURRENT_PENDING_WAKEUP_SCHEMA_VERSION, CURRENT_WORK_MAILBOX_SCHEMA_VERSION, executionLaneActiveRunKey, executionLaneActiveRunKeyParts, StorageConflictError, StorageCancelledError, StorageRecordError, FileTaskStore, storedCapabilityGrant, storedPublicationReference, storedReleaseWorkflow, isValidCapabilityGrantTransition, isValidReleaseWorkflowTransition, validateYuiConfig } from "./taskStore.js";
 import { publicationExternalKey } from "../task/publicationReference.js";
 import { gateArtifactKey, validateGateArtifact } from "../verification/gateArtifact.js";
-import { migrateSqliteSchema, SqliteSchemaMigrationError, SQLITE_AGGREGATE_VERSION, SQLITE_LAYOUT_VERSION, TELEMETRY_KEEP_PER_GENERATION, TELEMETRY_RUN_CAP } from "./sqliteSchema.js";
+import { inspectSqliteSchemaMigrations, migrateSqliteSchema, SqliteSchemaMigrationError, SQLITE_AGGREGATE_VERSION, SQLITE_LAYOUT_VERSION, TELEMETRY_KEEP_PER_GENERATION, TELEMETRY_RUN_CAP } from "./sqliteSchema.js";
 import { inspectStorageSchema } from "./storageSchema.js";
 /** Read the immutable Home identity without opening a writable Store connection. */
 export function readSqliteHomeIdentity(rootDir, databaseFilename = "yui.db") {
@@ -136,6 +136,7 @@ export class SqliteTaskStore {
     #db;
     #rootDir;
     #migration;
+    #openedSchemaHead;
     #inTransaction = false;
     #dirty = false;
     constructor(rootDir, _options = {}) {
@@ -159,6 +160,11 @@ export class SqliteTaskStore {
             migrateSqliteSchema(this.#db, {
                 mode: this.#migration || !databaseExisted ? "apply" : "validate"
             });
+            const schema = inspectSqliteSchemaMigrations(this.#db);
+            this.#openedSchemaHead = {
+                version: schema.currentVersion,
+                checksum: schema.currentChecksum
+            };
             this.#seedHomeMeta();
             this.#seedConfig();
         }
@@ -215,6 +221,13 @@ export class SqliteTaskStore {
         if (this.#migration)
             return;
         assertHomeWritable(this.#rootDir);
+        const current = this.#db.prepare("SELECT version, checksum FROM schema_migrations ORDER BY version DESC LIMIT 1").get();
+        if (current?.version !== this.#openedSchemaHead.version
+            || current.checksum !== this.#openedSchemaHead.checksum) {
+            throw new SqliteSchemaMigrationError(`open Store schema head ${this.#openedSchemaHead.version}/${this.#openedSchemaHead.checksum} `
+                + `changed to ${String(current?.version)}/${String(current?.checksum)}; `
+                + "reopen the Store with the active Yui version before writing", "admission");
+        }
     }
     #bumpRevision() {
         this.#db.prepare("UPDATE home_meta SET revision = revision + 1, updated_at = ? WHERE id = 1").run(this.#now());
@@ -230,9 +243,12 @@ export class SqliteTaskStore {
             this.#dirty = true;
             return result;
         }
-        this.#prepareWrite();
         this.#begin();
         try {
+            // Acquire SQLite's write reservation before checking the upgrade fence.
+            // A writer that started first commits before the upgrader's final
+            // transactional inventory; a writer that starts later sees the fence.
+            this.#prepareWrite();
             const result = fn();
             if (!this.#migration) {
                 this.#bumpRevision();
@@ -251,9 +267,12 @@ export class SqliteTaskStore {
             return run(this);
         this.#begin();
         try {
+            // Pin the Store's validated schema generation before user code runs.
+            // This prevents a long-lived pre-upgrade Store from executing old SQL
+            // after an in-place migration has committed.
+            this.#prepareWrite();
             const result = run(this);
             if (this.#dirty) {
-                this.#prepareWrite();
                 if (options?.requestId !== undefined) {
                     this.#insertOutbox(options.requestId, options.outboxCommand ?? null);
                 }
@@ -278,9 +297,9 @@ export class SqliteTaskStore {
     async transactionAsync(execute) {
         if (this.#inTransaction)
             return execute(this);
-        this.#prepareWrite();
         this.#begin();
         try {
+            this.#prepareWrite();
             const result = await execute(this);
             if (this.#dirty)
                 this.#bumpRevision();

package/dist/storage/upgrade/homeClassification.js

@@ -30,6 +30,7 @@ import { classifyStorage } from "../migration/index.js";
 import { inspectStorageSchema } from "../storageSchema.js";
 import { FileTaskStore, STORAGE_STATE_FILE, StorageRecordError } from "../taskStore.js";
 import { SqliteTaskStore } from "../sqliteStore.js";
+import { inspectSqliteSchemaMigrations } from "../sqliteSchema.js";
 import { inspectSourceVersionState } from "./homeMigrationTarget.js";
 import { readMigrationReceipt } from "./migrationReceipt.js";
 /**
@@ -111,6 +112,44 @@ export function classifyHome(options) {
             };
         }
     }
+    // Once the logical Home axes are current, the SQLite ledger is the only
+    // remaining upgrade coordinate. A valid applied prefix with pending entries
+    // is upgradeable in place, not corruption and not a reason to rebuild the
+    // database. Ordinary store opens still validate-and-refuse this state; only
+    // the explicit updater is allowed to apply it.
+    if (schema.status === "current"
+        && schema.currentLayoutVersion >= 7
+        && isFullyCurrent(source, latest)) {
+        let sqliteMigration;
+        try {
+            sqliteMigration = inspectLayout7SqliteMigrations(home);
+        }
+        catch (error) {
+            return {
+                ...base,
+                classification: {
+                    verdict: "CORRUPTED",
+                    status: "unsupported",
+                    detail: error instanceof Error ? error.message : String(error)
+                },
+                layoutVersion: schema.currentLayoutVersion,
+                aggregateVersion: schema.currentAggregateSchemaVersion
+            };
+        }
+        if (sqliteMigration.pendingVersions.length > 0) {
+            return {
+                ...base,
+                classification: {
+                    verdict: "MIGRATABLE",
+                    status: "migration-required",
+                    stepCount: sqliteMigration.pendingVersions.length
+                },
+                layoutVersion: schema.currentLayoutVersion,
+                aggregateVersion: schema.currentAggregateSchemaVersion,
+                sqliteMigration
+            };
+        }
+    }
     // The reference graph can only be validated by the strict loader, which only
     // understands the current versions. So run it exactly when every axis is
     // already current (the plan would be a no-op); a throw there is genuine
@@ -136,6 +175,19 @@ export function classifyHome(options) {
             : { incompatibleComponent: incompatibleComponentOf(schema) })
     };
 }
+function inspectLayout7SqliteMigrations(home) {
+    const db = new Database(join(home, "yui.db"), {
+        readonly: true,
+        fileMustExist: true
+    });
+    try {
+        db.pragma("query_only = ON");
+        return inspectSqliteSchemaMigrations(db);
+    }
+    finally {
+        db.close();
+    }
+}
 /**
  * Load a Home whose every axis is already current through the strict store
  * gate to detect real structural/reference corruption. This is only ever

package/dist/storage/upgrade/offlineUpgradeInventory.js

@@ -1,7 +1,10 @@
 import { existsSync, readFileSync, readdirSync } from "node:fs";
 import { join } from "node:path";
+import Database from "better-sqlite3";
 import { scanControllerResourceInventory } from "../../controller/resourceInventoryLinux.js";
 import { STORAGE_STATE_FILE } from "../taskStore.js";
+import { SQLITE_LAYOUT_VERSION } from "../sqliteSchema.js";
+import { inspectStorageSchema } from "../storageSchema.js";
 /** Pure blocker policy shared by synthetic tests and the real read-only scan. */
 export function classifyOfflineUpgradeFacts(facts) {
     const blockers = [];
@@ -45,14 +48,13 @@ export async function inspectOfflineUpgradeInventory(home, environment = process
     const raw = readRawFacts(home, inventory);
     return classifyOfflineUpgradeFacts(raw);
 }
-function readRawFacts(home, inventory) {
+function readRawFacts(home, inventory, state = readDurableStateObject(home), conservativeDurableSessions = false) {
     const runs = [];
     const sessions = [];
     const inFlight = [];
     const pendingCompletions = [];
     const lifecycle = [];
     const unknownRuntime = [];
-    const state = readStateObject(home);
     // Failure to enumerate processes, panes, or sockets means absence cannot be
     // proven. Ownership-load warnings are expected for an old record shape and
     // are handled by the raw durable scan plus unowned live-pane check below.
@@ -61,6 +63,8 @@ function readRawFacts(home, inventory) {
     }
     for (const [taskId, aggregateValue] of Object.entries(record(state.tasks))) {
         const aggregate = record(aggregateValue);
+        if (text(record(aggregate.task).status) === "retired")
+            continue;
         for (const runValue of Object.values(record(aggregate.agentRuns))) {
             const run = record(runValue);
             const roleName = text(run.roleName);
@@ -75,6 +79,7 @@ function readRawFacts(home, inventory) {
                 roleName,
                 set: record(setValue),
                 inventory,
+                conservativeDurableSessions,
                 sessions,
                 inFlight,
                 pendingCompletions
@@ -86,6 +91,7 @@ function readRawFacts(home, inventory) {
             roleName,
             set: record(setValue),
             inventory,
+            conservativeDurableSessions,
             sessions,
             inFlight,
             pendingCompletions
@@ -97,7 +103,7 @@ function readRawFacts(home, inventory) {
         const kind = text(target.kind);
         if (kind !== "role-runtime" && kind !== "global-role-runtime")
             continue;
-        if (mailbox.pending === null && mailbox.processing === null)
+        if (!mailboxHasRuntimeWork(mailbox))
             continue;
         lifecycle.push({
             ...(text(target.taskId) === undefined ? {} : { taskId: text(target.taskId) }),
@@ -170,7 +176,7 @@ function isUndeterminableNativeInventoryWarning(warning) {
         || warning.startsWith("Cannot inspect Unix sockets");
 }
 function collectSessionSet(options) {
-    const { taskId, roleName, set, inventory } = options;
+    const { taskId, roleName, set, inventory, conservativeDurableSessions } = options;
     const currentSessions = Object.entries(record(set.sessions));
     const activeAgentId = text(set.activeAgentId);
     const historyValue = set.history;
@@ -199,7 +205,7 @@ function collectSessionSet(options) {
         };
         options.sessions.push({
             ...base,
-            processState: processState(inventory, base)
+            processState: processState(inventory, base, conservativeDurableSessions)
         });
     }
     const flight = record(set.inFlight);
@@ -245,12 +251,22 @@ function activeCurrentSession(sessions, taskId, roleName) {
     return current.find(({ active }) => active === true)
         ?? (current.length === 1 ? current[0] : undefined);
 }
-function processState(inventory, session) {
+function processState(inventory, session, conservativeDurableSessions) {
     const matching = inventory.resources.filter((resource) => (resource.kind === "agent-session" && resourceMatchesSession(resource, session)));
     if (matching.some((resource) => resource.processes.length > 0))
         return "live";
     if (matching.some((resource) => resource.paneDead === false))
         return "unknown";
+    // The final SQLite gate runs after BEGIN IMMEDIATE, so no older writer can
+    // commit another Session record behind it. Treat every current non-terminal
+    // durable Session as active even when the earlier native-process snapshot
+    // did not contain it; this closes the inventory-scan -> transaction race.
+    if (conservativeDurableSessions
+        && !session.history
+        && session.status !== "stopped"
+        && session.status !== "broken") {
+        return "unknown";
+    }
     return "stopped";
 }
 function resourceMatchesSession(resource, session) {
@@ -272,13 +288,135 @@ function resourceMatchesSession(resource, session) {
         return false;
     return true;
 }
-function readStateObject(home) {
+function readDurableStateObject(home) {
+    const databasePath = join(home, "yui.db");
+    const schema = inspectStorageSchema(home);
+    const sqliteIsAuthoritative = (schema.status === "current" || schema.status === "unsupported")
+        && schema.currentLayoutVersion >= SQLITE_LAYOUT_VERSION;
+    if (sqliteIsAuthoritative && existsSync(databasePath)) {
+        return readSqliteStateObject(databasePath);
+    }
     const path = join(home, STORAGE_STATE_FILE);
     if (!existsSync(path))
         return {};
     const parsed = JSON.parse(readFileSync(path, "utf8"));
     return record(parsed);
 }
+/**
+ * Read only the durable runtime families needed by the offline gate. This raw
+ * adapter deliberately does not open SqliteTaskStore: a valid pending schema
+ * prefix is exactly the state the explicit upgrader must be able to inspect.
+ */
+function readSqliteStateObject(databasePath) {
+    const db = new Database(databasePath, { readonly: true, fileMustExist: true });
+    try {
+        db.pragma("query_only = ON");
+        return readSqliteStateObjectFromDatabase(db);
+    }
+    finally {
+        db.close();
+    }
+}
+/**
+ * Re-check the authoritative SQLite runtime families on an already-open
+ * connection. The in-place upgrader calls this inside its write transaction,
+ * after every older writer has either committed or rolled back.
+ */
+export function inspectSqliteDurableUpgradeInventory(home, db) {
+    return classifyOfflineUpgradeFacts(readRawFacts(home, { resources: [], warnings: [] }, readSqliteStateObjectFromDatabase(db), true));
+}
+function readSqliteStateObjectFromDatabase(db) {
+    const taskRows = db.prepare("SELECT task_id, status FROM tasks_catalog").all();
+    // Explicit retirement is the only isolation boundary for anomalous
+    // historical runtime state. Completed/archived/draft Tasks still fail
+    // closed if they retain an active Run or unfinished lifecycle record.
+    const retiredTaskIds = new Set(taskRows.filter(({ status }) => status === "retired").map(({ task_id }) => task_id));
+    const tasks = Object.fromEntries(taskRows
+        .filter(({ task_id }) => !retiredTaskIds.has(task_id))
+        .map(({ task_id, status }) => [task_id, {
+            task: { status },
+            agentRuns: {},
+            roleSessionSets: {}
+        }]));
+    const taskAggregate = (taskId) => {
+        const existing = tasks[taskId];
+        if (existing !== undefined)
+            return existing;
+        const created = { task: {}, agentRuns: {}, roleSessionSets: {} };
+        tasks[taskId] = created;
+        return created;
+    };
+    for (const row of db.prepare("SELECT task_id, run_id, role_name, status, payload FROM agent_runs").all()) {
+        if (retiredTaskIds.has(row.task_id))
+            continue;
+        const aggregate = taskAggregate(row.task_id);
+        record(aggregate.agentRuns)[row.run_id] = {
+            ...parseJsonRecord(row.payload, "agent_runs.payload"),
+            id: row.run_id,
+            roleName: row.role_name,
+            status: row.status
+        };
+    }
+    for (const row of db.prepare("SELECT task_id, role_name, payload FROM role_session_sets").all()) {
+        if (retiredTaskIds.has(row.task_id))
+            continue;
+        const aggregate = taskAggregate(row.task_id);
+        record(aggregate.roleSessionSets)[row.role_name] = parseJsonRecord(row.payload, "role_session_sets.payload");
+    }
+    const globalRoleSessionSets = {};
+    for (const row of db.prepare("SELECT name, payload FROM global_role_session_sets").all()) {
+        globalRoleSessionSets[row.name] = parseJsonRecord(row.payload, "global_role_session_sets.payload");
+    }
+    const mailboxColumns = new Set(db.pragma("table_info(mailboxes)").map(({ name }) => name));
+    const hasInputDelivery = mailboxColumns.has("input_delivery");
+    const mailboxes = {};
+    for (const row of db.prepare(`SELECT target_key, target_kind, task_id, role_name, processing, pending${hasInputDelivery ? ", input_delivery" : ""} FROM mailboxes`).all()) {
+        if (row.task_id !== null && retiredTaskIds.has(row.task_id))
+            continue;
+        mailboxes[row.target_key] = {
+            target: {
+                kind: row.target_kind,
+                ...(row.task_id === null ? {} : { taskId: row.task_id }),
+                ...(row.role_name === null ? {} : { roleName: row.role_name })
+            },
+            processing: parseNullableJson(row.processing, "mailboxes.processing"),
+            pending: parseNullableJson(row.pending, "mailboxes.pending"),
+            inputDelivery: parseNullableJson(row.input_delivery ?? null, "mailboxes.input_delivery")
+        };
+    }
+    return { tasks, globalRoleSessionSets, mailboxes };
+}
+function mailboxHasRuntimeWork(mailbox) {
+    if (mailbox.processing !== null && mailbox.processing !== undefined)
+        return true;
+    if (mailbox.inputDelivery !== null && mailbox.inputDelivery !== undefined)
+        return true;
+    if (mailbox.pending === null || mailbox.pending === undefined)
+        return false;
+    const pending = record(mailbox.pending);
+    if (Object.hasOwn(pending, "normal") || Object.hasOwn(pending, "userCorrection")) {
+        return pending.normal !== null || pending.userCorrection !== null;
+    }
+    // Pre-v14 mailboxes store one pending batch directly.
+    return Object.keys(pending).length > 0;
+}
+function parseJsonRecord(raw, label) {
+    const parsed = JSON.parse(raw);
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+        throw new Error(`${label} is not a JSON object.`);
+    }
+    return parsed;
+}
+function parseNullableJson(raw, label) {
+    if (raw === null)
+        return null;
+    try {
+        return JSON.parse(raw);
+    }
+    catch (error) {
+        throw new Error(`${label} is not valid JSON.`, { cause: error });
+    }
+}
 function countPendingInbox(home) {
     return [join(home, "runtime", "inbox"), join(home, "runtime", "inbox-invalid")]
         .reduce((total, directory) => total + countDirectory(directory), 0);