@zq-silk/yui 0.5.2 → 0.6.0

package/dist/storage/upgrade/sqliteMigrationTarget.js

@@ -0,0 +1,299 @@
+/**
+ * The SQLite-backed {@link MigrationTarget} for the layout 6 -> 7 staged
+ * migration (task-21 §8, work-item-4).
+ *
+ * This target is the seam between the generic, domain-free migration engine
+ * and the SQLite writer. It stages the migrated state into a sidecar
+ * `yui.db.staged` INSIDE the Home (not a sibling copy), verifies the staged
+ * database against an independent re-read of `state.json`, and commits by
+ * swapping `yui.db.staged` -> `yui.db` and advancing `schema.json` to layout 7
+ * in the same coordination critical section.
+ *
+ * Staged orchestration (§8.2):
+ *  - Snapshot:  `readSource` reads `schema.json` + `state.json` read-only.
+ *  - Stage:     `writeFreshOutput` populates `yui.db.staged` from the (possibly
+ *               transformed) snapshot; refuses to overwrite an existing stage.
+ *  - Verify:    `validateCurrentState` independently re-reads `state.json`,
+ *               re-derives the expected state through the registered transforms,
+ *               and compares per-family checksums against the staged database.
+ *  - Commit:    `atomicSwitchWithBackup` swaps the sidecar into `yui.db` (with
+ *               a timestamped backup of any prior database) and advances
+ *               `schema.json` to layout 7.
+ *  - Rollback:  `rollbackSqliteMigration` quarantines `yui.db` and flips
+ *               `schema.json` back to layout 6; `state.json` is never touched.
+ *
+ * The source `state.json` is retained read-only throughout: it is never
+ * overwritten, truncated, or deleted by the migration. This preserves the
+ * rollback path and the §8.4 invariants (no healthy Session reset, no evidence
+ * deleted).
+ */
+import { existsSync, readFileSync, renameSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { writeTextFileAtomically } from "../durableFile.js";
+import { STORAGE_SCHEMA_FILE } from "../storageSchema.js";
+import { STORAGE_STATE_FILE } from "../taskStore.js";
+import { planMigration } from "../migration/planner.js";
+import { AmbiguousSwitchError } from "../migration/index.js";
+import { describeActiveRuntime, homeRuntimeIsActive, inspectHomeRuntime, inspectSourceVersionState, inspectSnapshotVersionState } from "./homeMigrationTarget.js";
+import { COMMITTED_DATABASE_FILENAME, STAGED_DATABASE_FILENAME, computeDbFamilyChecksums, computeStateFamilyChecksums, populateSqliteFromState } from "./sqliteStateMigration.js";
+/** Build the SQLite-backed migration target. */
+export function createSqliteMigrationTarget(options) {
+    const home = options.home;
+    const latest = options.latest;
+    const registry = options.registry;
+    const now = options.now ?? (() => new Date());
+    const callerPid = options.callerPid ?? process.pid;
+    const stagedDbPath = join(home, STAGED_DATABASE_FILENAME);
+    const committedDbPath = join(home, COMMITTED_DATABASE_FILENAME);
+    // The transformed schema manifest is cached during writeFreshOutput so the
+    // switch can advance schema.json without re-reading or re-deriving it.
+    let stagedSchemaManifest = null;
+    return {
+        stagedDbPath,
+        inspectVersions() {
+            const inspected = inspectSourceVersionState(home, latest);
+            if ("corruption" in inspected) {
+                throw new Error(inspected.corruption.detail);
+            }
+            return inspected.source;
+        },
+        detectLiveRuntime() {
+            const signals = inspectHomeRuntime(home, callerPid);
+            if (!homeRuntimeIsActive(signals))
+                return { active: false };
+            return { active: true, detail: describeActiveRuntime(signals) };
+        },
+        readSource() {
+            const manifestRaw = readFileSync(join(home, STORAGE_SCHEMA_FILE), "utf8");
+            const schemaManifest = parseJsonObject(manifestRaw, STORAGE_SCHEMA_FILE);
+            const statePath = join(home, STORAGE_STATE_FILE);
+            const state = existsSync(statePath)
+                ? parseJsonObject(readFileSync(statePath, "utf8"), STORAGE_STATE_FILE)
+                : null;
+            return Object.freeze({ schemaManifest, state });
+        },
+        writeFreshOutput(snapshot) {
+            if (existsSync(stagedDbPath)) {
+                throw new Error(`Refusing to overwrite an existing staged SQLite database: ${stagedDbPath}. ` +
+                    "Discard it and retry.");
+            }
+            // Cache the transformed manifest for the switch's schema.json advancement.
+            // Ensure the layout version is the latest: the orchestrator applies the
+            // 6->7 transform before calling us, but direct callers (tests, drills)
+            // may pass an untransformed snapshot. Setting it here is idempotent.
+            stagedSchemaManifest = {
+                ...snapshot.schemaManifest,
+                storageVersion: latest.layout,
+                updatedAt: now().toISOString()
+            };
+            if (snapshot.state !== null) {
+                populateSqliteFromState(home, snapshot.state, STAGED_DATABASE_FILENAME);
+            }
+            else {
+                // An empty Home (no state.json) still gets a schema-ready database.
+                populateSqliteFromState(home, {}, STAGED_DATABASE_FILENAME);
+            }
+        },
+        rebuildDerivedState(effects) {
+            // The SQLite database is fully normalised by populateSqliteFromState;
+            // there is no separate derived index to rebuild. Echo the declared
+            // effects for the report, mirroring the file target.
+            return { rebuiltEffects: [...effects] };
+        },
+        validateCurrentState() {
+            // Independently re-read state.json from disk (not the in-memory snapshot
+            // used for staging) and re-derive the expected state by applying the
+            // registered transforms. This catches staging corruption, a torn read,
+            // or a concurrent writer that slipped past the quiesce gate.
+            const freshSnapshot = readSourceFresh();
+            const expectedState = deriveExpectedState(freshSnapshot);
+            if (expectedState !== null) {
+                verifyChecksums(expectedState);
+            }
+            const dbChecksums = computeDbFamilyChecksums(home, STAGED_DATABASE_FILENAME);
+            const familyCount = Object.keys(dbChecksums).length;
+            return {
+                checks: [
+                    {
+                        name: "SQLite staged-database checksum verification",
+                        outcome: "passed",
+                        detail: `verified ${familyCount} record families against an independent state.json re-read`
+                    }
+                ]
+            };
+        },
+        atomicSwitchWithBackup() {
+            if (!existsSync(stagedDbPath)) {
+                throw new Error(`No staged SQLite database to promote: ${stagedDbPath}.`);
+            }
+            const stamp = now().toISOString().replace(/[:.]/g, "-");
+            let backupPath;
+            // Phase 1: back up any existing yui.db, then promote the sidecar.
+            try {
+                if (existsSync(committedDbPath)) {
+                    backupPath = join(home, `${COMMITTED_DATABASE_FILENAME}.backup-${stamp}`);
+                    if (existsSync(backupPath)) {
+                        throw new Error(`Refusing to overwrite an existing database backup: ${backupPath}.`);
+                    }
+                    renameSync(committedDbPath, backupPath);
+                }
+                renameSync(stagedDbPath, committedDbPath);
+            }
+            catch (error) {
+                // Pre-promotion failure: restore the original database if we moved it.
+                if (backupPath !== undefined && existsSync(backupPath)) {
+                    try {
+                        renameSync(backupPath, committedDbPath);
+                    }
+                    catch {
+                        throw new AmbiguousSwitchError({
+                            homePath: home,
+                            backupPath,
+                            stagingPath: stagedDbPath,
+                            detail: `SQLite switch failed (${messageOf(error)}) and the automatic rollback also failed. ` +
+                                `The original database is at ${backupPath}; recover manually by renaming it to ${committedDbPath}.`
+                        });
+                    }
+                }
+                throw error;
+            }
+            // Phase 2: advance schema.json to layout 7 in the same critical section.
+            try {
+                if (stagedSchemaManifest !== null) {
+                    writeTextFileAtomically(join(home, STORAGE_SCHEMA_FILE), `${JSON.stringify(stagedSchemaManifest, null, 2)}\n`);
+                }
+            }
+            catch (error) {
+                // The database is promoted but schema.json could not be advanced.
+                // Attempt to restore the original database; if that fails, the Home
+                // is ambiguous and must be recovered manually.
+                try {
+                    if (backupPath !== undefined && existsSync(backupPath)) {
+                        renameSync(backupPath, committedDbPath);
+                    }
+                    else {
+                        rmSync(committedDbPath, { force: true });
+                    }
+                }
+                catch {
+                    throw new AmbiguousSwitchError({
+                        homePath: home,
+                        backupPath: backupPath ?? committedDbPath,
+                        stagingPath: stagedDbPath,
+                        detail: `SQLite database was promoted but schema.json could not be advanced (${messageOf(error)}), ` +
+                            `and the automatic rollback also failed. The database is at ${committedDbPath}; ` +
+                            `recover by advancing schema.json storageVersion to ${latest.layout} or restoring the backup.`
+                    });
+                }
+                throw error;
+            }
+            return {
+                status: "switched",
+                ...(backupPath === undefined ? {} : { backupPath }),
+                detail: `SQLite database promoted to ${committedDbPath} and schema.json advanced to layout ${latest.layout}.`
+            };
+        },
+        discardFreshOutput() {
+            rmSync(stagedDbPath, { force: true });
+            // Clean up WAL/SHM sidecars if the connection left them.
+            rmSync(`${stagedDbPath}-wal`, { force: true });
+            rmSync(`${stagedDbPath}-shm`, { force: true });
+            stagedSchemaManifest = null;
+        }
+    };
+    // -- helpers ---------------------------------------------------------------
+    function readSourceFresh() {
+        const manifestRaw = readFileSync(join(home, STORAGE_SCHEMA_FILE), "utf8");
+        const schemaManifest = parseJsonObject(manifestRaw, STORAGE_SCHEMA_FILE);
+        const statePath = join(home, STORAGE_STATE_FILE);
+        const state = existsSync(statePath)
+            ? parseJsonObject(readFileSync(statePath, "utf8"), STORAGE_STATE_FILE)
+            : null;
+        return Object.freeze({ schemaManifest, state });
+    }
+    /**
+     * Re-derive the expected post-migration state by reading the fresh snapshot,
+     * planning from its versions to `latest`, and applying every registered
+     * step transform. This mirrors the engine's apply phase but starts from an
+     * independent disk read.
+     */
+    function deriveExpectedState(snapshot) {
+        const inspected = inspectSnapshotVersionState(snapshot, latest);
+        if ("corruption" in inspected) {
+            throw new Error(inspected.corruption.detail);
+        }
+        const plan = planMigration(registry, inspected.source, latest);
+        if (plan.kind === "blocked") {
+            throw new Error(`SQLite migration verification cannot derive expected state: ${plan.blocker.message}`);
+        }
+        if (plan.kind === "no-op")
+            return snapshot.state;
+        let current = snapshot;
+        for (const planned of plan.steps) {
+            planned.step.preconditions(current);
+            current = planned.step.transform(current);
+        }
+        return current.state;
+    }
+    function verifyChecksums(expectedState) {
+        const expected = computeStateFamilyChecksums(expectedState);
+        const actual = computeDbFamilyChecksums(home, STAGED_DATABASE_FILENAME);
+        const families = new Set([...Object.keys(expected), ...Object.keys(actual)]);
+        const mismatches = [];
+        for (const family of families) {
+            const e = expected[family];
+            const a = actual[family];
+            if (e === undefined || a === undefined || e.count !== a.count || e.hash !== a.hash) {
+                mismatches.push(`${family} (expected ${e === undefined ? "absent" : `${e.count}/${e.hash.slice(0, 12)}`}, ` +
+                    `found ${a === undefined ? "absent" : `${a.count}/${a.hash.slice(0, 12)}`})`);
+            }
+        }
+        if (mismatches.length > 0) {
+            throw new Error(`SQLite migration checksum mismatch for ${mismatches.length} family/families: ${mismatches.join("; ")}`);
+        }
+    }
+}
+/**
+ * Roll back a committed layout-7 SQLite migration: quarantine `yui.db` and
+ * flip `schema.json` back to layout 6. The `state.json` document is never
+ * touched — it was retained read-only during the migration and remains the
+ * authoritative source after rollback.
+ *
+ * Returns the quarantine path. Throws if the Home is not at layout 7 or has
+ * no `yui.db` to quarantine.
+ */
+export function rollbackSqliteMigration(home, options = {}) {
+    const now = options.now ?? (() => new Date());
+    const schemaPath = join(home, STORAGE_SCHEMA_FILE);
+    const manifest = parseJsonObject(readFileSync(schemaPath, "utf8"), STORAGE_SCHEMA_FILE);
+    if (manifest.storageVersion !== 7) {
+        throw new Error(`Rollback requires a layout-7 Home; found layout ${manifest.storageVersion}.`);
+    }
+    const dbPath = join(home, COMMITTED_DATABASE_FILENAME);
+    if (!existsSync(dbPath)) {
+        throw new Error(`No SQLite database to quarantine: ${dbPath}.`);
+    }
+    const stamp = now().toISOString().replace(/[:.]/g, "-");
+    const quarantinePath = join(home, `${COMMITTED_DATABASE_FILENAME}.quarantine-${stamp}`);
+    if (existsSync(quarantinePath)) {
+        throw new Error(`Refusing to overwrite an existing quarantine: ${quarantinePath}.`);
+    }
+    renameSync(dbPath, quarantinePath);
+    // Clean up WAL/SHM sidecars.
+    rmSync(`${dbPath}-wal`, { force: true });
+    rmSync(`${dbPath}-shm`, { force: true });
+    // Flip schema.json back to layout 6.
+    const rolledBack = { ...manifest, storageVersion: 6 };
+    writeTextFileAtomically(schemaPath, `${JSON.stringify(rolledBack, null, 2)}\n`);
+    return quarantinePath;
+}
+function parseJsonObject(raw, label) {
+    const value = JSON.parse(raw);
+    if (typeof value !== "object" || value === null || Array.isArray(value)) {
+        throw new Error(`${label} must be a JSON object.`);
+    }
+    return value;
+}
+function messageOf(error) {
+    return error instanceof Error ? error.message : String(error);
+}