@deksden-com/dd-flow-cli 0.9.0-beta.143 → 0.9.0-beta.145

package/dist/storage/database.js

@@ -5,7 +5,7 @@ import { channel } from "node:diagnostics_channel";
 import { createRequire } from "node:module";
 import { ensureDir } from "./paths.js";
 import { AppError } from "../shared/errors.js";
-import { executeStoreSchema, readStoreWriterContract, registerStoreWriter, registerReconciliationWriter, STORE_WRITER_VERSION } from "./writer-contract.js";
+import { executeStoreSchema, readStoreWriterContract, registerStoreWriter, registerReconciliationWriter, bindStoreWriterAdmission, STORE_WRITER_VERSION } from "./writer-contract.js";
 import { assertImportAdmission } from "./import-guard.js";
 import { withStoreCleanup, rollbackStoreTransaction } from "./native-cleanup.js";
 const workContinuationStorageSql = `
@@ -25,7 +25,7 @@ function closeAfterOpenFailure(db, error) {
         db.close(); });
 }
 const require = createRequire(import.meta.url);
-const { DatabaseSync } = require("node:sqlite");
+const { DatabaseSync, constants } = require("node:sqlite");
 const resourceDatabases = new Map();
 const resourceDatabaseIdentities = new WeakMap();
 function assertResourceDatabaseIdentity(database) {
@@ -400,6 +400,8 @@ export function getDatabase(ddFlowHome, mode = "initialize", importPermitId, hoo
     if (!exists || mode === "migrate")
         ensureDir(ddFlowHome);
     const db = mode === "read_existing" ? new DatabaseSync(dbPath, { readOnly: true }) : new DatabaseSync(dbPath);
+    if (mode !== "read_existing")
+        installWriterAdmission(db, () => assertImportAdmission(ddFlowHome, importPermitId), () => db.isTransaction, true);
     try {
         db.exec(`PRAGMA busy_timeout = ${deadline?.("store_open") ?? 4000}`);
     }
@@ -408,10 +410,12 @@ export function getDatabase(ddFlowHome, mode = "initialize", importPermitId, hoo
     }
     if (mode !== "read_existing") {
         try {
+            const admission = () => { if (!db.isTransaction)
+                assertImportAdmission(ddFlowHome, importPermitId); };
             if (mode === "reconcile")
-                registerReconciliationWriter(db);
+                registerReconciliationWriter(db, admission);
             else
-                registerStoreWriter(db);
+                registerStoreWriter(db, admission);
         }
         catch (error) {
             return closeAfterOpenFailure(db, error);
@@ -464,7 +468,7 @@ export function getDatabase(ddFlowHome, mode = "initialize", importPermitId, hoo
         }
     }
     db.exec("PRAGMA foreign_keys = ON");
-    return databaseFacade(dbPath, mode !== "read_existing", db, undefined, deadline);
+    return databaseFacade(dbPath, mode !== "read_existing", db, undefined, deadline, mode === "read_existing" ? undefined : () => assertImportAdmission(ddFlowHome, importPermitId));
 }
 /**
  * Hook ingress is a hot, synchronous path.  It may write receipts, but it
@@ -526,13 +530,15 @@ export function withResourceReadSnapshot(resourceHome, read) {
     return withStoreCleanup(operation, () => { if (native.isOpen)
         native.close(); });
 }
-export function getResourceDatabase(resourceHome, mode = "initialize", deadlineAt) {
+export function getResourceDatabase(resourceHome, mode = "initialize", deadlineAt, importPermitId) {
     channel("dd-flow.sqlite.ingress").publish({ phase: "store_open", store_kind: "resource" });
     const deadline = requestDeadline(deadlineAt);
     deadline?.("resource_open");
     const dbPath = path.join(resourceHome, "runtime.sqlite");
+    if (mode !== "read_existing" && mode !== "read_isolated")
+        assertImportAdmission(resourceHome, importPermitId);
     if (mode === "retirement" && !fs.existsSync(dbPath))
-        return getResourceDatabase(resourceHome, "initialize", deadlineAt);
+        return getResourceDatabase(resourceHome, "initialize", deadlineAt, importPermitId);
     const cached = resourceDatabases.get(dbPath);
     if (mode === "create_new") {
         if (cached)
@@ -564,30 +570,29 @@ export function getResourceDatabase(resourceHome, mode = "initialize", deadlineA
     if (mode === "prepared_existing" && !fs.existsSync(dbPath))
         throw new AppError("resource_storage_unprepared", "Prepare the resource registry before maintenance", 1);
     if (cached) {
+        if (importPermitId && resourceDatabaseIdentities.get(cached)?.importPermitId !== importPermitId)
+            throw new AppError("resource_writer_reopen_required", "Close the owned cached resource writer before opening a privileged maintenance connection", 1);
         if (mode === "prepared_existing" || mode === "retirement")
             assertResourceDatabaseIdentity(cached);
         const bounded = deadlineAt === undefined ? cached : cached.withDeadline(deadlineAt);
         const reader = { prepare: (sql) => ({ get: (...params) => bounded.get(sql, params), all: (...params) => bounded.all(sql, params), run: (...params) => bounded.run(sql, params) }) };
-        if (mode === "prepared_existing" || mode === "retirement")
-            assertResourceStoreReady(reader, mode === "retirement");
-        else {
-            const required = readStoreWriterContract(reader);
-            if (required !== null && required !== STORE_WRITER_VERSION)
-                throw new AppError("store_writer_incompatible", "Retirement writer cannot admit productive resource operations", 1, { required, supported: STORE_WRITER_VERSION });
-        }
+        assertResourceStoreReady(reader, mode === "retirement");
         return bounded;
     }
     if (mode !== "prepared_existing" && mode !== "retirement")
         ensureDir(resourceHome);
     channel("dd-flow.sqlite.ingress").publish({ phase: "store_connection", store_kind: "resource" });
     const db = new DatabaseSync(dbPath);
+    installWriterAdmission(db, () => assertImportAdmission(resourceHome, importPermitId), () => db.isTransaction, true);
     try {
         channel("dd-flow.sqlite.ingress").publish({ phase: "store_writer_registration", store_kind: "resource" });
         db.exec(`PRAGMA busy_timeout = ${deadline?.("resource_open") ?? 4000}`);
+        const admission = () => { if (!db.isTransaction)
+            assertImportAdmission(resourceHome, importPermitId); };
         if (mode === "retirement")
-            registerReconciliationWriter(db);
+            registerReconciliationWriter(db, admission);
         else
-            registerStoreWriter(db);
+            registerStoreWriter(db, admission);
     }
     catch (error) {
         return closeAfterOpenFailure(db, error);
@@ -659,12 +664,12 @@ export function getResourceDatabase(resourceHome, mode = "initialize", deadlineA
     const database = databaseFacade(dbPath, true, db, () => {
         resourceDatabases.delete(dbPath);
         db.close?.();
-    });
+    }, undefined, () => assertImportAdmission(resourceHome, importPermitId));
     try {
         const identity = fs.lstatSync(dbPath);
         if (!identity.isFile())
             throw new AppError("resource_storage_changed", "Resource registry target is not a regular file", 1);
-        resourceDatabaseIdentities.set(database, { dev: identity.dev, ino: identity.ino });
+        resourceDatabaseIdentities.set(database, { dev: identity.dev, ino: identity.ino, ...(importPermitId ? { importPermitId } : {}) });
     }
     catch (error) {
         return closeAfterOpenFailure(db, error);
@@ -713,7 +718,25 @@ function emptyReadOnlyDatabase(dbPath) {
         all: () => []
     };
 }
-function databaseFacade(dbPath, writable, db, close = () => db.close?.(), deadline) {
+function installWriterAdmission(db, assertAdmission, reserved, initializing = false) {
+    const writes = new Set([constants.SQLITE_INSERT, constants.SQLITE_UPDATE, constants.SQLITE_DELETE,
+        constants.SQLITE_CREATE_INDEX, constants.SQLITE_CREATE_TABLE, constants.SQLITE_CREATE_TEMP_INDEX, constants.SQLITE_CREATE_TEMP_TABLE,
+        constants.SQLITE_CREATE_TEMP_TRIGGER, constants.SQLITE_CREATE_TEMP_VIEW, constants.SQLITE_CREATE_TRIGGER, constants.SQLITE_CREATE_VIEW,
+        constants.SQLITE_DROP_INDEX, constants.SQLITE_DROP_TABLE, constants.SQLITE_DROP_TEMP_INDEX, constants.SQLITE_DROP_TEMP_TABLE,
+        constants.SQLITE_DROP_TEMP_TRIGGER, constants.SQLITE_DROP_TEMP_VIEW, constants.SQLITE_DROP_TRIGGER, constants.SQLITE_DROP_VIEW,
+        constants.SQLITE_ATTACH, constants.SQLITE_DETACH, constants.SQLITE_ALTER_TABLE, constants.SQLITE_REINDEX, constants.SQLITE_ANALYZE,
+        constants.SQLITE_CREATE_VTABLE, constants.SQLITE_DROP_VTABLE]);
+    const filePragmas = new Set(["journal_mode", "wal_checkpoint", "optimize", "incremental_vacuum", "user_version", "application_id", "writable_schema", "auto_vacuum", "page_size", "max_page_count", "encoding", "locking_mode"]);
+    db.setAuthorizer((action, first, second) => {
+        if (initializing && action === constants.SQLITE_TRANSACTION && first === "BEGIN")
+            assertAdmission();
+        if (!reserved() && (writes.has(action) || action === constants.SQLITE_PRAGMA
+            && filePragmas.has(first?.toLowerCase() ?? "") && (second !== null || ["wal_checkpoint", "optimize", "incremental_vacuum"].includes(first?.toLowerCase() ?? ""))))
+            assertAdmission();
+        return constants.SQLITE_OK;
+    });
+}
+function databaseFacade(dbPath, writable, db, close = () => db.close?.(), deadline, assertAdmission) {
     let scopedDeadline = deadline;
     let reserved = false;
     let poisoned = false;
@@ -727,6 +750,16 @@ function databaseFacade(dbPath, writable, db, close = () => db.close?.(), deadli
     let manualFrames = [];
     let readDepth = 0;
     let ownedWriterToken = null;
+    if (writable && assertAdmission) {
+        // SQLite identifies DML/DDL even behind comments, CTEs or RETURNING.
+        // An already reserved writer may finish; maintenance must acquire its
+        // own barriers and recheck drain before mutating either store.
+        installWriterAdmission(db, assertAdmission, () => reserved && db.isTransaction);
+        // Existing writer triggers also check at execution time. This closes the
+        // prepare-before-fence / step-after-fence window for protected DML.
+        bindStoreWriterAdmission(db, () => { if (!(reserved && db.isTransaction))
+            assertAdmission(); });
+    }
     const writerTiming = channel("dd-flow.sqlite.writer");
     const ingressPhase = channel("dd-flow.sqlite.ingress");
     const storeKind = path.basename(dbPath) === "runtime.sqlite" ? "resource" : "run";
@@ -754,6 +787,7 @@ function databaseFacade(dbPath, writable, db, close = () => db.close?.(), deadli
             throw new AppError("read_snapshot_write", "Cannot reserve a writer inside a read snapshot", 1);
         if (!writable)
             throw new AppError("read_only_storage", "Read-only access cannot start a write transaction", 1, { path: dbPath });
+        assertAdmission?.();
         bound("writer_reservation");
         db.exec("BEGIN IMMEDIATE");
         reserved = true;

package/dist/storage/import-guard.js

@@ -2,6 +2,8 @@ import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
 import { AppError } from "../shared/errors.js";
+import { acquireInstallationLock } from "../services/installation-lock.js";
+import { withStoreCleanup } from "./native-cleanup.js";
 /** Outside the replaced runtime tree, so a copied source receipt cannot make
  * an incomplete target look ready. A failed target is diagnostic evidence. */
 export function importGuardPath(home) {
@@ -31,7 +33,7 @@ function readImportGuard(home) {
     if (guard.schema_id !== "dd-flow/import-guard@1" || guard.runtime_home !== canonicalHome(home)
         || !/^[0-9a-f-]{36}$/.test(guard.import_id) || !/^[a-f0-9]{64}$/.test(guard.source_manifest_sha256)
         || !path.isAbsolute(guard.project_root) || !path.isAbsolute(guard.workspace_root)
-        || !["in_progress", "completed"].includes(guard.status)) {
+        || !["in_progress", "completed"].includes(guard.status) || guard.operation !== undefined && guard.operation !== "storage-migration") {
         throw new AppError("snapshot_import_incomplete", "Snapshot import guard does not identify this target", 1, { guard_path: file });
     }
     return guard;
@@ -39,41 +41,96 @@ function readImportGuard(home) {
 export function assertImportAdmission(home, permit) {
     const guard = readImportGuard(home);
     if (guard?.status === "in_progress" && guard.import_id !== permit)
-        throw new AppError("snapshot_import_incomplete", "Snapshot import has not completed; preserve this target for diagnosis and restore into fresh targets", 1, { guard_path: importGuardPath(home), import_id: guard.import_id });
+        throw new AppError(guard.operation === "storage-migration" ? "storage_migration_incomplete" : "snapshot_import_incomplete", guard.operation === "storage-migration" ? "Storage maintenance requires reconciliation of its original request" : "Snapshot import has not completed; preserve this target for diagnosis and restore into fresh targets", 1, { guard_path: importGuardPath(home), import_id: guard.import_id });
 }
-function durableWrite(file, bytes, create) {
-    const target = create ? file : `${file}.tmp-${crypto.randomUUID()}`;
-    try {
-        fs.writeFileSync(target, bytes, { flag: "wx", mode: 0o600 });
-        const descriptor = fs.openSync(target, "r");
+function durableWrite(file, bytes, create, expected) {
+    const lease = acquireInstallationLock(file + ".lock", crypto.createHash("sha256").update(file).digest("hex"));
+    return withStoreCleanup(() => {
+        lease.assertOwnership();
+        // A different request may have selected maintenance since the caller read
+        // a completed guard. Never rename over its now-live admission fence.
+        if (!create && JSON.stringify(JSON.parse(fs.readFileSync(file, "utf8"))) !== JSON.stringify(expected))
+            throw new AppError("storage_guard_changed", "Re-read the current maintenance owner before continuing", 1);
+        const target = create ? file : `${file}.tmp-${crypto.randomUUID()}`;
         try {
-            fs.fsyncSync(descriptor);
+            fs.writeFileSync(target, bytes, { flag: "wx", mode: 0o600 });
+            const descriptor = fs.openSync(target, "r");
+            try {
+                fs.fsyncSync(descriptor);
+            }
+            finally {
+                fs.closeSync(descriptor);
+            }
+            if (!create)
+                fs.renameSync(target, file);
+            const directory = fs.openSync(path.dirname(file), "r");
+            try {
+                fs.fsyncSync(directory);
+            }
+            finally {
+                fs.closeSync(directory);
+            }
         }
         finally {
-            fs.closeSync(descriptor);
-        }
-        if (!create)
-            fs.renameSync(target, file);
-        const directory = fs.openSync(path.dirname(file), "r");
-        try {
-            fs.fsyncSync(directory);
+            if (!create)
+                fs.rmSync(target, { force: true });
         }
-        finally {
-            fs.closeSync(directory);
-        }
-    }
-    finally {
-        if (!create)
-            fs.rmSync(target, { force: true });
-    }
+    }, () => lease.release());
 }
 export function beginSnapshotImport(input) {
     const importId = crypto.randomUUID();
     const guard = { schema_id: "dd-flow/import-guard@1", import_id: importId, status: "in_progress", source_manifest_sha256: input.sourceManifestSha256,
-        runtime_home: canonicalHome(input.home), project_root: path.resolve(input.projectRoot), workspace_root: path.resolve(input.workspaceRoot) };
-    durableWrite(importGuardPath(input.home), `${JSON.stringify(guard)}\n`, true);
+        runtime_home: canonicalHome(input.home), project_root: path.resolve(input.projectRoot), workspace_root: path.resolve(input.workspaceRoot),
+        ...(input.operation ? { operation: input.operation } : {}) };
+    durableWrite(importGuardPath(input.home), `${JSON.stringify(guard)}\n`, true, null);
     return importId;
 }
+/** Reuse the import admission fence for one exact paired storage cutover.
+ * A completed older operation may be superseded; a live one never is. */
+export function beginStorageMaintenance(input) {
+    if (!/^[0-9a-f-]{36}$/.test(input.importId) || !/^[a-f0-9]{64}$/.test(input.sourceManifestSha256))
+        throw new AppError("storage_migration_request_invalid", "Invalid retained maintenance identity", 2);
+    const guard = readImportGuard(input.home);
+    const next = { schema_id: "dd-flow/import-guard@1", import_id: input.importId, status: "in_progress",
+        source_manifest_sha256: input.sourceManifestSha256, runtime_home: canonicalHome(input.home),
+        project_root: path.resolve(input.projectRoot), workspace_root: path.resolve(input.workspaceRoot), operation: "storage-migration" };
+    if (guard?.import_id === input.importId && guard.source_manifest_sha256 === input.sourceManifestSha256
+        && guard.project_root === next.project_root && guard.workspace_root === next.workspace_root && guard.operation === next.operation)
+        return;
+    if (guard?.import_id === input.importId)
+        throw new AppError("storage_migration_request_conflict", "A maintenance identity cannot select another request", 1);
+    if (guard?.status === "in_progress")
+        assertImportAdmission(input.home);
+    durableWrite(importGuardPath(input.home), `${JSON.stringify(next)}\n`, !guard, guard);
+}
+/** Explicit recovery may take over only its original paired maintenance scope.
+ * Keeping the permit identity makes an old apply unable to reopen recovery. */
+export function beginStorageRecovery(input) {
+    if (!/^[a-f0-9]{64}$/.test(input.originalSha256) || !/^[a-f0-9]{64}$/.test(input.recoverySha256))
+        throw new AppError("storage_recovery_scope_invalid", "Invalid retained recovery identity", 2);
+    const guard = readImportGuard(input.home);
+    if (!guard) {
+        beginStorageMaintenance({ ...input, sourceManifestSha256: input.originalSha256 });
+        return beginStorageRecovery(input);
+    }
+    if (guard.operation !== "storage-migration" || guard.import_id !== input.importId
+        || guard.project_root !== path.resolve(input.projectRoot) || guard.workspace_root !== path.resolve(input.workspaceRoot)
+        || ![input.originalSha256, input.recoverySha256].includes(guard.source_manifest_sha256))
+        throw new AppError("storage_recovery_scope_invalid", "Recovery cannot replace another maintenance scope", 1);
+    if (guard.source_manifest_sha256 === input.recoverySha256)
+        return;
+    durableWrite(importGuardPath(input.home), `${JSON.stringify({ ...guard, status: "in_progress", source_manifest_sha256: input.recoverySha256 })}\n`, false, guard);
+}
+export function completeStorageMaintenance(input) {
+    const guard = readImportGuard(input.home);
+    if (guard?.operation !== "storage-migration")
+        throw new AppError("storage_migration_incomplete", "Completion requires the original storage maintenance fence", 1);
+    if (guard?.status === "completed" && guard.operation === "storage-migration" && guard.import_id === input.importId
+        && guard.source_manifest_sha256 === input.sourceManifestSha256 && guard.project_root === path.resolve(input.projectRoot)
+        && guard.workspace_root === path.resolve(input.workspaceRoot))
+        return;
+    completeSnapshotImport(input.home, input.importId, input);
+}
 /** A fork with its own matching intent may rebuild a completed import when
  * its final fork receipt was lost. Fence the existing home before removing any
  * copied files; an interrupted rebuild remains closed to productive opens. */
@@ -87,7 +144,7 @@ export function rearmCompletedSnapshotImport(input) {
         throw new AppError("snapshot_import_incomplete", "Existing import does not match this completed fork; preserve it for diagnosis", 1, { guard_path: importGuardPath(input.home) });
     }
     const importId = crypto.randomUUID();
-    durableWrite(importGuardPath(input.home), `${JSON.stringify({ ...guard, import_id: importId, status: "in_progress" })}\n`, false);
+    durableWrite(importGuardPath(input.home), `${JSON.stringify({ ...guard, import_id: importId, status: "in_progress" })}\n`, false, guard);
     return importId;
 }
 export function completeSnapshotImport(home, importId, completed) {
@@ -98,5 +155,5 @@ export function completeSnapshotImport(home, importId, completed) {
         || guard.source_manifest_sha256 !== completed.sourceManifestSha256) {
         throw new AppError("snapshot_import_incomplete", "Import completion does not match its source and target paths", 1, { guard_path: importGuardPath(home) });
     }
-    durableWrite(importGuardPath(home), `${JSON.stringify({ ...guard, status: "completed" })}\n`, false);
+    durableWrite(importGuardPath(home), `${JSON.stringify({ ...guard, status: "completed" })}\n`, false, guard);
 }

package/dist/storage/memory-upgrade-schema-bootstrap.js

@@ -0,0 +1,38 @@
+import { getDatabase, getResourceDatabase } from "./database.js";
+import { withStoreCleanup } from "./native-cleanup.js";
+import { PreparedMemoryInitRuntimeStorage } from "../services/memory-init-storage.js";
+import { PreparedMemoryInitResourceStorage } from "../services/memory-init-resource-storage.js";
+import { AppError } from "../shared/errors.js";
+import fs from "node:fs";
+import path from "node:path";
+/** Fixed executing-engine recipe, never an operator command or a normal
+ * admission side effect. The paired migration owns backup, drain, permit,
+ * preservation verification and recovery for both stores around this step. */
+export function bootstrapMemoryUpgradeSchema(runHome, resourceHome, permitId) {
+    try {
+        for (const file of [path.join(runHome, "db.sqlite"), path.join(resourceHome, "runtime.sqlite")]) {
+            const owner = fs.lstatSync(file);
+            if (!owner.isFile() || owner.nlink !== 1)
+                throw new AppError("memory_schema_bootstrap_unavailable", "Schema bootstrap requires the existing paired stores", 1);
+        }
+        const db = getDatabase(runHome, "initialize", permitId);
+        withStoreCleanup(() => {
+            const resources = getResourceDatabase(resourceHome, "initialize", undefined, permitId);
+            withStoreCleanup(() => {
+                const context = { ddFlowHome: runHome, db,
+                    env: { DD_FLOW_HOME: runHome, DD_FLOW_RESOURCE_HOME: resourceHome }, now: () => new Date().toISOString() };
+                PreparedMemoryInitRuntimeStorage.prepareUpgrade(context);
+                PreparedMemoryInitResourceStorage.prepare(context);
+            }, () => resources.close?.());
+        }, () => db.close?.());
+    }
+    catch (error) {
+        // The writer cutover already happened. An admission failure does not mean
+        // no effect: retain both maintenance fences for exact replay or recovery.
+        if (error instanceof AppError) {
+            Object.assign(error.details, { effect: "unknown", next_action: "reconcile_exact_request" });
+            throw error;
+        }
+        throw new AppError("memory_schema_bootstrap_failed", "Reconcile the retained schema bootstrap before runtime admission", 1, { effect: "unknown", next_action: "reconcile_exact_request" });
+    }
+}