@deksden-com/dd-flow-cli 0.9.0-beta.103 → 0.9.0-beta.105

package/dist/services/runtime-budget.js

@@ -50,20 +50,14 @@ export async function waitForRuntimeScopeRecovery(context, input, assertOwner) {
     const db = getResourceDatabase(resourceHome(context));
     for (;;) {
         assertOwner();
-        const snapshot = (() => {
-            db.exec("SAVEPOINT scope_recovery_release");
-            try {
-                return {
-                    fence: db.get("SELECT 1 FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [input.scopeId]),
-                    latest: db.get("SELECT MAX(json_extract(metadata_json, '$.generation')) AS generation FROM managed_resources WHERE resource_kind = 'scope-control-request' AND owner_id = ?", [input.scopeId]),
-                    control: db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [input.scopeId]),
-                    release: db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-resume-release' AND resource_key = ?", [digest(JSON.stringify([input.scopeId, input.generation]))])
-                };
-            }
-            finally {
-                db.exec("RELEASE scope_recovery_release");
-            }
-        })();
+        const snapshot = db.readSnapshot(() => {
+            return {
+                fence: db.get("SELECT 1 FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [input.scopeId]),
+                latest: db.get("SELECT MAX(json_extract(metadata_json, '$.generation')) AS generation FROM managed_resources WHERE resource_kind = 'scope-control-request' AND owner_id = ?", [input.scopeId]),
+                control: db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [input.scopeId]),
+                release: db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-resume-release' AND resource_key = ?", [digest(JSON.stringify([input.scopeId, input.generation]))])
+            };
+        });
         if (snapshot.fence)
             throw new AppError("runtime_scope_stopped", "Terminal cancellation superseded scope recovery", 1);
         const { latest, control: row, release } = snapshot;
@@ -118,12 +112,10 @@ export function requestRuntimeScopeControl(context, scopeId, requestId, mode, ma
     const declaration = prepared.manifest;
     const db = getResourceDatabase(resourceHome(context));
     const requestKey = digest(JSON.stringify([scopeId, requestId]));
-    db.beginWriteTransaction();
-    try {
+    return db.writeTransaction(() => {
         const prior = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-request' AND resource_key = ?", [requestKey]);
         if (prior) {
             const receipt = scopeControlReceipt(prior.metadata_json, mode);
-            db.exec("COMMIT");
             return receipt;
         }
         if (declaration !== undefined) {
@@ -149,13 +141,8 @@ export function requestRuntimeScopeControl(context, scopeId, requestId, mode, ma
         const payload = JSON.stringify(receipt);
         db.run("INSERT INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control-request',?,?,?,?,NULL,?,?,?)", [requestKey, scopeId, requestKey, now, payload, now, now]);
         db.run("INSERT INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control',?,?,?,?,NULL,?,?,?) ON CONFLICT(resource_kind,resource_key) DO UPDATE SET lease_token = excluded.lease_token, metadata_json = excluded.metadata_json, updated_at = excluded.updated_at", [scopeId, scopeId, requestKey, now, payload, now, now]);
-        db.exec("COMMIT");
         return receipt;
-    }
-    catch (error) {
-        db.exec("ROLLBACK");
-        throw error;
-    }
+    });
 }
 function assertScopeAdmission(context, scopeId) {
     const db = getResourceDatabase(resourceHome(context));
@@ -245,21 +232,15 @@ export function prepareRuntimeScopeFence(scopeId, requestId) {
 export function fenceRuntimeBudget(context, scopeId, requestId) {
     prepareRuntimeScopeFence(scopeId, requestId);
     const db = getResourceDatabase(resourceHome(context));
-    db.beginWriteTransaction();
-    try {
+    return db.writeTransaction(() => {
         const existing = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [scopeId]);
         const now = context.now();
         const receipt = existing ? JSON.parse(existing.metadata_json)
             : { scope_id: scopeId, request_id: requestId, dispatch_blocked: true, fenced_at: now };
         if (!existing)
             db.run("INSERT INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('provider-fence',?,?,?,?,NULL,?,?,?)", [scopeId, scopeId, digest(requestId), now, JSON.stringify(receipt), now, now]);
-        db.exec("COMMIT");
         return receipt;
-    }
-    catch (error) {
-        db.exec("ROLLBACK");
-        throw error;
-    }
+    });
 }
 export function runtimeBudgetForRun(context, projectId, runId) {
     const retained = context.db.get("SELECT 1 FROM sqlite_schema WHERE name = 'run_controllers'")
@@ -304,16 +285,16 @@ export function reserveProviderTurn(context, record, input) {
     const db = getResourceDatabase(resourceHome(context));
     const key = digest(JSON.stringify([record.id, input.operationId]));
     const policy = JSON.stringify(budget), now = context.now();
-    db.beginWriteTransaction();
-    try {
+    const recoveryOwnerAlive = input.operation === "session.resume" && processIsAlive(record);
+    return db.writeTransaction(() => {
         const current = db.get("SELECT * FROM managed_processes WHERE id = ?", [record.id]);
-        if (current?.lease_token !== record.lease_token || current.state !== "running")
+        if (current?.lease_token !== record.lease_token || current.state !== "running" || current.lease_expires_at <= context.now() || managedProcessBinding(current, resourceHome(context)).registration_sha256 !== managedProcessBinding(record, resourceHome(context)).registration_sha256)
             throw new AppError("process_ownership_unknown", "Budget owner changed before admission", 1);
         try {
             assertScopeAdmission(context, budget.scope_id);
         }
         catch (error) {
-            if (!(error instanceof AppError) || error.code !== "runtime_scope_controlled" || !scopeRecoveryLoadAllowed(context, budget.scope_id, record, input))
+            if (!(error instanceof AppError) || error.code !== "runtime_scope_controlled" || !scopeRecoveryLoadAllowed(context, budget.scope_id, record, input, recoveryOwnerAlive))
                 throw error;
         }
         const frozen = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'provider-budget' AND resource_key = ?", [budget.scope_id]);
@@ -325,7 +306,6 @@ export function reserveProviderTurn(context, record, input) {
         if (prior) {
             if (prior.lease_token !== record.lease_token)
                 throw new AppError("process_ownership_unknown", "Retained budget claim belongs to another physical owner", 1);
-            db.exec("COMMIT");
             return { admitted: true, budget_scope: budget.scope_id, reused: true };
         }
         const activeTurns = db.all("SELECT process_id, metadata_json FROM managed_resources WHERE resource_kind = 'provider-turn' AND json_extract(metadata_json, '$.scope_id') = ? AND json_extract(metadata_json, '$.harness') = ? ORDER BY created_at, resource_key", [budget.scope_id, harness]);
@@ -335,21 +315,15 @@ export function reserveProviderTurn(context, record, input) {
                 const claim = JSON.parse(turn.metadata_json);
                 return { process_id: turn.process_id, operation_id: claim.operation_id ?? null, operation: claim.operation ?? null, session_id: claim.session_id ?? null };
             });
-            db.exec("COMMIT");
             return { admitted: false, reason: "provider_capacity", budget_scope: budget.scope_id, harness, limit, active, blocking_turns };
         }
         db.run("INSERT INTO managed_resources (resource_kind, resource_key, owner_id, lease_token, lease_expires_at, process_id, metadata_json, created_at, updated_at) VALUES ('provider-turn', ?, ?, ?, ?, ?, ?, ?, ?)", [key, record.owner_id, record.lease_token, record.lease_expires_at, record.id, JSON.stringify({ scope_id: budget.scope_id, harness, operation_id: input.operationId, operation: input.operation, session_id: input.sessionId ?? null, state_dir: metadata.state_dir }), now, now]);
-        db.exec("COMMIT");
         return { admitted: true, budget_scope: budget.scope_id, harness, limit, active: active + 1 };
-    }
-    catch (error) {
-        db.exec("ROLLBACK");
-        throw error;
-    }
+    });
 }
-function scopeRecoveryLoadAllowed(context, scopeId, record, input) {
+function scopeRecoveryLoadAllowed(context, scopeId, record, input, physicalOwnerAlive) {
     const metadata = JSON.parse(record.metadata_json);
-    if (input.operation !== "session.resume" || !input.daemonId || !metadata.state_dir || metadata.dd_flow_home !== context.ddFlowHome || !processIsAlive(record))
+    if (input.operation !== "session.resume" || !input.daemonId || !metadata.state_dir || metadata.dd_flow_home !== context.ddFlowHome || !physicalOwnerAlive)
         return false;
     const db = getResourceDatabase(resourceHome(context));
     const control = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);

package/dist/services/runtime-scope-capture.js

@@ -6,7 +6,7 @@ import { getResourceDatabase } from "../storage/database.js";
 import { resourceHome } from "./managed-processes.js";
 import { validateRetainedRuntimeScopeManifest } from "./runtime-budget.js";
 /** Archive a complete append-only prefix, not a claim that no later evidence
- * can arrive. The caller holds the registry transaction; never take the EVAL
+ * can arrive. Prepare before taking the registry writer; never take the EVAL
  * journal lock here (its writer may itself be waiting for the registry).
  */
 export function captureScopeJournal(context, control, manifest) {
@@ -41,10 +41,12 @@ export function captureScopeJournal(context, control, manifest) {
     const sha256 = hash(prefix), key = hash(JSON.stringify([control.scope_id, control.generation, manifest.manifest_sha256, sha256]));
     const payload = JSON.stringify({ scope_id: control.scope_id, generation: control.generation, manifest, manifest_bytes: manifestBytes, journal_source: source, journal_bytes: text });
     const registry = getResourceDatabase(resourceHome(context));
-    const prior = registry.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-capture' AND resource_key = ?", [key]);
-    if (prior && prior.metadata_json !== payload)
-        throw new AppError("runtime_scope_capture_changed", "Retained EVAL capture no longer matches its content identity", 1);
-    if (!prior)
-        registry.run("INSERT INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control-capture',?,?,?,?,NULL,?,?,?)", [key, control.scope_id, key, context.now(), payload, context.now(), context.now()]);
+    registry.writeTransaction(() => {
+        const prior = registry.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-capture' AND resource_key = ?", [key]);
+        if (prior && prior.metadata_json !== payload)
+            throw new AppError("runtime_scope_capture_changed", "Retained EVAL capture no longer matches its content identity", 1);
+        if (!prior)
+            registry.run("INSERT INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control-capture',?,?,?,?,NULL,?,?,?)", [key, control.scope_id, key, context.now(), payload, context.now(), context.now()]);
+    });
     return { artifact_key: key, source, sha256, byte_length: prefix.length, event_count: events.length, request_sequence: request.data.sequence, consistency: "append_only_prefix", trailing_bytes: bytes.length - prefix.length };
 }

package/dist/services/runtime-scope-control.js

@@ -2,7 +2,7 @@ import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
 import { createContext } from "../runtime/context.js";
-import { getResourceDatabase } from "../storage/database.js";
+import { getResourceDatabase, withDeferredPublications } from "../storage/database.js";
 import { AppError } from "../shared/errors.js";
 import { finishManagedProcess, managedLeaseReleased, managedProcessBinding, managedProcessStatus, processTreeIsAlive, resourceHome, stopManagedProcess } from "./managed-processes.js";
 import { requestRuntimeScopeControl, runtimeBudgetStatus, scopeLocalProcessRole, validateRetainedRuntimeScopeManifest } from "./runtime-budget.js";
@@ -14,6 +14,7 @@ import { verifyRecoverySnapshot } from "./eval-snapshots.js";
 import { captureScopeJournal } from "./runtime-scope-capture.js";
 import { historicalDaemonSettlement, assertHistoricalDaemonSettlement } from "./native-daemon-history.js";
 import { managedProviderBelongsToDaemon } from "./managed-daemon-binding.js";
+import { cleanupScopeWriters } from "./scope-writer-cleanup.js";
 const scopeInventoryIdentity = (records) => JSON.stringify(records.map(record => managedProcessBinding(record, "")).sort((a, b) => a.process_id.localeCompare(b.process_id)));
 /** Persist the all-role drain boundary, separately from recovery readiness.
  * Registry -> RUN lock order matches scope dispatch. No await/provider call is
@@ -22,21 +23,17 @@ const scopeInventoryIdentity = (records) => JSON.stringify(records.map(record =>
 export function retainScopeDrain(context, control, processes, runs, native, probes) {
     const registry = getResourceDatabase(resourceHome(context));
     const targets = new Map();
+    const heldTargets = new Set();
+    const workspaces = new Map();
     const captures = [];
-    registry.beginWriteTransaction();
+    let registryHeld = false, primaryFailure;
+    let committedReceipt;
     try {
         for (const run of [...runs].sort((a, b) => String(a.runtime_home).localeCompare(String(b.runtime_home)))) {
             const home = String(run.runtime_home);
             let target = targets.get(home);
             if (!target) {
                 target = createContext({ ...context.env, DD_FLOW_HOME: home, DD_FLOW_RESOURCE_HOME: resourceHome(context), DD_FLOW_ENGINE_MODE: "1" });
-                try {
-                    target.db.beginWriteTransaction();
-                }
-                catch (error) {
-                    target.db.close?.();
-                    throw error;
-                }
                 targets.set(home, target);
             }
             const project = target.db.get("SELECT root FROM projects WHERE id = ?", [run.project_id]);
@@ -51,6 +48,7 @@ export function retainScopeDrain(context, control, processes, runs, native, prob
                     throw new AppError("runtime_scope_inventory_changed", "RUN no longer proves its captured process binding", 1);
             }
             run.receipt = fresh;
+            workspaces.set(run, target.db.get("SELECT workspace_root FROM runs WHERE project_id = ? AND id = ?", [run.project_id, run.run_id]).workspace_root);
             // Capture failure does not undo physical drain. Retain the failure beside
             // the stop evidence, and keep resume fenced until capture is reconciled.
             try {
@@ -104,22 +102,52 @@ export function retainScopeDrain(context, control, processes, runs, native, prob
         const capture = { manifest, journal, run_captures: captures, native_bindings: resumableNative.map(owner => ({ process_id: owner.process_id, role: owner.role, state_dir: owner.state_dir, daemon_id: owner.daemon_id, runtime_owner: owner.runtime_owner, inventory_version: owner.inventory_version, sessions: owner.sessions, operations: owner.operations })), probe_operations: probes, pending_reasons: [...new Set([manifestReason, ...captures.filter(item => item.status !== "verified").map(item => String(item.reason)), ...(resumableNative.length ? ["non_work_recovery_required"] : []), ...(probes.length ? ["probe_reconciliation_required"] : [])])] };
         const receipt = { scope_id: control.scope_id, generation: control.generation, requested_mode: control.requested_mode, observed_at: context.now(), physical_settled: true, recovery_ready: false, process_bindings: processes.map(record => managedProcessBinding(record, resourceHome(context))), runs, native, probes, capture };
         const key = crypto.createHash("sha256").update(JSON.stringify([control.scope_id, control.generation])).digest("hex");
+        // Captures are immutable evidence, not release authority. Only this short
+        // registry -> RUN boundary publishes the all-role drain grant.
+        registry.beginWriteTransaction();
+        registryHeld = true;
+        for (const target of targets.values()) {
+            target.db.beginWriteTransaction();
+            heldTargets.add(target);
+        }
+        const current = runtimeBudgetStatus(context, control.scope_id);
+        if (current.fence || current.control?.generation !== control.generation
+            || scopeInventoryIdentity(processes) !== scopeInventoryIdentity(current.processes) || current.provider_turns.length
+            || JSON.stringify(current.manifest) !== JSON.stringify(latest.manifest))
+            throw new AppError("runtime_scope_inventory_changed", "Scope changed while its drain evidence was prepared", 1);
+        for (const run of runs) {
+            const target = targets.get(String(run.runtime_home));
+            const project = target.db.get("SELECT root FROM projects WHERE id = ?", [run.project_id]);
+            const saved = target.db.get("SELECT workspace_root FROM runs WHERE project_id = ? AND id = ?", [run.project_id, run.run_id]);
+            const previous = run.receipt;
+            if (!project || saved?.workspace_root !== workspaces.get(run)
+                || JSON.stringify(runControlStatus(target, { projectRoot: project.root, runId: String(run.run_id), controlId: previous.control.control_id })) !== JSON.stringify(run.receipt))
+                throw new AppError("runtime_scope_inventory_changed", "RUN changed while its drain evidence was prepared", 1);
+        }
         registry.run("INSERT INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control-drain',?,?,?,?,NULL,?,?,?) ON CONFLICT(resource_kind,resource_key) DO UPDATE SET metadata_json = excluded.metadata_json, updated_at = excluded.updated_at", [key, control.scope_id, key, context.now(), JSON.stringify(receipt), context.now(), context.now()]);
         registry.exec("COMMIT");
+        registryHeld = false;
+        committedReceipt = receipt;
         return receipt;
     }
     catch (error) {
-        registry.exec("ROLLBACK");
+        primaryFailure = error;
         throw error;
     }
     finally {
-        for (const target of targets.values()) {
+        const errors = cleanupScopeWriters([
+            ...[...targets.values()].reverse().map(target => ({ name: target.ddFlowHome, db: target.db, rollback: heldTargets.has(target), close: "always" })),
+            { name: resourceHome(context), db: registry, rollback: registryHeld, close: "on_error" }
+        ]);
+        if (errors.length && committedReceipt)
+            committedReceipt.cleanup_errors = errors;
+        else if (errors.length && primaryFailure instanceof AppError)
+            primaryFailure.details.scope_cleanup_errors = errors;
+        else if (errors.length) {
             try {
-                target.db.exec("ROLLBACK");
-            }
-            finally {
-                target.db.close?.();
+                process.stderr.write(`${JSON.stringify({ phase: "scope_drain_cleanup", primary: String(primaryFailure), errors })}\n`);
             }
+            catch { /* Cleanup must not replace the primary outcome. */ }
         }
     }
 }
@@ -157,16 +185,10 @@ export async function controlRuntimeScopeLocalProbes(context, scopeId, generatio
             if (mode === "pause" && processTreeIsAlive(current))
                 return { process_id: record.id, physical_settled: false, reason: "local_probe_running" };
             const operationId = `scope-probe:${crypto.createHash("sha256").update(JSON.stringify([scopeId, generation, record.id, record.lease_token, record.pid_started_at])).digest("hex")}`;
-            registry.beginWriteTransaction();
-            try {
+            registry.writeTransaction(() => {
                 current = validate();
                 registry.run("INSERT OR IGNORE INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control-operation',?,?,?,?,NULL,?,?,?)", [operationId, scopeId, record.lease_token, context.now(), JSON.stringify({ scope_id: scopeId, generation, process_id: record.id, operation: mode === "stop" ? "probe.stop" : "probe.observe", status: "dispatching" }), context.now(), context.now()]);
-                registry.exec("COMMIT");
-            }
-            catch (error) {
-                registry.exec("ROLLBACK");
-                throw error;
-            }
+            });
             // A prior owner may have died after recording intent but before signalling.
             // Repeating an identity-checked stop never redispatches productive work.
             if (mode === "stop" && processTreeIsAlive(current))
@@ -240,16 +262,10 @@ export async function controlRuntimeScopeNativeOwners(context, scopeId, generati
                         const operationId = `scope-cancel:${crypto.createHash("sha256").update(JSON.stringify([scopeId, generation, record.id, state.daemon_id, id])).digest("hex")}`;
                         const now = context.now();
                         let inserted = false;
-                        registry.beginWriteTransaction();
-                        try {
+                        registry.writeTransaction(() => {
                             assertOwner();
                             inserted = registry.run("INSERT OR IGNORE INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control-operation',?,?,?,?,NULL,?,?,?)", [operationId, scopeId, record.lease_token, now, JSON.stringify({ scope_id: scopeId, generation, process_id: record.id, daemon_id: state.daemon_id, session_id: id, status: "dispatching" }), now, now]).changes === 1;
-                            registry.exec("COMMIT");
-                        }
-                        catch (error) {
-                            registry.exec("ROLLBACK");
-                            throw error;
-                        }
+                        });
                         try {
                             const saved = JSON.parse(registry.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-operation' AND resource_key = ?", [operationId]).metadata_json);
                             if (saved.status === "completed") {
@@ -331,8 +347,7 @@ export async function controlRuntimeScopeRuns(context, scopeId, requestId, mode,
             let input = { projectRoot: project.root, runId: record.run_id, requestId: runRequestId, action: mode };
             // Lock order is resource registry then RUN database. No provider or await
             // occurs here; a newer scope request cannot interleave RUN intent creation.
-            registry.beginWriteTransaction();
-            try {
+            withDeferredPublications(() => registry.writeTransaction(() => {
                 const current = registry.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);
                 if (!current || JSON.parse(current.metadata_json).generation !== control.generation || registry.get("SELECT 1 FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [scopeId]))
                     throw new AppError("runtime_scope_control_superseded", "Scope request no longer owns admission", 1);
@@ -346,12 +361,7 @@ export async function controlRuntimeScopeRuns(context, scopeId, requestId, mode,
                         ...(active.requested_mode === "stop" ? { force: active.request.force ?? false } : { graceMs: active.request.grace_ms ?? 30_000, ...(active.request.escalate ? { escalate: active.request.escalate } : {}) }) };
                 }
                 recordRunControlIntent(target, input);
-                registry.exec("COMMIT");
-            }
-            catch (error) {
-                registry.exec("ROLLBACK");
-                throw error;
-            }
+            }));
             seen.add(key);
             const receipt = await requestRunControl(target, { ...input, waitMs: 0, ...(reconcileId ? { reconcileId: crypto.createHash("sha256").update(JSON.stringify([scopeId, control.generation, reconcileId])).digest("hex") } : {}) });
             runs.push({ runtime_home: metadata.dd_flow_home, project_id: record.project_id, run_id: record.run_id, request_id: input.requestId, receipt });