@deksden-com/dd-flow-cli 0.9.0-beta.136 → 0.9.0-beta.138

package/dist/services/run-control.js

@@ -292,6 +292,34 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
     const nativeVersions = new Map();
     for (const [directory, { state, records }] of daemonInventories) {
         let failedStart = null;
+        // Explicit stop can reconcile a retired owner whose last bookkeeping ACK
+        // failed. This is physical retirement, not a fabricated clean shutdown.
+        if (control?.action === "stop" && state.shutdown_state === "cleanup_failed")
+            try {
+                const daemonId = state.daemon_id ?? state.config?.daemonId;
+                const daemon = records.filter(record => record.kind.endsWith("-daemon"));
+                const provider = records.filter(record => record.kind.endsWith("-provider"));
+                const operations = retainedDaemonOperations(directory);
+                const owner = state.config?.runtime_owner;
+                if (daemonId && !conflicts.has(directory) && !unsafeDaemons.has(canonicalPath(directory, false))
+                    && state.active_tree === false && !state.active_operation && state.shutdown?.daemon_id === daemonId
+                    && state.shutdown.phases?.tree === true && state.shutdown.phases.provider === true
+                    && daemon.length === 1 && provider.length === 1 && records.length === 2
+                    && daemon[0].id === state.resource_process?.id && daemon[0].lease_token === state.resource_process.lease_token && daemon[0].pid === state.pid
+                    && provider[0].id === state.provider_process?.id && provider[0].lease_token === state.provider_process.lease_token && provider[0].pid === state.provider_pid
+                    && JSON.parse(daemon[0].metadata_json).dispatch_barrier === "run-generation@1"
+                    && records.every(record => record.pid && record.pid_started_at && ["stopped", "failed"].includes(record.state) && record.finished_at && !processTreeIsAlive(record))
+                    && operations.length > 0 && operations.every(operation => ["completed", "failed", "interrupted"].includes(String(operation.state))
+                    && (operation.operation === "daemon.start" ? operation.daemon_id === null && operation.launch?.runtime_owner === JSON.stringify(owner)
+                        : operation.daemon_id === daemonId && ["daemon.stop", "daemon.status", "session.create", "session.prompt", "session.start", "session.fork", "session.resume", "session.cancel", "session.inspect"].includes(String(operation.operation))
+                            && (operation.session_id == null || state.sessions.some(item => (typeof item === "string" ? item : adapterSessionId(item)) === operation.session_id))))) {
+                    nativeVersions.set(directory, nativeInventoryVersion(directory, operations));
+                    closedDaemons.set(directory, { source: "observed_owned_retirement", daemon_id: daemonId, stopped: true, settled: true,
+                        session_ids: state.sessions.map(item => typeof item === "string" ? item : adapterSessionId(item)),
+                        processes: records.map(record => ({ process_id: record.id, pid: record.pid, pid_started_at: record.pid_started_at })) });
+                }
+            }
+            catch { /* Invalid or unfinished evidence remains on the existing recovery path. */ }
         if (!interrupt) {
             try {
                 const nativeOperations = retainedDaemonOperations(directory);
@@ -309,16 +337,48 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
                     const saved = JSON.parse(operation.input_json);
                     return saved.owner === owner?.owner_id && saved.executable === directoryAdapters.get(directory) && saved.args?.[0] === "daemon" && saved.args?.[1] === "start" && saved.args.includes("--state-dir") && saved.args[saved.args.indexOf("--state-dir") + 1] === directory;
                 });
+                const cleanup = start?.error?.details;
+                const proof = cleanup?.startup_child_cleanup;
+                let ownedChildGone = false;
+                if (proof && !cleanup?.provider_cleanup_failed && !cleanup?.cleanup_failed && provider.length === 0 && !state.provider_process && state.provider_pid == null
+                    && daemon.length === 1 && daemon[0].pid === null && daemon[0].pid_started_at === null
+                    && proof.settled === true && proof.physical_exit === "ESRCH" && proof.daemon_id === state.daemon_id
+                    && proof.process_id === daemon[0].id
+                    && proof.pid === state.pid && Number.isSafeInteger(proof.pid) && Number(proof.pid) > 0
+                    && proof.pgid === (process.platform === "win32" ? null : proof.pid)
+                    && typeof proof.observed_at === "string" && Number.isFinite(Date.parse(proof.observed_at))
+                    && proof.owner_id === owner?.owner_id && proof.project_id === project.id && proof.run_id === input.runId
+                    && proof.operation_id === start?.operation_id && proof.operation_id === owner?.operation_id && proof.state_dir === directory) {
+                    let leaderGone = false;
+                    try {
+                        process.kill(Number(proof.pid), 0);
+                    }
+                    catch (error) {
+                        leaderGone = error.code === "ESRCH";
+                    }
+                    if (leaderGone) {
+                        if (process.platform === "win32")
+                            ownedChildGone = true;
+                        else {
+                            try {
+                                process.kill(-Number(proof.pgid), 0);
+                            }
+                            catch (error) {
+                                ownedChildGone = error.code === "ESRCH";
+                            }
+                        }
+                    }
+                }
                 if (state.shutdown_state === "starting" && state.sessions?.length === 0 && state.active_tree !== true && !state.active_operation
                     && start?.operation === "daemon.start" && start.state === "failed" && start.predecessor_daemon_id === null && start.daemon_id === null
                     && typeof state.daemon_id === "string" && controllerStarts.length === 1 && starts.length === 1 && starts[0].operation_id === start.operation_id
                     && launch?.runtime_owner === JSON.stringify(owner)
                     && daemon.length === 1 && provider.length <= 1 && records.length === daemon.length + provider.length
                     && daemon[0].id === state.resource_process?.id && daemon[0].lease_token === state.resource_process.lease_token
-                    && (state.pid == null || state.pid === daemon[0].pid)
+                    && (state.pid == null || state.pid === daemon[0].pid || ownedChildGone)
                     && (!state.provider_process && state.provider_pid == null && provider.length === 0 || provider.length === 1 && provider[0].id === state.provider_process?.id && provider[0].lease_token === state.provider_process.lease_token && state.provider_pid === provider[0].pid)
-                    && records.every(record => record.pid && record.pid_started_at && !processTreeIsAlive(record) && ["stopped", "failed"].includes(record.state) && Boolean(record.finished_at))) {
-                    failedStart = { source: "failed_start_without_create", daemon_id: state.daemon_id, operation_id: start.operation_id, session_ids: [], processes: records.map(record => ({ process_id: record.id, pid: record.pid, pid_started_at: record.pid_started_at })) };
+                    && records.every(record => (record.pid && record.pid_started_at && !processTreeIsAlive(record) || ownedChildGone && record === daemon[0]) && ["stopped", "failed"].includes(record.state) && Boolean(record.finished_at))) {
+                    failedStart = { source: "failed_start_without_create", daemon_id: state.daemon_id, operation_id: start.operation_id, session_ids: [], processes: records.map(record => ({ process_id: record.id, pid: record.pid, pid_started_at: record.pid_started_at })), ...(ownedChildGone ? { startup_child_cleanup: proof } : {}) };
                     closedDaemons.set(directory, failedStart);
                 }
                 for (const operation of nativeOperations)
@@ -331,6 +391,8 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
                 unresolved.push({ state_dir: directory, status: "pending", reason: "daemon_operation_inventory_invalid" });
             }
         }
+        if (closedDaemons.get(directory)?.source === "observed_owned_retirement")
+            continue;
         if (state.shutdown_state !== "clean") {
             if (state.shutdown_state === "unclean")
                 unresolved.push({ state_dir: directory, status: "pending", reason: "native_shutdown_unclean" });
@@ -634,11 +696,17 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
         const nativeOperations = retainedDaemonOperations(directory);
         const targetList = [...targets.values()];
         const sessionsSettled = ids.every(id => nodes.some((node, index) => targetList[index]?.session.state_dir === directory && targetList[index]?.session.id === id && node.status === "completed"));
-        // A confirmed failed tree guard admits a new shutdown attempt only after
-        // fresh tree settlement. Unknown outcomes retain their original ID.
+        // A confirmed failure may retry the same incarnation's settled tree or
+        // remaining idempotent resource finish. Unknown outcomes keep their ID.
         if (sessionsSettled) {
+            const remainingFinish = state.shutdown_state === "cleanup_failed" && state.active_tree === false
+                && state.shutdown?.daemon_id === daemonId && state.shutdown.phases?.tree === true
+                && state.shutdown.required_phases?.includes("daemon_resource")
+                && state.shutdown.phases.daemon_resource !== true
+                && state.shutdown.required_phases.every(phase => phase === "daemon_resource" || state.shutdown.phases[phase] === true);
             while (nativeOperations.some(operation => operation.operation_id === operationId && operation.daemon_id === daemonId && operation.operation === "daemon.stop"
-                && operation.state === "failed" && operation.error?.code === "tree_not_settled"))
+                && operation.state === "failed" && (operation.error?.code === "tree_not_settled"
+                || remainingFinish && operation.error?.code === "process_maintenance_timeout")))
                 operationId += ":finalize";
         }
         assertCurrent();
@@ -697,11 +765,13 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
             const executable = directoryAdapters.get(directory);
             if (!inventory)
                 continue;
+            const pending = retainedDaemonOperations(directory).filter(operation => operation.daemon_id === daemonId && ["running", "awaiting_provider"].includes(String(operation.state)) && ["session.create", "session.prompt", "session.start", "session.fork", "session.resume"].includes(String(operation.operation)) && typeof operation.operation_id === "string");
+            if (!pending.length)
+                continue;
             if (!daemonId || !executable || !node.operation_id) {
                 terminalizedOperations.push({ state_dir: directory, status: "pending", reason: "operation_terminalization_binding_unknown" });
                 continue;
             }
-            const pending = retainedDaemonOperations(directory).filter(operation => operation.daemon_id === daemonId && ["running", "awaiting_provider"].includes(String(operation.state)) && ["session.create", "session.prompt", "session.start", "session.fork", "session.resume"].includes(String(operation.operation)) && typeof operation.operation_id === "string");
             terminalizedOperations.push(...await Promise.all(pending.map(async (operation) => {
                 try {
                     const operationId = operation.operation_id, sessionId = typeof operation.session_id === "string" ? operation.session_id : null;
@@ -795,7 +865,8 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
             verifyInventory();
         }
         catch (error) {
-            unresolved.push({ status: "pending", reason: error instanceof AppError && error.code === "owned_inventory_changed" ? error.code : "owned_inventory_recheck_failed" });
+            unresolved.push({ status: "pending", reason: error instanceof AppError && error.code === "owned_inventory_changed" ? error.code : "owned_inventory_recheck_failed",
+                ...(error instanceof AppError && error.code === "owned_inventory_changed" ? { components: error.details.components } : {}) });
         }
     }
     assertCurrent();
@@ -818,7 +889,7 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
 }
 export function nativeInventoryVersion(directory, operations = retainedDaemonOperations(directory)) {
     const state = JSON.parse(fs.readFileSync(path.join(directory, "daemon.json"), "utf8"));
-    const keys = ["schema_id", "daemon_id", "config", "pid", "provider_pid", "shutdown_state", "active_operation", "active_tree", "sessions", "descendants", "root_session_id", "resource_process", "provider_process", "owned_processes"];
+    const keys = ["schema_id", "daemon_id", "config", "pid", "provider_pid", "shutdown_state", "shutdown", "active_operation", "active_tree", "sessions", "descendants", "root_session_id", "resource_process", "provider_process", "owned_processes"];
     // Only these native verbs are read-only. Unknown verbs remain guarded.
     const productive = operations.filter(operation => !isObservationOperation(operation.operation));
     return crypto.createHash("sha256").update(JSON.stringify([Object.fromEntries(keys.map(key => [key, state[key]])), productive])).digest("hex");

package/dist/services/run-controller-capture.js

@@ -6,7 +6,7 @@ import { createEvalRunSnapshot, verifyControllerBoundarySnapshot } from "./eval-
 import { observeControllerBoundary } from "./run-control.js";
 import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
 import { appendEvent, digest, errorRecord, requireController, requireRun } from "./run-controller-state.js";
-import { getFlowRunStatus } from "./runs.js";
+import { getFlowRunStatus, readFlowRunExecutionBundle } from "./runs.js";
 import { recoveryObservationBudget, withRecoveryObservation } from "./recovery-observation-budget.js";
 import { getResourceDatabase } from "../storage/database.js";
 import { managedProcessBinding, resourceHome } from "./managed-processes.js";
@@ -60,6 +60,7 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
             throw error;
         }
     }
+    const preparedBundle = readFlowRunExecutionBundle(context, scope);
     const assertStartOwner = prepareControllerPublication(context, row, assertPhysicalOwner);
     context.db.writeTransaction(() => {
         assertStartOwner();
@@ -82,7 +83,7 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
         if (!current || !["done", "skipped"].includes(current.status) || digest(identity(current)) !== boundaryKey)
             throw new AppError("controller_boundary_changed", "Stage identity changed after its capture fence", 1);
         if (physical) {
-            const continuation = getFlowRunStatus(context, scope).continuation;
+            const continuation = getFlowRunStatus(context, scope, preparedBundle ?? undefined).continuation;
             const stoppedAtTarget = nextStage === null && manifest.request.stop_after === stage.stage && continuation.kind === "start_stage";
             if (nextStage ? continuation.kind !== "start_stage" || continuation.stage !== nextStage : continuation.kind !== "terminal" && !stoppedAtTarget)
                 throw new AppError("controller_boundary_changed", "Stage successor changed after its capture fence", 1);
@@ -123,7 +124,13 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
                 assertCurrent();
                 context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), context.now(), operationId]);
                 let verify = () => { throw new AppError("controller_boundary_unsettled", "Boundary has no stable writer observation", 1); };
-                const observation = await withRecoveryObservation(observationBudget, () => observeControllerBoundary(context, { ...scope, controllerId: row.controller_id, operationId, assertCurrent, retainVerification: value => { verify = value; } }));
+                const observation = await withRecoveryObservation(observationBudget, () => observeControllerBoundary(context, { ...scope, controllerId: row.controller_id, operationId, assertCurrent, retainVerification: value => {
+                        verify = () => {
+                            value();
+                            if (readFlowRunExecutionBundle(context, scope)?.sha256 !== preparedBundle?.sha256)
+                                throw new AppError("controller_boundary_changed", "Frozen bundle changed before snapshot publication", 1);
+                        };
+                    } }));
                 if (observation.settled === true) {
                     try {
                         createEvalRunSnapshot(context, { ...scope, output, ...(nextStage ? { stageEntry: nextStage } : { candidate: true }), boundary: { controllerId: row.controller_id, operationId, boundaryKey, observation, verify } });

package/dist/services/run-controller-process.js

@@ -48,6 +48,22 @@ export function assertControllerRecoveryAllowed(context, row) {
         || !context.db.get("SELECT 1 FROM run_controls WHERE control_id = ? AND recovery_id = ? AND status = 'resuming'", [state.recovery.control_id, state.recovery.recovery_id]))
         throw new AppError("run_control_superseded", "Controller launch no longer owns the accepted recovery", 1);
 }
+/** An admitted physical owner may renew while closing a terminal RUN. This
+ * does not admit a new owner or release a recovery/generation fence. */
+export function assertControllerLeaseAllowed(context, row) {
+    const current = requireController(context, row.controller_id);
+    if (current.owner_token !== row.owner_token || current.generation !== row.generation
+        || current.owner_pid !== process.pid || current.process_id !== row.process_id
+        || current.process_lease_token !== row.process_lease_token || !current.process_id || !current.process_lease_token
+        || !["running", "capturing", "waiting_for_user", "waiting_for_context", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(current.status))
+        throw new AppError("controller_owner_mismatch", "Lease renewal no longer owns this physical controller", 1);
+    const run = requireRun(context, current.project_id, current.run_id);
+    if ((["done", "failed", "cancelled"].includes(run.status) || current.status === "capturing") && launchRecord(context, current)?.value.dispatch_ready === true) {
+        assertControllerRecoveryAllowed(context, current);
+        return;
+    }
+    assertLaunchAllowed(context, current);
+}
 function retire(context, row, identity) {
     const record = managedProcessStatus(context).find(item => item.id === identity.id);
     if (!record || record.kind !== "run-controller" || record.owner_id !== row.controller_id || record.project_id !== row.project_id || record.run_id !== row.run_id
@@ -123,6 +139,10 @@ export function claimControllerProcess(context, row) {
         throw new AppError("controller_owner_mismatch", "Managed registration is not authorized", 1);
     const owner = birth();
     context.db.writeTransaction(() => {
+        // Recovery fences are reconciled by serve's retained failure boundary;
+        // only terminal admission must reject before claiming a physical owner.
+        if (["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
+            throw new AppError("invalid_run_state", "A terminal RUN cannot start a physical controller", 1);
         const claimed = context.db.run("UPDATE run_controllers SET owner_pid = ?, status = 'running', updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [process.pid, context.now(), row.controller_id, row.owner_token]);
         if (claimed.changes !== 1)
             throw new AppError("controller_already_owned", "Managed owner has already claimed this launch", 1);
@@ -142,6 +162,7 @@ export function claimControllerProcess(context, row) {
     }
 }
 export function markControllerDispatchReady(context, row) {
+    assertLaunchAllowed(context, row);
     context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.dispatch_ready', json('true')), updated_at = ? WHERE operation_id = ?", [context.now(), launchId(row)]);
 }
 export async function spawnController(context, row) {

package/dist/services/run-controller-recovery.js

@@ -10,6 +10,7 @@ import { verifyRecoverySnapshot } from "./eval-snapshots.js";
 import { recoveryGuard, resumeRunRecovery } from "./run-recovery.js";
 import { prepareRunRecoveryDispatch, retainRecoveryCapacityIntent } from "./run-recovery-runtime.js";
 import { runtimeHarness } from "./execution-policy.js";
+import { selectControllerExecution } from "./controller-execution.js";
 import { managedProcessBinding, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processTreeIsAlive, processStartedAt } from "./managed-processes.js";
 import { requireRun, requireController, controllerReceipt, appendEvent, saveState, digest, errorRecord } from "./run-controller-state.js";
 import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
@@ -74,7 +75,8 @@ export function prepareControllerResume(context, input) {
         if (!guard || guard.recovery_id !== control?.recovery_id || guard.status !== "sealed" || control.status !== "settled" || !guard.capture_path || JSON.parse(guard.settlement_json ?? "{}").settled !== true)
             throw new AppError("run_control_not_resumable", "Resume requires the current settled control and its immutable recovery capture", 1);
         const manifest = JSON.parse(row.manifest_json), state = JSON.parse(row.state_json);
-        if (!manifest.execution_profile?.settings.execution || !manifest.execution_profile.agent_profiles || !path.isAbsolute(manifest.resource_home) || !path.isAbsolute(manifest.flow_executable))
+        selectControllerExecution(context, run, manifest);
+        if (!path.isAbsolute(manifest.resource_home) || !path.isAbsolute(manifest.flow_executable))
             throw new AppError("controller_manifest_invalid", "Resume requires the original frozen controller manifest", 1);
         if (state.current_session === null || !state.sessions[state.current_session]?.id)
             throw new AppError("recovery_owner_missing", "Resume requires the retained coordinator Session", 1);

package/dist/services/run-controller-state.js

@@ -3,7 +3,7 @@ import { AppError, errorRecord as structuredErrorRecord } from "../shared/errors
 import { runtimeHarness } from "./execution-policy.js";
 import { managedProcessRuntimeHome, managedProcessStatus, processTreeIsAlive } from "./managed-processes.js";
 export function requireRun(context, projectId, runId) {
-    const run = context.db.get("SELECT id, project_id, project_root, workspace_root, run_root, status, index_json FROM runs WHERE project_id = ? AND id = ?", [projectId, runId]);
+    const run = context.db.get("SELECT id, project_id, flow_kind, project_root, workspace_root, run_root, status, index_json FROM runs WHERE project_id = ? AND id = ?", [projectId, runId]);
     if (!run?.run_root)
         throw new AppError("not_found", "Managed RUN has no materialized root", 1, { run_id: runId });
     return run;