@deksden-com/dd-flow-cli 0.9.0-beta.144 → 0.9.0-beta.149

package/dist/services/run-controller-process.js

@@ -2,8 +2,8 @@ import fs from "node:fs";
 import crypto from "node:crypto";
 import path from "node:path";
 import { spawn } from "node:child_process";
-import { fileURLToPath } from "node:url";
 import { AppError } from "../shared/errors.js";
+import { validateCheckEnvironment } from "./check-environment.js";
 import { getResourceDatabase } from "../storage/database.js";
 import { finishManagedProcess, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processStartedAt, processTreeIsAlive, registerManagedProcess } from "./managed-processes.js";
 import { setFlowRunRecoveryPaused } from "./runs.js";
@@ -133,11 +133,10 @@ function prepareSpawn(context, row) {
     });
 }
 /** The physical child records its own birth before resource registration. */
-export function claimControllerProcess(context, row) {
+export function claimControllerProcess(context, row, owner = birth()) {
     const launch = launchRecord(context, row);
     if (!launch && (!row.process_id || !row.process_lease_token))
         throw new AppError("controller_owner_mismatch", "Managed registration is not authorized", 1);
-    const owner = birth();
     context.db.writeTransaction(() => {
         // Recovery fences are reconciled by serve's retained failure boundary;
         // only terminal admission must reject before claiming a physical owner.
@@ -173,13 +172,11 @@ export async function spawnController(context, row) {
     const manifest = JSON.parse(row.manifest_json);
     const run = requireRun(context, row.project_id, row.run_id);
     const controllerId = row.controller_id, root = row.state_dir, token = row.owner_token;
-    const entry = fileURLToPath(new URL("../cli.js", import.meta.url));
     try {
-        if (!fs.existsSync(entry))
-            throw new AppError("harness_runtime_unbuilt", "Build the CLI before launching its detached controller", 1);
+        const entry = (await import("./engines.js")).boundRunEngineEntrypoint(context, { projectRoot: run.project_root, runId: run.id });
         const output = fs.openSync(path.join(root, "controller.log"), "a", 0o600);
         try {
-            const child = spawn(process.execPath, [entry, "run", "drive", "serve", "--run", run.id, "--project-root", run.project_root, "--controller-id", controllerId, "--json"], { cwd: run.workspace_root, detached: true, stdio: ["ignore", output, output], env: { ...context.env, DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_ENGINE_MODE: "1", DD_FLOW_CONTROLLER_TOKEN: token } });
+            const child = spawn(process.execPath, [entry, "run", "drive", "serve", "--run", run.id, "--project-root", run.project_root, "--controller-id", controllerId, "--json"], { cwd: run.workspace_root, detached: true, stdio: ["ignore", output, output], env: { ...context.env, ...validateCheckEnvironment(manifest.check_environment), DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_ENGINE_MODE: "1", DD_FLOW_CONTROLLER_TOKEN: token } });
             await new Promise((resolve, reject) => { child.once("spawn", resolve); child.once("error", reject); });
             context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.child', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify({ pid: child.pid, pid_started_at: child.pid ? processStartedAt(child.pid) : null }), context.now(), launchId(row)]);
             child.unref();

package/dist/services/run-controller-recovery.js

@@ -1,4 +1,5 @@
 import crypto from "node:crypto";
+import { nativeShutdownConfirmed } from "../harness-runtime/lib/native-children.mjs";
 import fs from "node:fs";
 import path from "node:path";
 import { setTimeout as delay } from "node:timers/promises";
@@ -148,7 +149,7 @@ export async function resumeControllerSession(context, row, manifest, state, ass
     const call = (args, operationId) => controllerAdapter(executionContext, row, manifest, session, args, assertRecoveryOwner, operationId);
     if (processTreeIsAlive(record)) {
         const stopped = await call(["daemon", "stop", "--state-dir", session.state_dir, "--json"]);
-        if (stopped.stopped !== true || stopped.clean !== true)
+        if (!retired.daemon_id || stopped.stopped !== true || !nativeShutdownConfirmed(stopped, { daemonId: retired.daemon_id }))
             throw new AppError("recovery_native_shutdown_unproven", "Recovery requires a clean native close before replacing its daemon", 1);
         const deadline = Date.now() + 30_000;
         while (processTreeIsAlive(record) && Date.now() < deadline) {
@@ -255,7 +256,12 @@ export async function resumeControllerSession(context, row, manifest, state, ass
                     || current.process.lease_expires_at <= context.now() || metadata.run_generation !== row.generation || !processIsAlive(current.process))
                     throw new AppError("recovery_capacity_owner_changed", "Replacement native owner changed during capacity wait", 1);
             },
-            decision: () => recoveryGuard(context, row.project_id, row.run_id)?.status === "recovered" ? "completed" : "continue",
+            decision: (boundary) => {
+                if (boundary && (boundary.lifecycle_operation !== "recovery_accept" || boundary.recovery_id !== recovery.recovery_id
+                    || !context.db.get("SELECT 1 FROM run_recovery_bindings b JOIN hook_events h ON h.event_key = b.hook_event_key WHERE b.recovery_id = ? AND b.status = 'accepted' AND b.daemon_id = ? AND b.provider_session_id = ? AND h.project_id = ? AND h.status = 'claimed' AND h.event_key = ?", [recovery.recovery_id, daemonId, session.id, row.project_id, boundary.native_hook_event_key])))
+                    return "reconciliation_blocked";
+                return recoveryGuard(context, row.project_id, row.run_id)?.status === "recovered" ? "completed" : "continue";
+            },
             retainIntent: intent => retainRecoveryCapacityIntent(executionContext, { projectId: row.project_id, runId: row.run_id, recoveryId: recovery.recovery_id,
                 controllerId: row.controller_id, generation: row.generation, daemonId, intent, prompt: continuationText }),
             prompt: (operationId, prompt) => sendAck(operationId, prompt) });

package/dist/services/run-controller.js

@@ -1,15 +1,18 @@
-import { assertLifecycleOutcomes, currentLifecycleFailure, managedInvocationContext, managedLifecycleCommand } from "./lifecycle-invocations.js";
+import { assertLifecycleOutcomes, currentLifecycleFailure, managedInvocationContext, managedLifecycleCommand, awaitLifecycleInvocation, assertLifecycleInvocationCurrent } from "./lifecycle-invocations.js";
+import { isDiagnosticFlow } from "../domain/diagnostic-flows.js";
 export function recoveryPacketMatchesStage(packet, scope) {
     return packet.schema_id === 'dd-flow/root-recovery-packet@1' && packet.continuation === 'unfinished_work'
         && packet.run_id === scope.runId && packet.generation === scope.generation && packet.stage === scope.stage
         && packet.work_id === scope.workId && packet.paused === false;
 }
 import crypto from "node:crypto";
+import { nativeShutdownConfirmed } from "../harness-runtime/lib/native-children.mjs";
 import fs from "node:fs";
 import path from "node:path";
 import { setTimeout as delay } from "node:timers/promises";
 import { fileURLToPath } from "node:url";
 import { quote } from "shell-quote";
+import { retainedBrowserEnvironment, validateCheckEnvironment } from "./check-environment.js";
 import { ensureRunControllerStorage, getDatabase } from "../storage/database.js";
 import { canonicalPath, resolveProjectRoot } from "../storage/paths.js";
 import { publishImmutableInput } from "../storage/immutable-input.js";
@@ -20,17 +23,21 @@ import { requireProjectByRoot } from "./projects.js";
 import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
 import { getFlowRunStatus, setFlowRunRecoveryPaused, assertPreparedRunBundleSqlAuthority } from "./runs.js";
 import { loadExternalStageContext } from "./stage-context.js";
-import { flowCommand } from "./stage-pause.js";
+import { flowCommand, stageResumeSourcePath } from "./stage-pause.js";
 import { sameSessionProfile, runtimeHarness } from "./execution-policy.js";
 import { selectControllerExecution } from "./controller-execution.js";
 import { resolveHarnessCommand, harnessRuntimeArguments } from "./harness-config.js";
 import { adapterSessionId, prepareHarnessFlowExecutable, runHarnessAdapter } from "./harness-adapter.js";
-import { assertManagedProcessLease, confirmManagedProcess, renewManagedProcess, finishManagedProcess, processStartedAt, processTreeIsAlive, resourceHome } from "./managed-processes.js";
+import { assertManagedProcessLease, confirmManagedProcess, prepareManagedProcessConfirmation, renewManagedProcess, finishManagedProcess, processStartedAt, processTreeIsAlive, resourceHome } from "./managed-processes.js";
 import { managedLeaseMonitor } from "./managed-lease-monitor.js";
+import { physicalStartup } from "./physical-startup.js";
 import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
 import { initCodexHome } from "./hooks.js";
 import { launchExternalWork } from "./external-work-launch.js";
+import { runControllerMemoryTask } from "./controller-memory-task.js";
 import { controllerNativeChildren, controllerStageEntryPrompt, controllerStageResumePrompt, nextControllerFanout } from "./controller-fanout.js";
+import { publishPromptDelivery } from "./prompt-delivery.js";
+import { publishStagePacket } from "./stage-report-renderer.js";
 import { claimQueuedMergeRequest } from "./merge-server.js";
 import { prepareVnextCodeWorkspace } from "./vnext-code.js";
 import { requireRun, requireController, controllerReceipt, appendEvent, saveState, retainControllerContinuation, retainControllerCapacityIntent, digest, digestBytes, errorRecord } from "./run-controller-state.js";
@@ -38,7 +45,7 @@ import { assertNoCorrelatedRejection } from "./continuation-outcome.js";
 import { spawnController, recordControllerLauncher, claimControllerProcess, markControllerDispatchReady, assertControllerLeaseAllowed } from "./run-controller-process.js";
 import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
 import { retainedChildObserver } from "./controller-child-observer.js";
-import { capacityDelay, continueProviderCapacity, eligibleProviderChildOverload, providerChildFailure } from "./provider-capacity.js";
+import { capacityDelay, continueProviderCapacity, eligibleProviderChildOverload, providerChildFailure, ownedProviderBoundary } from "./provider-capacity.js";
 import { resumeControllerSession, failControllerRecovery } from "./run-controller-recovery.js";
 import { captureControllerBoundary } from "./run-controller-capture.js";
 import { requestRunControl } from "./run-control-worker.js";
@@ -133,7 +140,8 @@ export function retainRunControllerLaunch(context, input, prepared = prepareRunC
     }
     const manifest = { schema_id: routing.kind === "descriptor" ? "dd-flow/run-controller-manifest@2" : "dd-flow/run-controller-manifest@1", request,
         ...(routing.kind === "descriptor" ? { execution_binding: routing.binding } : { execution_profile: routing.legacy }),
-        contexts: frozenContexts, adapters, flow_executable: flowExecutable, resource_home: resourceHome(context), budget };
+        contexts: frozenContexts, adapters, flow_executable: flowExecutable, resource_home: resourceHome(context), budget,
+        ...(retainedBrowserEnvironment(context.env) ? { check_environment: retainedBrowserEnvironment(context.env) } : {}) };
     const token = crypto.randomUUID();
     const now = context.now();
     const state = { stage: null, sessions: [], current_session: null, turns: 0, last_continuation: null };
@@ -215,92 +223,120 @@ export async function serveRunController(context, input) {
     const row = requireController(context, input.controllerId);
     if (row.project_id !== project.id || row.run_id !== input.runId || row.owner_token !== context.env.DD_FLOW_CONTROLLER_TOKEN)
         throw new AppError("controller_owner_mismatch", "Managed serve does not own this launch", 1);
-    claimControllerProcess(context, row);
-    if (!row.process_id || !row.process_lease_token)
-        throw new AppError("controller_owner_mismatch", "Managed owner has no resource lease", 1);
-    const confirmed = confirmManagedProcess(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid });
-    const lease = controllerLeaseMonitor(attemptMs => renewManagedProcess({ ...context, resourceDatabaseMode: "prepared_existing", databaseDeadlineAt: Date.now() + attemptMs }, { id: row.process_id, leaseToken: row.process_lease_token }), confirmed.lease_expires_at);
-    const state = JSON.parse(row.state_json);
-    // Resource and RUN commits are distinct effects. A RUN keepalive failure must
-    // neither erase a resource receipt nor grant RUN dispatch authority.
-    const mergeLease = controllerLeaseMonitor(attemptMs => {
-        const expiry = new Date(Date.now() + 120_000).toISOString();
-        const db = context.db.withDeadline(Date.now() + attemptMs);
-        if (db.get("SELECT 1 FROM merge_requests WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status = 'dispatching' LIMIT 1", [row.project_id, row.run_id, row.controller_id]))
-            db.writeTransaction(() => {
-                const guard = recoveryGuard({ ...context, db }, row.project_id, row.run_id);
-                if ((guard?.generation ?? 0) !== row.generation || ["draining", "sealed", "resuming"].includes(guard?.status ?? ""))
-                    throw new AppError("controller_lease_lost", "RUN generation changed before MERGE keepalive", 1);
-                db.run("UPDATE merge_requests SET dispatch_lease_expires_at = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status = 'dispatching'", [expiry, context.now(), row.project_id, row.run_id, row.controller_id]);
-            });
-        return { owned: true, lease_expires_at: expiry };
-    });
-    const assertLeaseFence = () => {
-        assertControllerLeaseAllowed(context, row);
-    };
-    const heartbeat = setInterval(() => {
-        lease.background(assertLeaseFence, () => { });
-        if (state.merge_claim)
-            mergeLease.background(assertLeaseFence, () => { });
-    }, 30_000);
-    heartbeat.unref();
-    const manifest = JSON.parse(row.manifest_json);
-    const assertPhysicalOwner = () => {
-        lease.assertNoRenewal();
-        if (state.merge_claim)
-            mergeLease.assertNoRenewal();
-        // This isolated read already bounds SQLite contention. A wall deadline
-        // before opening the reader mistakes scheduler delay for heartbeat loss;
-        // the observed row's actual expiry/token still fences productive dispatch.
-        assertManagedProcessLease(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid });
-        const current = requireController(context, row.controller_id);
-        if (current.owner_pid !== process.pid || current.owner_token !== row.owner_token || !["running", "capturing", "waiting_for_user", "waiting_for_context", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(current.status))
-            throw new AppError("controller_owner_mismatch", "Controller no longer owns productive dispatch", 1);
-    };
-    const assertOwner = () => { assertPhysicalOwner(); assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation); };
+    let lease;
+    let mergeLease;
+    let heartbeat;
+    let state;
+    let claimedProcess = false;
     let primaryFailure;
     const retainSecondary = (phase, error) => {
         process.stderr.write(`${JSON.stringify({ controller_id: row.controller_id, phase, primary: errorRecord(primaryFailure), secondary: errorRecord(error) })}\n`);
     };
     try {
+        const currentState = state = JSON.parse(row.state_json);
+        const startup = physicalStartup(context.databaseDeadlineAt);
+        const assertStartingOwner = () => {
+            const current = requireController(context, row.controller_id);
+            if (current.owner_token !== row.owner_token || current.generation !== row.generation || current.owner_pid !== null || current.status !== "starting")
+                throw new AppError("controller_owner_mismatch", "Controller startup no longer owns this launch", 1);
+            if (["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
+                throw new AppError("invalid_run_state", "A terminal RUN cannot start a physical controller", 1);
+        };
+        const owner = await startup.birth(assertStartingOwner);
+        try {
+            claimControllerProcess(context, row, owner);
+        }
+        finally {
+            claimedProcess = row.owner_pid === process.pid;
+        }
+        if (!row.process_id || !row.process_lease_token)
+            throw new AppError("controller_owner_mismatch", "Managed owner has no resource lease", 1);
+        const confirmation = { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid };
+        const originalLease = prepareManagedProcessConfirmation(context, confirmation).lease_expires_at;
+        const confirmed = await startup.observe(() => confirmManagedProcess(context, confirmation), () => assertControllerLeaseAllowed(context, row), originalLease);
+        lease = controllerLeaseMonitor(attemptMs => renewManagedProcess({ ...context, resourceDatabaseMode: "prepared_existing", databaseDeadlineAt: Date.now() + attemptMs }, { id: row.process_id, leaseToken: row.process_lease_token }), confirmed.lease_expires_at);
+        // Resource and RUN commits are distinct effects. A RUN keepalive failure must
+        // neither erase a resource receipt nor grant RUN dispatch authority.
+        mergeLease = controllerLeaseMonitor(attemptMs => {
+            const expiry = new Date(Date.now() + 120_000).toISOString();
+            const db = context.db.withDeadline(Date.now() + attemptMs);
+            if (db.get("SELECT 1 FROM merge_requests WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status = 'dispatching' LIMIT 1", [row.project_id, row.run_id, row.controller_id]))
+                db.writeTransaction(() => {
+                    const guard = recoveryGuard({ ...context, db }, row.project_id, row.run_id);
+                    if ((guard?.generation ?? 0) !== row.generation || ["draining", "sealed", "resuming"].includes(guard?.status ?? ""))
+                        throw new AppError("controller_lease_lost", "RUN generation changed before MERGE keepalive", 1);
+                    db.run("UPDATE merge_requests SET dispatch_lease_expires_at = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status = 'dispatching'", [expiry, context.now(), row.project_id, row.run_id, row.controller_id]);
+                });
+            return { owned: true, lease_expires_at: expiry };
+        });
+        const assertLeaseFence = () => {
+            assertControllerLeaseAllowed(context, row);
+        };
+        heartbeat = setInterval(() => {
+            lease.background(assertLeaseFence, () => { });
+            if (currentState.merge_claim)
+                mergeLease.background(assertLeaseFence, () => { });
+        }, 30_000);
+        heartbeat.unref();
+        const manifest = JSON.parse(row.manifest_json);
+        const assertPhysicalOwner = () => {
+            lease.assertNoRenewal();
+            if (currentState.merge_claim)
+                mergeLease.assertNoRenewal();
+            // This isolated read already bounds SQLite contention. A wall deadline
+            // before opening the reader mistakes scheduler delay for heartbeat loss;
+            // the observed row's actual expiry/token still fences productive dispatch.
+            assertManagedProcessLease(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid });
+            const current = requireController(context, row.controller_id);
+            if (current.owner_pid !== process.pid || current.owner_token !== row.owner_token || !["running", "capturing", "waiting_for_user", "waiting_for_context", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(current.status))
+                throw new AppError("controller_owner_mismatch", "Controller no longer owns productive dispatch", 1);
+        };
+        const assertOwner = () => { assertPhysicalOwner(); assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation); };
         await lease.ensureConfirmed(assertLeaseFence);
-        if (state.merge_claim)
+        if (currentState.merge_claim)
             await mergeLease.ensureConfirmed(assertLeaseFence);
         context.db.writeTransaction(() => {
             markControllerDispatchReady(context, row);
-            if (!state.recovery && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation && requireRun(context, row.project_id, row.run_id).status === "paused")
+            if (!currentState.recovery && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation && requireRun(context, row.project_id, row.run_id).status === "paused")
                 setFlowRunRecoveryPaused(context, row.project_id, row.run_id, false);
         });
         appendEvent(context, row.controller_id, "owner_started", { pid: process.pid, generation: row.generation });
-        if (state.recovery && !["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
-            await resumeControllerSession(context, row, manifest, state, assertPhysicalOwner);
-        await executeController(context, row, manifest, state, assertOwner, assertPhysicalOwner, async () => { await lease.ensureConfirmed(assertLeaseFence); if (state.merge_claim)
+        if (currentState.recovery && !["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
+            await resumeControllerSession(context, row, manifest, currentState, assertPhysicalOwner);
+        await executeController(context, row, manifest, currentState, assertOwner, assertPhysicalOwner, async () => { await lease.ensureConfirmed(assertLeaseFence); if (currentState.merge_claim)
             await mergeLease.ensureConfirmed(assertLeaseFence); });
         return controllerReceipt(requireController(context, row.controller_id));
     }
     catch (error) {
         primaryFailure = error;
         try {
-            try {
-                failControllerRecovery(context, row, state, error);
-            }
-            catch (secondary) {
-                retainSecondary("fail_controller_recovery", secondary);
-            }
-            if ((error instanceof AppError && ["run_recovery_guarded", "run_recovery_generation_stale"].includes(error.code) || (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) !== row.generation)
-                && context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controls'")) {
-                const guard = recoveryGuard(context, row.project_id, row.run_id);
-                const control = guard && ["draining", "sealed"].includes(guard.status) && context.db.get("SELECT control_id, action FROM run_controls WHERE project_id = ? AND run_id = ? AND recovery_id = ? AND status IN ('requested','settled')", [row.project_id, row.run_id, guard.recovery_id]);
-                if (control) {
-                    // Control is a later state transition, not evidence that the causal
-                    // failure which fenced admission disappeared.
-                    context.db.run("UPDATE run_controllers SET status = CASE WHEN last_error_json IS NULL THEN 'control_requested' ELSE status END, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [context.now(), row.controller_id, row.owner_token]);
-                    appendEvent(context, row.controller_id, "control_boundary_reached", { control_id: control.control_id, action: control.action, generation: guard.generation, settled: false });
-                    return controllerReceipt(requireController(context, row.controller_id));
+            const controlled = context.db.writeTransaction(() => {
+                const currentOwner = requireController(context, row.controller_id);
+                const sameBinding = currentOwner.owner_token === row.owner_token && currentOwner.generation === row.generation
+                    && currentOwner.process_id === row.process_id && currentOwner.process_lease_token === row.process_lease_token;
+                const ownsFailure = claimedProcess ? currentOwner.owner_pid === process.pid : row.owner_pid === null && currentOwner.owner_pid === null && row.status === "starting" && currentOwner.status === "starting";
+                if (!sameBinding || !ownsFailure || ["completed", "cancelled", "superseded"].includes(currentOwner.status))
+                    throw error;
+                try {
+                    if (state)
+                        failControllerRecovery(context, row, state, error);
                 }
-            }
-            try {
-                context.db.writeTransaction(() => {
+                catch (secondary) {
+                    retainSecondary("fail_controller_recovery", secondary);
+                }
+                if ((error instanceof AppError && ["run_recovery_guarded", "run_recovery_generation_stale"].includes(error.code) || (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) !== row.generation)
+                    && context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controls'")) {
+                    const guard = recoveryGuard(context, row.project_id, row.run_id);
+                    const control = guard && ["draining", "sealed"].includes(guard.status) && context.db.get("SELECT control_id, action FROM run_controls WHERE project_id = ? AND run_id = ? AND recovery_id = ? AND status IN ('requested','settled')", [row.project_id, row.run_id, guard.recovery_id]);
+                    if (control) {
+                        // Control is a later state transition, not evidence that the causal
+                        // failure which fenced admission disappeared.
+                        context.db.run("UPDATE run_controllers SET status = CASE WHEN last_error_json IS NULL THEN 'control_requested' ELSE status END, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [context.now(), row.controller_id, row.owner_token]);
+                        appendEvent(context, row.controller_id, "control_boundary_reached", { control_id: control.control_id, action: control.action, generation: guard.generation, settled: false });
+                        return controllerReceipt(requireController(context, row.controller_id));
+                    }
+                }
+                try {
                     // A failed capture must not reopen admission when this process exits.
                     const changed = context.db.run("UPDATE run_controllers SET status = CASE WHEN status IN ('capturing','capture_failed') THEN 'capture_failed' ELSE 'recovery_required' END, last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status NOT IN ('completed','cancelled','superseded')", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token]);
                     context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status IN ('dispatching','active')", [JSON.stringify(errorRecord(error)), context.now(), row.project_id, row.run_id, row.controller_id]);
@@ -308,11 +344,14 @@ export async function serveRunController(context, input) {
                         setFlowRunRecoveryPaused(context, row.project_id, row.run_id, true);
                     }
                     appendEvent(context, row.controller_id, "recovery_required", errorRecord(error));
-                });
-            }
-            catch (failure) {
-                retainSecondary("persist_controller_failure", failure);
-            }
+                }
+                catch (failure) {
+                    retainSecondary("persist_controller_failure", failure);
+                }
+                return null;
+            });
+            if (controlled)
+                return controlled;
         }
         catch (secondary) {
             retainSecondary("controller_failure_boundary", secondary);
@@ -320,12 +359,15 @@ export async function serveRunController(context, input) {
         throw error;
     }
     finally {
-        lease.close();
-        mergeLease.close();
-        clearInterval(heartbeat);
+        lease?.close();
+        mergeLease?.close();
+        if (heartbeat)
+            clearInterval(heartbeat);
         finishOwner();
     }
     function finishOwner() {
+        if (!claimedProcess || row.owner_pid !== process.pid || !row.process_id || !row.process_lease_token)
+            return;
         try {
             finishManagedProcess(context, { id: row.process_id, leaseToken: row.process_lease_token, state: "stopped", reason: "controller_owner_exited" });
         }
@@ -497,7 +539,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
     const routing = selectControllerExecution(context, requireRun(context, row.project_id, row.run_id), manifest);
     const policy = routing.policy;
     const profiles = routing.profiles;
-    let executionContext = { ...context, env: { ...context.env, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_RUNTIME_BUDGET: manifest.budget ? JSON.stringify(manifest.budget) : undefined, DD_FLOW_CODE_READINESS_OWNER: row.controller_id } };
+    let executionContext = { ...context, env: { ...context.env, ...validateCheckEnvironment(manifest.check_environment), DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_RUNTIME_BUDGET: manifest.budget ? JSON.stringify(manifest.budget) : undefined, DD_FLOW_CODE_READINESS_OWNER: row.controller_id } };
     const bindInvocationScope = (session) => {
         executionContext = managedInvocationContext(executionContext, { harness: runtimeHarness(session.profile), stateDir: session.state_dir, sessionId: session.id,
             projectRoot: manifest.request.project_root, runId: row.run_id, generation: row.generation });
@@ -598,9 +640,9 @@ async function executeController(context, row, manifest, state, assertOwnership,
     const call = (session, args, productive = true) => controllerAdapter(executionContext, row, manifest, session, args, productive ? assertOwner : undefined, undefined, productive ? cancellation.signal : undefined, productive ? renewLease : undefined);
     const confirmStopped = async (stateDir, receipt) => {
         const fail = () => new AppError("controller_native_shutdown_unproven", "The managed daemon is not proven clean and closed", 1, { state_dir: stateDir, receipt });
-        if (receipt.stopped !== true || receipt.clean !== true)
-            throw fail();
         const binding = requireManagedDaemonBinding(executionContext, { stateDir, projectId: row.project_id, runId: row.run_id });
+        if (!binding.state.daemon_id || receipt.stopped !== true || !nativeShutdownConfirmed(receipt, { daemonId: binding.state.daemon_id }))
+            throw fail();
         const deadline = performance.now() + 30_000;
         while (processTreeIsAlive(binding.process) && performance.now() < deadline)
             await delay(100);
@@ -645,6 +687,45 @@ async function executeController(context, row, manifest, state, assertOwnership,
         if (!pending)
             await reconcilePromptPredecessor(session);
         const firstOperationId = pending?.root_operation_id ?? operationId ?? `${row.controller_id}:prompt:${crypto.randomUUID()}`;
+        const hitlText = async () => {
+            const answerId = firstOperationId.slice(0, -`:prompt:${row.generation}`.length);
+            const answer = context.db.get("SELECT input_json,receipt_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND status IN ('accepted','completed')", [answerId, row.controller_id]);
+            const accepted = answer && JSON.parse(answer.input_json);
+            const packet = answer?.receipt_json && JSON.parse(answer.receipt_json).hitl_packet;
+            if (!packet || !accepted || packet.generation !== row.generation || packet.pause_id !== accepted.pause_id || packet.stage !== accepted.stage || packet.work_id !== accepted.work_id || packet.answer_file !== accepted.file || packet.answer_sha256 !== accepted.sha256)
+                throw new AppError("controller_hitl_packet_invalid", "Accepted HITL continuation lost its exact binding", 1);
+            const issued = context.db.get("SELECT id,status,scope_json,command FROM lifecycle_invocations WHERE id = ?", [packet.invocation_id ?? null]);
+            if (!issued || issued.scope_json !== executionContext.env.DD_FLOW_INVOCATION_SCOPE)
+                throw new AppError("controller_hitl_packet_invalid", "Accepted HITL command belongs to another scope", 1);
+            if (issued.status === "executing" || issued.status === "settled") {
+                const recovered = await awaitLifecycleInvocation(executionContext, { id: issued.id, command: issued.command, assertCurrent: scope => assertLifecycleInvocationCurrent(executionContext, scope, issued.command) });
+                if (!recovered.replay)
+                    throw new AppError("controller_hitl_reconciliation_required", "Accepted resume is not committed", 1);
+                const outcome = recovered.outcome;
+                if (!outcome.result)
+                    throw new AppError("controller_hitl_reconciliation_required", "Accepted resume has no committed continuation", 1);
+                if (outcome.result.resume_source_path !== packet.resume_source_path || typeof outcome.result.answer_path !== "string")
+                    throw new AppError("controller_hitl_packet_invalid", "Committed resume differs from the accepted packet locator", 1);
+                publishStagePacket(executionContext, row.project_id, row.run_id, packet.stage, outcome.result.answer_path);
+                publishStagePacket(executionContext, row.project_id, row.run_id, packet.stage, packet.resume_source_path);
+                const bytes = fs.readFileSync(outcome.result.answer_path);
+                if (digestBytes(bytes) !== packet.answer_sha256)
+                    throw new AppError("context_checksum_mismatch", "Committed answer differs from its accepted bytes", 1);
+                fs.mkdirSync(path.dirname(packet.answer_file), { recursive: true, mode: 0o700 });
+                publishImmutableInput(packet.answer_file, bytes, "hitl_answer_conflict");
+                publishPromptDelivery(outcome.result);
+                const delivery = outcome.result.prompt_delivery;
+                const promptPath = delivery?.path ?? outcome.result.prompt_path;
+                if (typeof promptPath !== "string" || !path.isAbsolute(promptPath))
+                    throw new AppError("controller_hitl_packet_invalid", "Committed resume has no absolute prompt delivery", 1);
+                // Keep continuation bytes stable across a retained overload episode.
+                // Its conditional instructions point at this restored exact custody.
+                return controllerStageResumePrompt(packet);
+            }
+            if (issued.status !== "issued" || digestBytes(fs.readFileSync(packet.answer_file)) !== packet.answer_sha256)
+                throw new AppError("context_checksum_mismatch", "Accepted HITL answer or command changed before resume", 1);
+            return controllerStageResumePrompt(packet);
+        };
         const send = (value, id) => {
             const retained = context.db.get("SELECT input_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [id, row.controller_id, row.generation]);
             if (retained) {
@@ -678,7 +759,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
                 promptPending = true;
                 observedPromptOperation = firstOperationId;
                 try {
-                    receipt = await send(text, firstOperationId);
+                    receipt = await send(hitl ? await hitlText() : text, firstOperationId);
                 }
                 finally {
                     promptPending = false;
@@ -686,9 +767,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
                 }
             }
             catch (error) {
-                const resumeCommand = hitl ? text.split("\n\n")[1] : undefined;
-                if (hitl && !resumeCommand?.includes(" stage resume "))
-                    throw error;
+                const hitlContinuationText = hitl && terminalProviderOverload(error, session.id) ? await hitlText() : undefined;
                 const inspect = (args, id) => runHarnessAdapter({ executable: manifest.adapters[runtimeHarness(session.profile)].executable,
                     args, env: controllerEnvironment(context, manifest, session), runtimeOwner: session.owner, operationId: id, timeoutMs: 30_000, signal: cancellation.signal });
                 const resumed = await continueProviderCapacity({ error, sessionId: session.id, operationId: firstOperationId, signal: cancellation.signal, allowSettledDescendants: true, atomicIntent: true,
@@ -706,7 +785,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
                     assertCurrent: () => {
                         assertOwner();
                     },
-                    decision: async () => {
+                    decision: async (boundary) => {
                         if (kind === "child") {
                             const target = Object.entries(state.child_capacity ?? {}).find(([, retained]) => retained.operation_id === firstOperationId);
                             if (!target)
@@ -718,6 +797,11 @@ async function executeController(context, row, manifest, state, assertOwnership,
                         }
                         const view = getFlowRunStatus(executionContext, { projectRoot: manifest.request.project_root, runId: row.run_id });
                         const next = view.continuation;
+                        if (boundary) {
+                            if (!ownedProviderBoundary(context, { projectId: row.project_id, runId: row.run_id, stage: state.stage, sessionId: session.id, generation: row.generation, scope: "stage" }, boundary))
+                                return "reconciliation_blocked";
+                            return boundary.outcome === "paused" ? "paused" : "completed";
+                        }
                         if (next.kind === "blocked")
                             return "reconciliation_blocked";
                         if (next.kind === "paused" && next.stage !== state.stage)
@@ -740,7 +824,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
                     },
                     retainIntent: intent => retainControllerCapacityIntent(context, row, state, intent, session.id, kind ?? "stage", assertOwner),
                     prompt: (id, value) => send(value, id),
-                    ...(resumeCommand ? { continuationText: `Continue the same accepted HITL task in this Session. First inspect current durable RUN and pause state. Only if the exact pause still awaits the accepted answer, run this previously issued command once:\n\n${resumeCommand}\n\nNever replay an already committed answer. Otherwise use only the current issued successor.` } : {}) });
+                    ...(hitlContinuationText ? { continuationText: hitlContinuationText } : {}) });
                 if (!resumed)
                     throw error;
                 receipt = resumed;
@@ -804,6 +888,17 @@ async function executeController(context, row, manifest, state, assertOwnership,
                 await prompt(session, fs.readFileSync(file, "utf8"), pending.root_operation_id, pending.kind === "stage" ? undefined : pending.kind);
                 continue;
             }
+            if (next.kind !== "paused" && state.current_session !== null) {
+                const committed = context.db.get("SELECT operation_id,receipt_json FROM run_controller_operations WHERE controller_id = ? AND status = 'accepted' AND json_extract(input_json,'$.action') = 'user_answer' AND json_extract(receipt_json,'$.hitl_packet.generation') = ? AND json_extract(receipt_json,'$.hitl_packet.stage') = ? ORDER BY created_at LIMIT 1", [row.controller_id, row.generation, state.stage]);
+                if (committed) {
+                    const packet = JSON.parse(committed.receipt_json).hitl_packet;
+                    const receipt = await prompt(state.sessions[state.current_session], controllerStageResumePrompt(packet), `${committed.operation_id}:prompt:${row.generation}`, "hitl");
+                    context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = json_set(receipt_json,'$.delivered',json('true'),'$.session_id',?), updated_at = ? WHERE operation_id = ? AND status = 'accepted'", [adapterSessionId(receipt), context.now(), committed.operation_id]);
+                    state.last_continuation = null;
+                    saveState(context, row, state);
+                    continue;
+                }
+            }
             const completed = state.stage && view.index.stage_runs.find(stage => stage.stage === state.stage && ["done", "skipped"].includes(stage.status));
             if (state.current_session !== null && (completed || next.stage !== state.stage) && childObserver(state.sessions[state.current_session])?.failed().length)
                 throw new AppError("fanout_dispatch_fenced", "A retained child failure forbids completing or advancing its Stage", 1);
@@ -849,13 +944,29 @@ async function executeController(context, row, manifest, state, assertOwnership,
                 assertOwner();
                 // The accepted answer is logical intent; only its generation-bound
                 // adapter operation represents a physical dispatch that stop must drain.
-                const command = managedLifecycleCommand(executionContext, `${flowCommand(executionContext)} stage resume ${run.id} --stage ${next.stage} --work ${input.work_id} --project-root ${quote([run.project_root])} --answer-file ${quote([input.file])} --json`, { replaceUnobservedAnswer: true });
-                const receipt = await prompt(state.sessions[state.current_session], controllerStageResumePrompt(pause.id, command), `${answer.operation_id}:prompt:${row.generation}`, "hitl");
+                let invocationId;
+                const capture = executionContext.captureLifecycleAssignment;
+                executionContext.captureLifecycleAssignment = id => { invocationId = id; capture?.(id); };
+                let command;
+                try {
+                    command = managedLifecycleCommand(executionContext, `${flowCommand(executionContext)} stage resume ${run.id} --stage ${next.stage} --work ${input.work_id} --project-root ${quote([run.project_root])} --answer-file ${quote([input.file])} --json`, { replaceUnobservedAnswer: true });
+                }
+                finally {
+                    if (capture)
+                        executionContext.captureLifecycleAssignment = capture;
+                    else
+                        delete executionContext.captureLifecycleAssignment;
+                }
+                if (!invocationId)
+                    throw new AppError("controller_hitl_packet_invalid", "Accepted resume did not capture its issued identity", 1);
+                const packet = { pause_id: pause.id, stage: next.stage, work_id: input.work_id, answer_file: input.file, answer_sha256: input.sha256, command, generation: row.generation, invocation_id: invocationId, resume_source_path: stageResumeSourcePath(pause, fs.readFileSync(input.file, "utf8")) };
+                context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'accepted'", [JSON.stringify({ hitl_packet: packet }), context.now(), answer.operation_id]);
+                const receipt = await prompt(state.sessions[state.current_session], controllerStageResumePrompt(packet), `${answer.operation_id}:prompt:${row.generation}`, "hitl");
                 const afterAnswer = getFlowRunStatus(executionContext, { projectRoot: run.project_root, runId: run.id });
                 if (afterAnswer.index.stage_runs.some(stage => stage.status === "paused" && stage.pause?.id === pause.id)) {
                     throw new AppError("controller_answer_not_applied", "The answer Turn completed without resuming the assigned HITL pause", 1, { pause_id: pause.id, operation_id: answer.operation_id, stage: next.stage, effect: "no_progress" });
                 }
-                context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify({ delivered: true, session_id: adapterSessionId(receipt) }), context.now(), answer.operation_id]);
+                context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = json_set(receipt_json,'$.delivered',json('true'),'$.session_id',?), updated_at = ? WHERE operation_id = ?", [adapterSessionId(receipt), context.now(), answer.operation_id]);
                 state.last_continuation = null;
                 saveState(context, row, state);
                 setStatus("running");
@@ -1088,7 +1199,12 @@ async function executeController(context, row, manifest, state, assertOwnership,
                     continue;
                 }
             }
-            const fanout = await nextControllerFanout(executionContext, { projectRoot: run.project_root, runId: run.id, stage: next.stage, harness: runtimeHarness(session.profile), sessionId: session.id, children, policy, profiles, dispatchFence: retainedFailures.length > 0 });
+            const memoryFlow = ["mb-init", "mb-upgrade"].includes(run.flow_kind) || isDiagnosticFlow(run.flow_kind);
+            const fanoutInput = { projectRoot: run.project_root, runId: run.id, stage: next.stage, harness: runtimeHarness(session.profile), sessionId: session.id, children, policy, profiles, dispatchFence: retainedFailures.length > 0 };
+            const fanout = memoryFlow ? await runControllerMemoryTask(executionContext, { operation: "fanout", input: fanoutInput }, cancellation.signal)
+                : await nextControllerFanout(executionContext, fanoutInput);
+            if (memoryFlow)
+                await renewLease();
             assertOwner();
             if (fanout.kind === "none") {
                 // An ACK-only recovery Turn may leave the original root Work unfinished.
@@ -1129,7 +1245,12 @@ async function executeController(context, row, manifest, state, assertOwnership,
                     try {
                         await renewLease();
                         assertOwner();
-                        const receipt = await launchExternalWork(executionContext, { workId: work.work_id, stage: next.stage, signal: cancellation.signal });
+                        const receipt = memoryFlow ? await runControllerMemoryTask(executionContext, { operation: "external", input: { projectRoot: run.project_root, runId: run.id, workId: work.work_id, stage: next.stage } }, cancellation.signal)
+                            : await launchExternalWork(executionContext, { workId: work.work_id, stage: next.stage, signal: cancellation.signal });
+                        if (memoryFlow) {
+                            await renewLease();
+                            assertOwner();
+                        }
                         if (fatal)
                             throw fatal;
                         appendEvent(context, row.controller_id, "external_work_observed", { work_id: work.work_id, receipt });

package/dist/services/run-fork.js

@@ -28,7 +28,7 @@ export function prepareRunFork(input) {
     }
     prepareForkEngine(engineVersion, input.integrityChecksum);
     const manifest = prepareEvalRunSnapshot(snapshot);
-    if (manifest.schema_id !== "dd-flow/eval-run-snapshot@5" || manifest.purpose !== "stage_entry" || typeof manifest.stage_entry !== "string" || typeof manifest.run_id !== "string") {
+    if (!["dd-flow/eval-run-snapshot@5", "dd-flow/eval-run-snapshot@7"].includes(manifest.schema_id) || manifest.purpose !== "stage_entry" || typeof manifest.stage_entry !== "string" || typeof manifest.run_id !== "string") {
         throw new AppError("fork_checkpoint_invalid", "Fork requires a sealed stage-entry snapshot", 2, { snapshot });
     }
     const operational = prepareSnapshotOperationalDecision({ snapshot, runId: manifest.run_id, projectRoot: path.join(output, "project"), now: new Date().toISOString(), ...(input.operationalDecisionFile ? { file: input.operationalDecisionFile } : {}) });

package/dist/services/run-recovery-guard.js

@@ -1,6 +1,36 @@
+import crypto from "node:crypto";
 import { AppError } from "../shared/errors.js";
 import { CAPACITY_POLICY, MAX_OVERLOAD_RETRIES, overloadExhausted } from "../harness-runtime/lib/provider-overload-policy.mjs";
 import { assertRunStartupSettled } from "./run-start-custody.js";
+export function recoveryPacketReceipt(binding) {
+    if (!binding.packet_json || !binding.packet_sha256 || crypto.createHash("sha256").update(binding.packet_json).digest("hex") !== binding.packet_sha256)
+        throw new AppError("recovery_packet_missing", "Recovery requires its original verified Work packet", 1);
+    return { packet_json: binding.packet_json, packet_sha256: binding.packet_sha256, packet: JSON.parse(binding.packet_json) };
+}
+/** A cooperative drain permits only the WorkSession present at its fence. */
+export function assertWorkSettlementAllowed(context, projectId, runId, workId) {
+    const guard = recoveryGuard(context, projectId, runId);
+    if (guard?.status === "draining" && context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controls'")) {
+        const admission = context.db.get("SELECT a.work_session_id FROM run_control_admissions a JOIN run_controls c ON c.control_id = a.control_id JOIN work_sessions ws ON ws.id = a.work_session_id JOIN works w ON w.work_id = a.work_id WHERE c.project_id = ? AND c.run_id = ? AND c.recovery_id = ? AND c.action = 'pause' AND c.status = 'requested' AND a.work_id = ? AND ws.work_id = a.work_id AND ws.session_id = a.session_id AND ws.status = 'running' AND w.status = 'running' AND w.project_id = c.project_id AND w.run_id = c.run_id LIMIT 1", [projectId, runId, guard.recovery_id, workId]);
+        if (admission)
+            return;
+    }
+    assertRunMutationAllowed(context, projectId, runId);
+}
+/** Completion rights are pinned to the Stage attempt present at the pause fence. */
+export function assertStageSettlementAllowed(context, projectId, runId, stage, expectedGeneration) {
+    const guard = recoveryGuard(context, projectId, runId);
+    const generation = guard?.generation ?? 0;
+    if (expectedGeneration !== undefined && generation !== expectedGeneration)
+        throw new AppError("run_recovery_generation_stale", "RUN execution generation changed while the operation was in flight", 1, { run_id: runId, expected_generation: expectedGeneration, generation });
+    if (guard?.status === "draining" && context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_control_stage_admissions'")) {
+        const admitted = context.db.get(`SELECT a.work_session_id FROM run_control_stage_admissions a JOIN run_controls c ON c.control_id = a.control_id JOIN run_control_admissions w ON w.control_id = a.control_id AND w.work_session_id = a.work_session_id JOIN work_sessions ws ON ws.id = w.work_session_id JOIN runs r ON r.project_id = c.project_id AND r.id = c.run_id, json_each(r.index_json, '$.stage_runs') live
+      WHERE c.project_id = ? AND c.run_id = ? AND c.recovery_id = ? AND c.action = 'pause' AND c.status = 'requested' AND a.stage = ? AND json_extract(live.value, '$.stage') = a.stage AND COALESCE(json_extract(live.value, '$.attempt'), 'try-001') = a.attempt AND ws.work_id = w.work_id AND ws.session_id = w.session_id AND NOT EXISTS (SELECT 1 FROM work_sessions newer WHERE newer.work_id = w.work_id AND newer.status = 'running' AND newer.id <> ws.id) LIMIT 1`, [projectId, runId, guard.recovery_id, stage]);
+        if (admitted)
+            return generation;
+    }
+    return assertRunMutationAllowed(context, projectId, runId, expectedGeneration);
+}
 export function recoveryGuard(context, projectId, runId) { return context.db.get("SELECT recovery_id, project_id, run_id, generation, status, interruption_json, settlement_json, capture_path, created_at, updated_at, sealed_at, resumed_at FROM run_recovery_guards WHERE project_id = ? AND run_id = ? ORDER BY generation DESC LIMIT 1", [projectId, runId]) ?? null; }
 export function assertRunRecoveryAllowed(context, projectId, runId, expectedGeneration) {
     const guard = recoveryGuard(context, projectId, runId);