@deksden-com/dd-flow-cli 0.9.0-beta.122 → 0.9.0-beta.124

package/dist/services/codex-capacity.js

@@ -1,34 +1,211 @@
 import { setTimeout as delay } from "node:timers/promises";
-import { terminalCodexOverload } from "../harness-runtime/lib/dd-codex.mjs";
-/** Only a terminal, effect-free native Turn may get another productive Turn. */
+import crypto from "node:crypto";
+import { CAPACITY_POLICY, capacityBackoff, nativeItemsSettled, normalizeCodexFailure, overloadBurst, parseCapacityOperation, refusalObservation, terminalCodexOverload } from "../harness-runtime/lib/codex-capacity-policy.mjs";
+import { AppError } from "../shared/errors.js";
+/** Permit acquisition must still belong to the original task, not merely an
+ * open RUN. Controls/inspection and ordinary successor prompts are unaffected. */
+export function assertCapacityOwnerAdmission(context, record, operationId, binding) {
+    if (!operationId || !record.project_id || !record.run_id)
+        return;
+    const root = parseCapacityOperation(operationId).root;
+    const childOrdinal = /:child:[a-f0-9]+:followup:(\d+)$/.exec(root)?.[1];
+    if (childOrdinal && (!Number.isSafeInteger(Number(childOrdinal)) || Number(childOrdinal) < 1 || String(Number(childOrdinal)) !== childOrdinal))
+        throw new AppError("capacity_continuation_conflict", "Invalid native child continuation ordinal", 1);
+    if (root === operationId && !childOrdinal)
+        return;
+    const reject = () => { throw new AppError("capacity_owner_boundary_changed", "Queued continuation no longer belongs to an unfinished task; reconcile its durable state", 1, { operation_id: operationId, work_id: record.work_id, run_id: record.run_id }); };
+    const metadata = JSON.parse(record.metadata_json);
+    const hasTable = (name) => Boolean(context.db.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", [name]));
+    const parsed = parseCapacityOperation(operationId);
+    if (parsed.ordinal > 0 && !["judge", "probe"].includes(metadata.role ?? "")) {
+        if (!binding?.nativeSessionId || !binding.promptSha256)
+            reject();
+        let intent;
+        const predecessor = parsed.ordinal === 1 ? root : `${root}:capacity:${parsed.ordinal - 1}:session.prompt`;
+        if (hasTable("run_controller_operations")) {
+            const saved = context.db.get("SELECT receipt_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'failed'", [predecessor, record.owner_id, metadata.run_generation ?? 0]);
+            if (saved)
+                intent = JSON.parse(saved.receipt_json).capacity_intent;
+        }
+        if (!intent && record.work_id)
+            for (const table of ["work_launches", "merge_requests"]) {
+                if (!hasTable(table))
+                    continue;
+                const rows = table === "work_launches"
+                    ? context.db.all("SELECT l.adapter_receipt_json FROM work_launches l JOIN works w ON w.work_id = l.work_id WHERE l.work_id = ? AND w.project_id = ? AND w.run_id = ?", [record.work_id, record.project_id, record.run_id])
+                    : context.db.all("SELECT adapter_receipt_json FROM merge_requests WHERE executor_work_id = ? AND project_id = ? AND run_id = ?", [record.work_id, record.project_id, record.run_id]);
+                for (const saved of rows) {
+                    const retained = JSON.parse(saved.adapter_receipt_json ?? "null");
+                    intent = retained?.capacity_intents?.find((item) => item.operation_id === operationId);
+                    if (intent)
+                        break;
+                }
+                if (intent)
+                    break;
+            }
+        if (!intent || intent.policy !== CAPACITY_POLICY || intent.root_operation_id !== root || intent.ordinal !== parsed.ordinal
+            || intent.predecessor !== predecessor || intent.operation_id !== operationId || intent.refusal?.turn_id !== intent.turn_id
+            || intent.refusal?.ordinal !== parsed.ordinal - 1 || intent.refusal?.session_id !== intent.session_id
+            || !Number.isFinite(Date.parse(intent.not_before)) || Date.parse(intent.not_before) > Date.parse(context.now())
+            || intent.deadline !== null && (typeof intent.deadline !== "number" || intent.deadline <= Date.parse(context.now()))
+            || binding?.nativeSessionId !== undefined && intent.session_id !== binding.nativeSessionId
+            || binding?.promptSha256 !== undefined && intent.prompt_sha256 !== binding.promptSha256)
+            reject();
+    }
+    if (context.db.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'run_recovery_bindings'")) {
+        if (context.db.get("SELECT 1 FROM run_recovery_bindings b JOIN run_recovery_guards g ON g.recovery_id = b.recovery_id WHERE g.project_id = ? AND g.run_id = ? AND g.status = 'recovered' AND b.status = 'accepted' AND json_extract(b.runtime_dispatch_json, '$.operation_id') = ?", [record.project_id, record.run_id, root]))
+            reject();
+    }
+    if (record.work_id) {
+        const work = context.db.get("SELECT status FROM works WHERE work_id = ? AND project_id = ? AND run_id = ?", [record.work_id, record.project_id, record.run_id]);
+        const merge = context.db.get("SELECT status FROM merge_requests WHERE executor_work_id = ? AND project_id = ? AND run_id = ?", [record.work_id, record.project_id, record.run_id]);
+        if (merge && !["dispatching", "active"].includes(merge.status))
+            reject();
+        // MERGE owns the Stage/request boundary: finishing its Work does not
+        // necessarily commit stage finish yet. External workers own only Work.
+        const mergeOwner = metadata.role === "merge";
+        if (!work || !(merge && mergeOwner ? ["created", "running", "completed"] : ["created", "running"]).includes(work.status))
+            reject();
+    }
+    if (!context.db.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'run_controllers'"))
+        return;
+    const controller = context.db.get("SELECT state_json FROM run_controllers WHERE controller_id = ? AND project_id = ? AND run_id = ?", [record.owner_id, record.project_id, record.run_id]);
+    if (!controller)
+        return;
+    const state = JSON.parse(controller.state_json);
+    const run = context.db.get("SELECT index_json FROM runs WHERE project_id = ? AND id = ?", [record.project_id, record.run_id]);
+    const index = JSON.parse(run?.index_json ?? "{}");
+    if (state.stage && (index.current_stage && index.current_stage !== state.stage || index.stage_runs?.some(stage => stage.stage === state.stage && ["done", "skipped"].includes(stage.status))))
+        reject();
+    const child = Object.entries(state.child_capacity ?? {}).find(([, intent]) => intent.operation_id === root)?.[0];
+    if (childOrdinal && !child)
+        reject();
+    if (child && context.db.get("SELECT 1 FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id JOIN sessions s ON s.session_id = ws.session_id WHERE w.project_id = ? AND w.run_id = ? AND s.provider_session_id = ? AND w.status <> 'running'", [record.project_id, record.run_id, child]))
+        reject();
+}
+export function capacityDelay(error, ordinal, now = Date.now()) {
+    return capacityBackoff(error, ordinal, now);
+}
+/** Continue a confirmed terminal Turn, never replay its completed effects. */
 export async function continueCodexCapacity(input) {
     let { error, operationId } = input;
-    for (let ordinal = 1; ordinal <= 2; ordinal += 1) {
-        if (!terminalCodexOverload(error, input.sessionId)) {
-            if (ordinal === 1)
-                return null;
+    const history = [...(input.retainedIntents?.() ?? [])];
+    const parsed = parseCapacityOperation(input.operationId);
+    const rootId = parsed.root;
+    const text = input.continuationText ?? "Continue the same assigned task in this Session. Check current durable RUN, Work and Stage state first. Never repeat an observed lifecycle command, accepted HITL answer, check or merge apply; use only an exact issued successor. Stop at the assigned boundary.";
+    const promptSha = crypto.createHash("sha256").update(text).digest("hex");
+    let ordinal = parsed.ordinal + 1;
+    const conflict = (message) => { throw new AppError("capacity_continuation_conflict", message, 1, { operation_id: operationId, root_operation_id: rootId, session_id: input.sessionId }); };
+    const refusals = [...(input.retainedFailures?.() ?? [])];
+    if (parsed.ordinal > history.length)
+        conflict("Continuation has no complete retained predecessor chain");
+    for (let index = 0; index < history.length; index++) {
+        const intent = history[index];
+        const expected = index + 1;
+        if (intent.ordinal !== expected || intent.operation_id !== `${rootId}:capacity:${expected}:session.prompt`
+            || intent.predecessor !== (expected === 1 ? rootId : `${rootId}:capacity:${expected - 1}:session.prompt`)
+            || intent.prompt_sha256 !== promptSha || intent.policy !== CAPACITY_POLICY || intent.root_operation_id !== rootId || intent.session_id !== input.sessionId
+            || intent.deadline !== (input.deadline ?? null) || !intent.refusal || intent.refusal.ordinal !== expected - 1 || intent.refusal.turn_id !== intent.turn_id
+            || !Number.isFinite(Date.parse(intent.not_before)))
+            conflict("Retained capacity chain is incomplete or belongs to another policy/task");
+        const previous = refusals.find(item => item.ordinal === intent.refusal.ordinal);
+        if (previous && JSON.stringify(previous) !== JSON.stringify(intent.refusal))
+            conflict("Retained failure observation changed");
+        if (!previous)
+            refusals.push(intent.refusal);
+    }
+    refusals.sort((a, b) => a.ordinal - b.ordinal);
+    for (let index = 0; index < refusals.length; index++) {
+        const refusal = refusals[index];
+        if (refusal.ordinal !== index || refusal.policy !== CAPACITY_POLICY || refusal.session_id !== input.sessionId || !refusal.turn_id
+            || refusals.slice(0, index).some(item => item.turn_id === refusal.turn_id) || !Number.isFinite(Date.parse(refusal.first_observed_at)))
+            conflict("Retained refusal chain is invalid");
+    }
+    // Reconcile a retained dispatch before preparing another. Native absence is
+    // the only safe way to reuse a prepared intent; a lost reply is not absence.
+    for (const intent of history) {
+        if (intent.ordinal < ordinal)
+            continue;
+        if (intent.predecessor !== operationId || intent.ordinal !== ordinal || intent.operation_id !== `${rootId}:capacity:${ordinal}:session.prompt`)
+            conflict("Retained successor skips its predecessor");
+        if (intent.prompt_sha256 !== promptSha || !intent.turn_id || !Number.isFinite(Date.parse(intent.not_before)))
             throw error;
-        }
-        let prior;
+        let observed;
         try {
-            prior = await input.inspectOperation(operationId);
+            observed = await input.inspectOperation(intent.operation_id);
+        }
+        catch (cause) {
+            let code = cause;
+            for (let depth = 0; depth < 4 && code?.code === "harness_adapter_failed"; depth++)
+                code = code.details?.cause;
+            if (code?.code !== "operation_not_found")
+                throw cause;
+            break;
+        }
+        if (observed.operation_id !== intent.operation_id || observed.session_id !== input.sessionId || observed.operation !== "session.prompt" || observed.primary_error)
+            throw error;
+        if (observed.state === "completed" && observed.result && typeof observed.result === "object") {
+            input.signal?.throwIfAborted();
+            input.assertCurrent();
+            const boundary = await input.decision?.() ?? "continue";
+            input.signal?.throwIfAborted();
+            input.assertCurrent();
+            if (boundary === "reconciliation_blocked")
+                throw new AppError("capacity_continuation_blocked", "Retained completion cannot be reconciled with its owner boundary", 1);
+            return boundary === "paused" || boundary === "completed" ? { provider_session_id: input.sessionId, capacity_decision: boundary } : observed.result;
         }
-        catch {
+        if (observed.state !== "failed" || !terminalCodexOverload(observed.error, input.sessionId))
+            throw error;
+        operationId = intent.operation_id;
+        error = observed.error;
+        ordinal += 1;
+    }
+    const decide = async () => {
+        input.signal?.throwIfAborted();
+        input.assertCurrent();
+        const decision = await input.decision?.() ?? "continue";
+        input.signal?.throwIfAborted();
+        input.assertCurrent();
+        if (decision === "reconciliation_blocked")
+            throw new AppError("capacity_continuation_blocked", "Native refusal cannot be reconciled with its durable owner boundary", 1, { primary_error: error, root_operation_id: rootId, provider_session_id: input.sessionId });
+        if (decision === "continue" && input.deadline !== undefined && Date.now() >= input.deadline)
+            throw new AppError("capacity_owner_deadline_exceeded", "Continuation exhausted its original owner deadline", 1, { primary_error: error, deadline: input.deadline, root_operation_id: rootId, provider_session_id: input.sessionId });
+        return decision === "continue" ? null : { provider_session_id: input.sessionId, capacity_decision: decision };
+    };
+    for (;; ordinal += 1) {
+        if (!Number.isSafeInteger(ordinal))
+            conflict("Continuation ordinal overflow");
+        if (!terminalCodexOverload(error, input.sessionId)) {
             if (ordinal === 1)
                 return null;
             throw error;
         }
+        const completed = await decide();
+        if (completed)
+            return completed;
+        const prior = await input.inspectOperation(operationId);
         const failure = prior.error;
-        const turnId = failure?.details?.turn_id;
+        const confirmed = normalizeCodexFailure(failure, input.sessionId);
+        const turnId = confirmed?.turn_id;
+        input.signal?.throwIfAborted();
+        input.assertCurrent();
         if (prior.operation_id !== operationId || prior.operation !== "session.prompt" || prior.session_id !== input.sessionId
-            || prior.state !== "failed" || prior.primary_error || !turnId || !terminalCodexOverload(failure, input.sessionId, turnId)
-            || failure.details?.native_turn_items?.observed !== true || failure.details.native_turn_items.possible_effects !== false) {
+            || prior.state !== "failed" || prior.primary_error || !turnId || !terminalCodexOverload(failure, input.sessionId, turnId)) {
             if (ordinal === 1)
                 return null;
             throw error;
         }
         const settled = async () => {
-            const seen = await input.inspectSession();
+            input.signal?.throwIfAborted();
+            input.assertCurrent();
+            const seen = await input.inspectSession(turnId);
+            input.signal?.throwIfAborted();
+            input.assertCurrent();
+            if (seen.terminal_turn_id !== undefined && seen.terminal_turn_id !== turnId)
+                return false;
+            const items = seen.terminal_turn_id === turnId && seen.terminal_status === "failed" ? seen.native_turn_items : confirmed?.native_turn_items;
+            if (!nativeItemsSettled(items))
+                return false;
             const settlement = seen.settlement;
             const ids = Array.isArray(seen.tree_session_ids) ? seen.tree_session_ids : [];
             const observations = Array.isArray(seen.tree_observations) ? seen.tree_observations : [];
@@ -38,34 +215,77 @@ export async function continueCodexCapacity(input) {
                     && Array.isArray(seen.unsettled_sessions) && seen.unsettled_sessions.length === 0
                 : ids.length === 1 && ids[0] === input.sessionId;
             return seen.provider_session_id === input.sessionId && seen.settled === true && treeProven
-                && (!settlement || ["settled", "not_required"].includes(settlement.state ?? ""));
+                && (!settlement || settlement.state === "not_required" || settlement.state === "settled" && settlement.operation_ids?.includes(operationId) === true);
         };
-        input.assertCurrent();
-        try {
-            if (!await settled()) {
-                if (ordinal === 1)
-                    return null;
-                throw error;
-            }
-        }
-        catch {
+        if (!await settled()) {
             if (ordinal === 1)
                 return null;
             throw error;
         }
-        const nextId = `${input.operationId}:capacity:${ordinal}:session.prompt`;
-        input.retainIntent({ predecessor: operationId, operation_id: nextId, turn_id: turnId, ordinal, not_before: new Date(Date.now() + (ordinal === 1 ? 5_000 : 15_000)).toISOString() });
-        await delay(ordinal === 1 ? 5_000 : 15_000);
-        input.assertCurrent();
-        if (!await settled().catch(() => false))
+        const native = normalizeCodexFailure(failure, input.sessionId, turnId);
+        let refusal = refusals.find(item => item.ordinal === ordinal - 1);
+        if (refusal && (refusal.turn_id !== turnId || refusal.session_id !== input.sessionId || refusal.policy !== CAPACITY_POLICY))
+            conflict("Failure ordinal identifies a different native Turn");
+        if (refusal) {
+            const expected = refusalObservation(failure, ordinal - 1, refusal.first_observed_at);
+            if (refusal.native_terminal_at !== expected.native_terminal_at || refusal.native_terminal_basis !== expected.native_terminal_basis)
+                conflict("Retained refusal time differs from its native receipt");
+        }
+        if (!refusal) {
+            if (refusals.some(item => item.turn_id === turnId))
+                conflict("A cached native Turn cannot be a second refusal");
+            // A cached legacy failure cannot be assigned a new time during reattach.
+            if (ordinal > 1 && parsed.ordinal > 0 && !native.observed_at)
+                conflict("Retained continuation has no original failure time");
+            const observedAt = typeof native.observed_at === "string" && Number.isFinite(Date.parse(native.observed_at)) ? native.observed_at : new Date().toISOString();
+            refusal = refusalObservation(failure, ordinal - 1, observedAt);
+            input.retainFailure?.(operationId, refusal);
+            refusals.push(refusal);
+        }
+        const burst = overloadBurst(refusals);
+        if (burst)
+            throw new AppError("provider_overload_burst", "Two consecutive continuation Turns were overloaded within two minutes", 1, { ...burst, root_operation_id: rootId, provider_session_id: input.sessionId, continuation_count: ordinal - 1, primary_error: failure });
+        const nextId = `${rootId}:capacity:${ordinal}:session.prompt`;
+        const existing = history.find(item => item.operation_id === nextId);
+        if (existing && (existing.predecessor !== operationId || existing.turn_id !== turnId || existing.prompt_sha256 !== promptSha || existing.ordinal !== ordinal))
             throw error;
+        const wait = existing ? Math.max(0, Date.parse(existing.not_before) - Date.now()) : capacityDelay(failure, ordinal);
+        if (!Number.isFinite(wait) || Date.now() + wait > 8.64e15 || input.deadline !== undefined && Date.now() + wait >= input.deadline)
+            throw new AppError("capacity_owner_deadline_exceeded", "Continuation wait exceeds its original owner deadline", 1, { primary_error: failure, deadline: input.deadline ?? null });
+        if (!existing) {
+            const intent = { predecessor: operationId, operation_id: nextId, turn_id: turnId, ordinal, prompt_sha256: promptSha, not_before: new Date(Date.now() + wait).toISOString(), policy: CAPACITY_POLICY, root_operation_id: rootId, session_id: input.sessionId, deadline: input.deadline ?? null, refusal };
+            input.retainIntent(intent);
+            history.push(intent);
+        }
+        // Node timers truncate large delays. Bounded slices preserve cancellation
+        // and owner checks; monotonic elapsed time does not renew a Retry-After.
+        const until = performance.now() + wait;
+        while (performance.now() < until) {
+            await delay(Math.min(250, Math.max(0, until - performance.now())), undefined, { signal: input.signal });
+            const boundary = await decide();
+            if (boundary)
+                return boundary;
+        }
+        const afterWait = await decide();
+        if (afterWait)
+            return afterWait;
+        if (!await settled())
+            throw new AppError("capacity_continuation_blocked", "Native predecessor settlement changed during capacity wait", 1, { primary_error: failure });
+        const beforeDispatch = await decide();
+        if (beforeDispatch)
+            return beforeDispatch;
         try {
-            return await input.prompt(nextId, input.continuationText ?? "Continue the same assigned task in this Session. Check current durable RUN, Work and Stage state first. Never repeat an observed lifecycle command, accepted HITL answer, check or merge apply; use only an exact issued successor. Stop at the assigned boundary.", ordinal);
+            return await input.prompt(nextId, text, ordinal);
         }
         catch (nextError) {
+            // A task can complete/pause while the daemon queues for its permit.
+            // Admission then refuses the stale prompt; accept that owner decision
+            // before treating the transport's rejection as another provider failure.
+            const finished = await decide();
+            if (finished)
+                return finished;
             error = nextError;
             operationId = nextId;
         }
     }
-    throw error;
 }

package/dist/services/controller-fanout.js

@@ -138,6 +138,7 @@ export function controllerNativeChildren(receipt, rootSessionId) {
         const settled = rootChild && status === "unknown" && settledTree.has(id) ? "settled_by_root" : status;
         candidates.push({ session_id: id, parent_session_id: parent ?? "", status: settled, source, ...(rootChild ? {} : { provenance: "parent_mismatch" }),
             ...(text(value, ["terminal_turn_id"]) ? { terminal_turn_id: text(value, ["terminal_turn_id"]) } : {}),
+            ...(text(value, ["terminal_observed_at"]) ? { terminal_observed_at: text(value, ["terminal_observed_at"]) } : {}),
             ...(value.provider_error !== undefined ? { provider_error: value.provider_error } : {}),
             ...(value.native_turn_items && typeof value.native_turn_items === "object" ? { native_turn_items: value.native_turn_items } : {}) });
     };

package/dist/services/eval-snapshots.js

@@ -409,7 +409,12 @@ export function restoreEvalRunSnapshot(context, input, prepared = prepareEvalRun
     let result, primary, failed = false;
     try {
         const runtimeLineage = recovery ? null : detachSnapshotRuntimeOwners(restored, snapshot, manifest);
+        const originalRunRoot = restored.db.get("SELECT run_root FROM runs WHERE id = ?", [manifest.run_id])?.run_root;
         rebaseRuntime(restored, manifest.project_root, projectRoot, sourceWorkspaceRoot(manifest), workspaceRoot, manifest.dd_flow_home, context.ddFlowHome, recovery);
+        const publicationRun = restored.db.get("SELECT project_id, run_root FROM runs WHERE id = ?", [manifest.run_id]);
+        if (!originalRunRoot || !publicationRun?.run_root || !isInside(manifest.dd_flow_home, originalRunRoot) || !isInside(context.ddFlowHome, publicationRun.run_root))
+            throw new AppError("snapshot_invalid", "Restored publication roots are not bound to their runtime", 1);
+        restored.db.run("INSERT INTO audit_events (project_id, event_type, forced, payload_json, created_at) VALUES (?, 'flow_run.publication_relocation', 0, ?, ?)", [publicationRun.project_id, JSON.stringify({ run_id: manifest.run_id, from: originalRunRoot, to: publicationRun.run_root, source_manifest_sha256: sourceManifestSha256, import_id: importId }), restored.now()]);
         registerProject(restored, { root: projectRoot });
         requireProjectByRoot(restored, projectRoot);
         if (selectedProfile && routingInput && runtimeLineage) {
@@ -423,14 +428,6 @@ export function restoreEvalRunSnapshot(context, input, prepared = prepareEvalRun
                 fs.writeFileSync(artifact, routingInput.bytes, { flag: "wx", mode: 0o600 });
             importFlowRunExecutionProfile(restored, { projectRoot, runId: manifest.run_id, stage: manifest.stage_entry, lineageFile: runtimeLineage, profile: selectedProfile });
         }
-        const status = getFlowRunStatus(restored, { projectRoot, runId: manifest.run_id });
-        if (!recovery)
-            assertStageEntry(status.index.stage_runs ?? [], manifest.stage_entry);
-        else
-            restored.db.run("UPDATE run_recovery_guards SET capture_path = ? WHERE recovery_id = ? AND status = 'sealed'", [snapshot, input.recoveryId]);
-        if (!status.run.run_root)
-            throw new AppError("runtime_missing", "Restored RUN has no artifact root", 1, { run_id: manifest.run_id });
-        const runHome = status.run.run_root;
         const relocation = recovery ? {
             schema_id: "dd-flow/recovery-relocation@1", physical_runtime_id: crypto.randomUUID(), recovery_id: input.recoveryId, run_id: manifest.run_id,
             path_mapping: [{ from: manifest.dd_flow_home, to: context.ddFlowHome }, { from: sourceWorkspaceRoot(manifest), to: workspaceRoot }, { from: manifest.project_root, to: projectRoot }]
@@ -440,6 +437,14 @@ export function restoreEvalRunSnapshot(context, input, prepared = prepareEvalRun
             fs.mkdirSync(path.dirname(relocationFile), { recursive: true });
             writeJson(relocationFile, relocation);
         }
+        const status = getFlowRunStatus(restored, { projectRoot, runId: manifest.run_id });
+        if (!recovery)
+            assertStageEntry(status.index.stage_runs ?? [], manifest.stage_entry);
+        else
+            restored.db.run("UPDATE run_recovery_guards SET capture_path = ? WHERE recovery_id = ? AND status = 'sealed'", [snapshot, input.recoveryId]);
+        if (!status.run.run_root)
+            throw new AppError("runtime_missing", "Restored RUN has no artifact root", 1, { run_id: manifest.run_id });
+        const runHome = status.run.run_root;
         const expectedWorkspaceRoot = restoredWorkspaceRoot(manifest, projectRoot, context.ddFlowHome);
         if (physicalPath(workspaceRoot) !== physicalPath(expectedWorkspaceRoot))
             throw new AppError("snapshot_workspace_restore_failed", "Restored workspace differs from the import target", 1, { workspace_root: workspaceRoot, expected_workspace_root: expectedWorkspaceRoot });

package/dist/services/external-work-launch.js

@@ -93,7 +93,7 @@ export async function launchExternalWork(context, input) {
     const common = ["--dd-flow-bin", flowExecutable, "--state-dir", root, "--journal", journal, "--cwd", isolatedWorkspace ?? run.workspace_root, "--provider", resolved.worker.provider, "--model", resolved.worker.model, "--reasoning", resolved.worker.reasoning, "--mode", resolved.worker.mode, "--permission", resolved.worker.permission, "--project-root", run.project_root, "--dd-flow-home", context.ddFlowHome, ...harnessRuntimeArguments(context.ddFlowHome, runtimeHarness(resolved.worker)), "--json"];
     const evidence = (value) => fs.appendFileSync(journal, `${JSON.stringify({ type: "adapter_call", at: context.now(), ...value })}\n`);
     try {
-        const call = (args, timeoutMs, id = `${operationId}:${args[0]}.${args[1]}`) => runHarnessAdapter({ executable, args, env: runtimeContext.env, operationId: id, runtimeOwner, ...(timeoutMs ? { timeoutMs } : {}), ...(input.progress ? { progress: input.progress } : {}), onEvidence: evidence });
+        const call = (args, timeoutMs, id = `${operationId}:${args[0]}.${args[1]}`) => runHarnessAdapter({ executable, args, env: runtimeContext.env, operationId: id, runtimeOwner, ...(timeoutMs ? { timeoutMs } : {}), ...(input.signal ? { signal: input.signal } : {}), ...(input.progress ? { progress: input.progress } : {}), onEvidence: evidence });
         await call(["daemon", "start", ...common]);
         assertRunMutationAllowed(context, work.project_id, work.run_id, generation);
         const created = await call(["session", "create", ...common], runtimeHarness(resolved.worker) === "zcode-acp" ? 180_000 : undefined);
@@ -111,6 +111,8 @@ export async function launchExternalWork(context, input) {
         const promptFile = path.join(root, "launch.md");
         fs.writeFileSync(promptFile, `Start the assigned Work now. Your first tool call must be this exact standalone command:\n\n${workStartCommand(runtimeContext, work)}\n\nUse the packet returned by that command. Do not claim another Work or run this command more than once.\n\n${lifecycleRetryInstruction}${isolatedWorkspace ? `\n\nThe review copy has no .git. Read the captured status/diff or gitless note in ${path.join(isolatedWorkspace, ".dd-flow", "review-evidence")}; do not run Git inside the copy.\n` : "\n"}`);
         const promptOperationId = `${operationId}:session.prompt`;
+        const promptDeadline = Date.now() + 45 * 60_000;
+        context.db.run("UPDATE work_launches SET adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'launching'", [JSON.stringify({ create: created, prompt_deadline: promptDeadline }), context.now(), operationId]);
         let prompted;
         try {
             prompted = await call(["session", "prompt", "--session-id", sessionId, "--prompt-file", promptFile, ...common], 45 * 60_000);
@@ -118,37 +120,60 @@ export async function launchExternalWork(context, input) {
         catch (error) {
             if (runtimeHarness(resolved.worker) !== "codex-desktop")
                 throw error;
-            const resumed = await continueCodexCapacity({ error, sessionId, operationId: promptOperationId,
+            const retained = context.db.get("SELECT adapter_receipt_json FROM work_launches WHERE operation_id = ?", [operationId]);
+            const capacityIntents = retained?.adapter_receipt_json ? JSON.parse(retained.adapter_receipt_json).capacity_intents ?? [] : [];
+            const capacityFailures = retained?.adapter_receipt_json ? JSON.parse(retained.adapter_receipt_json).capacity_failures ?? [] : [];
+            const retainCapacity = () => {
+                const changed = context.db.run("UPDATE work_launches SET adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'launching'", [JSON.stringify({ create: created, prompt_deadline: promptDeadline, capacity_intents: capacityIntents, capacity_failures: capacityFailures }), context.now(), operationId]);
+                if (changed.changes !== 1)
+                    throw new AppError("capacity_continuation_blocked", "External Work launch owner changed", 1);
+            };
+            const resumed = await continueCodexCapacity({ error, sessionId, operationId: promptOperationId, allowSettledDescendants: true,
+                ...(input.signal ? { signal: input.signal } : {}), deadline: promptDeadline,
+                continuationText: `Continue only Work ${work.work_id} in RUN ${work.run_id}, Stage ${stage ?? "current"}, in the same Session. Inspect its current durable state and the retained Work packet. Never repeat Work start or a completed tool, accepted answer, check or lifecycle command. Use only the current exact issued successor and finish at the original Work boundary.`,
+                retainedIntents: () => capacityIntents,
+                retainedFailures: () => capacityFailures,
+                retainFailure: (_id, refusal) => { capacityFailures.push(refusal); retainCapacity(); },
                 inspectOperation: id => call(["daemon", "operation", "--state-dir", root, "--operation-id", id, "--json"], 30_000, `${id}:capacity-observe`),
-                inspectSession: () => call(["session", "inspect", "--session-id", sessionId, ...common], 30_000, `${operationId}:capacity-inspect`),
+                inspectSession: turnId => call(["session", "inspect", "--session-id", sessionId, ...(turnId ? ["--turn-id", turnId] : []), ...common], 30_000, `${operationId}:capacity-inspect`),
                 assertCurrent: () => {
                     assertRunMutationAllowed(context, work.project_id, work.run_id, generation);
                     const current = context.db.get("SELECT w.status, l.status AS launch_status FROM works w JOIN work_launches l ON l.work_id = w.work_id WHERE w.work_id = ? AND l.operation_id = ?", [work.work_id, operationId]);
-                    if (current?.status !== "created" || current.launch_status !== "launching")
+                    if (!current || !["created", "running", "paused", "completed"].includes(current.status) || current.launch_status !== "launching")
                         throw new AppError("capacity_continuation_blocked", "External Work changed before its native continuation", 1, { work_id: work.work_id, state: current });
                     if (reviewCopy && reviewSourceHash(run.workspace_root, reviewInputs, reviewBase) !== reviewCopy.source_sha256)
                         throw new AppError("read_only_input_changed", "Authoritative review workspace changed during capacity wait", 2);
                 },
+                decision: () => {
+                    const status = context.db.get("SELECT status FROM works WHERE work_id = ?", [work.work_id])?.status;
+                    return status === "completed" ? "completed" : status === "paused" ? "paused" : "continue";
+                },
                 retainIntent: intent => {
-                    const changed = context.db.run("UPDATE work_launches SET adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'launching'", [JSON.stringify({ create: created, capacity_intent: intent }), context.now(), operationId]);
-                    if (changed.changes !== 1)
-                        throw new AppError("capacity_continuation_blocked", "External Work launch owner changed", 1);
+                    capacityIntents.push(intent);
+                    retainCapacity();
                 },
                 prompt: (id, text, ordinal) => {
                     const file = path.join(root, `capacity-${ordinal}.md`);
-                    fs.writeFileSync(file, `${text}\n`, { flag: "wx", mode: 0o600 });
-                    return call(["session", "prompt", "--session-id", sessionId, "--prompt-file", file, ...common], 45 * 60_000, id);
+                    if (fs.existsSync(file)) {
+                        if (fs.readFileSync(file, "utf8") !== text)
+                            throw new AppError("capacity_continuation_conflict", "Retained Work continuation prompt differs", 1);
+                    }
+                    else
+                        fs.writeFileSync(file, text, { flag: "wx", mode: 0o600 });
+                    return call(["session", "prompt", "--session-id", sessionId, "--prompt-file", file, ...common], Math.max(1, promptDeadline - Date.now()), id);
                 } });
             if (!resumed)
                 throw error;
             prompted = resumed;
         }
         assertRunMutationAllowed(context, work.project_id, work.run_id, generation);
-        context.db.run("UPDATE work_launches SET status = 'start_requested', provider_session_id = ?, adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ?", [sessionId, JSON.stringify({ create: created, prompt: prompted }), context.now(), operationId]);
+        const saved = context.db.get("SELECT adapter_receipt_json FROM work_launches WHERE operation_id = ?", [operationId]);
+        context.db.run("UPDATE work_launches SET status = 'start_requested', provider_session_id = ?, adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ?", [sessionId, JSON.stringify({ ...(saved?.adapter_receipt_json ? JSON.parse(saved.adapter_receipt_json) : {}), create: created, prompt: prompted }), context.now(), operationId]);
         return { ok: true, work_id: work.work_id, operation_id: operationId, status: "start_requested", session: { harness_id: runtimeHarness(resolved.worker), session_id: sessionId }, profile_hash: resolved.profile_hash, journal_path: journal };
     }
     catch (error) {
-        context.db.run("UPDATE work_launches SET status = 'failed', adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify({ error: errorRecord(error) }), context.now(), operationId]);
+        const saved = context.db.get("SELECT adapter_receipt_json FROM work_launches WHERE operation_id = ?", [operationId]);
+        context.db.run("UPDATE work_launches SET status = 'failed', adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify({ ...(saved?.adapter_receipt_json ? JSON.parse(saved.adapter_receipt_json) : {}), error: errorRecord(error) }), context.now(), operationId]);
         throw error;
     }
 }

package/dist/services/harness-adapter.js

@@ -2,7 +2,8 @@ import { spawn } from "node:child_process";
 import fs from "node:fs";
 import path from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
-import { AppError } from "../shared/errors.js";
+import { AppError, errorRecord } from "../shared/errors.js";
+import { stopProcessGroup } from "../harness-runtime/lib/managed-daemon.mjs";
 import { recoveryRpcTimeout } from "./recovery-observation-budget.js";
 const maxAdapterOutputBytes = 1_000_000;
 /** Native hooks need a real executable, not a PATH-dependent shell expression. */
@@ -64,18 +65,26 @@ export function runHarnessAdapter(input) {
         let outputTooLarge = false;
         let timedOut = false;
         let aborted = false;
-        let killTimer;
+        let cleanup;
         let settled = false;
         let callbackError;
         let spawnError;
-        const terminate = (signal) => { try {
-            if (detached && child.pid)
-                process.kill(-child.pid, signal);
-            else
-                child.kill(signal);
-        }
-        catch { /* The process already settled. */ } };
-        const stopTransport = () => { terminate("SIGTERM"); killTimer ??= setTimeout(() => terminate("SIGKILL"), 5_000); };
+        const stopTransport = () => {
+            cleanup ??= stopProcessGroup(child, 5_000, { detached }).catch(error => {
+                if (timer)
+                    clearTimeout(timer);
+                if (heartbeat)
+                    clearInterval(heartbeat);
+                input.signal?.removeEventListener("abort", abort);
+                const details = { cleanup_error: errorRecord(error) };
+                if (callbackError instanceof Error) {
+                    Object.assign(callbackError, { cleanup_error: details.cleanup_error, details: { ...callbackError.details, ...details } });
+                    reject(callbackError);
+                }
+                else
+                    reject(new AppError(outputTooLarge ? "harness_adapter_output_limit" : timedOut ? "harness_adapter_timeout" : aborted ? "harness_adapter_aborted" : "harness_adapter_cleanup_unconfirmed", "Adapter transport cleanup could not be confirmed", 1, details));
+            });
+        };
         const deliver = (callback) => {
             try {
                 const result = callback();
@@ -109,7 +118,7 @@ export function runHarnessAdapter(input) {
         child.stdout.setEncoding("utf8").on("data", (chunk) => { stdout = append(stdout, chunk); });
         child.stderr.setEncoding("utf8").on("data", (chunk) => { stderr = append(stderr, chunk); });
         child.once("error", (error) => { spawnError = error; });
-        child.once("close", (status, signal) => {
+        child.once("close", async (status, signal) => {
             if (settled)
                 return;
             settled = true;
@@ -118,10 +127,7 @@ export function runHarnessAdapter(input) {
             input.signal?.removeEventListener("abort", abort);
             if (heartbeat)
                 clearInterval(heartbeat);
-            if (killTimer) {
-                clearTimeout(killTimer);
-                terminate("SIGKILL");
-            }
+            await cleanup;
             const progressFailure = callbackError;
             deliver(() => evidence({ status, signal, timed_out: timedOut, aborted, stdout, stderr, ...(spawnError ? { error: spawnError.message } : {}) }));
             if (spawnError)

package/dist/services/managed-processes.js

@@ -4,7 +4,7 @@ import path from "node:path";
 import { spawnSync } from "node:child_process";
 import net from "node:net";
 import { getResourceDatabase } from "../storage/database.js";
-import { AppError } from "../shared/errors.js";
+import { AppError, errorRecord } from "../shared/errors.js";
 export function managedProcessBinding(record, home) {
     return { resource_home: home, process_id: record.id, registration_sha256: crypto.createHash("sha256").update(JSON.stringify([home, record.id, record.kind, record.owner_id, record.lease_token, record.pid, record.pid_started_at, record.project_id, record.run_id, record.work_id, record.check_id, record.operation_id, record.metadata_json])).digest("hex") };
 }
@@ -339,14 +339,19 @@ export async function stopManagedProcess(context, input) {
         return record;
     db.run("UPDATE managed_processes SET state = 'stopping', updated_at = ? WHERE id = ? AND lease_token = ? AND state IN ('starting','running','orphaned')", [context.now(), record.id, record.lease_token]);
     if (processTreeIsAlive(record)) {
-        terminateOwnedProcess(record, "SIGTERM");
+        const signalErrors = [];
+        const termError = terminateOwnedProcess(record, "SIGTERM");
+        if (termError)
+            signalErrors.push(errorRecord(termError));
         await delay(graceMs);
-        if (input.force !== false && processTreeIsAlive(record)) {
-            terminateOwnedProcess(record, "SIGKILL");
+        if (!termError && input.force !== false && processTreeIsAlive(record)) {
+            const killError = terminateOwnedProcess(record, "SIGKILL");
+            if (killError)
+                signalErrors.push(errorRecord(killError));
             await delay(250);
         }
         if (processTreeIsAlive(record))
-            throw Object.assign(new Error("Owned process tree could not be safely stopped; resources retained"), { code: "process_tree_not_settled", details: { process_id: record.id } });
+            throw Object.assign(new Error("Owned process tree could not be safely stopped; resources retained"), { code: "process_tree_not_settled", details: { process_id: record.id, signal_errors: signalErrors } });
     }
     if (!finishManagedProcess(context, { id: record.id, leaseToken: record.lease_token, state: "stopped", reason: input.reason ?? "requested_stop" }))
         throw Object.assign(new Error("Managed process ownership changed before settlement"), { code: "process_lease_lost" });
@@ -460,7 +465,10 @@ function terminateOwnedProcess(record, signal) {
     try {
         process.kill(process.platform === "win32" || group !== record.pid ? record.pid : -group, signal);
     }
-    catch { /* inspect again below */ }
+    catch (error) {
+        if (error.code !== "ESRCH")
+            return Object.assign(error, { details: { process_id: record.id, pid: record.pid, pgid: group ?? null, signal, phase: "owned_process_stop" } });
+    }
 }
 function parseMetadata(value) { try {
     const parsed = JSON.parse(value);