@wardby/cli 0.6.0 → 0.7.0

package/dist/core/runner.js

@@ -18,19 +18,24 @@
  */
 import { createHash, randomBytes } from "node:crypto";
 import { z } from "zod";
-import { RunOwnershipLostError, runStepInline } from "../providers/engine/types.js";
+import { Prisma } from "#prisma";
+import { RunOwnershipLostError, STEPS_CHANGED_REASON, StepReplayMismatchError, runStepInline, } from "../providers/engine/types.js";
 import { isLlmEffort } from "../providers/llm/types.js";
 import { createPrivilegedHost } from "../sandbox/host-functions.js";
 import { consoleRedactionValues } from "../sandbox/secret-broker.js";
 import { runUserToolCall } from "../sandbox/user-tool.js";
-import { runSandboxedEngine, SandboxRunCancelledError, } from "../native-worker/gateway.js";
+import { logUnexpectedError, runSandboxedEngine, SandboxRunCancelledError, } from "../native-worker/gateway.js";
 import { NATIVE_WORKER_PROTOCOL_VERSION } from "../native-worker/protocol.js";
 import { asStringArray, asPrefixMap } from "../sandbox/tool-capabilities.js";
 import { scopeDatastore } from "../providers/datastore/scoped.js";
 import { buildSecretsAccessor, scopeSecretsAccessor } from "./secrets.js";
+import { loadAgentSecretValues, maskNamedSecrets, POSTING_TOOL_NAMES, SECRET_CHECK_UNAVAILABLE_MESSAGE, WITHHELD_TEXT_NOTICE, } from "./secret-guard.js";
 import { buildSharedDatastoreAccessor, scopeSharedDatastoreAccessor } from "./datastores.js";
 import { effectiveBudgetForRun } from "./budget-groups.js";
+import { stopChildrenOfLostRun } from "./lost-run-children.js";
 import { createDelegationSiblings } from "./delegation-siblings.js";
+import { findDelegationChild } from "./delegation-child.js";
+import { DRIVABLE } from "./run-status.js";
 import { createSerialGate } from "./serial-gate.js";
 import { withRunHeartbeat } from "./run-heartbeat.js";
 import { ContinuationRefusedError, dispatchRun } from "./dispatch.js";
@@ -40,6 +45,7 @@ import { DELEGATE_TOOL_PREFIX, duplicateToolNames } from "./tool-names.js";
 import { SUBAGENT_MEMORY_GET_TOOL, PARENT_MEMORY_GET_TOOL, handleSubAgentMemoryGet, handleParentMemoryGet, } from "./subagent-memory-tools.js";
 import { prisma as defaultDb } from "./db.js";
 import { logger } from "./logger.js";
+import { CHILD_RUN_FAILURE_CODES, declaredCodeOf } from "./error-code.js";
 import { loadCodingConcurrencyConfig } from "../config/providers.js";
 import { REVIEW_HOST_TOOL_DEFS, REVIEW_HOST_TOOL_NAMES, handleReviewHostTool, } from "./review-host-tools.js";
 import { closeOpenHostCheck } from "./review-host-checks.js";
@@ -55,14 +61,17 @@ import { emitRunFinishedEvents } from "./workflow-run-events.js";
 import { startReviewFixAfterReview } from "./review-fix.js";
 import { updateRelatedPullRequests } from "./related-pull-requests.js";
 import { syncMergeOrderChecksAfterRun } from "./merge-order-check.js";
+import { markReadyAfterRun } from "./mark-ready.js";
 import { startDeferredForRequest } from "./host-events.js";
 import { recordNativeModelUsage } from "./model-usage.js";
 import { pinNativeRunPricing } from "./run-pricing.js";
 import { RoutingLlmProvider } from "../providers/llm/routing.js";
 import { ModelUnavailableError } from "../providers/llm/catalog-types.js";
 import { trackRun } from "./in-flight-runs.js";
+import { ReviewReadLedger } from "./review-reads.js";
 import { createRepoAccessGate, requiredLevel } from "./repo-access.js";
-const runnerLog = logger.child({ module: "runner" });
+import { loadBudgetEvent, ownerBudgetSentence } from "./budget-events.js";
+export const runnerLog = logger.child({ module: "runner" });
 /**
  * Sub-agent dispatch uses one synthetic tool per declared child, named by the
  * boundName it's attached
@@ -145,7 +154,7 @@ const DelegateArgs = z
     grantParentMemoryKeys: z.array(z.string()).optional(),
     continuePriorRun: z.string().optional(),
     mergeOrder: z
-        .number({ invalid_type_error: MERGE_ORDER_RANGE_MESSAGE })
+        .number({ error: MERGE_ORDER_RANGE_MESSAGE })
         .int(MERGE_ORDER_RANGE_MESSAGE)
         .min(1, MERGE_ORDER_RANGE_MESSAGE)
         .max(99, MERGE_ORDER_RANGE_MESSAGE)
@@ -325,18 +334,7 @@ function toIssueProjectLink(row) {
         maxNewIssuesPerRun: row.maxNewIssuesPerRun ?? null,
     };
 }
-/**
- * The two states a Run can still be driven out of. Every write `executeRun`
- * makes is filtered on these: a durable executor can have two attempts of the
- * same run in flight at once (a still-live original parked in a slow LLM step
- * and an adopted attempt the reconciler resumed elsewhere), and whichever
- * reaches a terminal state first must win. A conditional `updateMany` makes
- * that a database-level CAS rather than a race — the loser's WHERE matches
- * zero rows. It also stops a *later* attempt resurrecting a run the
- * reconciler already reaped: a `lost` row stays `lost`. A step-level
- * ownership loss writes nothing at all (RunOwnershipLostError).
- */
-const DRIVABLE = ["pending", "running"];
+// DRIVABLE (the two states a Run can still be driven out of) lives in run-status.ts.
 /**
  * Raised when a run's step boundary observed a cancellation (DbosExecutor's
  * `stop`). `executeRun`'s backstop persists these as `cancelled` with the
@@ -370,6 +368,37 @@ async function finishRunClaimed(db, runId, data) {
     const updated = await db.run.updateMany({ where: { id: runId, status: { in: [...DRIVABLE] } }, data });
     return { run: await db.run.findUniqueOrThrow({ where: { id: runId } }), claimed: updated.count > 0 };
 }
+/**
+ * A step runner made to notice a stop. Before each step this reads the run's own row: once anything
+ * else has made it terminal (stop_run, an MCP task's cancel, a parent run's stop, the reconciler),
+ * the run stops at this boundary with a RunCancelledError. The terminal row is never rewritten
+ * (finishRunClaimed matches zero rows), so the stop keeps its reason. Inline runs need it because
+ * nothing else interrupts their steps. DbosExecutor wraps its own step runner in it too: DBOS stops
+ * only a workflow it was told to cancel, and stop() can name only the workflow the run records.
+ */
+export function stopAwareStep(db, runId, inner = runStepInline, options = {}) {
+    const check = async (name, own) => {
+        const read = db.run.findUnique({ where: { id: runId }, select: { status: true, executionWorkflowId: true } });
+        const row = options.failOpen ? await read.catch(() => null) : await read;
+        if (row?.executionWorkflowId && own && options.ownership?.forkedFrom(row.executionWorkflowId, own)) {
+            throw new RunOwnershipLostError(name);
+        }
+        if (row && !DRIVABLE.includes(row.status)) {
+            throw new RunCancelledError(`Run ${row.status}: stopped before step "${name}".`);
+        }
+    };
+    // Each check waits for the one before it, so steps reach `inner` in the order they were called,
+    // as they did without a read in front: a durable runner numbers a step when it is called (DBOS
+    // assigns its function id then), and a replay of concurrent delegations matches steps by number.
+    let previous = Promise.resolve();
+    return async (name, fn) => {
+        const own = options.ownership?.workflowId();
+        const checked = previous.then(() => check(name, own));
+        previous = checked.catch(() => undefined);
+        await checked;
+        return inner(name, fn);
+    };
+}
 /**
  * Why a sandbox-mode native run is not executed here. The runner only ever
  * executes native runs in the control plane; a run whose snapshot says
@@ -399,12 +428,16 @@ export const CODING_CHILD_WAIT_GRACE_SEC = 60;
  * a parent left waiting on a child that never finishes.
  */
 export const SANDBOX_CHILD_WAIT_SEC = 60 * 60;
+/** The leading snake_case code of an error message (`code: sentence`); messages can carry internals. */
 /**
  * What a coding sub-agent's run tells the parent run's model. A run refused
  * over its services carries the host sentence (never the raw Run.error), so
- * the parent can pass it on — for a mention, that is the requester.
+ * the parent can pass it on — for a mention, that is the requester. A failed
+ * run carries only its error code, so the parent can say why without seeing internals.
  */
-export function codingChildResult(run, codingResult) {
+export function codingChildResult(run, codingResult, 
+/** The child's recorded budget refusal (loadBudgetEvent), if any. */
+budgetEvent) {
     // Checked before the "refused" branch: a continuation whose PR closed
     // before the run ever spent anything is stored as "refused" (a preflight
     // refusal, container.ts), not "failed" -- but it is the same
@@ -415,6 +448,10 @@ export function codingChildResult(run, codingResult) {
         : run.status === "refused"
             ? serviceRefusalSentence(run.error)
             : null;
+    const errorCode = run.status === "failed" ? declaredCodeOf(run.error, CHILD_RUN_FAILURE_CODES) : undefined;
+    const budget = budgetEvent && (run.status === "budget_exhausted" || run.status === "refused")
+        ? ownerBudgetSentence(budgetEvent.event, budgetEvent.agentName)
+        : undefined;
     return JSON.stringify({
         status: run.status,
         finalText: run.finalText,
@@ -422,6 +459,8 @@ export function codingChildResult(run, codingResult) {
         tokensIn: run.tokensIn,
         tokensOut: run.tokensOut,
         ...(refusal ? { refusal } : {}),
+        ...(errorCode ? { errorCode } : {}),
+        ...(budget ? { budget } : {}),
         ...pullRequestOf(codingResult),
     });
 }
@@ -668,6 +707,18 @@ export async function loadNativeRun(options) {
         throw err;
     });
 }
+/**
+ * Every value of the secrets `names` (readable or brokered; a SigV4 secret's key and token alone
+ * too, the same set the in-process host redacts), as a redaction list. Rejects when ANY read
+ * fails: a list missing one secret would let that secret through, so a caller treats a rejection
+ * as "no list" and fails closed. A secret with no value contributes nothing.
+ */
+export async function readRedactionValues(accessor, names) {
+    const values = await Promise.all(names.map((secret) => accessor.resolve
+        ? accessor.resolve(secret).then((e) => (e ? consoleRedactionValues(e) : []))
+        : accessor.get(secret).then((v) => (v === undefined ? [] : [v]))));
+    return values.flat().filter((value) => typeof value === "string");
+}
 /**
  * One attempt's tool surface for a native run. `builtinHandler(name)` is the
  * single source of truth for which names are trusted built-ins (memory,
@@ -683,6 +734,8 @@ export function createNativeRunTools(options) {
     // jira_create_issue's per-run cap counter: shared by every tool call of this attempt (a resumed attempt
     // starts a fresh one, floored by the run's recorded fingerprint creates).
     const issueCreationCounters = new Map();
+    // What this attempt's review tools have read, for repo_publish_review's unread_files gate.
+    const reviewReads = options.reviewReads ?? new ReviewReadLedger();
     // One per attempt: admits this run's delegations one at a time (see the delegate branch).
     const delegationGate = createSerialGate();
     // The children this attempt's delegations have running, and the sub-agents whose delegation is
@@ -690,7 +743,125 @@ export function createNativeRunTools(options) {
     // place in the same-child and limit checks while the gate is open to the others.
     const delegationSiblings = createDelegationSiblings();
     const waitingDelegations = new Set();
-    const builtinHandler = (name) => {
+    /**
+     * Waits on a dispatched coding or sandbox-mode native child until it is terminal, its wait
+     * bound passes (it is then stopped), or this run is cancelled, and returns what the model sees.
+     * The bound is measured from the child's creation, so a re-executed delegate step (a resume, or
+     * a takeover under a new version) never extends how long this run waits for it.
+     */
+    const awaitDelegatedChild = async (wait) => {
+        const { executor, childRunId, kind, boundName } = wait;
+        const startedAt = Date.now();
+        let boundMs;
+        if (kind === "coding") {
+            const codingRun = await db.codingRun.findUnique({
+                where: { runId: childRunId },
+                select: { timeoutSec: true },
+            });
+            const { queueTimeoutSec } = loadCodingConcurrencyConfig();
+            boundMs = (queueTimeoutSec + (codingRun?.timeoutSec ?? 0) + CODING_CHILD_WAIT_GRACE_SEC) * 1000;
+        }
+        else {
+            boundMs = SANDBOX_CHILD_WAIT_SEC * 1000;
+        }
+        const elapsedMs = Math.max(0, Date.now() - wait.createdAtMs);
+        // Executor.start may resolve before the child is terminal, so wait on the row.
+        const waited = await waitForCodingChild({
+            db,
+            executor,
+            childRunId,
+            parentRunId: runId,
+            boundMs: Math.max(0, boundMs - elapsedMs),
+        });
+        wait.onSettled();
+        runnerLog.info({
+            runId,
+            childRunId,
+            boundName,
+            outcome: waited.kind === "terminal" ? waited.run.status : waited.kind,
+            durationMs: Date.now() - startedAt,
+        }, "delegation finished");
+        if (waited.kind === "timed_out") {
+            return JSON.stringify({
+                error: "subagent_wait_timed_out",
+                runId: childRunId,
+                message: kind === "coding"
+                    ? "The coding sub-agent did not finish within its queue timeout plus run timeout; it was stopped and produced no result."
+                    : "The sandbox sub-agent did not finish in time; it was stopped and produced no result.",
+            });
+        }
+        if (waited.kind === "parent_cancelled") {
+            return JSON.stringify({
+                error: "parent_cancelled",
+                runId: childRunId,
+                message: kind === "coding"
+                    ? "This run was cancelled while waiting for the coding sub-agent; the sub-agent was stopped."
+                    : "This run was cancelled while waiting for the sandbox sub-agent; the sub-agent was stopped.",
+            });
+        }
+        if (kind === "coding") {
+            const stored = await db.codingRun
+                .findUnique({ where: { runId: childRunId }, select: { result: true } })
+                .catch((err) => {
+                runnerLog.warn({ err, childRunId }, "could not read the coding sub-run's result");
+                return null;
+            });
+            return codingChildResult(waited.run, stored?.result, await loadBudgetEvent(db, childRunId));
+        }
+        return nativeChildResult(waited.run, wait.mergeOrderNote);
+    };
+    /**
+     * A re-executed delegate call whose child already exists (findDelegationChild): wait on it
+     * again rather than refuse it as a duplicate. A control-plane native child ran inline in the
+     * attempt that died, so nothing drives it any more: it is stopped (or its result returned, if
+     * it had finished) and no new child is started.
+     */
+    const reattachDelegatedChild = async (child, boundName, argsJson) => {
+        const row = await db.run.findUniqueOrThrow({ where: { id: child.id } });
+        const childAgentRow = await db.agent.findUnique({ where: { id: row.agentId }, select: { kind: true } });
+        const parsedArgs = parseDelegateArgs(argsJson);
+        const mergeOrderNote = "args" in parsedArgs && parsedArgs.args.mergeOrder !== undefined ? MERGE_ORDER_IGNORED_NOTE : undefined;
+        const coding = childAgentRow?.kind === "coding";
+        if (!coding && row.nativeExecutionMode !== "sandbox") {
+            if (!DRIVABLE.includes(row.status))
+                return nativeChildResult(row, mergeOrderNote);
+            await providers.executor?.stop(child.id, "parent resumed without it").catch((err) => {
+                runnerLog.warn({ err, childRunId: child.id, runId }, "failed to stop an interrupted sub-agent run");
+            });
+            return JSON.stringify({
+                error: "subagent_interrupted",
+                message: "The sub-agent was interrupted by a restart and cannot be resumed; no new sub-agent run was started.",
+            });
+        }
+        if (!providers.executor) {
+            return coding
+                ? JSON.stringify({
+                    error: "coding_dispatch_unavailable",
+                    message: "This execution context has no Executor wired in, so a coding-kind sub-agent cannot be dispatched.",
+                })
+                : JSON.stringify({
+                    error: "sandbox_dispatch_unavailable",
+                    message: "This execution context has no Executor wired in, so a sandbox-mode sub-agent cannot be dispatched.",
+                });
+        }
+        // Counted as a running sibling while it is waited on, as a fresh dispatch is.
+        const finishSibling = delegationSiblings.start();
+        try {
+            return await awaitDelegatedChild({
+                executor: providers.executor,
+                childRunId: child.id,
+                createdAtMs: child.createdAt.getTime(),
+                kind: coding ? "coding" : "native-sandbox",
+                boundName,
+                mergeOrderNote: coding ? undefined : mergeOrderNote,
+                onSettled: finishSibling,
+            });
+        }
+        finally {
+            finishSibling();
+        }
+    };
+    const unguardedBuiltinHandler = (name) => {
         if (loaded.memoryEnabled && MEMORY_TOOL_NAMES.has(name)) {
             return async (argsJson) => {
                 return handleMemoryTool(name, argsJson, loaded.agentId, providers.memory);
@@ -703,6 +874,7 @@ export function createNativeRunTools(options) {
                     agentId: loaded.agentId,
                     links: loaded.repositoryLinks,
                     hosts: reviewHosts,
+                    reads: reviewReads,
                     db,
                     runCheck: check && !check.completedAt
                         ? {
@@ -791,7 +963,9 @@ export function createNativeRunTools(options) {
             };
         }
         if (name.startsWith(DELEGATE_TOOL_PREFIX)) {
-            return async (argsJson) => {
+            return async (argsJson, toolCallId) => {
+                // An empty id (an adapter that had none) is no key: every such call would share it.
+                const callId = toolCallId || undefined;
                 const boundName = name.slice(DELEGATE_TOOL_PREFIX.length);
                 const edge = loaded.subAgentEdges.find((e) => e.boundName === boundName);
                 if (!edge) {
@@ -800,8 +974,36 @@ export function createNativeRunTools(options) {
                         message: `No sub-agent is bound to name "${boundName}".`,
                     });
                 }
+                // A re-executed delegate step (a resume, or a takeover under a new version) finds the
+                // child this very call created and waits on it again, instead of refusing it as a duplicate.
+                const existingChild = await findDelegationChild(db, runId, edge.childAgentId, callId);
+                if (existingChild) {
+                    runnerLog.info({ runId, childRunId: existingChild.id, boundName }, "delegation re-attached");
+                    return reattachDelegatedChild(existingChild, boundName, argsJson);
+                }
                 let release = await delegationGate.acquire();
                 let finishSibling = () => { };
+                /**
+                 * A child create that lost the (parentRunId, parentCallId) unique key to another execution of
+                 * this same delegate step (a stalled owner overlapping an adopted attempt): wait on the child
+                 * that execution created, as a re-executed step does. A key held by no child of this agent
+                 * (an id colliding with another sub-agent's call) is the model's to see, never a run failure.
+                 */
+                const afterDelegationKeyConflict = async (err) => {
+                    if (!(err instanceof Prisma.PrismaClientKnownRequestError && err.code === "P2002"))
+                        throw err;
+                    release();
+                    const raced = await findDelegationChild(db, runId, edge.childAgentId, callId);
+                    if (raced) {
+                        runnerLog.info({ runId, childRunId: raced.id, boundName }, "delegation re-attached after a concurrent create");
+                        return reattachDelegatedChild(raced, boundName, argsJson);
+                    }
+                    runnerLog.warn({ err, runId, boundName }, "delegation child create hit a unique key held by no child of this agent");
+                    return JSON.stringify({
+                        error: "delegation_conflict",
+                        message: "The sub-agent run could not be created: its call id is already used by another sub-agent run.",
+                    });
+                };
                 try {
                     // At most maxDelegationsPerRun dispatches per run (default one: a
                     // classifier deciding plan-vs-implement commits to exactly one child),
@@ -839,6 +1041,7 @@ export function createNativeRunTools(options) {
                         while (delegationSiblings.inFlight > 0) {
                             // Taken before the budget read, so a sibling that finishes during it still wakes this wait.
                             const finishes = delegationSiblings.finishes;
+                            // What dispatch will reserve: a coding child's base too, its overrun is never reserved.
                             const { exhaustedBy } = await effectiveBudgetForRun(db, childAgent, new Date(), runId);
                             const remainingMs = deadline - Date.now();
                             if (!exhaustedBy || remainingMs <= 0)
@@ -898,6 +1101,7 @@ export function createNativeRunTools(options) {
                                 continuesCodingRunId: args.continuePriorRun,
                                 mergeOrder: args.mergeOrder,
                                 parentRunId: runId,
+                                parentCallId: callId,
                                 // The child's result flows back into this run, which the
                                 // triggerer sees, so the child is visible to them too.
                                 triggeredById: existingRun.triggeredById,
@@ -917,7 +1121,7 @@ export function createNativeRunTools(options) {
                             // a pull request another deployment opened, say) is the model's to
                             // report, not a reason to fail the whole run.
                             if (!(err instanceof ContinuationRefusedError))
-                                throw err;
+                                return await afterDelegationKeyConflict(err);
                             return JSON.stringify({
                                 error: "continuation_refused",
                                 message: `${err.message} No sub-agent run was started. Do not open a new pull request in its place: ` +
@@ -928,49 +1132,16 @@ export function createNativeRunTools(options) {
                             return JSON.stringify({ error: "dispatch_failed", message: "The sub-agent run could not be created." });
                         }
                         const childRunId = dispatched.run.id;
-                        const startedAt = Date.now();
                         runnerLog.info({ runId, childRunId, boundName, kind: "coding" }, "delegation started");
-                        const codingRun = await db.codingRun.findUnique({
-                            where: { runId: childRunId },
-                            select: { timeoutSec: true },
-                        });
-                        const { queueTimeoutSec } = loadCodingConcurrencyConfig();
-                        const waited = await waitForCodingChild({
-                            db,
+                        return await awaitDelegatedChild({
                             executor: providers.executor,
                             childRunId,
-                            parentRunId: runId,
-                            boundMs: (queueTimeoutSec + (codingRun?.timeoutSec ?? 0) + CODING_CHILD_WAIT_GRACE_SEC) * 1000,
-                        });
-                        finishSibling();
-                        runnerLog.info({
-                            runId,
-                            childRunId,
+                            createdAtMs: dispatched.run.startedAt.getTime(),
+                            kind: "coding",
                             boundName,
-                            outcome: waited.kind === "terminal" ? waited.run.status : waited.kind,
-                            durationMs: Date.now() - startedAt,
-                        }, "delegation finished");
-                        if (waited.kind === "timed_out") {
-                            return JSON.stringify({
-                                error: "subagent_wait_timed_out",
-                                runId: childRunId,
-                                message: "The coding sub-agent did not finish within its queue timeout plus run timeout; it was stopped and produced no result.",
-                            });
-                        }
-                        if (waited.kind === "parent_cancelled") {
-                            return JSON.stringify({
-                                error: "parent_cancelled",
-                                runId: childRunId,
-                                message: "This run was cancelled while waiting for the coding sub-agent; the sub-agent was stopped.",
-                            });
-                        }
-                        const stored = await db.codingRun
-                            .findUnique({ where: { runId: childRunId }, select: { result: true } })
-                            .catch((err) => {
-                            runnerLog.warn({ err, childRunId }, "could not read the coding sub-run's result");
-                            return null;
+                            mergeOrderNote: undefined,
+                            onSettled: finishSibling,
                         });
-                        return codingChildResult(waited.run, stored?.result);
                     }
                     const taskOverride = delegationTaskOverride(args);
                     // mergeOrder is a coding-only concept (it orders CodingRuns within a merge); a native
@@ -988,73 +1159,64 @@ export function createNativeRunTools(options) {
                                 message: "This execution context has no Executor wired in, so a sandbox-mode sub-agent cannot be dispatched.",
                             });
                         }
-                        const dispatched = await dispatchRun({
-                            db,
-                            executor: providers.executor,
-                            selfDefects: { db, issueTrackers },
-                            agentId: edge.childAgentId,
-                            trigger: "subagent",
-                            taskOverride,
-                            grantedParentMemoryKeys: args.grantParentMemoryKeys ?? [],
-                            parentRunId: runId,
-                            triggeredById: existingRun.triggeredById,
-                            awaitExecution: true,
-                            onPersisted: (persistedRun) => {
-                                if (persistedRun.status !== "refused" && persistedRun.status !== "failed") {
-                                    finishSibling = delegationSiblings.start();
-                                }
-                                release();
-                            },
-                        });
+                        let dispatched;
+                        try {
+                            dispatched = await dispatchRun({
+                                db,
+                                executor: providers.executor,
+                                selfDefects: { db, issueTrackers },
+                                agentId: edge.childAgentId,
+                                trigger: "subagent",
+                                taskOverride,
+                                grantedParentMemoryKeys: args.grantParentMemoryKeys ?? [],
+                                parentRunId: runId,
+                                parentCallId: callId,
+                                triggeredById: existingRun.triggeredById,
+                                awaitExecution: true,
+                                onPersisted: (persistedRun) => {
+                                    if (persistedRun.status !== "refused" && persistedRun.status !== "failed") {
+                                        finishSibling = delegationSiblings.start();
+                                    }
+                                    release();
+                                },
+                            });
+                        }
+                        catch (err) {
+                            return await afterDelegationKeyConflict(err);
+                        }
                         if (!dispatched) {
                             return JSON.stringify({ error: "dispatch_failed", message: "The sub-agent run could not be created." });
                         }
                         const childRunId = dispatched.run.id;
-                        const startedAt = Date.now();
                         runnerLog.info({ runId, childRunId, boundName, kind: "native-sandbox" }, "delegation started");
-                        // Executor.start may resolve before the child is terminal, so wait on the row like a coding child.
-                        const waited = await waitForCodingChild({
-                            db,
+                        return await awaitDelegatedChild({
                             executor: providers.executor,
                             childRunId,
-                            parentRunId: runId,
-                            boundMs: SANDBOX_CHILD_WAIT_SEC * 1000,
-                        });
-                        finishSibling();
-                        runnerLog.info({
-                            runId,
-                            childRunId,
+                            createdAtMs: dispatched.run.startedAt.getTime(),
+                            kind: "native-sandbox",
                             boundName,
-                            outcome: waited.kind === "terminal" ? waited.run.status : waited.kind,
-                            durationMs: Date.now() - startedAt,
-                        }, "delegation finished");
-                        if (waited.kind === "timed_out") {
-                            return JSON.stringify({
-                                error: "subagent_wait_timed_out",
-                                runId: childRunId,
-                                message: "The sandbox sub-agent did not finish in time; it was stopped and produced no result.",
-                            });
-                        }
-                        if (waited.kind === "parent_cancelled") {
-                            return JSON.stringify({
-                                error: "parent_cancelled",
-                                runId: childRunId,
-                                message: "This run was cancelled while waiting for the sandbox sub-agent; the sub-agent was stopped.",
-                            });
-                        }
-                        return nativeChildResult(waited.run, mergeOrderNote);
+                            mergeOrderNote,
+                            onSettled: finishSibling,
+                        });
+                    }
+                    let childRun;
+                    try {
+                        childRun = await db.run.create({
+                            data: {
+                                agentId: edge.childAgentId,
+                                trigger: "subagent",
+                                parentRunId: runId,
+                                parentCallId: callId,
+                                grantedParentMemoryKeys: args.grantParentMemoryKeys ?? [],
+                                taskOverride,
+                                triggeredById: existingRun.triggeredById,
+                                nativeExecutionMode: childAgent.nativeExecutionMode,
+                            },
+                        });
+                    }
+                    catch (err) {
+                        return await afterDelegationKeyConflict(err);
                     }
-                    const childRun = await db.run.create({
-                        data: {
-                            agentId: edge.childAgentId,
-                            trigger: "subagent",
-                            parentRunId: runId,
-                            grantedParentMemoryKeys: args.grantParentMemoryKeys ?? [],
-                            taskOverride,
-                            triggeredById: existingRun.triggeredById,
-                            nativeExecutionMode: childAgent.nativeExecutionMode,
-                        },
-                    });
                     finishSibling = delegationSiblings.start();
                     // The child row exists: admit the next delegation while this child runs.
                     release();
@@ -1081,6 +1243,26 @@ export function createNativeRunTools(options) {
         }
         return undefined;
     };
+    /**
+     * The only route to a built-in tool, in both execution modes (the in-process loop and the native
+     * gateway's builtin.call). A tool that posts model-written text to a host, and a delegate_to_*
+     * call (its task text becomes the child run's stored task), has every attached secret's value
+     * masked in its arguments first; when the secrets cannot be read the call is refused and
+     * neither the host nor a child run is reached. (The native gateway serves delegate_to_* through
+     * durableDelegate, which masks the same way.)
+     */
+    const builtinHandler = (name) => {
+        const handler = unguardedBuiltinHandler(name);
+        if (!handler || !(POSTING_TOOL_NAMES.has(name) || name.startsWith(DELEGATE_TOOL_PREFIX)))
+            return handler;
+        return async (argsJson, toolCallId) => {
+            const masked = await maskModelWrittenArgs({ db, secrets: providers.secrets, agentId: loaded.agentId, runId }, name, argsJson);
+            if (masked === undefined) {
+                return JSON.stringify({ error: "secret_check_unavailable", message: SECRET_CHECK_UNAVAILABLE_MESSAGE });
+            }
+            return handler(masked, toolCallId);
+        };
+    };
     /**
      * The privileged host for one call of the user tool `name`, scoped to that attachment's
      * consented capabilities. Undefined for a name that is not an attached user tool. In-process it
@@ -1101,17 +1283,15 @@ export function createNativeRunTools(options) {
         const tool = toolsByName.get(name);
         return tool ? scopedHost(name, tool, signal, redactSecretValues) : undefined;
     };
-    /** Every secret value the user tool `name` may use (readable or brokered), for console redaction by a stateless gateway. */
+    /**
+     * Every secret value the user tool `name` may use (readable or brokered), for console redaction
+     * by a stateless gateway. Rejects when any read fails (readRedactionValues).
+     */
     const readableSecretValues = async (name) => {
         const tool = toolsByName.get(name);
         if (!tool)
             return [];
-        const scoped = scopeSecretsAccessor(secretsAccessor, tool.allowedSecrets);
-        const values = await Promise.all(tool.allowedSecrets.map((secret) => (scoped.resolve
-            ? scoped.resolve(secret).then((e) => (e ? consoleRedactionValues(e) : []))
-            : scoped.get(secret).then((v) => (v === undefined ? [] : [v]))).catch(() => [])));
-        // A SigV4 secret's key and token alone, too: the same set the in-process host redacts.
-        return values.flat().filter((value) => typeof value === "string");
+        return readRedactionValues(scopeSecretsAccessor(secretsAccessor, tool.allowedSecrets), tool.allowedSecrets);
     };
     const runUserTool = async (name, argsJson) => {
         const tool = toolsByName.get(name);
@@ -1123,9 +1303,9 @@ export function createNativeRunTools(options) {
         }
         return runUserToolCall(tool, argsJson, (signal) => scopedHost(name, tool, signal));
     };
-    const runSandboxTool = async (name, argsJson) => {
+    const runSandboxTool = async (name, argsJson, callId) => {
         const builtin = builtinHandler(name);
-        return builtin ? builtin(argsJson) : runUserTool(name, argsJson);
+        return builtin ? builtin(argsJson, callId) : runUserTool(name, argsJson);
     };
     return { toolsByName, builtinHandler, privilegedHostFor, readableSecretValues, runUserTool, runSandboxTool };
 }
@@ -1135,11 +1315,13 @@ export function createNativeRunTools(options) {
  * tree's shared budget (computeRunTreeSpend) then counts a running parent's spend so far, as it
  * already does for coding runs, whose proxy ledger writes their totals live. Best-effort: a
  * failed write only delays what observers see, and finishRun writes the final totals anyway.
+ * A cancelled run takes them too: a stop lands mid-step, and that step's spend is real, while the
+ * cancelled run's terminal write (failNativeRun) records no totals of its own.
  */
 export async function recordNativeRunProgress(db, runId, progress) {
     try {
         await db.run.updateMany({
-            where: { id: runId, status: "running" },
+            where: { id: runId, status: { in: ["running", "cancelled"] } },
             data: {
                 turns: progress.turns,
                 tokensIn: progress.usage.tokensIn,
@@ -1153,6 +1335,35 @@ export async function recordNativeRunProgress(db, runId, progress) {
         runnerLog.warn({ err, runId }, "failed to record run progress");
     }
 }
+/**
+ * `texts` with every value of every secret attached to the run's agent masked, for a terminal
+ * write: the stored row is what get_run, the viewer, host and issue status comments and run
+ * notifications all read. Fails closed: when the secrets cannot be read, every non-null text is
+ * replaced by WITHHELD_TEXT_NOTICE (the log line carries no text and no error message). A pure
+ * function of the stored secrets, so a durable replay writes the same row.
+ */
+async function maskRunTexts(ctx, texts) {
+    const entries = Object.entries(texts);
+    if (!entries.some(([, text]) => typeof text === "string" && text.length > 0))
+        return texts;
+    let mask;
+    try {
+        const row = await ctx.db.run.findUnique({ where: { id: ctx.runId }, select: { agentId: true } });
+        if (!row)
+            throw new Error("run not found");
+        const named = await loadAgentSecretValues(ctx.db, ctx.providers.secrets, row.agentId);
+        mask = (text) => maskNamedSecrets(text, named);
+    }
+    catch (err) {
+        runnerLog.warn({
+            runId: ctx.runId,
+            event: "run_text_secret_check_failed",
+            errorName: err instanceof Error ? err.name : typeof err,
+        }, "could not read the agent's secrets to check the run's final text and error; storing the withheld notice");
+        mask = () => WITHHELD_TEXT_NOTICE;
+    }
+    return Object.fromEntries(entries.map(([key, text]) => [key, typeof text === "string" && text.length > 0 ? mask(text) : text]));
+}
 /** Everything after the terminal write, shared by the normal and backstop paths. */
 async function settleFinishedNativeRun(ctx, finished, claimed) {
     const { runId, db, providers, reviewHosts, repoAccess, issueTrackers } = ctx;
@@ -1192,17 +1403,22 @@ async function settleFinishedNativeRun(ctx, finished, claimed) {
     // host reads). Only for a tree root that delegated. Not gated on `claimed`: the sync is
     // idempotent (an unchanged check writes nothing). Never throws.
     await syncMergeOrderChecksAfterRun({ db, hosts: reviewHosts }, finished);
+    // A reviewer run's end may be the last thing a pull request waited for: mark it ready for review
+    // when its builder link opted in (#287). Last, so nothing above waits on its host reads; idempotent
+    // (claimed once per pull request); no host reads unless a link opted in. Never throws.
+    await markReadyAfterRun({ db, hosts: reviewHosts, log: runnerLog }, runId);
     return finished;
 }
 /** The terminal write for an engine result, then its usage, host/issue status, and follow-ups. */
 export async function finishNativeRun(ctx, model, result) {
+    const texts = await maskRunTexts(ctx, { error: result.error ?? null, finalText: result.finalText || null });
     const { run: finished, claimed } = await finishRunClaimed(ctx.db, ctx.runId, {
         status: result.status,
         tokensIn: result.usage.tokensIn,
         tokensOut: result.usage.tokensOut,
         costUsd: result.usage.costUsd,
-        error: result.error ?? null,
-        finalText: result.finalText || null,
+        error: texts.error,
+        finalText: texts.finalText,
         turns: result.turns,
         finishedAt: new Date(),
     });
@@ -1212,11 +1428,24 @@ export async function finishNativeRun(ctx, model, result) {
 }
 /** The backstop terminal write for a run that threw. A cancellation is not a failure: it carries the operator's own reason. */
 export async function failNativeRun(ctx, err) {
+    const { error } = await maskRunTexts(ctx, { error: err instanceof Error ? err.message : String(err) });
     const { run: finished, claimed } = await finishRunClaimed(ctx.db, ctx.runId, {
         status: err instanceof RunCancelledError || err instanceof SandboxRunCancelledError ? "cancelled" : "failed",
-        error: err instanceof Error ? err.message : String(err),
+        error,
+        finishedAt: new Date(),
+    });
+    return settleFinishedNativeRun(ctx, finished, claimed);
+}
+/** The terminal write for a run that cannot continue (it ends `lost` with `reason`), then the same settle path. */
+export async function loseNativeRun(ctx, reason) {
+    const { error } = await maskRunTexts(ctx, { error: reason });
+    const { run: finished, claimed } = await finishRunClaimed(ctx.db, ctx.runId, {
+        status: "lost",
+        error,
         finishedAt: new Date(),
     });
+    if (claimed)
+        await stopChildrenOfLostRun(ctx.db, ctx.providers.executor, ctx.runId);
     return settleFinishedNativeRun(ctx, finished, claimed);
 }
 /** Where a run stands for the native gateway: still drivable, cancelled (its task), or otherwise ended. */
@@ -1250,6 +1479,25 @@ export function sandboxWorkerInput(runId, loaded, tools) {
         pricing: { ...loaded.pricing.entry, efforts: [...loaded.pricing.entry.efforts] },
     };
 }
+/**
+ * A tool call's model-written arguments with every secret value attached to the agent masked, or
+ * undefined (logged by error name only) when the secrets cannot be read and the call must be refused.
+ */
+async function maskModelWrittenArgs(ctx, tool, argsJson) {
+    try {
+        const named = await loadAgentSecretValues(ctx.db, ctx.secrets, ctx.agentId);
+        return maskNamedSecrets(argsJson, named);
+    }
+    catch (err) {
+        runnerLog.warn({
+            runId: ctx.runId,
+            tool,
+            event: "post_secret_check_failed",
+            errorName: err instanceof Error ? err.name : typeof err,
+        }, "could not read the agent's secrets to check a posted text; refusing the call");
+        return undefined;
+    }
+}
 /** Long-poll window: well under common proxy and load-balancer idle timeouts. */
 export const DELEGATION_POLL_WINDOW_MS = 20_000;
 /**
@@ -1308,13 +1556,24 @@ export async function durableDelegate(ctx, callId, name, argsJson) {
     if (!childRunId && state?.childAgentId === edge.childAgentId) {
         // A replica that died between dispatching the child and recording it left the child row:
         // adopt it rather than refusing this very delegation as a duplicate.
-        const orphan = await db.run.findFirst({
-            where: { parentRunId: runId, agentId: edge.childAgentId },
-            select: { id: true },
-        });
-        childRunId = orphan?.id ?? null;
+        childRunId = (await findDelegationChild(db, runId, edge.childAgentId, callId))?.id ?? null;
+        // Recorded, so later calls wait on it by id.
+        if (childRunId) {
+            await ledger.setDelegation(sessionId, callId, { status: "pending", childAgentId: edge.childAgentId, childRunId });
+        }
     }
     if (!childRunId) {
+        // About to create a child: its task is model-written and stored, so mask the parent's secret
+        // values first. Only here, never on a poll of an existing child (a transient secret-read
+        // failure must not orphan a running child). Recorded as the call's result, which also completes a waiting row; the model's retry comes with a new callId.
+        const maskedArgsJson = await maskModelWrittenArgs({ db, secrets: providers.secrets, agentId: loaded.agentId, runId }, name, argsJson);
+        if (maskedArgsJson === undefined) {
+            return finish(JSON.stringify({ error: "secret_check_unavailable", message: SECRET_CHECK_UNAVAILABLE_MESSAGE }));
+        }
+        const maskedArgs = parseDelegateArgs(maskedArgsJson);
+        if ("refusal" in maskedArgs)
+            return finish(maskedArgs.refusal);
+        const dispatchArgs = maskedArgs.args;
         const limit = loaded.maxDelegationsPerRun ?? 1;
         const refusalNow = async (reader) => {
             const prior = await reader.run.findMany({ where: { parentRunId: runId }, select: { agentId: true } });
@@ -1356,9 +1615,17 @@ export async function durableDelegate(ctx, callId, name, argsJson) {
                 agentId: edge.childAgentId,
                 trigger: "subagent",
                 ...(coding
-                    ? { codingTask: args.task, continuesCodingRunId: args.continuePriorRun, mergeOrder: args.mergeOrder }
-                    : { taskOverride: delegationTaskOverride(args), grantedParentMemoryKeys: args.grantParentMemoryKeys ?? [] }),
+                    ? {
+                        codingTask: dispatchArgs.task,
+                        continuesCodingRunId: dispatchArgs.continuePriorRun,
+                        mergeOrder: dispatchArgs.mergeOrder,
+                    }
+                    : {
+                        taskOverride: delegationTaskOverride(dispatchArgs),
+                        grantedParentMemoryKeys: dispatchArgs.grantParentMemoryKeys ?? [],
+                    }),
                 parentRunId: runId,
+                parentCallId: callId,
                 // The child's result flows back into this run, which the triggerer sees, so the child is visible to them too.
                 triggeredById: existingRun.triggeredById,
                 beforePersist: async (tx) => {
@@ -1370,12 +1637,21 @@ export async function durableDelegate(ctx, callId, name, argsJson) {
             });
         }
         catch (err) {
-            if (!(err instanceof ContinuationRefusedError))
-                throw err;
+            if (err instanceof ContinuationRefusedError) {
+                return finish(JSON.stringify({
+                    error: "continuation_refused",
+                    message: `${err.message} No sub-agent run was started. Do not open a new pull request in its place: ` +
+                        "tell the requester that this wardby deployment cannot continue that pull request's branch.",
+                }));
+            }
+            // Recorded as the call's result, so the model can report it and a retried callId never
+            // dispatches twice. Only the leading code is exposed: messages can carry internals.
+            const code = declaredCodeOf(err);
+            const values = await ctx.secretValues?.().catch(() => undefined);
+            logUnexpectedError(runnerLog, err, values, "delegation dispatch failed", { runId, boundName });
             return finish(JSON.stringify({
-                error: "continuation_refused",
-                message: `${err.message} No sub-agent run was started. Do not open a new pull request in its place: ` +
-                    "tell the requester that this wardby deployment cannot continue that pull request's branch.",
+                error: "dispatch_failed",
+                message: `The sub-agent run could not be created${code ? ` (${code})` : ""}.`,
             }));
         }
         if (!dispatched) {
@@ -1403,7 +1679,7 @@ export async function durableDelegate(ctx, callId, name, argsJson) {
             const stored = await db.codingRun
                 .findUnique({ where: { runId: childRunId }, select: { result: true } })
                 .catch(() => null);
-            return finish(codingChildResult(child, stored?.result));
+            return finish(codingChildResult(child, stored?.result, await loadBudgetEvent(db, childRunId)));
         }
         if (now() - startedAt >= boundMs) {
             await executor.stop(childRunId, "sub-agent wait timed out").catch((err) => {
@@ -1505,8 +1781,17 @@ export async function createSandboxSession(options) {
         sessionId: session.id,
         capability,
         input: { ...input, gateway: { url: options.gatewayUrl, capability } },
+        secretValues: () => userToolSecretValues(tools, Object.keys(input.userTools)),
     };
 }
+/**
+ * Every secret value the sandbox run's user tools `names` may read (readable or brokered): what its
+ * worker can be handed, so what the gateway checks its cause codes against and a launcher masks
+ * from the worker's output tail. Rejects when any read fails (readRedactionValues).
+ */
+export async function userToolSecretValues(tools, names) {
+    return (await Promise.all(names.map((name) => tools.readableSecretValues(name)))).flat();
+}
 /** Run.executionBackend for a sandbox-mode run whose worker is (or was) launched. */
 export const NATIVE_SANDBOX_BACKEND = "native-sandbox";
 /** The stored form of a sandbox capability: never the token itself. */
@@ -1539,13 +1824,18 @@ async function executeTrackedRun(runId, providers, db, onText, step) {
         ? (providers.repoAccess ?? createRepoAccessGate({ db, hosts: reviewHosts }))
         : undefined;
     const issueTrackers = configuredIssueTrackers(providers.issueTrackers);
+    // Inline execution checks for a stop before every step (stopAwareStep); a durable executor passes
+    // its own step runner, which does its own check (DbosExecutor wraps it in stopAwareStep too).
+    if (step === runStepInline)
+        step = stopAwareStep(db, runId);
     // Fails a run that ends before its engine starts, closing what its trigger opened (the review
     // check, the host and issue status comments) exactly as the normal and catch paths below do.
     // Deliberately no self-defect: every caller is a configuration state, not this run's defect (an
     // admin disabled or removed the model, or this deployment has no coding or native sandbox
     // executor), and filing would open one defect per affected agent rather than describe a failure
     // of that agent.
-    const finishEarly = async (error) => {
+    const finishEarly = async (reason) => {
+        const { error } = await maskRunTexts({ runId, db, providers }, { error: reason });
         const finished = await finishRun(db, runId, { status: "failed", error, finishedAt: new Date() });
         await closeOpenHostCheck(db, finished, reviewHosts);
         await completeHostStatus(db, finished, reviewHosts);
@@ -1601,6 +1891,7 @@ async function executeTrackedRun(runId, providers, db, onText, step) {
                 ctx: { providers: { llm }, onText, onProgress },
                 builtinHandler: tools.builtinHandler,
                 privilegedHostFor: tools.privilegedHostFor,
+                readableSecretValues: tools.readableSecretValues,
                 drivability: () => runDrivability(db, runId),
                 launcher: providers.nativeSandbox,
             });
@@ -1634,6 +1925,10 @@ async function executeTrackedRun(runId, providers, db, onText, step) {
             logRunOwnershipLost(runId, err);
             throw err;
         }
+        // A replay that no longer matches its record (a run recorded by a release whose steps differ)
+        // cannot continue: it ends lost, not failed, through the same settle path.
+        if (err instanceof StepReplayMismatchError)
+            return loseNativeRun(finishContext, STEPS_CHANGED_REASON);
         return failNativeRun(finishContext, err);
     }
 }