@humwick/server 0.0.0-stage → 0.130.0

package/src/services/run-command.ts

@@ -0,0 +1,510 @@
+// #2038 (leaf of #2031) — `relay_run`, the INLINE bounded command executor.
+//
+// THE PROBLEM THIS EXISTS FOR. Agents fall back to raw harness Bash for one decisive reason: Bash
+// returns the result INLINE, in the same turn. `relay_schedule_timer{command}` returns a timer id
+// and the result arrives later as a Relay message — a NEW turn, with the reasoning context gone. No
+// instruction beats that gradient. The executor itself was already ~90% built (dispatchCommand →
+// orchestrator `scheduler.command` → execProcess with a hard timeout, a cgroup process-group reap
+// and an output cap); what was missing is an inline-returning FRONT DOOR. That is all this module
+// adds. There is deliberately NO new execution substrate — if this file ever grows one, that is the
+// design smell #2038 says to raise before building it.
+//
+// WHAT IS STRUCTURALLY GUARANTEED (the property #2031 asks for — a property of the executor, not of
+// agent discipline):
+//   • The CALL is bounded by `waitMs` (default 0, cap 10s — below provider MCP timeouts, so the caller
+//     receives a runId before its transport gives up).
+//   • The COMMAND is bounded by `maxMs` (default 45min, cap 24h — the existing command-timer
+//     ceiling). It is enforced by the orchestrator as a real SIGKILL of the whole process GROUP
+//     (cgroup v2 `cgroup.kill`, falling back to a detached-pgid + descendant sweep), so a
+//     `setsid`-detached grandchild — the actual shape of the observed hang — cannot survive it.
+//   • Captured output is CAPPED by the orchestrator before it is ever reported
+//     (`schedulerCommandMaxOutputBytes`). Generic scheduler commands still abort a `yes`-style
+//     flood; this trusted relay_run path instead retains a bounded tail and the process's real
+//     terminal verdict. The tails stored/returned here are additionally clipped to
+//     {@link RUN_OUTPUT_TAIL_CHARS} characters per stream. See {@link RunCommandResult.truncated}.
+//   • Neither bound is reachable-around: a run that outlives `waitMs` returns `still-running` and
+//     KEEPS running under the orchestrator with its completion wake armed, so no state is lost and
+//     no turn is blocked.
+//
+// WHAT IT DOES NOT GRANT. The command string is caller-supplied, but it runs under the caller's own
+// already-authorized identity, in the caller's own authorized cwd, via the SAME authorization path
+// as `relay_schedule_timer{command}` (`normalizeScheduleCommand` → agent-cwd/constraint resolution →
+// `isPathWithinBase` against the orchestrator baseDir → the component `command:write` cwd-constraint
+// check), and the orchestrator re-checks containment against its own baseDir before spawning. So it
+// is exactly the reach the agent already had through Bash — no wider cwd, no extra privilege, no env
+// beyond what the caller could already set. Every containment check is REUSED, never re-implemented;
+// this module adds no authorization logic of its own beyond run OWNERSHIP on the poll path.
+import { errMessage, maskCommandCredentials } from "@humwick/sdk";
+import { GATE_RUN_SUPERSEDED_NOTICE, isSupersededRunResult } from "@humwick/sdk/gate-run-lease";
+import { HEAVY_GATE_RUN_CAP_MS } from "@humwick/sdk/land-gates";
+import { getCommandAsync } from "../commands-write";
+import { ValidationError, disableTimerAsync, getTimerAsync, updateTimerCommandAsync, type ScheduledTimer, type ScheduledTimerCommand } from "../db";
+import { ServiceAuthError } from "./errors";
+import { scheduleTimerAsync } from "./scheduler";
+import { RUN_QUEUE_BUDGET_MS } from "./scheduler-command";
+import { readRunLog, type RunLogPage, type RunLogQuery } from "./run-log";
+import { cancelScheduledCommand } from "./scheduler-command-cancel";
+import { describeRunQueue, readRunQueue, type RunQueueView } from "./run-queue";
+import type { AuthContext } from "./auth-context";
+import type { Command } from "../types";
+
+/** Detached requests get a short inline window before returning a pending run id. */
+export const RUN_DEFAULT_WAIT_MS = 0;
+/** Blocking waits above 10s are clamped below provider MCP tool-call timeouts (#2790). */
+export const RUN_MAX_WAIT_MS = 10_000;
+// Leave the final read and consumption write room after the blocking window (#2790).
+const RUN_RESPONSE_SLACK_MS = 1_000;
+/** How long the COMMAND may run before the orchestrator SIGKILLs its process group. */
+export const RUN_DEFAULT_MAX_MS = 45 * 60_000;
+/** #2776 — the ORDER every queued/still-running result carries. Codex agents treated the result as a
+ * blocking call and polled hundreds of times per turn; the wake is the only supported way back. */
+export const RUN_END_TURN_ORDER = "Unless this run's completion is already in your context, END YOUR TURN NOW. Relay wakes you once on finish, failure or timeout. If the result is already present, handle it and continue working. Do not poll, sleep, tail or ps.";
+/** #2776 — a `relay_run{runId}` poll sooner than this after the previous contact is refused at once. */
+export const RUN_POLL_MIN_INTERVAL_MS = 60_000;
+/** #2776 — `maxMs` defaults to this multiple of `expectedMs` when the caller gives only the estimate. */
+export const RUN_EXPECTED_MAX_FACTOR = 4;
+/** Ceiling on `maxMs` — the same 24h ceiling `relay_schedule_timer` command payloads carry. */
+export const RUN_MAX_MAX_MS = 24 * 60 * 60 * 1000;
+/** Per-stream character cap on the tails returned here and persisted on the run row. The
+ * orchestrator preserves relay_run's terminal verdict while its bounded byte capture drops older
+ * output; this is the second, always-applied clip. Truncated tails keep the END of the stream
+ * (where errors are) and are flagged via {@link RunCommandResult.truncated}. */
+export const RUN_OUTPUT_TAIL_CHARS = 8_000;
+/** Marker prefixed to a tail this module clipped, so a truncated tail is never mistaken for the
+ * whole stream. The full stream is always in the log artifact. */
+export const RUN_TRUNCATION_MARKER = "…[truncated — full output in the log artifact]\n";
+
+/** Poll cadence, backed off by how long this call has already been waiting. A short command has to
+ * feel immediate, so the first seconds are polled tightly; a 10-minute wait must not spend that
+ * whole time hammering the DB on the agent's behalf, so it settles to once a second. Latency added
+ * at the tail is bounded by the last step and is irrelevant next to the runtime that earned it. */
+function pollIntervalMs(waitedMs: number): number {
+  if (waitedMs < 2_000) return 100;
+  if (waitedMs < 15_000) return 250;
+  return 1_000;
+}
+/** After the COMMAND row goes terminal, the settle path still has to store the log artifact and
+ * patch the run row. We wait this long for that to land (so the returned result carries its
+ * `logArtifactId`) before reporting from the command row alone. Bounded, so a settle that never
+ * happens degrades the result rather than the call. */
+const SETTLE_GRACE_MS = 3_000;
+
+export interface RunCommandInput {
+  /** Shell command to run. Required unless `runId` is given. */
+  command?: string;
+  /** Working directory. Must resolve inside an online orchestrator baseDir and inside the caller's
+   * own cwd constraint. Defaults to the caller agent's registered cwd (its worktree, for an
+   * isolated worker — see #2033). */
+  cwd?: string;
+  env?: Record<string, string>;
+  waitMs?: number;
+  /** Same as `waitMs: 0`: return fast results inline, otherwise return the runId and wake once. */
+  detach?: boolean;
+  maxMs?: number;
+  /** Expected run time; when `maxMs` is not given it defaults to {@link RUN_EXPECTED_MAX_FACTOR}x this. */
+  expectedMs?: number;
+  /** Poll an existing run instead of starting a new one. */
+  runId?: string;
+  /** With `runId`: read the run's own log artifact instead of polling it (#3114). */
+  log?: RunLogQuery;
+  /** With `runId`: stop the caller's own queued or running run (#3630). */
+  cancel?: boolean;
+}
+
+export interface RunCommandResult {
+  runId: string;
+  /** `exited` — the command reached a terminal state and this result is final (check `exitCode` /
+   * `timedOut`). `queued` — waiting for admission, no process has started. `still-running` — the command is alive under the orchestrator; END THE
+   * TURN and be woken. `superseded` (#2461) — this gate run was
+   * KILL-REPLACED by a newer gate run you started yourself; it produced NO verdict, so it is
+   * neither a pass nor a failure and must not be reported as one. `canceled` (#3630) — its owner stopped it. */
+  status: "exited" | "queued" | "still-running" | "superseded" | "canceled";
+  command: string;
+  cwd: string;
+  /** null while running, and null when the run was killed at `maxMs` (no code was ever produced). */
+  exitCode: number | null;
+  signal: string | null;
+  /** True when the orchestrator killed the run at `maxMs` rather than it exiting on its own. */
+  timedOut: boolean;
+  stdoutTail: string;
+  stderrTail: string;
+  /** True when either stream was clipped for reporting. The complete output is in `logArtifactId`. */
+  truncated: boolean;
+  /** True when the run was KILLED for flooding — it blew the orchestrator's output byte cap instead
+   * of finishing. Reported separately from `truncated` because a cap-kill and a clean exit both land
+   * with a null exit code; without this the only way to tell them apart is reading `stderrTail`. */
+  outputLimitExceeded: boolean;
+  /** How long the COMMAND has run (final when `exited`, elapsed-so-far when `still-running`). */
+  durationMs: number;
+  queueDurationMs?: number;
+  runDurationMs?: number;
+  /** Queue creation and host admission are separate clocks; no admission means zero run duration. */
+  queuedAt: number;
+  admittedAt?: number;
+  runDeadlineAt?: number;
+  /** How long THIS CALL blocked. Bounded by the requested wait, or the short detached window when it is zero. */
+  waitedMs: number;
+  logArtifactId?: string;
+  /** Present on `still-running`: what happens next, so the contract needs no out-of-band prose. */
+  note?: string;
+  /** Present when the call passed `log` (#3114). */
+  log?: RunLogPage;
+  /** Present on `queued`: the run's place in its host lane and what holds the lane (#3630). */
+  queue?: RunQueueView;
+}
+
+/**
+ * THE entry point. Starts a bounded run (or polls one) and returns its state INLINE.
+ *
+ * The run is created through the ordinary command-timer path (`scheduleTimerAsync`), which is what
+ * buys: authorization, cwd/orchestrator resolution, the dispatched `scheduler.command`, the
+ * agent-busy overlay while it runs, the log artifact, and the completion wake. The ONE thing this
+ * module adds on top is the inline WAIT, plus the two-step handoff that keeps the wake honest:
+ * `inlineUntil` (a lease saying a caller is EXPECTED to report this result, so DEFER the wake) and
+ * `consumedAt` (the fact that one DID, so retire the run). A run consumed inline therefore never
+ * also wakes its agent into a redundant turn — and a caller that dies mid-wait gets a late wake
+ * rather than a silently stranded run.
+ */
+export async function runCommandInline(input: RunCommandInput, ctx: AuthContext, signal?: AbortSignal): Promise<RunCommandResult> {
+  if (input.detach && input.waitMs !== undefined && input.waitMs !== 0) throw new ValidationError("detach cannot be combined with a non-zero waitMs");
+  const waitMs = input.detach ? 0 : Math.min(cleanBoundedMs(input.waitMs, "waitMs", RUN_DEFAULT_WAIT_MS, Number.MAX_SAFE_INTEGER, 0), RUN_MAX_WAIT_MS);
+  if ((input.log || input.cancel) && !input.runId) throw new ValidationError(`${input.cancel ? "cancel" : "log"} requires a runId`);
+  if (input.cancel && (input.log || input.command)) throw new ValidationError("cancel takes only a runId");
+  if (input.cancel) return cancelRun(input.runId!.trim(), ctx, signal);
+  if (input.log) return readOwnRunLog(input.runId!.trim(), input.log, ctx);
+  return input.runId ? pollRun(input.runId.trim(), ctx, signal) : startRun(input, waitMs, ctx, signal);
+}
+
+async function startRun(input: RunCommandInput, waitMs: number, ctx: AuthContext, signal?: AbortSignal): Promise<RunCommandResult> {
+  if (typeof input.command !== "string" || !input.command.trim()) {
+    throw new ValidationError("command is required (or pass runId to poll an existing run)");
+  }
+  const expectedMs = input.expectedMs === undefined ? undefined : cleanBoundedMs(input.expectedMs, "expectedMs", 0, RUN_MAX_MAX_MS);
+  const maxMsGiven = input.maxMs !== undefined && input.maxMs !== null;
+  const maxMs = maxMsGiven
+    ? cleanBoundedMs(input.maxMs, "maxMs", RUN_DEFAULT_MAX_MS, RUN_MAX_MAX_MS)
+    : expectedMs ? Math.min(expectedMs * RUN_EXPECTED_MAX_FACTOR, RUN_MAX_MAX_MS) : RUN_DEFAULT_MAX_MS;
+  if (maxMsGiven && maxMs < waitMs) throw new ValidationError("maxMs must be at least waitMs");
+  // Detached callers still get a short inline window so fast completions need no wake (#3305).
+  const callWaitMs = Math.min(waitMs || RUN_MAX_WAIT_MS, maxMs);
+  const startedAt = Date.now();
+  const timer = await scheduleTimerAsync({
+    target: "me",
+    // `maxMs` is the RUN budget: the orchestrator restarts its SIGKILL clock at admission (#2776). The
+    // timer's own deadline is the queue ceiling; the admission report re-arms it to the run end.
+    afterMs: RUN_QUEUE_BUDGET_MS,
+    runTimeoutMs: maxMs,
+    command: input.command,
+    cwd: input.cwd,
+    env: input.env,
+    subject: "relay_run",
+    body: "relay_run command completion",
+    via: "relay_run",
+    inlineUntil: startedAt + callWaitMs,
+  }, ctx);
+  return awaitRun(timer, startedAt + callWaitMs, signal, startedAt + RUN_MAX_WAIT_MS + RUN_RESPONSE_SLACK_MS);
+}
+
+/** Status read of a run this caller started (#2776). It never blocks and never re-arms the inline
+ * lease: a blocking poll is the polling loop the tool exists to prevent, and re-arming deferred the wake
+ * into a redundant extra message. Reads inside {@link RUN_POLL_MIN_INTERVAL_MS} are refused. */
+async function pollRun(runId: string, ctx: AuthContext, signal?: AbortSignal): Promise<RunCommandResult> {
+  if (!runId) throw new ValidationError("runId must not be empty");
+  const timer = await getOwnedRun(runId, ctx);
+  const now = Date.now();
+  if (isTerminalRunStatus(timer.command.status)) return await markConsumed(settledResult(timer, now, 0), signal);
+  const lastContact = timer.command.lastPolledAt;
+  const commandRow = lastContact !== undefined && timer.command.commandId ? await getCommandAsync(timer.command.commandId) : null;
+  if (lastContact !== undefined && now - lastContact < RUN_POLL_MIN_INTERVAL_MS && !(commandRow && isTerminalCommandStatus(commandRow.status))) {
+    const secs = Math.round((now - lastContact) / 1_000);
+    return await pendingResult(timer, now, 0, `poll refused: last contact was ${secs}s ago (minimum ${RUN_POLL_MIN_INTERVAL_MS / 1_000}s). `, false);
+  }
+  return awaitRun(timer, now, signal, now + RUN_MAX_WAIT_MS);
+}
+
+/** Ownership, not admin reach: a run is the creating agent's own shell, and nothing about this tool
+ * needs to widen who may read another agent's command output or log. */
+async function getOwnedRun(runId: string, ctx: AuthContext): Promise<ScheduledTimer & { command: ScheduledTimerCommand }> {
+  const timer = await getTimerAsync(runId);
+  if (!timer?.command) throw new ValidationError(`run ${runId} not found`);
+  if (timer.createdBy !== (ctx.callerAgentId ?? ctx.actor.id)) {
+    throw new ServiceAuthError("cannot read or cancel a run started by another caller");
+  }
+  return timer as ScheduledTimer & { command: ScheduledTimerCommand };
+}
+
+/** #3630 — stop the caller's own run: a queued run leaves its queue at once, a running one has its
+ * process group killed by its host. Waits at most {@link RUN_MAX_WAIT_MS} for the host's witness; past
+ * that the run settles as canceled on its own and Relay wakes the caller once. */
+async function cancelRun(runId: string, ctx: AuthContext, signal?: AbortSignal): Promise<RunCommandResult> {
+  if (!runId) throw new ValidationError("runId must not be empty");
+  const owned = await getOwnedRun(runId, ctx);
+  const began = Date.now();
+  if (isTerminalRunStatus(owned.command.status)) return await markConsumed(settledResult(owned, began, 0), signal);
+  const deadline = began + RUN_MAX_WAIT_MS;
+  // This call reports the outcome, so hold the wake while it waits (the same lease a start takes).
+  await updateTimerCommandAsync(runId, { inlineUntil: deadline });
+  const reason = "relay_run canceled by its owner";
+  let timer: ScheduledTimer = owned;
+  for (;;) {
+    const done = await cancelScheduledCommand(runId, { reason });
+    timer = await getTimerAsync(runId) ?? timer;
+    // Settled is enough: cancel's own success also wants the log artifact, which must not cost a retry loop here.
+    if (done || isTerminalRunStatus(timer.command?.status) || Date.now() >= deadline || signal?.aborted) break;
+    await sleep(250);
+  }
+  const now = Date.now();
+  if (isTerminalRunStatus(timer.command?.status)) return await markConsumed(settledResult(timer, now, now - began), signal, deadline + RUN_RESPONSE_SLACK_MS);
+  return await pendingResult(timer, now, now - began, "cancel requested; the host is stopping the run. ", false);
+}
+
+/** #3114 — read a page of the caller's own run log. Not a poll: no contact stamp, no rate guard, and
+ * the run is never retired (`markConsumed`), so the completion wake is unaffected. The artifact id
+ * is taken from the owned run row, never from the caller. */
+async function readOwnRunLog(runId: string, query: RunLogQuery, ctx: AuthContext): Promise<RunCommandResult> {
+  if (!runId) throw new ValidationError("runId must not be empty");
+  const timer = await getOwnedRun(runId, ctx);
+  const now = Date.now();
+  const base = isTerminalRunStatus(timer.command!.status) ? settledResult(timer, now, 0) : await pendingResult(timer, now, 0, "", false);
+  const artifactId = timer.command!.logArtifactId ?? base.logArtifactId;
+  const page = artifactId ? await readRunLog(artifactId, query) : undefined;
+  if (page) return { ...base, log: page };
+  return { ...base, note: `run ${runId} has no readable log artifact yet${base.status === "exited" ? "" : " (it has not finished)"}.` };
+}
+
+/**
+ * Block until the run settles or the inline lease lapses, then report.
+ *
+ * WHY THE LOOP READS THE RUN ROW AND NOT THE COMMAND ROW. The run row's terminal status is written
+ * by the settle path AFTER it has stored the log artifact, so keying on it makes the returned result
+ * complete by construction. The command row is consulted only as the liveness fallback: if it goes
+ * terminal but the settle patch does not follow within {@link SETTLE_GRACE_MS}, we report from the
+ * command row anyway rather than blocking on a settle that may never come.
+ *
+ * WHY IT KEEPS POLLING TO THE DEADLINE AND THEN READS ONCE MORE. Wake suppression is decided at
+ * settle time by `now < inlineUntil`. Any settle strictly before the lease lapses is withheld — and
+ * this final read (taken at/after the lapse) is guaranteed to observe it, so a withheld completion
+ * is never lost. Note the lease only DEFERS the wake; nothing this loop does can drop one. Only
+ * `markConsumed` below — reached exactly when the result is on its way back to the agent — retires a
+ * run for good. See {@link import("./scheduler-command").inlineWakeSuppressed}.
+ */
+async function awaitRun(timer: ScheduledTimer, deadline: number, signal?: AbortSignal, responseDeadline = Infinity): Promise<RunCommandResult> {
+  const callBegan = Date.now();
+  let current = timer;
+  let commandTerminalAt: number | undefined;
+  for (;;) {
+    const now = Date.now();
+    if (isTerminalRunStatus(current.command?.status)) return await markConsumed(settledResult(current, now, now - callBegan), signal, responseDeadline);
+    const command = current.command?.commandId ? await getCommandAsync(current.command.commandId) : null;
+    if (command && isTerminalCommandStatus(command.status)) {
+      commandTerminalAt ??= now;
+      if (now - commandTerminalAt >= SETTLE_GRACE_MS) return await markConsumed(commandRowResult(current, command, now - callBegan), signal, responseDeadline);
+    }
+    if (now >= deadline || signal?.aborted) break;
+    await sleep(Math.min(pollIntervalMs(now - callBegan), Math.max(1, deadline - now)));
+    current = await getTimerAsync(timer.id) ?? current;
+  }
+  // Past the lease: one last read decides. From here on any settle wakes normally, so the worst case
+  // is a redundant wake — never a lost one.
+  const settled = await getTimerAsync(timer.id) ?? current;
+  const now = Date.now();
+  if (isTerminalRunStatus(settled.command?.status)) return await markConsumed(settledResult(settled, now, now - callBegan), signal, responseDeadline);
+  const command = settled.command?.commandId ? await getCommandAsync(settled.command.commandId) : null;
+  if (command && isTerminalCommandStatus(command.status)) return await markConsumed(commandRowResult(settled, command, now - callBegan), signal, responseDeadline);
+  return pendingResult(settled, now, now - callBegan, "", true);
+}
+
+/** Non-final report (queued / still-running). Stamps the contact time that feeds the poll-rate guard,
+ * except when the caller is being refused, which must not extend its own window. */
+async function pendingResult(timer: ScheduledTimer, now: number, waitedMs: number, prefix: string, stamp: boolean): Promise<RunCommandResult> {
+  const command = timer.command?.commandId ? await getCommandAsync(timer.command.commandId) : null;
+  const queued = !command || command.status === "pending" || command.status === "accepted" || command.result?.phase === "queued";
+  const result = command?.result ?? {};
+  const admittedAt = typeof result.admittedAt === "number" ? result.admittedAt : timer.command?.admittedAt;
+  const stdoutTail = stringOr(result.stdoutTail, timer.command?.stdoutTail ?? "");
+  const stderrTail = stringOr(result.stderrTail, timer.command?.stderrTail ?? "");
+  const queueReason = queued ? stringOr(command?.result?.queueReason, "orchestrator admission") : undefined;
+  const queue = queued && command ? await readRunQueue(command, now).catch((error) => { console.warn(`relay_run queue view for ${timer.id}: ${errMessage(error)}`); return undefined; }) : undefined;
+  if (stamp) {
+    try { await updateTimerCommandAsync(timer.id, { lastPolledAt: now }); }
+    catch (error) { console.warn(`relay_run could not stamp contact for run ${timer.id}: ${errMessage(error)}`); }
+  }
+  return {
+    ...baseResult(timer, waitedMs),
+    status: queued ? "queued" : "still-running",
+    exitCode: null,
+    signal: null,
+    timedOut: false,
+    admittedAt,
+    runDeadlineAt: typeof result.runDeadlineAt === "number" ? result.runDeadlineAt : timer.command?.runDeadlineAt,
+    durationMs: queued || admittedAt === undefined ? 0 : Math.max(0, now - admittedAt),
+    stdoutTail: clip(stdoutTail),
+    stderrTail: clip(stderrTail),
+    truncated: result.outputTruncated === true || clipped(stdoutTail) || clipped(stderrTail),
+    ...(queue ? { queue } : {}),
+    note: prefix + (queued
+      ? `queued for ${queueReason}${queue ? ` (${describeRunQueue(queue)})` : ""}; the command has not started. Queue time does not count toward maxMs. Cancel: relay_run {runId, cancel:true}. `
+      : `still running; it keeps running under the orchestrator and is killed at runDeadlineAt (maxMs of execution excluding gate waits; overall ceiling at runDeadlineAt; at most ${HEAVY_GATE_RUN_CAP_MS / 60_000} min while it holds the heavy-gate slot). `)
+      + RUN_END_TURN_ORDER,
+  };
+}
+
+/**
+ * Record that this run's terminal result is going back to the agent NOW, and retire the run.
+ *
+ * This is the fact that closes the loop, and it is deliberately the ONLY thing that ends a run for
+ * good. Suppression on the `inlineUntil` lease alone is a PREDICTION that a caller will report; if
+ * that caller dies mid-wait — a relay restart during a self-upgrade, a killed turn — the prediction
+ * is wrong and the settle path has already deferred the wake rather than dropping it. Stamping
+ * `consumedAt` converts prediction into fact, which is what makes retiring safe.
+ *
+ * Ordering matters in both directions and is why this runs BEFORE the result is returned:
+ *   • settle already happened — the run row is terminal, so retire it and no deferred wake fires;
+ *   • settle has NOT happened yet (the {@link SETTLE_GRACE_MS} fallback) — leave the timer alone so
+ *     the settle can still write the terminal patch and the log artifact, and let it see
+ *     `consumedAt` and retire itself. Disabling here would abort that write-back and leave a later
+ *     `relay_run{runId}` poll looking at a run that never finished.
+ *
+ * Best-effort: a bookkeeping failure must never turn a completed run into a failed tool call. The
+ * cost of losing it is at worst one redundant wake, never a lost result.
+ */
+async function markConsumed(result: RunCommandResult, signal?: AbortSignal, responseDeadline = Infinity): Promise<RunCommandResult> {
+  // A canceled or overdue call cannot attest delivery; keep its durable completion wake armed (#2790).
+  if (signal?.aborted || Date.now() >= responseDeadline) return result;
+  try {
+    const updated = await updateTimerCommandAsync(result.runId, { consumedAt: Date.now() });
+    if (updated?.command && isTerminalRunStatus(updated.command.status)) {
+      await disableTimerAsync(result.runId, "completion consumed inline by relay_run");
+    }
+  } catch (error) {
+    console.warn(`relay_run could not retire run ${result.runId}: ${errMessage(error)}`);
+  }
+  return result;
+}
+
+/** Report a run whose terminal state has been written back onto the run row (the complete case:
+ * exit code, tails and log artifact are all present). */
+function settledResult(timer: ScheduledTimer, now: number, waitedMs: number): RunCommandResult {
+  const command = timer.command!;
+  const superseded = isSupersededRunResult(command.exitCode, command.stderrTail ?? "");
+  return {
+    ...baseResult(timer, waitedMs),
+    status: command.status === "canceled" ? "canceled" : superseded ? "superseded" : "exited",
+    ...(superseded ? { note: GATE_RUN_SUPERSEDED_NOTICE } : {}),
+    exitCode: command.exitCode ?? null,
+    signal: command.signal ?? null,
+    timedOut: command.status === "timed_out",
+    stdoutTail: clip(command.stdoutTail ?? ""),
+    stderrTail: clip(command.stderrTail ?? ""),
+    truncated: wasTruncated(command),
+    outputLimitExceeded: command.outputLimitExceeded === true,
+    queueDurationMs: command.queueDurationMs,
+    runDurationMs: command.runDurationMs,
+    durationMs: command.durationMs ?? (command.admittedAt === undefined ? 0 : Math.max(0, (command.finishedAt ?? now) - command.admittedAt)),
+    ...(command.logArtifactId ? { logArtifactId: command.logArtifactId } : {}),
+  };
+}
+
+/** Fallback report built from the COMMAND row when the settle write-back did not land in time. Same
+ * data, minus the log artifact the settle path would have attached. */
+function commandRowResult(timer: ScheduledTimer, command: Command, waitedMs: number): RunCommandResult {
+  const result = (command.result ?? {}) as Record<string, unknown>;
+  const stdoutTail = stringOr(result.stdoutTail ?? result.stdout, "");
+  const stderrTail = stringOr(result.stderrTail ?? result.stderr, command.error ?? "");
+  const exitCode = typeof result.exitCode === "number" ? result.exitCode : null;
+  const superseded = isSupersededRunResult(exitCode, stderrTail);
+  return {
+    ...baseResult(timer, waitedMs),
+    status: command.status === "canceled" ? "canceled" : superseded ? "superseded" : "exited",
+    ...(superseded ? { note: GATE_RUN_SUPERSEDED_NOTICE } : {}),
+    exitCode,
+    signal: typeof result.signal === "string" ? result.signal : null,
+    timedOut: result.timedOut === true || command.status === "timed_out",
+    stdoutTail: clip(stdoutTail),
+    stderrTail: clip(stderrTail),
+    truncated: result.outputTruncated === true || clipped(stdoutTail) || clipped(stderrTail),
+    outputLimitExceeded: result.outputLimitExceeded === true,
+    queueDurationMs: typeof result.queueDurationMs === "number" ? result.queueDurationMs : undefined,
+    runDurationMs: typeof result.runDurationMs === "number" ? result.runDurationMs : undefined,
+    durationMs: typeof result.durationMs === "number" ? result.durationMs : 0,
+    admittedAt: typeof result.admittedAt === "number" ? result.admittedAt : undefined,
+    runDeadlineAt: typeof result.runDeadlineAt === "number" ? result.runDeadlineAt : undefined,
+    ...(stringOr(result.logArtifactId, "") ? { logArtifactId: String(result.logArtifactId) } : {}),
+  };
+}
+
+function baseResult(timer: ScheduledTimer, waitedMs: number): RunCommandResult {
+  const command = timer.command!;
+  return {
+    runId: timer.id,
+    queuedAt: timer.createdAt,
+    admittedAt: command.admittedAt,
+    runDeadlineAt: command.runDeadlineAt,
+    status: "still-running",
+    command: maskCommandCredentials(command.command),
+    cwd: command.cwd,
+    exitCode: null,
+    signal: null,
+    timedOut: false,
+    stdoutTail: clip(command.stdoutTail ?? ""),
+    stderrTail: clip(command.stderrTail ?? ""),
+    truncated: wasTruncated(command),
+    outputLimitExceeded: command.outputLimitExceeded === true,
+    queueDurationMs: command.queueDurationMs,
+    runDurationMs: command.runDurationMs,
+    durationMs: command.durationMs ?? 0,
+    waitedMs,
+  };
+}
+
+function isTerminalRunStatus(status: ScheduledTimerCommand["status"] | undefined): boolean {
+  return status === "succeeded" || status === "failed" || status === "timed_out" || status === "canceled";
+}
+
+function isTerminalCommandStatus(status: Command["status"]): boolean {
+  return status === "succeeded" || status === "failed" || status === "timed_out" || status === "canceled" || status === "rejected";
+}
+
+/** Did this run lose output? READ, never re-derived from tail length: the settle path clips on the
+ * way in, so by the time we see `stdoutTail` a clipped stream and a whole one are the same string.
+ * `outputTruncated` is recorded at the point the clip (or the orchestrator's byte-cap abort) actually
+ * happened. `clipped()` below then covers only the belt-and-braces case where a tail somehow reaches
+ * us longer than our own cap. */
+function wasTruncated(command: ScheduledTimerCommand): boolean {
+  return command.outputTruncated === true || clipped(command.stdoutTail ?? "") || clipped(command.stderrTail ?? "");
+}
+
+function clipped(value: string): boolean {
+  return value.length > RUN_OUTPUT_TAIL_CHARS;
+}
+
+function clip(value: string): string {
+  if (!clipped(value)) return value;
+  return RUN_TRUNCATION_MARKER + value.slice(value.length - RUN_OUTPUT_TAIL_CHARS);
+}
+
+function stringOr(value: unknown, fallback: string): string {
+  return typeof value === "string" ? value : fallback;
+}
+
+function cleanBoundedMs(value: unknown, field: string, fallback: number, cap: number, min = 1): number {
+  if (value === undefined || value === null) return fallback;
+  if (typeof value !== "number" || !Number.isSafeInteger(value) || value < min) {
+    throw new ValidationError(`${field} must be ${min === 0 ? "a non-negative" : "a positive"} integer number of milliseconds`);
+  }
+  if (value > cap) throw new ValidationError(`${field} cannot exceed ${cap}`);
+  return value;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/** Human-readable one-liner for logs/messages. Exported so callers do not re-derive the phrasing. */
+export function describeRun(result: RunCommandResult): string {
+  if (result.status === "queued") return `run ${result.runId} queued: ${result.note}`;
+  if (result.status === "still-running") return `run ${result.runId} still running for ${result.durationMs}ms since admission`;
+  if (result.status === "canceled") return `run ${result.runId} canceled${result.admittedAt === undefined ? " before it started" : ` after ${result.durationMs}ms`}`;
+  const outcome = result.timedOut ? "killed at its run deadline" : `exit ${result.exitCode ?? "?"}`;
+  return `run ${result.runId} ${outcome} in ${result.durationMs}ms`;
+}

package/src/services/run-completion-message.ts

@@ -0,0 +1,70 @@
+// #3114 — the `relay_run` completion wake body. The Claude monitor delivers a message whole only under
+// ~2800 chars total / 480 per line, otherwise the agent pays an extra get-message. So the body carries
+// the verdict (exit code, timedOut, duration), the last test-runner summary lines, and how to read
+// more; the full tails stay in the payload and the full log in the artifact.
+import type { ScheduledTimerCommand } from "../db";
+
+const COMMAND_MAX_CHARS = 120;
+const SUMMARY_LINE_MAX_CHARS = 160;
+const SUMMARY_MAX_LINES = 8;
+/** Only the end of each stream is scanned: a summary is printed last. */
+const SUMMARY_SCAN_LINES = 80;
+
+// Lines test runners print at the end (bun, jest/vitest, pytest, go, cargo, tsc, mocha).
+const SUMMARY_LINE = new RegExp([
+  /^\s*\d+\s+(pass|fail|skip|todo|expect\(\))/,
+  /^\s*Ran \d+ tests?/,
+  /^\s*(Tests|Test Suites|Test Files|Snapshots|Duration|Time):/,
+  /^\s*=+ .*\b(passed|failed|error|errors)\b.* =+\s*$/,
+  /^\s*test result:/,
+  /^\s*(ok|FAIL)\s+\S+\s+[\d.]+s/,
+  /^\s*(FAIL|PASS)\s+\S+\.(test|spec)\./,
+  /^\s*---\s*FAIL:/,
+  /^\s*Found \d+ errors?/,
+  /^\s*\d+\s+(passing|failing|pending)\b/,
+  /^\s*\(fail\)/,
+  /^\s*error: (script|command) .* exited with code/,
+].map((re) => re.source).join("|"));
+
+function clipLine(line: string, max = SUMMARY_LINE_MAX_CHARS): string {
+  const collapsed = line.replace(/\s+/g, " ").trim();
+  return collapsed.length > max ? `${collapsed.slice(0, max - 1)}…` : collapsed;
+}
+
+function endLines(stream: string): string[] {
+  return stream.split("\n").slice(-SUMMARY_SCAN_LINES).map((line) => line.replace(/\x1b\[[0-9;]*m/g, "")).filter((line) => line.trim());
+}
+
+/** The last test-runner summary lines when the output has any, else the last few non-empty lines
+ * (stderr first: that is where errors are). Raw tails are often docker/compose substrate noise, so
+ * recognised summary lines win over them. */
+export function runSummaryLines(stdoutTail: string, stderrTail: string): string[] {
+  const stderr = endLines(stderrTail);
+  const stdout = endLines(stdoutTail);
+  const summary = [...stdout, ...stderr].filter((line) => SUMMARY_LINE.test(line));
+  const picked = summary.length ? summary : (stderr.length ? stderr : stdout);
+  return [...new Set(picked.map((line) => clipLine(line)))].slice(-SUMMARY_MAX_LINES);
+}
+
+function formatDuration(ms: number): string {
+  return ms < 1_000 ? `${ms}ms` : `${(ms / 1_000).toFixed(1)}s`;
+}
+
+export function relayRunCompletionBody(runId: string, command: ScheduledTimerCommand): string {
+  const label = command.status === "succeeded" ? "succeeded" : command.status === "timed_out" ? "timed out" : command.status === "canceled" ? "canceled" : "failed";
+  const timedOut = command.status === "timed_out";
+  const exit = command.exitCode === undefined || command.exitCode === null ? "none" : String(command.exitCode);
+  const duration = command.durationMs === undefined ? "?" : formatDuration(command.durationMs);
+  const summary = runSummaryLines(command.stdoutTail ?? "", command.stderrTail ?? "");
+  const lines = [
+    `Background command ${label}: exit ${exit}, in ${duration} ${clipLine(command.command, COMMAND_MAX_CHARS)}`,
+    `runId: ${runId} | exitCode: ${exit} | timedOut: ${timedOut} | duration: ${duration}`
+      + (command.outputLimitExceeded ? " | killed: output limit exceeded" : command.outputTruncated ? " | output truncated" : ""),
+    ...(timedOut ? ["FINAL: killed at the execution or overall deadline. The command has stopped; a null exitCode is expected after a kill. This run did not pass."] : []),
+    ...(command.runDurationMs !== undefined ? [`Timing: run ${formatDuration(command.runDurationMs)} | queue ${formatDuration(command.queueDurationMs ?? 0)} (gate waits excluded from run)`] : []),
+    ...(summary.length ? [timedOut ? "Partial output before kill (not a passing result):" : "Summary:", ...summary.map((line) => `  ${line}`)] : []),
+    `Logs for diagnosis only (the final result is above): relay_run {runId:"${runId}", log:{tail:100}} (or log:{grep:"fail"}, log:{offset:N})`
+      + (command.logArtifactId ? `; full log artifact ${command.logArtifactId}` : ""),
+  ];
+  return lines.join("\n");
+}