@cat-factory/executor-harness 1.96.0 → 1.98.0

package/src/runner.ts

@@ -6,11 +6,13 @@ import type { SliceReview } from './subagents.js'
 import type { ObservedMcpServer } from './agent-capabilities.js'
 import type { HarnessCallMetric, TodoProgress, ToolSpan } from './pi.js'
 import { log, type Logger } from './logger.js'
+import { ToolSilenceWatchdog, type ToolProgressWindow } from './tool-silence.js'
 import {
   type FailureCause,
   failureCauseOf,
   inactivityAbortMessage,
   maxDurationAbortMessage,
+  toolSilenceAbortMessage,
 } from './failure.js'
 
 /** Non-secret correlation fields a job carries on every log line (jobId, repo, branch, …). */
@@ -32,6 +34,22 @@ export interface RunOptions {
   onProgress?: (progress: TodoProgress) => void
   /** Receives one compact {@link ToolSpan} per completed tool call (observability). */
   onSpan?: (span: ToolSpan) => void
+  /**
+   * Opens the tool-silence window (stuck-run audit F13) for ONE agent stream, returning the
+   * handle that stream beats on every completed tool call and closes when it ends.
+   *
+   * Called by the agent-CLI runners themselves rather than by the phase marker, because the
+   * window is only meaningful while something able to RESET it is running and only the runner
+   * knows whether its CLI reports completed tool calls at all. A caller that runs no tool loop
+   * (the inline one-shot completion) simply does not forward this, which is a statement, not an
+   * omission: the run stays bounded by the inactivity and max-duration watchdogs, and a window
+   * nothing could ever beat would only be able to expire.
+   *
+   * Absent ⇒ no window is opened and this watchdog is silent for that work. It fails toward NOT
+   * killing on purpose: this audit exists as much to stop recovery machinery ending healthy runs
+   * as to bound wedged ones, and the wall-clock cap is underneath either way.
+   */
+  beginToolWindow?: () => ToolProgressWindow
   /** Receives the forward-looking follow-up / question items the Coder streamed since the last poll. */
   onFollowUp?: (items: FollowUpLine[]) => void
   /**
@@ -281,6 +299,20 @@ export interface RunnerLimits {
    * progress, which counts as activity). Set to 0 to disable.
    */
   coldStartMs: number
+  /**
+   * Stuck-run audit F13: force-fail the job if a running agent stream completes no tool call for
+   * this long. The gap the other two watchdogs structurally cannot see — a model that keeps
+   * talking (or thinking out loud) resets the inactivity timer on every chunk while completing
+   * nothing, so the only remaining bound was the full wall-clock cap and the engine's
+   * ~70-minute poll budget behind it.
+   *
+   * Armed only for as long as a stream that REPORTS completed tool calls is running (see
+   * {@link RunOptions.beginToolWindow}), so the activity-silent stretches — clone, dependency
+   * install, push, a validation loop's check commands — are outside it by construction: they
+   * legitimately complete no tool calls, and they are bounded by their own per-command timeouts.
+   * Set to 0 to disable.
+   */
+  toolSilenceMs: number
 }
 
 function intEnv(value: string | undefined, fallback: number): number {
@@ -296,23 +328,48 @@ function intEnvAllowZero(value: string | undefined, fallback: number): number {
 }
 
 export function loadRunnerLimits(env: NodeJS.ProcessEnv = process.env): RunnerLimits {
+  const maxDurationMs = intEnv(env.JOB_MAX_DURATION_MS, 60 * 60_000)
+  const inactivityMs = intEnv(env.JOB_INACTIVITY_MS, 10 * 60_000)
   return {
     // 60 minutes: generous headroom for serious multi-file coding tasks while
     // still bounding a runaway container.
-    maxDurationMs: intEnv(env.JOB_MAX_DURATION_MS, 60 * 60_000),
+    maxDurationMs,
     // 10 minutes of zero output is treated as hung (a single long LLM/tool call
     // is far shorter; Pi streams events as it works). The per-git command ceiling
     // (`GIT_TIMEOUT_MS` in git.ts) is DERIVED from this value — a fixed margin below
     // it — so a slow clone/push (which emits no activity events) always times out
     // with git's own clear reason rather than this watchdog's "likely hung" message,
     // for any configured window. See the invariant note in git.ts.
-    inactivityMs: intEnv(env.JOB_INACTIVITY_MS, 10 * 60_000),
+    inactivityMs,
     // 2 minutes: comfortably longer than a warm agent's time-to-first-token yet far
     // under the 10-minute inactivity kill, so a truly output-less start is flagged early.
     coldStartMs: intEnvAllowZero(env.JOB_COLD_START_MS, 2 * 60_000),
+    toolSilenceMs: intEnvAllowZero(
+      env.JOB_TOOL_SILENCE_MS,
+      toolSilenceDefault(maxDurationMs, inactivityMs),
+    ),
   }
 }
 
+/**
+ * The tool-silence window when the operator has not set one: HALF the configured wall-clock cap,
+ * DERIVED rather than a constant so lowering `JOB_MAX_DURATION_MS` tightens it too (a fixed
+ * 30 minutes would sit past the whole budget of a deployment that runs 20-minute jobs, i.e. be
+ * silently disabled).
+ *
+ * Floored at the inactivity window so the default never races the gone-quiet diagnostic more
+ * often than it has to. The floor is a sizing choice, NOT the thing that keeps the two watchdogs
+ * apart: they anchor on different events (the last completed tool call vs the last byte of
+ * output), so the tool-silence anchor is always the earlier of the two and equal windows would
+ * still have it firing first. What actually keeps this watchdog off a hang is the expiry test in
+ * {@link ToolSilenceWatchdog}, which fires only when output arrived DURING the window that
+ * elapsed — and which also holds for an operator who sets `JOB_TOOL_SILENCE_MS` below
+ * `JOB_INACTIVITY_MS`, where no default-side clamp applies at all.
+ */
+function toolSilenceDefault(maxDurationMs: number, inactivityMs: number): number {
+  return Math.max(Math.round(maxDurationMs / 2), inactivityMs)
+}
+
 function toView<TResult extends JobResultBase>(entry: JobEntry<TResult>): JobView<TResult> {
   const {
     promise: _promise,
@@ -475,7 +532,7 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
 
   private async drive(entry: JobEntry<TResult>, job: TJob): Promise<void> {
     const controller = new AbortController()
-    let killReason: 'inactivity' | 'max-duration' | undefined
+    let killReason: 'inactivity' | 'max-duration' | 'no-tool-progress' | undefined
 
     const jobLog = log.child({ jobId: entry.id, ...this.describe(job) })
 
@@ -519,6 +576,21 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
     // that never got going at all.
     let lastActivityAt: number | undefined
 
+    // Stuck-run audit F13: the third watchdog, and the only one that can see a model which keeps
+    // TALKING while completing nothing — its output resets the inactivity timer on every chunk,
+    // and it is nowhere near the wall-clock cap. It is armed by the agent stream itself rather
+    // than by the phase marker (see `RunOptions.beginToolWindow`) and reads `lastActivityAt` at
+    // expiry, which is what keeps the gone-quiet case with the inactivity watchdog that owns it.
+    const toolSilence = new ToolSilenceWatchdog({
+      windowMs: this.limits.toolSilenceMs,
+      lastActivityAt: () => lastActivityAt,
+      onExpired: () => {
+        // First watchdog to fire wins the reason (see `resetInactivity` above).
+        killReason ??= 'no-tool-progress'
+        controller.abort(new Error('no tool progress'))
+      },
+    })
+
     // ADR 0026 D4: a one-shot cold-start watchdog. If the job produces no activity within
     // `coldStartMs`, record a structured diagnostic (a likely onboarding/auth wedge) so it
     // is legible early — it does NOT abort the run (the inactivity watchdog still owns
@@ -556,6 +628,7 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
           entry.spanBuffer.push(span)
           lastTool = { name: span.tool, at: span.endedAt }
         },
+        beginToolWindow: () => toolSilence.open(),
         onFollowUp: (items) => {
           entry.followUpBuffer.push(...items)
         },
@@ -630,6 +703,7 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
       clearTimeout(inactivity)
       clearTimeout(cap)
       clearTimeout(coldStart)
+      toolSilence.stop()
       entry.abort = undefined
       entry.heartbeatAt = Date.now()
     }
@@ -679,6 +753,19 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
         ),
       }
     }
+    if (ctx.killReason === 'no-tool-progress') {
+      return {
+        // The breadcrumb carries the last completed tool, which is the whole diagnostic here:
+        // it names what the agent was doing when it stopped doing anything.
+        message: redactSecrets(
+          `${toolSilenceAbortMessage(this.limits.toolSilenceMs)} (${breadcrumb})`,
+        ),
+        cause: 'no-tool-progress',
+        detail: redactSecrets(
+          `Phase timings: ${phaseBreakdown || '(none)'}. ${breadcrumb}.${cold}`,
+        ),
+      }
+    }
     const raw = ctx.error instanceof Error ? ctx.error.message : String(ctx.error)
     // A thrown error tagged with a structured cause (a git op / an upstream API call) keeps
     // it; an untagged throw is a generic agent failure.
@@ -699,7 +786,7 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
  */
 interface FailureContext {
   /** Which watchdog killed it; unset when the run threw on its own. */
-  killReason: 'inactivity' | 'max-duration' | undefined
+  killReason: 'inactivity' | 'max-duration' | 'no-tool-progress' | undefined
   error: unknown
   /** The phase the job was IN when it failed (captured before the `failed` transition). */
   phase: string

package/src/structured-output.ts

@@ -1,6 +1,7 @@
 import { redact, redactSecrets, secretsToRedact } from './redact.js'
 import { log } from './logger.js'
-import { PI_MAX_OUTPUT_TOKENS, phasedProxyBaseUrl } from './pi.js'
+import { phasedProxyBaseUrl } from './pi.js'
+import { PI_MAX_OUTPUT_TOKENS } from './pi-reduction.js'
 
 // A reusable abstraction for the "agent returns a structured JSON document as its
 // final assistant message" pattern (requirements, blueprint, merger — and any future

package/src/tool-silence.ts

@@ -0,0 +1,125 @@
+// The tool-silence watchdog (stuck-run audit F13): the third bound on a job, and the only one
+// that can see a model which keeps TALKING while completing nothing. Its output resets the
+// inactivity timer on every chunk, and it is nowhere near the wall-clock cap.
+//
+// WHY THIS IS ITS OWN MODULE — the window is only meaningful while something that REPORTS
+// completed tool calls is running, and only that producer knows whether it does. Keying the
+// window on the job's coarse phase label instead looked equivalent and was not: `agent` is a
+// telemetry breadcrumb several call sites mark for several different things (a Codex pass, a
+// tool-less inline completion, the label restored around a repair loop's shell commands), so a
+// phase-armed window spent most of its time armed over work that could not possibly reset it.
+// The producer opens its own window instead, which makes "armed" and "can beat" the same fact by
+// construction rather than by two call sites agreeing.
+
+/**
+ * The live window for ONE tool-reporting agent stream. Opened by the stream, beaten by each
+ * completed tool call, closed when the stream ends (cleanly or not) — so the window can never
+ * outlive the only thing able to reset it.
+ */
+export interface ToolProgressWindow {
+  /** A tool call completed: the only evidence this watchdog accepts as progress. */
+  toolCompleted(): void
+  /** The stream ended; the window closes with it. Idempotent. */
+  close(): void
+}
+
+/**
+ * A window that measures nothing: what the watchdog hands out when disabled, and what a producer
+ * substitutes when its caller wired no watchdog at all. Having one means every producer holds a
+ * real window and states its tool progress unconditionally, rather than guarding each call site
+ * with a `?.` that reads as though beating the window were optional.
+ */
+export const NO_TOOL_WINDOW: ToolProgressWindow = { toolCompleted: () => {}, close: () => {} }
+
+export interface ToolSilenceDeps {
+  /** The window length. `<= 0` disables the watchdog entirely (every window is inert). */
+  windowMs: number
+  /**
+   * When the run last produced ANY output, or undefined if it never has — the same clock the
+   * inactivity watchdog resets on. Read at EXPIRY (see {@link ToolSilenceWatchdog.open}), which
+   * is what keeps this watchdog off the gone-quiet case.
+   */
+  lastActivityAt: () => number | undefined
+  /** Called when a window expires with the run demonstrably still talking. */
+  onExpired: () => void
+}
+
+/**
+ * Hands out {@link ToolProgressWindow}s and fires `onExpired` for one that goes a full window
+ * without a completed tool call while the run keeps producing output.
+ *
+ * ONE window is open at a time (a job runs one agent stream at a time, and a repair loop runs
+ * them in sequence). Opening a second supersedes the first, and a superseded handle's calls are
+ * ignored rather than reaching back into the live window — a stale `close()` from a stream that
+ * finished after its successor started would otherwise disarm the watchdog for the rest of the job.
+ */
+export class ToolSilenceWatchdog {
+  private timer: ReturnType<typeof setTimeout> | undefined
+  /** Identity of the window currently open; every handout takes the next number. */
+  private openId = 0
+  /** When the live window was last armed — the start of the span `onExpired` is a verdict about. */
+  private armedAt = 0
+
+  constructor(private readonly deps: ToolSilenceDeps) {}
+
+  /** Open a window for one agent stream. Returns an inert handle when the watchdog is disabled. */
+  open(): ToolProgressWindow {
+    if (this.deps.windowMs <= 0) return NO_TOOL_WINDOW
+    const id = ++this.openId
+    this.arm()
+    const live = (): boolean => this.openId === id
+    return {
+      toolCompleted: () => {
+        if (live()) this.arm()
+      },
+      close: () => {
+        if (!live()) return
+        // Retire the id as well as the timer, so a late `toolCompleted()` from this stream
+        // cannot re-arm a window whose producer has already gone.
+        this.openId++
+        this.stop()
+      },
+    }
+  }
+
+  /** Disarm for good (the job settled). Safe to call with no window open. */
+  stop(): void {
+    clearTimeout(this.timer)
+    this.timer = undefined
+  }
+
+  private arm(): void {
+    clearTimeout(this.timer)
+    this.armedAt = Date.now()
+    this.timer = setTimeout(() => this.expire(), this.deps.windowMs)
+  }
+
+  /**
+   * A window elapsed with no completed tool call. Fire ONLY if the run was talking through it.
+   *
+   * `no-tool-progress` claims something specific — output arrived, but nothing got done — and
+   * that is only a truthful reading when output actually arrived DURING the window that just
+   * expired. A window that passed in total silence is the INACTIVITY watchdog's fact, whose
+   * diagnostic ("the container went quiet") is the one an operator can act on; relabelling it as
+   * a rabbit-hole would send them looking at the model instead of at the hang.
+   *
+   * This is a structural guard, not a tie-breaker: the two timers anchor on different events (the
+   * last tool call vs the last byte of output), so no arithmetic between the two window LENGTHS
+   * can order them. Equal windows put the tool-silence anchor strictly earlier — it fires first —
+   * and an operator setting `JOB_TOOL_SILENCE_MS` below `JOB_INACTIVITY_MS` gets that on every
+   * quiet run. Deciding from what the expired window actually SAW is independent of both numbers.
+   *
+   * A deferred window re-arms rather than standing down, so a run that goes quiet and then
+   * resumes its monologue is still caught, one full window later. The deferral terminates:
+   * either output resumes (the next expiry has output in its window and fires) or it does not
+   * (inactivity fires).
+   */
+  private expire(): void {
+    const lastActivityAt = this.deps.lastActivityAt()
+    if (lastActivityAt === undefined || lastActivityAt <= this.armedAt) {
+      this.arm()
+      return
+    }
+    this.deps.onExpired()
+  }
+}