@cat-factory/executor-harness 1.94.0 → 1.98.0

package/src/jsonl-stream.ts

@@ -0,0 +1,149 @@
+// BOUNDED reading of an agent CLI's child streams: LF-framing its JSONL stdout, and holding a
+// capped tail of raw output for diagnostics.
+//
+// WHY THIS MODULE EXISTS — the harness's two watchdog timers and its `/health` + `/jobs` poll
+// endpoints share ONE Node event loop with the stream-parsing hot path, so the advertised "a
+// container can never run forever" guarantee only holds while that loop stays live (stuck-run
+// audit F6). Both CLI readers (`runPi`, `runSubscriptionAgent`) had grown the same unbounded
+// framing loop: a record with no terminating newline accumulated without limit, so a runaway
+// producer could drive the buffer until a single `JSON.parse` (or the allocation behind it)
+// stalled the loop past the abort timers and the poll handlers alike. The container then stops
+// answering polls while its own watchdogs never fire — the exact wedge the timers exist to
+// prevent, with only the engine-side poll-failure tolerance and the reaper left underneath.
+//
+// One definition of "how much of a child's output we are willing to hold" therefore serves both
+// harnesses, for the same reason `ProgressGuard` does: two copies of a bound are two bounds.
+
+/**
+ * Longest single JSONL record either CLI may emit before the reader stops buffering it.
+ *
+ * Deliberately far above the largest LEGITIMATE record — the terminal `agent_end`, which carries
+ * the run's whole message transcript including tool results — because dropping that one costs the
+ * run its summary and stats. The cap is not a size policy, it is the ceiling that keeps a
+ * runaway producer from growing the buffer until parsing it wedges the event loop, so it only
+ * has to be low enough that one parse of it stays well inside the poll cadence.
+ */
+export const MAX_JSONL_LINE_CHARS = 32 * 1024 * 1024
+
+/**
+ * A fixed-size tail of a text stream, for output kept ONLY to quote back on a failure.
+ *
+ * Retaining a whole run's stdout to slice the last 2 KB off it at close is the memory half of
+ * F6: a chatty agent's output is unbounded, and the container OOMing is another way for a job to
+ * stop answering polls with no watchdog having fired. The tail is trimmed lazily — only once it
+ * has grown past twice the bound — so a run that streams thousands of chunks pays an amortized
+ * O(1) copy per chunk rather than an O(maxChars) slice on every one of them.
+ */
+export class BoundedTail {
+  private text = ''
+  private total = 0
+
+  constructor(private readonly maxChars: number) {}
+
+  push(chunk: string): void {
+    this.text += chunk
+    this.total += chunk.length
+    if (this.text.length > this.maxChars * 2) this.text = this.text.slice(-this.maxChars)
+  }
+
+  /** The last `maxChars` characters seen. */
+  toString(): string {
+    return this.text.length > this.maxChars ? this.text.slice(-this.maxChars) : this.text
+  }
+
+  /** Everything ever pushed, whether or not it is still retained. */
+  get totalChars(): number {
+    return this.total
+  }
+
+  /**
+   * Characters dropped off the FRONT because the tail is bounded; 0 while everything still fits.
+   *
+   * A caller that renders the tail to a human owes them this: a bounded tail is the opposite of a
+   * prefix, so a reader who assumes one concludes the producer stopped where the text begins.
+   * Diagnostic quotes (a stderr tail) need no such note — being a tail is what they are for.
+   */
+  get droppedChars(): number {
+    return this.total - this.toString().length
+  }
+}
+
+/**
+ * Frames a child's LF-delimited JSONL stdout into complete records, bounding what it will buffer
+ * for any one of them.
+ *
+ * `onLine` is invoked per complete record with `final: false`, and once more from {@link flush}
+ * with `final: true` for a trailing record that arrived without its newline (a clean exit can
+ * leave the last event unterminated). `final` is what lets a caller deliver the record's
+ * progress/telemetry signal while suppressing any decision that would KILL the run: the process
+ * has already exited, so a guard tripping on that last buffered record would turn a clean exit
+ * into a spurious failure.
+ *
+ * A record that outgrows {@link MAX_JSONL_LINE_CHARS} is DROPPED, not truncated: a partial JSON
+ * document is not a record, and handing the parser half of one would report it as corrupt output
+ * rather than as the bound firing. The reader then resynchronises on the next newline, so the
+ * oversized record costs its own signal and nothing after it. Callers report {@link droppedLines}
+ * at close (never per line) so the loss is diagnosable instead of silent.
+ */
+export class JsonlLineReader {
+  private buffer = ''
+  /** True while discarding the tail of a record that already blew the cap. */
+  private skipping = false
+  private dropped = 0
+
+  constructor(
+    private readonly onLine: (line: string, final: boolean) => void,
+    private readonly maxLineChars: number = MAX_JSONL_LINE_CHARS,
+  ) {}
+
+  /** Feed one stdout chunk, emitting every complete record it finishes. */
+  push(text: string): void {
+    // Framing scans the incoming CHUNK, never the accumulated buffer. `buffer += chunk` is a
+    // cheap rope in V8 and `.length` reads off it in constant time, but ANY search over it
+    // flattens the rope — so scanning the buffer once per chunk costs O(record) per chunk, i.e.
+    // quadratic in a runaway record, paid on the very event loop this class exists to keep
+    // answering polls. Measured, a 32 MB unterminated record cost ~6s of solid blocking that
+    // way: the cap bounded the memory and handed back the stall in its place.
+    let rest = text
+    for (;;) {
+      const nl = rest.indexOf('\n')
+      if (nl === -1) break
+      if (this.skipping) {
+        // The newline that ends an oversized record ends the skip with it: everything buffered
+        // for that record is already gone, and what follows is a fresh one.
+        this.skipping = false
+      } else {
+        // The only place the buffer is materialised, and only for a record that COMPLETED —
+        // which the branch below has already kept under the cap.
+        const raw = this.buffer + rest.slice(0, nl)
+        this.buffer = ''
+        // The cap is on the RECORD, not on the leftover buffer: a record that arrived whole
+        // inside one chunk was never buffered across pushes, and dropping it only when it
+        // straddles a chunk boundary would make the bound depend on how the OS split the reads.
+        if (raw.length > this.maxLineChars) this.dropped++
+        else this.onLine(raw.trim(), false)
+      }
+      rest = rest.slice(nl + 1)
+    }
+    if (this.skipping) return
+    this.buffer += rest
+    if (this.buffer.length > this.maxLineChars) {
+      this.buffer = ''
+      // Count the record once, however many chunks it goes on to spill.
+      this.dropped++
+      this.skipping = true
+    }
+  }
+
+  /** Emit any trailing unterminated record (see the class doc); call once, after the child exits. */
+  flush(): void {
+    const line = this.buffer.trim()
+    this.buffer = ''
+    if (line && !this.skipping) this.onLine(line, true)
+  }
+
+  /** Records dropped for exceeding the line cap; 0 on every ordinary run. */
+  get droppedLines(): number {
+    return this.dropped
+  }
+}

package/src/pi-reduction.ts

@@ -0,0 +1,359 @@
+import { BoundedTail } from './jsonl-stream.js'
+
+// Reducing a Pi `--mode json` event stream to what the run PRODUCED: the assistant's answer, what
+// it actually did, its output-quality signals, and whether it ended in a hard error.
+//
+// WHY THIS IS ITS OWN MODULE — every one of these answers used to be computed by scanning a
+// retained array of every record the run emitted, and `runPi` held that array for the whole run
+// (stuck-run audit F6). Bounding the JSONL FRAMING while retaining an unbounded number of parsed
+// records only moves the heap-exhaustion mode: a parsed object is typically larger than the raw
+// text it replaced, and a container that OOMs is another way for a job to stop answering polls
+// with no watchdog having fired.
+//
+// So the reduction FOLDS instead. {@link PiRunReducer} observes each record as it streams and
+// retains only what the close-of-run answers actually need: the terminal record, the one
+// transcript they read, running counters, and a bounded tail of streamed assistant text. Its
+// memory is O(largest single record), not O(records).
+//
+// The array-shaped entry points below (used by offline tooling over a captured stdout) are
+// DEFINED in terms of the same reducer rather than keeping their own scans, so the live path and
+// the offline one cannot drift into disagreeing about what a run produced.
+
+/**
+ * Per-completion output-token ceiling Pi requests (its model-entry `maxTokens`).
+ * Generous on purpose: a reasoning model (e.g. GLM-5.2) spends tokens on its
+ * `<think>` trace before the answer + tool calls, so a tight cap truncates it
+ * mid-reasoning and the agent never commits edits. It is a ceiling, not a target
+ * — unused output tokens are not billed and Workers AI clamps the request to the
+ * model's real max — so erring high is safe. Raised to 32k after a spec-writer run
+ * truncated an intermediate tool call at the old 16k cap; the document itself
+ * stopped well under it, so this is headroom for larger specs/diffs, with
+ * {@link runDiagnostics} flagging the rare case where even 32k is not enough.
+ */
+export const PI_MAX_OUTPUT_TOKENS = 32_768
+
+/**
+ * How much streamed assistant text the fallback summary holds when a run emitted no terminal
+ * transcript. Far above any real answer, because this is a bound on a runaway producer rather
+ * than a size policy — and what it drops is REPORTED (see {@link PiRunReducer.reduce}), since a
+ * tail read as a whole answer would look like a model that stopped mid-sentence.
+ */
+const FALLBACK_SUMMARY_CHARS = 256 * 1024
+
+export function isObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null
+}
+
+/**
+ * What the agent actually did this run, independent of any file changes. Used to
+ * tell a genuine no-op (the agent never reached the model / never acted) apart
+ * from a real run, so a bootstrap that produced nothing is failed rather than
+ * pushed as an empty repo. `toolCalls === 0 && assistantChars === 0` is the
+ * signature of a run where Pi never made a successful model call.
+ */
+export interface PiRunStats {
+  /** Tool calls the assistant emitted across the transcript (0 ⇒ it never acted). */
+  toolCalls: number
+  /** Total characters of assistant text (0 ⇒ the model produced nothing). */
+  assistantChars: number
+}
+
+/**
+ * Output-quality signals lifted from the agent's transcript, so the harness can fail
+ * LOUDLY on a malformed run instead of silently handing a half-baked artifact to the
+ * structured-output repair (which would manufacture a doc from garbage — the trap
+ * behind the spec-writer ⇄ companion rework loop). Two distinct invalid states, both
+ * seen in production from `kimi-k2.7-code`:
+ *  - a completion that hit the output ceiling (its answer/tool call was cut off), and
+ *  - a FINAL turn that carried no text at all (an empty `content: []` despite spending
+ *    output tokens), so there is no answer to parse.
+ */
+export interface RunDiagnostics {
+  /** Some completion ended at the output-token ceiling — its content was cut off. */
+  truncated: boolean
+  /** The agent's FINAL completion hit the ceiling: its ANSWER (not a mid-run step) was cut off. */
+  finalTruncated: boolean
+  /** The agent's final turn carried no text content (e.g. an empty `content: []`). */
+  finalAnswerEmpty: boolean
+}
+
+/** What a Pi run's event stream reduces to (the run's product, before any process-level detail). */
+export interface PiRunReduction {
+  summary: string
+  stats: PiRunStats
+  diagnostics: RunDiagnostics
+}
+
+/**
+ * Folds a Pi event stream into {@link PiRunReduction} plus the run's terminal-failure signal.
+ *
+ * Feed every parsed record to {@link observe} in stream order, then read the answers at close.
+ * What it keeps, and why that is all of it:
+ *  - the LAST `agent_end` / `auto_retry_end` record, which is exactly what a scan-from-the-end
+ *    for the terminal signal would have stopped on;
+ *  - the LAST `agent_end` transcript, the canonical source for the summary, the stats and the
+ *    diagnostics alike (all three scanned back to the same record);
+ *  - running counters and a bounded text tail, which are the FALLBACKS those three use when a
+ *    run emitted no terminal transcript at all.
+ */
+export class PiRunReducer {
+  /** The last terminal record seen (`agent_end` or `auto_retry_end`), whichever came last. */
+  private terminal: Record<string, unknown> | undefined
+  /** Messages of the last `agent_end` that carried a transcript. */
+  private transcript: unknown[] | undefined
+  private streamedToolCalls = 0
+  private streamedToolResults = 0
+  private streamedAssistantChars = 0
+  private readonly streamedText = new BoundedTail(FALLBACK_SUMMARY_CHARS)
+
+  /** Fold one parsed record. */
+  observe(event: Record<string, unknown>): void {
+    const type = event.type
+    if (type === 'agent_end' || type === 'auto_retry_end') {
+      this.terminal = event
+      if (type === 'agent_end' && Array.isArray(event.messages)) {
+        this.transcript = event.messages as unknown[]
+      }
+      return
+    }
+    if (type === 'tool_execution_end') {
+      this.streamedToolCalls++
+      return
+    }
+    if (type === 'message_end' && isObject(event.message)) {
+      const message = event.message
+      if (message.role === 'assistant') {
+        const text = messageText(message)
+        this.streamedAssistantChars += text.length
+        if (text) this.streamedText.push(this.streamedText.totalChars ? `\n${text}` : text)
+      } else if (message.role === 'toolResult') {
+        this.streamedToolResults++
+      }
+    }
+  }
+
+  /**
+   * Whether the run emitted a terminal record at all. False means {@link terminalError} answered
+   * from nothing rather than from a clean ending, which a caller deciding whether the run
+   * SUCCEEDED has to tell apart (see `runPi`'s exit-0 path).
+   */
+  get sawTerminalRecord(): boolean {
+    return this.terminal !== undefined
+  }
+
+  /**
+   * The terminal-failure message when the run ended in a hard error (the model was unreachable /
+   * refused, and Pi exhausted its auto-retries), else undefined. Only the FINAL outcome counts: a
+   * mid-run hiccup the agent recovered from leaves a clean terminal `agent_end`.
+   */
+  terminalError(): string | undefined {
+    const e = this.terminal
+    if (!e) return undefined
+    if (e.type === 'auto_retry_end') {
+      if (e.success === false) {
+        return typeof e.finalError === 'string'
+          ? e.finalError
+          : 'the agent failed after exhausting its retries'
+      }
+      return undefined
+    }
+    return e.stopReason === 'error' && typeof e.errorMessage === 'string'
+      ? e.errorMessage
+      : undefined
+  }
+
+  /**
+   * The run's product. `stdoutTail` backs the last-resort summary for a run whose output matched
+   * nothing structured, and a TAIL is all that fallback ever wanted: it slices the final 2 KB.
+   */
+  reduce(stdoutTail: string, cap: number = PI_MAX_OUTPUT_TOKENS): PiRunReduction {
+    return {
+      summary: this.summary(stdoutTail),
+      stats: this.stats(),
+      diagnostics: this.diagnostics(cap),
+    }
+  }
+
+  /**
+   * Preferred: the last assistant message of the terminal transcript. Falls back to the streamed
+   * assistant text, then to a raw tail, so a schema tweak never loses output.
+   */
+  private summary(stdoutTail: string): string {
+    if (this.transcript) {
+      const text = lastAssistantText(this.transcript)
+      if (text) return text
+    }
+    const streamed = this.streamedText.toString().trim()
+    if (streamed) {
+      const dropped = this.streamedText.droppedChars
+      // Say so when this is a tail rather than the whole answer: a reader who took it for a
+      // prefix would conclude the model stopped where the text begins.
+      return dropped > 0
+        ? `[earlier assistant output omitted: ${dropped} characters]\n${streamed}`
+        : streamed
+    }
+    return stdoutTail.trim().slice(-2000)
+  }
+
+  /**
+   * Count what the agent actually did. Prefers the terminal transcript (assistant `toolCall`
+   * parts + text); falls back to the streamed `tool_execution_end` / `message_end` counters, so a
+   * no-op is never mistaken for a real run because of a schema tweak.
+   */
+  private stats(): PiRunStats {
+    if (this.transcript) return statsFromMessages(this.transcript)
+    return {
+      // The same call can surface as both a `tool_execution_end` and a toolResult `message_end`;
+      // prefer the former and only fall back to toolResult counts.
+      toolCalls: this.streamedToolCalls || this.streamedToolResults,
+      assistantChars: this.streamedAssistantChars,
+    }
+  }
+
+  /**
+   * Output-quality signals over the terminal transcript: whether any completion hit the output
+   * ceiling (its content was cut off), whether the FINAL completion did, and whether that final
+   * turn carried no text at all. Defaults to all-false with no terminal transcript (a no-op run
+   * is already caught by `agentNeverActed`).
+   *
+   * `cap` is the per-completion ceiling Pi requested ({@link PI_MAX_OUTPUT_TOKENS}); truncation is
+   * detected by an assistant message whose `usage.output` reached it, which is reliable even when
+   * the model reports a non-`length` stop reason (Workers AI labelled a cut-off tool call
+   * `tool_calls`, not `length`).
+   */
+  private diagnostics(cap: number): RunDiagnostics {
+    if (!this.transcript) {
+      return { truncated: false, finalTruncated: false, finalAnswerEmpty: false }
+    }
+    const assistants = this.transcript.filter(
+      (m): m is Record<string, unknown> => isObject(m) && m.role === 'assistant',
+    )
+    const last = assistants.at(-1)
+    return {
+      truncated: assistants.some((m) => assistantOutputTokens(m) >= cap),
+      finalTruncated: last ? assistantOutputTokens(last) >= cap : false,
+      finalAnswerEmpty: last ? messageText(last) === '' : false,
+    }
+  }
+}
+
+/** Fold an already-parsed event array through a fresh {@link PiRunReducer}. */
+function reducerOver(events: Record<string, unknown>[]): PiRunReducer {
+  const reducer = new PiRunReducer()
+  for (const event of events) reducer.observe(event)
+  return reducer
+}
+
+/** Parse Pi's LF-framed JSONL stdout into its event records, skipping noise. */
+export function parsePiEvents(stdout: string): Record<string, unknown>[] {
+  const events: Record<string, unknown>[] = []
+  for (const line of stdout.split('\n')) {
+    const trimmed = line.trim()
+    if (!trimmed.startsWith('{')) continue
+    try {
+      events.push(JSON.parse(trimmed) as Record<string, unknown>)
+    } catch {
+      // Skip a corrupted/truncated record; the surrounding stream is still usable.
+    }
+  }
+  return events
+}
+
+/** {@link PiRunReducer.terminalError} over Pi's raw `--mode json` stdout. */
+export function terminalRunError(stdout: string): string | undefined {
+  return terminalErrorFromEvents(parsePiEvents(stdout))
+}
+
+/** {@link PiRunReducer.terminalError} over records already parsed from the stream. */
+export function terminalErrorFromEvents(events: Record<string, unknown>[]): string | undefined {
+  return reducerOver(events).terminalError()
+}
+
+/** {@link PiRunReducer.reduce} over Pi's raw `--mode json` stdout. */
+export function summarizePiRun(stdout: string): PiRunReduction {
+  return summarizeFromEvents(parsePiEvents(stdout), stdout)
+}
+
+/** {@link PiRunReducer.reduce} over records already parsed from the stream. */
+export function summarizeFromEvents(
+  events: Record<string, unknown>[],
+  stdoutTail: string,
+): PiRunReduction {
+  return reducerOver(events).reduce(stdoutTail)
+}
+
+/** {@link RunDiagnostics} over records already parsed from the stream. */
+export function diagnosticsFromEvents(
+  events: Record<string, unknown>[],
+  cap: number = PI_MAX_OUTPUT_TOKENS,
+): RunDiagnostics {
+  return reducerOver(events).reduce('', cap).diagnostics
+}
+
+/** {@link RunDiagnostics} over Pi's raw `--mode json` stdout. */
+export function runDiagnostics(stdout: string, cap: number = PI_MAX_OUTPUT_TOKENS): RunDiagnostics {
+  return diagnosticsFromEvents(parsePiEvents(stdout), cap)
+}
+
+/**
+ * Extract the assistant's final summary from Pi's JSON-lines output. Pi emits a terminal
+ * `agent_end` event whose `messages` is the full transcript, so the last assistant message there
+ * is the canonical answer (see {@link PiRunReducer.reduce} for the fallbacks).
+ */
+export function parsePiOutput(stdout: string): string {
+  return summarizePiRun(stdout).summary
+}
+
+/** `usage.output` (completion tokens) reported on a Pi assistant message, or 0. */
+function assistantOutputTokens(message: Record<string, unknown>): number {
+  const usage = message.usage
+  if (!isObject(usage)) return 0
+  const output = usage.output
+  return typeof output === 'number' ? output : 0
+}
+
+/** {@link PiRunStats} from a transcript: assistant `toolCall` parts + text length. */
+function statsFromMessages(messages: unknown[]): PiRunStats {
+  let toolCalls = 0
+  let assistantChars = 0
+  for (const m of messages) {
+    if (!isObject(m) || m.role !== 'assistant') continue
+    const content = m.content
+    if (typeof content === 'string') {
+      assistantChars += content.trim().length
+    } else if (Array.isArray(content)) {
+      for (const part of content) {
+        if (!isObject(part)) continue
+        if (part.type === 'toolCall') toolCalls++
+        else if (typeof part.text === 'string') assistantChars += part.text.length
+      }
+    }
+  }
+  return { toolCalls, assistantChars }
+}
+
+/** The text of the last assistant message in a transcript, or '' if none. */
+function lastAssistantText(messages: unknown[]): string {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const m = messages[i]
+    if (isObject(m) && m.role === 'assistant') {
+      const text = messageText(m)
+      if (text) return text
+    }
+  }
+  return ''
+}
+
+/** Join the text parts of a Pi message whose content is a string or parts array. */
+export function messageText(message: unknown): string {
+  if (!isObject(message)) return ''
+  const content = message.content
+  if (typeof content === 'string') return content.trim()
+  if (Array.isArray(content)) {
+    return content
+      .map((part) => (isObject(part) && typeof part.text === 'string' ? part.text : ''))
+      .join('')
+      .trim()
+  }
+  return ''
+}

package/src/pi-workspace.ts

@@ -8,8 +8,6 @@ import { log } from './logger.js'
 import {
   type ContextFileInfo,
   type PiRunOutcome,
-  type PiRunStats,
-  type RunDiagnostics,
   CONTEXT_DIR,
   materializeContextFiles,
   materializeSkillResources,
@@ -21,6 +19,7 @@ import {
   writePiModelsConfig,
   writeWebToolsConfig,
 } from './pi.js'
+import type { PiRunStats, RunDiagnostics } from './pi-reduction.js'
 import {
   type ProgressGuardLimits,
   mergeGuardLimits,
@@ -328,10 +327,19 @@ export async function runAgentInWorkspace(
       // The run's tool-call trajectory, the same hook the Pi path feeds — so a subscription run
       // and a proxied one produce the same evidence rather than one of them producing none.
       onSpan: opts.onSpan,
+      // The tool-silence window (stuck-run audit F13), opened by whichever CLI actually runs.
+      // Wired for BOTH subscription harnesses: each reports tool activity on its own stream, so
+      // each can beat the window it opens.
+      beginToolWindow: opts.beginToolWindow,
       // Per-slice review capture, so a parallel review's finished slices are persisted as they
       // land rather than only in the terminal output. Only the subscription runners fan work out
       // across subagents, so this is the only path that can produce it.
       onSliceReviews: opts.onSliceReviews,
+      // What the CLI reported about the tool servers it loaded. Wired for BOTH subscription
+      // harnesses even though only claude-code's stream carries the report today: the hook is a
+      // pass-through, and a codex run that never calls it leaves the backend's record honestly
+      // absent rather than claiming every server it wired failed to start.
+      onToolServers: opts.onToolServers,
       // Stream this run's per-call telemetry to the job's live drain. The subscription
       // harnesses are the only producers of `callMetrics` (Pi's calls are metered by the LLM
       // proxy as they happen), so this is the only path that needs the hook.
@@ -379,7 +387,7 @@ export async function runAgentInWorkspace(
     model: spec.model,
     proxyBaseUrl: phasedProxyBaseUrl(proxyBaseUrl, opts.currentPhase?.(), spec.proxyPhasePath),
   })
-  const { signal, onActivity, onProgress, onSpan } = opts
+  const { signal, onActivity, onProgress, onSpan, beginToolWindow } = opts
   const piOutcome = await runPi({
     cwd: spec.dir,
     model: spec.model,
@@ -389,6 +397,7 @@ export async function runAgentInWorkspace(
     onActivity,
     onProgress,
     onSpan,
+    beginToolWindow,
     expectsEdits: spec.expectsEdits ?? true,
     // Start from the env/built-in defaults and apply only the per-knob overrides the
     // backend set for this kind (loosen-only), so an unspecified knob keeps its default.