@cat-factory/executor-harness 1.88.0 → 1.92.0

package/src/agent-runner.ts

@@ -4,6 +4,11 @@ import { tmpdir } from 'node:os'
 import { dirname, join } from 'node:path'
 import { claudeAssistantContent, isObject, numberOf, redactBody } from './claude-stream.js'
 import { createClaudeRunTelemetry, subagentDispatchId } from './claude-call-aggregator.js'
+import {
+  ToolCallTracker,
+  type TrackedToolCall,
+  recordClaudeToolResults,
+} from './tool-trajectory.js'
 import type { Logger } from './logger.js'
 import {
   createCallMetricPublisher,
@@ -13,6 +18,7 @@ import {
   type PiRunOutcome,
   type PiRunStats,
   type TodoProgress,
+  type ToolSpan,
 } from './pi.js'
 import {
   claudeAllowedToolPatterns,
@@ -123,6 +129,13 @@ export interface SubscriptionRunOptions {
   onActivity?: () => void
   /** Called with the latest subtask counts each time the CLI updates its todo/plan list. */
   onProgress?: (progress: TodoProgress) => void
+  /**
+   * Called once per completed tool call with a {@link ToolSpan}: the run's TRAJECTORY. The CLI's
+   * tool loop is internal to the CLI and never touches our proxy, so its own event stream is the
+   * only place these exist — without this hook a subscription-harness run's account of what it
+   * DID dies with the container.
+   */
+  onSpan?: (span: ToolSpan) => void
   /**
    * Called with the FULL set of per-slice reviews each time one lands, so the backend can persist
    * a parallel review's completed work as it happens instead of only from the terminal result.
@@ -538,6 +551,36 @@ function createClaudeProgressGuard(opts: SubscriptionRunOptions): {
   }
 }
 
+/**
+ * The run's TRAJECTORY, on the claude-code stream: each `tool_use` block paired with the
+ * `tool_result` that answers it on the following user turn, numbered and captured (scrubbed +
+ * capped). The CLI's stream is the only place this loop is visible at all — its tool calls never
+ * touch our proxy — so without this a subscription-harness run's account of what it DID dies with
+ * the container.
+ *
+ * Both halves are no-ops when the caller wants no spans, so a driver that only needs the run's
+ * output never pays to serialise a body nothing will read. Split out of {@link runClaudeCode} for
+ * the per-function line budget, like {@link createClaudeProgressGuard}.
+ */
+function createClaudeToolTrajectory(
+  opts: SubscriptionRunOptions,
+  secrets: readonly string[],
+): {
+  onToolUse: (id: string, name: string, input: unknown) => void
+  onToolResults: (content: unknown[]) => void
+} {
+  if (!opts.onSpan) return { onToolUse: () => {}, onToolResults: () => {} }
+  const onSpan = opts.onSpan
+  const tracker = new ToolCallTracker(secrets)
+  return {
+    onToolUse: (id, name, input) => tracker.started(id, name, input),
+    onToolResults: (content) =>
+      recordClaudeToolResults(tracker, content, (call: TrackedToolCall) =>
+        onSpan({ ...call, bodies: 'stored' }),
+      ),
+  }
+}
+
 export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRunOutcome> {
   const stats: PiRunStats = { toolCalls: 0, assistantChars: 0 }
   let summary = ''
@@ -628,6 +671,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
   // diagnostic. Disabled when the caller supplies no limits (only the external watchdog bounds it).
   const progressGuard = createClaudeProgressGuard(opts)
   const { rememberTool, feedGuard, guardAbort } = progressGuard
+  const trajectory = createClaudeToolTrajectory(opts, secrets)
 
   const onEvent = (event: Record<string, unknown>, meta?: { final?: boolean }): void => {
     const type = event.type
@@ -650,6 +694,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
         // `is_error` its `tool_result` carries on the next `user` turn.
         if (typeof block.id === 'string' && typeof block.name === 'string') {
           rememberTool(block.id, block.name)
+          trajectory.onToolUse(block.id, block.name, block.input)
         }
         if (block.name === 'TodoWrite') {
           const progress = todosToProgress((block.input as Record<string, unknown>)?.todos)
@@ -672,6 +717,9 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
         // Not on the at-close flush: the CLI has already exited, so tripping the guard there
         // would kill nothing and only convert a clean exit into a spurious failure.
         if (!meta?.final) feedGuard(content)
+        // The trajectory's other half — fed on the FINAL flush too, unlike the guard, since a
+        // CLI that has exited is exactly when the last calls' results matter.
+        trajectory.onToolResults(content)
         telemetry.onToolResult(dispatchId, content)
       }
     } else if (type === 'result') {

package/src/job.ts

@@ -181,9 +181,13 @@ function parseGuardLimits(value: unknown): GuardLimitsSpec | undefined {
   const noEdit = posInt(o.maxToolCallsWithoutEdit)
   const errors = posInt(o.maxConsecutiveErrors)
   const web = posInt(o.maxConsecutiveWebCalls)
+  const mcp = posInt(o.maxConsecutiveMcpCalls)
+  const nonAction = posInt(o.maxConsecutiveNonActionCalls)
   if (noEdit !== undefined) spec.maxToolCallsWithoutEdit = noEdit
   if (errors !== undefined) spec.maxConsecutiveErrors = errors
   if (web !== undefined) spec.maxConsecutiveWebCalls = web
+  if (mcp !== undefined) spec.maxConsecutiveMcpCalls = mcp
+  if (nonAction !== undefined) spec.maxConsecutiveNonActionCalls = nonAction
   return Object.keys(spec).length > 0 ? spec : undefined
 }
 
@@ -874,6 +878,8 @@ export interface GuardLimitsSpec {
   maxToolCallsWithoutEdit?: number
   maxConsecutiveErrors?: number
   maxConsecutiveWebCalls?: number
+  maxConsecutiveMcpCalls?: number
+  maxConsecutiveNonActionCalls?: number
 }
 
 /**

package/src/pi-workspace.ts

@@ -325,6 +325,9 @@ export async function runAgentInWorkspace(
       expectsEdits: spec.expectsEdits ?? true,
       onActivity: opts.onActivity,
       onProgress: opts.onProgress,
+      // The run's tool-call trajectory, the same hook the Pi path feeds — so a subscription run
+      // and a proxied one produce the same evidence rather than one of them producing none.
+      onSpan: opts.onSpan,
       // Per-slice review capture, so a parallel review's finished slices are persisted as they
       // land rather than only in the terminal output. Only the subscription runners fan work out
       // across subagents, so this is the only path that can produce it.

package/src/pi.ts

@@ -4,7 +4,7 @@ import { homedir } from 'node:os'
 import { dirname, join } from 'node:path'
 import { killChildProcess, spawnDetached } from './process.js'
 import { pathExists } from './fs-utils.js'
-import { redactSecrets } from './redact.js'
+import { redactSecrets, secretsToRedact } from './redact.js'
 import { HarnessFailure } from './failure.js'
 import { log } from './logger.js'
 import type { EffortReport } from './effort.js'
@@ -14,6 +14,12 @@ import {
   toolCallSignal,
   type ProgressGuardLimits,
 } from './progress-guard.js'
+import {
+  ToolCallTracker,
+  readToolCallId,
+  toolCallResult,
+  toolCallStart,
+} from './tool-trajectory.js'
 
 // Drives the Pi coding-agent CLI. Pi is pointed at the Worker's OpenAI-compatible
 // proxy via a custom provider in ~/.pi/agent/models.json, authenticated with the
@@ -501,18 +507,39 @@ export interface TodoProgress {
 }
 
 /**
- * One tool invocation in Pi's loop, captured for the run's observability trace.
- * Metadata only (name + timing + ok) — never the tool's args or result — so the
- * harness buffer stays tiny. The backend drains these on its existing job poll and
- * emits them as child spans under the run trace.
+ * One tool invocation in an agent's loop, captured for the run's TRAJECTORY: the ordered
+ * account of what the agent did, drained by the backend on its existing job poll and
+ * both persisted and emitted as a child span under the run trace.
+ *
+ * It carries the call's arguments and result (scrubbed and capped at capture — see
+ * `tool-trajectory.ts`), because the question asked of a finished run is which command
+ * ran against what, not how long a tool named `bash` took. Whether those bodies are
+ * RETAINED is the backend's decision, taken against the deployment switch and the
+ * workspace's opt-out; the harness's job is to capture them bounded and scrubbed.
  */
 export interface ToolSpan {
   tool: string
-  /** Epoch ms the tool call started (approximated as the previous tool's end). */
+  /**
+   * The call's 0-based ordinal within this job. Two calls routinely land in the same
+   * millisecond, so this is the only thing that orders the trajectory — and it is what
+   * makes the backend's stored row id deterministic, so a replayed poll re-records
+   * instead of duplicating.
+   */
+  seq: number
+  /** Epoch ms the tool call started (the previous call's end when no start was seen). */
   startedAt: number
   /** Epoch ms the tool call ended (when its `tool_execution_end` event arrived). */
   endedAt: number
   ok: boolean
+  /** Whether the bodies below were captured at all — always `'stored'` from this harness. */
+  bodies: 'stored' | 'withheld'
+  /** The call's arguments, serialised, scrubbed and capped. `''` when it took none. */
+  args: string
+  /** What the tool returned, scrubbed and capped. `''` when it returned nothing. */
+  result: string
+  /** Characters the cap dropped from {@link args} / {@link result}; 0 when nothing was cut. */
+  argsDropped: number
+  resultDropped: number
 }
 
 function isObject(value: unknown): value is Record<string, unknown> {
@@ -918,10 +945,17 @@ export function runPi(opts: {
       opts.guardLimits ?? progressGuardLimitsFromEnv(),
       opts.expectsEdits ?? true,
     )
-    // Start boundary for the next tool span: each tool's slice runs from the previous
-    // tool's end (or the run start) to its own `tool_execution_end`. Approximate but
-    // contiguous — enough for the trace tree, and metadata-only.
-    let toolBoundary = Date.now()
+    // Pairs each tool call's start with its result, numbers the pairs and captures the two
+    // bodies (scrubbed + capped). A call whose start Pi never emitted still gets an entry,
+    // timed from the previous call's end — see `ToolCallTracker`.
+    //
+    // The known-secret list is DERIVED from the token this function itself hands the child
+    // (`PI_PROXY_TOKEN` / `SEARXNG_API_KEY`) rather than taken as a parameter: the bodies
+    // travel to a store and to external trace sinks, and a caller that forgets to pass the
+    // list produces bodies scrubbed of credential SHAPES only, with no signal that the
+    // narrower rule ever ran. Deriving it here means the one place that knows the child's
+    // credentials is the place that scrubs them.
+    const tools = new ToolCallTracker(secretsToRedact(opts.sessionToken))
 
     // SIGTERM first, then SIGKILL if Pi ignores it. Shared by the watchdog abort
     // and the no-progress guard; the `close` handler turns it into a rejection.
@@ -958,21 +992,22 @@ export function runPi(opts: {
         if (progress) opts.onProgress(progress)
       }
       if (opts.onSpan) {
+        const start = toolCallStart(event)
+        if (start) tools.started(start.id, start.name, start.args)
         const signal = toolCallSignal(event)
         if (signal && signal.name) {
-          const endedAt = Date.now()
+          const call = tools.finished(
+            readToolCallId(event),
+            signal.name,
+            toolCallResult(event),
+            signal.isError,
+          )
           try {
-            opts.onSpan({
-              tool: signal.name,
-              startedAt: toolBoundary,
-              endedAt,
-              ok: !signal.isError,
-            })
+            opts.onSpan({ ...call, bodies: 'stored' })
           } catch {
             // A faulty observer must never break the run.
             observerErrors++
           }
-          toolBoundary = endedAt
         }
       }
       if (runGuard && !guardReason && !aborted) {

package/src/progress-guard.ts

@@ -50,6 +50,31 @@ export interface ProgressGuardLimits {
    * without it.
    */
   maxConsecutiveWebCalls?: number
+  /**
+   * Abort after this many consecutive MCP tool-server calls (`mcp__*`) with no other
+   * tool call in between: the tool-server analogue of `maxConsecutiveWebCalls`, and
+   * present for the same reason. An `mcp__*` call is exempt from the no-edit bound (see
+   * `isMcpToolCall`), so without a streak of its own a run could query a tool server
+   * indefinitely without tripping any guard. Any non-MCP tool call resets the streak.
+   * Optional: defaults to {@link DEFAULT_PROGRESS_GUARD_LIMITS}.
+   */
+  maxConsecutiveMcpCalls?: number
+  /**
+   * Abort after this many consecutive calls that are EXEMPT from the no-edit bound
+   * (planning, read-only exploration, subagent dispatch, `mcp__*`) with no action call
+   * in between. The backstop that makes each individual exemption mean "not counted"
+   * rather than "unbounded": every per-family streak above resets on any call outside
+   * its own family, so a run alternating `web_search` with `mcp__issues__search` (or
+   * with `read`) trips none of them and, having never made an action call, never
+   * reaches `maxToolCallsWithoutEdit` either. Only the job's wall-clock ceiling
+   * bounded that.
+   *
+   * Deliberately far above every family cap, because it is not a research bound and
+   * must not become one: reading a hundred files before the first edit is legitimate
+   * work-up, and any `bash`/edit/action call resets the streak. Optional: defaults to
+   * {@link DEFAULT_PROGRESS_GUARD_LIMITS}.
+   */
+  maxConsecutiveNonActionCalls?: number
 }
 
 // `satisfies` (not a type annotation) so each property keeps its concrete `number`
@@ -63,6 +88,16 @@ export const DEFAULT_PROGRESS_GUARD_LIMITS = {
   // A genuine research burst is a handful of searches; an uninterrupted run of this
   // many web calls (with no read/edit/bash between) is a search loop, not progress.
   maxConsecutiveWebCalls: 25,
+  // Looser than the web cap: a tool server is usually the agent's route to the SYSTEM OF
+  // RECORD (the issue tracker, the advisory database, the design source), and reading a
+  // list and then each of its items is a normal opening move, not a rabbit-hole. A run
+  // that makes this many in a row with no read, edit or bash between is looping.
+  maxConsecutiveMcpCalls: 40,
+  // Well clear of every family cap above, and of any plausible read-up: a run that makes
+  // this many exempt calls with not one action call between them has stopped converging,
+  // whatever mix of reads, searches and lookups it is cycling through. Sized as a
+  // backstop rather than a judgement, because the families are where judgement belongs.
+  maxConsecutiveNonActionCalls: 200,
 } satisfies ProgressGuardLimits
 
 // Tool names that mutate files, so a call to one clears the no-edit suspicion. Kept
@@ -139,6 +174,32 @@ const EXPLORATION_TOOLS = new Set([
 // Pi's `web_search`/`web_fetch` and Claude Code's `WebSearch`/`WebFetch`.
 const WEB_TOOLS = new Set(['web_search', 'web_fetch', 'websearch', 'webfetch'])
 
+// A call to a tool server (MCP). Every MCP client names these `mcp__<server>__<tool>`, and
+// the prefix is the ONLY thing the harness can know about them: what a given server's tools
+// do is a backend registration this image has never seen, so the guard classifies by shape.
+//
+// Matched, rather than enumerated in EXPLORATION_TOOLS, because the set is open: it is
+// whatever tool servers the running kind was wired with. A prefix test is also why this is a
+// function, `name.startsWith` on the already-lower-cased name, so `MCP__Issues__search`
+// classifies the same as `mcp__issues__search`.
+function isMcpToolCall(loweredName: string): boolean {
+  return loweredName.startsWith('mcp__')
+}
+
+// Whether a call is EXEMPT from the no-edit bound: planning and bookkeeping, read-only
+// exploration, a subagent dispatch, or a tool-server call. One predicate rather than the
+// four tests inlined at the branch, because the combined non-action streak and the no-edit
+// exemption must be the SAME set: a family exempted in one place and missed in the other is
+// either an unbounded loop or a run killed for a call the bound says it may make.
+function isNonActionToolCall(loweredName: string): boolean {
+  return (
+    PLANNING_TOOLS.has(loweredName) ||
+    EXPLORATION_TOOLS.has(loweredName) ||
+    SUBAGENT_DISPATCH_TOOLS.has(loweredName) ||
+    isMcpToolCall(loweredName)
+  )
+}
+
 /** Read {@link ProgressGuardLimits} from the environment, falling back to the defaults. */
 export function progressGuardLimitsFromEnv(
   env: NodeJS.ProcessEnv = process.env,
@@ -160,6 +221,14 @@ export function progressGuardLimitsFromEnv(
       env.JOB_MAX_CONSECUTIVE_WEB_CALLS,
       DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveWebCalls,
     ),
+    maxConsecutiveMcpCalls: num(
+      env.JOB_MAX_CONSECUTIVE_MCP_CALLS,
+      DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveMcpCalls,
+    ),
+    maxConsecutiveNonActionCalls: num(
+      env.JOB_MAX_CONSECUTIVE_NON_ACTION_CALLS,
+      DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveNonActionCalls,
+    ),
   }
 }
 
@@ -186,12 +255,21 @@ export function mergeGuardLimits(
       overrides.maxToolCallsWithoutEdit,
     ),
     maxConsecutiveErrors: loosen(base.maxConsecutiveErrors, overrides.maxConsecutiveErrors),
-    // `maxConsecutiveWebCalls` is optional on the interface (callers may omit it), so
-    // fall back to the default before loosening — keeps `loosen`'s base a concrete number.
+    // The streak knobs are optional on the interface (callers may omit them), so fall back
+    // to the default before loosening: it keeps `loosen`'s base a concrete number.
     maxConsecutiveWebCalls: loosen(
       base.maxConsecutiveWebCalls ?? DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveWebCalls,
       overrides.maxConsecutiveWebCalls,
     ),
+    maxConsecutiveMcpCalls: loosen(
+      base.maxConsecutiveMcpCalls ?? DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveMcpCalls,
+      overrides.maxConsecutiveMcpCalls,
+    ),
+    maxConsecutiveNonActionCalls: loosen(
+      base.maxConsecutiveNonActionCalls ??
+        DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveNonActionCalls,
+      overrides.maxConsecutiveNonActionCalls,
+    ),
   }
 }
 
@@ -207,6 +285,8 @@ export class ProgressGuard {
   private edits = 0
   private consecutiveErrors = 0
   private consecutiveWebCalls = 0
+  private consecutiveMcpCalls = 0
+  private consecutiveNonActionCalls = 0
 
   constructor(
     private readonly limits: ProgressGuardLimits,
@@ -257,16 +337,55 @@ export class ProgressGuard {
       this.consecutiveWebCalls = 0
     }
 
-    // Planning, read-only exploration and subagent-dispatch calls don't count toward the
-    // no-edit bound (see PLANNING_TOOLS / EXPLORATION_TOOLS / SUBAGENT_DISPATCH_TOOLS) —
-    // only "action" calls without an edit do.
-    if (
-      PLANNING_TOOLS.has(name) ||
-      EXPLORATION_TOOLS.has(name) ||
-      SUBAGENT_DISPATCH_TOOLS.has(name)
-    ) {
+    // Tool-server (MCP) calls: bounded as their own streak for exactly the reason the web
+    // streak exists. They are exempt from the no-edit bound below, and an exemption with no
+    // counter-bound is a loop the guard cannot see. Any non-MCP call resets it.
+    if (isMcpToolCall(name)) {
+      this.consecutiveMcpCalls++
+      const mcpCap =
+        this.limits.maxConsecutiveMcpCalls ?? DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveMcpCalls
+      if (this.consecutiveMcpCalls >= mcpCap) {
+        return (
+          `no progress: ${this.consecutiveMcpCalls} consecutive tool-server (MCP) calls without ` +
+          `any other action. The agent is stuck querying its tools instead of doing the work. ` +
+          `Aborting.`
+        )
+      }
+    } else {
+      this.consecutiveMcpCalls = 0
+    }
+
+    // Planning, read-only exploration, subagent-dispatch and tool-server calls don't count
+    // toward the no-edit bound (see `isNonActionToolCall`): only "action" calls without an
+    // edit do.
+    //
+    // An `mcp__*` call is exempt for the same reason a `read` is: the bound targets the
+    // credential rabbit-hole (endless `bash` probing with nothing implemented), and reaching
+    // a registered tool server is the platform TELLING the agent to look something up
+    // ("prefer them over guessing"). Counting them would abort an edits-expected kind for
+    // consulting the issue tracker the deployment wired for it, punishing the run for
+    // following its own prompt. They are neutral rather than edit-satisfying, exactly like a
+    // subagent dispatch: a read-only lookup must not clear the suspicion the bound holds.
+    //
+    // The exempt calls carry ONE streak of their own, and it is what keeps every exemption
+    // above from adding up to an unbounded run: each per-family cap resets on any call
+    // outside its family, so alternating two exempt families trips neither, and a run that
+    // never makes an action call never reaches the no-edit bound either.
+    if (isNonActionToolCall(name)) {
+      this.consecutiveNonActionCalls++
+      const nonActionCap =
+        this.limits.maxConsecutiveNonActionCalls ??
+        DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveNonActionCalls
+      if (this.consecutiveNonActionCalls >= nonActionCap) {
+        return (
+          `no progress: ${this.consecutiveNonActionCalls} consecutive read-only calls (searching, ` +
+          `reading, tool-server lookups, subagent dispatches) with no action call between them. ` +
+          `The agent is cycling through research instead of doing the work. Aborting.`
+        )
+      }
       return null
     }
+    this.consecutiveNonActionCalls = 0
     this.toolCalls++
     if (FILE_EDIT_TOOLS.has(name)) this.edits++