@cat-factory/executor-harness 1.86.2 → 1.90.0

package/src/job.ts

@@ -20,6 +20,11 @@ import {
   type SkillResourceSpec,
   type SkillSpec,
 } from './agent-capabilities.js'
+import { type TestSecretSpec, parseInfraEnv, parseSecretEnvPairs, str } from './job-env.js'
+
+// Re-exported so a handler describing a job keeps ONE import site (the env-pair shape is a job
+// body field like any other; only its VALIDATION moved out).
+export type { TestSecretSpec }
 
 // Re-exported so the job body stays the one import site for a harness handler describing a job.
 export type { McpServerSpec, SkillResourceSpec, SkillSpec }
@@ -142,13 +147,6 @@ export interface ReferenceRepoSpec {
   ghToken?: string
 }
 
-function str(value: unknown, path: string): string {
-  if (typeof value !== 'string' || value.length === 0) {
-    throw new Error(`Invalid job: '${path}' must be a non-empty string`)
-  }
-  return value
-}
-
 /** A positive finite integer, or undefined for any other input (silently ignored). */
 function posInt(value: unknown): number | undefined {
   return typeof value === 'number' && Number.isFinite(value) && value > 0
@@ -183,9 +181,13 @@ function parseGuardLimits(value: unknown): GuardLimitsSpec | undefined {
   const noEdit = posInt(o.maxToolCallsWithoutEdit)
   const errors = posInt(o.maxConsecutiveErrors)
   const web = posInt(o.maxConsecutiveWebCalls)
+  const mcp = posInt(o.maxConsecutiveMcpCalls)
+  const nonAction = posInt(o.maxConsecutiveNonActionCalls)
   if (noEdit !== undefined) spec.maxToolCallsWithoutEdit = noEdit
   if (errors !== undefined) spec.maxConsecutiveErrors = errors
   if (web !== undefined) spec.maxConsecutiveWebCalls = web
+  if (mcp !== undefined) spec.maxConsecutiveMcpCalls = mcp
+  if (nonAction !== undefined) spec.maxConsecutiveNonActionCalls = nonAction
   return Object.keys(spec).length > 0 ? spec : undefined
 }
 
@@ -494,49 +496,6 @@ export function parsePackageRegistries(
   return entries
 }
 
-/**
- * One sensitive test credential the tester receives: an env-var name + its (secret) value.
- * The backend seals these at rest and decrypts them at dispatch; the harness injects each as an
- * environment variable the tester's shell can read (out of band — the value is NEVER in the
- * prompt/telemetry). See {@link parseTestSecrets}.
- */
-export interface TestSecretSpec {
-  key: string
-  value: string
-}
-
-/** A valid POSIX shell variable name (letters, digits, underscore; not starting with a digit). */
-const ENV_VAR_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/
-
-/**
- * Validate the optional tester `testSecrets` list — `{ key, value }` env pairs the harness
- * injects into the run environment. Keys must be valid env-var names; toolchain-critical /
- * reserved names ({@link isReservedEnvName}) and duplicates are dropped so a drifted body can't
- * clobber PATH/NODE_OPTIONS/etc. Absent ⇒ no secrets injected.
- */
-export function parseTestSecrets(value: unknown): TestSecretSpec[] {
-  if (value === undefined || value === null) return []
-  if (!Array.isArray(value)) throw new Error("Invalid job: 'testSecrets' must be an array")
-  const entries: TestSecretSpec[] = []
-  const seen = new Set<string>()
-  for (const [i, raw] of value.entries()) {
-    if (typeof raw !== 'object' || raw === null) {
-      throw new Error(`Invalid job: 'testSecrets[${i}]' must be an object`)
-    }
-    const entry = raw as Record<string, unknown>
-    const key = str(entry.key, `testSecrets[${i}].key`).trim()
-    if (!ENV_VAR_NAME_PATTERN.test(key)) {
-      throw new Error(
-        `Invalid job: 'testSecrets[${i}].key' must be a valid environment variable name`,
-      )
-    }
-    if (isReservedEnvName(key) || seen.has(key)) continue
-    seen.add(key)
-    entries.push({ key, value: str(entry.value, `testSecrets[${i}].value`) })
-  }
-  return entries
-}
-
 // ---- Shared repo-bootstrap target ---------------------------------------
 
 /** The new repository a repo-bootstrap run force-pushes its fresh history to. */
@@ -782,6 +741,16 @@ export interface AgentJob extends HarnessAuthFields {
    * Absent ⇒ no secrets injected.
    */
   testSecrets?: TestSecretSpec[]
+  /**
+   * The resolved credentials of the step's GENERATIVE BINARY INTEGRATIONS (the image / music /
+   * video generation APIs its `binaryOutput` selection named), as env pairs the harness injects
+   * into the agent's own process — where the agent's brief has already told it to read them from.
+   * Distinct from {@link testSecrets} because the two have different producers and different
+   * lifetimes: tester secrets are workspace state a human stored, these are a deployment's
+   * registration resolved per dispatch. Absent ⇒ no integration declared a credential, or none
+   * resolved (which the agent is told to report rather than work around).
+   */
+  generatorSecrets?: TestSecretSpec[]
   /**
    * Explore mode: stand the service's dependencies up before the agent runs (the
    * tester). Brings the docker-compose infra up on localhost for the duration of the
@@ -909,6 +878,8 @@ export interface GuardLimitsSpec {
   maxToolCallsWithoutEdit?: number
   maxConsecutiveErrors?: number
   maxConsecutiveWebCalls?: number
+  maxConsecutiveMcpCalls?: number
+  maxConsecutiveNonActionCalls?: number
 }
 
 /**
@@ -1110,66 +1081,6 @@ function parseStringMap(value: unknown): Record<string, string> | undefined {
   return Object.keys(out).length ? out : undefined
 }
 
-/**
- * Env-var names never injected from a frontend binding: spread over `process.env` at build
- * time, so any of these would break the toolchain (or enable code execution / cert overrides)
- * rather than name an upstream URL. Matched exactly (Linux env is case-sensitive); the
- * {@link RESERVED_ENV_PREFIXES} below cover whole families (`npm_config_*`, `GIT_*`, …).
- */
-const RESERVED_ENV_NAMES = new Set([
-  'PATH',
-  'HOME',
-  'NODE_OPTIONS',
-  'NODE_PATH',
-  'NODE_EXTRA_CA_CERTS',
-  'LD_PRELOAD',
-  'LD_LIBRARY_PATH',
-  'BASH_ENV',
-  'ENV',
-  'SHELL',
-  'IFS',
-])
-
-/**
- * Env-var name PREFIXES never injected from a frontend binding. `npm_config_*` reconfigures the
- * package manager (registry, scripts, prefix), and `GIT_*` reconfigures git — both run during a
- * frontend install/build, so a binding in either family is toolchain control, not an upstream URL.
- * Compared case-INSENSITIVELY (lower-cased here, matched lower-cased below): npm reads its config
- * env with a case-insensitive `/^npm_config_/i`, so `NPM_CONFIG_REGISTRY` is honoured just like
- * `npm_config_registry` — a case-sensitive prefix match would let the upper-cased form slip through.
- */
-const RESERVED_ENV_PREFIXES = ['npm_config_', 'git_']
-
-/**
- * Whether an env-var name is reserved (an exact name, or a reserved family prefix). The exact
- * names are canonical upper-case env vars matched verbatim (Linux env is case-sensitive, so a
- * distinct lower-cased `home` is a different, harmless var); the family PREFIXES are matched
- * case-insensitively because npm interprets `npm_config_*` regardless of case (see above).
- */
-function isReservedEnvName(key: string): boolean {
-  if (RESERVED_ENV_NAMES.has(key)) return true
-  const lower = key.toLowerCase()
-  return RESERVED_ENV_PREFIXES.some((p) => lower.startsWith(p))
-}
-
-/**
- * Collect only string→string entries from a raw `env` bag. A non-string value is dropped so a
- * malformed binding can't inject `[object Object]` (or undefined) as an upstream URL. Reserved
- * names that would break the toolchain or enable injection (PATH, NODE_OPTIONS, LD_PRELOAD, …) are
- * dropped too: they are spread over `process.env` at build time, so a binding named `PATH` would
- * replace it with a URL and the build would no longer find its tools. Extracted from the infra
- * parsers to keep their cyclomatic complexity down.
- */
-function parseInfraEnv(raw: unknown): Record<string, string> {
-  const env: Record<string, string> = {}
-  if (typeof raw === 'object' && raw !== null) {
-    for (const [key, val] of Object.entries(raw as Record<string, unknown>)) {
-      if (key && !isReservedEnvName(key) && typeof val === 'string') env[key] = val
-    }
-  }
-  return env
-}
-
 /** Parse the frontend UI-test infra spec (`kind: 'frontend'`), tolerating missing knobs. */
 function parseFrontendInfraSpec(o: Record<string, unknown>): FrontendInfraSpec {
   const packageManager =
@@ -1321,7 +1232,8 @@ export function parseAgentJob(input: unknown): AgentJob {
     packageRegistries: parsePackageRegistries(o.packageRegistries),
     skills: parseSkillSpecs(o.skills),
     mcpServers: parseMcpServerSpecs(o.mcpServers),
-    testSecrets: parseTestSecrets(o.testSecrets),
+    testSecrets: parseSecretEnvPairs(o.testSecrets, 'testSecrets'),
+    generatorSecrets: parseSecretEnvPairs(o.generatorSecrets, 'generatorSecrets'),
     guardLimits: parseGuardLimits(o.guardLimits),
     validation: parseValidationSpec(o.validation),
     validationChecks: parseValidationChecksSpec(o.validationChecks),
@@ -1362,7 +1274,8 @@ interface ParsedAgentJobParts {
   packageRegistries: ReturnType<typeof parsePackageRegistries>
   skills: ReturnType<typeof parseSkillSpecs>
   mcpServers: ReturnType<typeof parseMcpServerSpecs>
-  testSecrets: ReturnType<typeof parseTestSecrets>
+  testSecrets: ReturnType<typeof parseSecretEnvPairs>
+  generatorSecrets: ReturnType<typeof parseSecretEnvPairs>
   guardLimits: ReturnType<typeof parseGuardLimits>
   validation: ReturnType<typeof parseValidationSpec>
   validationChecks: ReturnType<typeof parseValidationChecksSpec>
@@ -1425,6 +1338,7 @@ function assembleAgentJob(
     reproduction,
     dependencyInstall,
     reviewPrNumber,
+    generatorSecrets,
   } = parts
   const repo = (o.repo ?? {}) as Record<string, unknown>
   return {
@@ -1445,6 +1359,7 @@ function assembleAgentJob(
     ...(skills ? { skills } : {}),
     ...(mcpServers ? { mcpServers } : {}),
     ...(testSecrets.length ? { testSecrets } : {}),
+    ...(generatorSecrets.length ? { generatorSecrets } : {}),
     ...(infra ? { infra } : {}),
     ...(pr ? { pr } : {}),
     ...(peerRepos.length ? { peerRepos } : {}),

package/src/progress-guard.ts

@@ -50,6 +50,31 @@ export interface ProgressGuardLimits {
    * without it.
    */
   maxConsecutiveWebCalls?: number
+  /**
+   * Abort after this many consecutive MCP tool-server calls (`mcp__*`) with no other
+   * tool call in between: the tool-server analogue of `maxConsecutiveWebCalls`, and
+   * present for the same reason. An `mcp__*` call is exempt from the no-edit bound (see
+   * `isMcpToolCall`), so without a streak of its own a run could query a tool server
+   * indefinitely without tripping any guard. Any non-MCP tool call resets the streak.
+   * Optional: defaults to {@link DEFAULT_PROGRESS_GUARD_LIMITS}.
+   */
+  maxConsecutiveMcpCalls?: number
+  /**
+   * Abort after this many consecutive calls that are EXEMPT from the no-edit bound
+   * (planning, read-only exploration, subagent dispatch, `mcp__*`) with no action call
+   * in between. The backstop that makes each individual exemption mean "not counted"
+   * rather than "unbounded": every per-family streak above resets on any call outside
+   * its own family, so a run alternating `web_search` with `mcp__issues__search` (or
+   * with `read`) trips none of them and, having never made an action call, never
+   * reaches `maxToolCallsWithoutEdit` either. Only the job's wall-clock ceiling
+   * bounded that.
+   *
+   * Deliberately far above every family cap, because it is not a research bound and
+   * must not become one: reading a hundred files before the first edit is legitimate
+   * work-up, and any `bash`/edit/action call resets the streak. Optional: defaults to
+   * {@link DEFAULT_PROGRESS_GUARD_LIMITS}.
+   */
+  maxConsecutiveNonActionCalls?: number
 }
 
 // `satisfies` (not a type annotation) so each property keeps its concrete `number`
@@ -63,6 +88,16 @@ export const DEFAULT_PROGRESS_GUARD_LIMITS = {
   // A genuine research burst is a handful of searches; an uninterrupted run of this
   // many web calls (with no read/edit/bash between) is a search loop, not progress.
   maxConsecutiveWebCalls: 25,
+  // Looser than the web cap: a tool server is usually the agent's route to the SYSTEM OF
+  // RECORD (the issue tracker, the advisory database, the design source), and reading a
+  // list and then each of its items is a normal opening move, not a rabbit-hole. A run
+  // that makes this many in a row with no read, edit or bash between is looping.
+  maxConsecutiveMcpCalls: 40,
+  // Well clear of every family cap above, and of any plausible read-up: a run that makes
+  // this many exempt calls with not one action call between them has stopped converging,
+  // whatever mix of reads, searches and lookups it is cycling through. Sized as a
+  // backstop rather than a judgement, because the families are where judgement belongs.
+  maxConsecutiveNonActionCalls: 200,
 } satisfies ProgressGuardLimits
 
 // Tool names that mutate files, so a call to one clears the no-edit suspicion. Kept
@@ -139,6 +174,32 @@ const EXPLORATION_TOOLS = new Set([
 // Pi's `web_search`/`web_fetch` and Claude Code's `WebSearch`/`WebFetch`.
 const WEB_TOOLS = new Set(['web_search', 'web_fetch', 'websearch', 'webfetch'])
 
+// A call to a tool server (MCP). Every MCP client names these `mcp__<server>__<tool>`, and
+// the prefix is the ONLY thing the harness can know about them: what a given server's tools
+// do is a backend registration this image has never seen, so the guard classifies by shape.
+//
+// Matched, rather than enumerated in EXPLORATION_TOOLS, because the set is open: it is
+// whatever tool servers the running kind was wired with. A prefix test is also why this is a
+// function, `name.startsWith` on the already-lower-cased name, so `MCP__Issues__search`
+// classifies the same as `mcp__issues__search`.
+function isMcpToolCall(loweredName: string): boolean {
+  return loweredName.startsWith('mcp__')
+}
+
+// Whether a call is EXEMPT from the no-edit bound: planning and bookkeeping, read-only
+// exploration, a subagent dispatch, or a tool-server call. One predicate rather than the
+// four tests inlined at the branch, because the combined non-action streak and the no-edit
+// exemption must be the SAME set: a family exempted in one place and missed in the other is
+// either an unbounded loop or a run killed for a call the bound says it may make.
+function isNonActionToolCall(loweredName: string): boolean {
+  return (
+    PLANNING_TOOLS.has(loweredName) ||
+    EXPLORATION_TOOLS.has(loweredName) ||
+    SUBAGENT_DISPATCH_TOOLS.has(loweredName) ||
+    isMcpToolCall(loweredName)
+  )
+}
+
 /** Read {@link ProgressGuardLimits} from the environment, falling back to the defaults. */
 export function progressGuardLimitsFromEnv(
   env: NodeJS.ProcessEnv = process.env,
@@ -160,6 +221,14 @@ export function progressGuardLimitsFromEnv(
       env.JOB_MAX_CONSECUTIVE_WEB_CALLS,
       DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveWebCalls,
     ),
+    maxConsecutiveMcpCalls: num(
+      env.JOB_MAX_CONSECUTIVE_MCP_CALLS,
+      DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveMcpCalls,
+    ),
+    maxConsecutiveNonActionCalls: num(
+      env.JOB_MAX_CONSECUTIVE_NON_ACTION_CALLS,
+      DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveNonActionCalls,
+    ),
   }
 }
 
@@ -186,12 +255,21 @@ export function mergeGuardLimits(
       overrides.maxToolCallsWithoutEdit,
     ),
     maxConsecutiveErrors: loosen(base.maxConsecutiveErrors, overrides.maxConsecutiveErrors),
-    // `maxConsecutiveWebCalls` is optional on the interface (callers may omit it), so
-    // fall back to the default before loosening — keeps `loosen`'s base a concrete number.
+    // The streak knobs are optional on the interface (callers may omit them), so fall back
+    // to the default before loosening: it keeps `loosen`'s base a concrete number.
     maxConsecutiveWebCalls: loosen(
       base.maxConsecutiveWebCalls ?? DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveWebCalls,
       overrides.maxConsecutiveWebCalls,
     ),
+    maxConsecutiveMcpCalls: loosen(
+      base.maxConsecutiveMcpCalls ?? DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveMcpCalls,
+      overrides.maxConsecutiveMcpCalls,
+    ),
+    maxConsecutiveNonActionCalls: loosen(
+      base.maxConsecutiveNonActionCalls ??
+        DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveNonActionCalls,
+      overrides.maxConsecutiveNonActionCalls,
+    ),
   }
 }
 
@@ -207,6 +285,8 @@ export class ProgressGuard {
   private edits = 0
   private consecutiveErrors = 0
   private consecutiveWebCalls = 0
+  private consecutiveMcpCalls = 0
+  private consecutiveNonActionCalls = 0
 
   constructor(
     private readonly limits: ProgressGuardLimits,
@@ -257,16 +337,55 @@ export class ProgressGuard {
       this.consecutiveWebCalls = 0
     }
 
-    // Planning, read-only exploration and subagent-dispatch calls don't count toward the
-    // no-edit bound (see PLANNING_TOOLS / EXPLORATION_TOOLS / SUBAGENT_DISPATCH_TOOLS) —
-    // only "action" calls without an edit do.
-    if (
-      PLANNING_TOOLS.has(name) ||
-      EXPLORATION_TOOLS.has(name) ||
-      SUBAGENT_DISPATCH_TOOLS.has(name)
-    ) {
+    // Tool-server (MCP) calls: bounded as their own streak for exactly the reason the web
+    // streak exists. They are exempt from the no-edit bound below, and an exemption with no
+    // counter-bound is a loop the guard cannot see. Any non-MCP call resets it.
+    if (isMcpToolCall(name)) {
+      this.consecutiveMcpCalls++
+      const mcpCap =
+        this.limits.maxConsecutiveMcpCalls ?? DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveMcpCalls
+      if (this.consecutiveMcpCalls >= mcpCap) {
+        return (
+          `no progress: ${this.consecutiveMcpCalls} consecutive tool-server (MCP) calls without ` +
+          `any other action. The agent is stuck querying its tools instead of doing the work. ` +
+          `Aborting.`
+        )
+      }
+    } else {
+      this.consecutiveMcpCalls = 0
+    }
+
+    // Planning, read-only exploration, subagent-dispatch and tool-server calls don't count
+    // toward the no-edit bound (see `isNonActionToolCall`): only "action" calls without an
+    // edit do.
+    //
+    // An `mcp__*` call is exempt for the same reason a `read` is: the bound targets the
+    // credential rabbit-hole (endless `bash` probing with nothing implemented), and reaching
+    // a registered tool server is the platform TELLING the agent to look something up
+    // ("prefer them over guessing"). Counting them would abort an edits-expected kind for
+    // consulting the issue tracker the deployment wired for it, punishing the run for
+    // following its own prompt. They are neutral rather than edit-satisfying, exactly like a
+    // subagent dispatch: a read-only lookup must not clear the suspicion the bound holds.
+    //
+    // The exempt calls carry ONE streak of their own, and it is what keeps every exemption
+    // above from adding up to an unbounded run: each per-family cap resets on any call
+    // outside its family, so alternating two exempt families trips neither, and a run that
+    // never makes an action call never reaches the no-edit bound either.
+    if (isNonActionToolCall(name)) {
+      this.consecutiveNonActionCalls++
+      const nonActionCap =
+        this.limits.maxConsecutiveNonActionCalls ??
+        DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveNonActionCalls
+      if (this.consecutiveNonActionCalls >= nonActionCap) {
+        return (
+          `no progress: ${this.consecutiveNonActionCalls} consecutive read-only calls (searching, ` +
+          `reading, tool-server lookups, subagent dispatches) with no action call between them. ` +
+          `The agent is cycling through research instead of doing the work. Aborting.`
+        )
+      }
       return null
     }
+    this.consecutiveNonActionCalls = 0
     this.toolCalls++
     if (FILE_EDIT_TOOLS.has(name)) this.edits++