@cat-factory/executor-harness 1.64.4 → 1.68.0

package/src/job.ts

@@ -12,6 +12,16 @@ import {
   type ReproductionReport,
   type ReproductionSpec,
 } from './reproduction-proof.js'
+import {
+  parseMcpServerSpecs,
+  parseSkillSpecs,
+  type McpServerSpec,
+  type SkillResourceSpec,
+  type SkillSpec,
+} from './agent-capabilities.js'
+
+// Re-exported so the job body stays the one import site for a harness handler describing a job.
+export type { McpServerSpec, SkillResourceSpec, SkillSpec }
 
 // The job the Worker's ContainerAgentExecutor POSTs to /run. Kept as plain
 // types with a hand-rolled validator so the image needs no schema dependency.
@@ -632,26 +642,6 @@ export interface ContextFileSpec {
   content: string
 }
 
-/** One materialisable resource file of a skill (repo-sourced Claude Skills). */
-export interface SkillResourceSpec {
-  /** Path within the skill directory, e.g. `templates/report.md` (subdirs preserved, no traversal). */
-  relPath: string
-  content: string
-}
-
-/**
- * A repo-sourced Claude Skill to make available for a `skill` step. Materialised HARNESS-AWARE:
- * `CLAUDE_CONFIG_DIR/skills/<name>/SKILL.md` (+ resources) for the claude-code CLI to load
- * natively, or `.cat-context/skill/<relPath>` for the Pi/codex checkout (their prompt carries the
- * instructions). A dedicated top-level body field (like `packageRegistries`), never a context file.
- */
-export interface SkillSpec {
-  name: string
-  description: string
-  instructions: string
-  resources: SkillResourceSpec[]
-}
-
 /** How an explore agent's reply is consumed. */
 export interface AgentOutputSpec {
   /** `prose` keeps the reply text; `structured` parses (and optionally repairs) it to JSON. */
@@ -739,11 +729,20 @@ export interface AgentJob extends HarnessAuthFields {
    */
   packageRegistries?: PackageRegistrySpec[]
   /**
-   * A repo-sourced Claude Skill to make available for a `skill` step (see {@link SkillSpec}).
-   * Materialised harness-aware before the run: natively into `CLAUDE_CONFIG_DIR/skills/<name>/`
-   * for claude-code, or `.cat-context/skill/<relPath>` for Pi/codex. Absent ⇒ no skill installed.
+   * The skills to make available for this run (see {@link SkillSpec}) — a `skill` step's picked
+   * skill and/or the playbooks the running agent kind declares. Materialised harness-aware before
+   * the run: natively into `CLAUDE_CONFIG_DIR/skills/<name>/` for claude-code, or
+   * `.cat-context/skill/<name>/<relPath>` for Pi/codex. Absent ⇒ no skills installed.
    */
-  skill?: SkillSpec
+  skills?: SkillSpec[]
+  /**
+   * Tool servers (MCP) to wire into the agent CLI for this run (see {@link McpServerSpec}). The
+   * backend has already dropped anything this harness cannot serve, so every entry here is
+   * expected to work. SECRET-BEARING (`env`/`headers` carry resolved credentials), so the config
+   * files written from it live outside the checkout and are never logged. Absent ⇒ the CLI's
+   * built-in tools only.
+   */
+  mcpServers?: McpServerSpec[]
   /**
    * Tester kinds only: sensitive test credentials injected into the run's ENVIRONMENT (out of
    * band) as `{ key, value }` env pairs, so the tester's shell can read `$KEY` without the value
@@ -942,6 +941,12 @@ export interface AgentResult {
     exitCode: number
     validationOutputTail?: string
     iteration?: number
+    /**
+     * The work-branch HEAD the command was judged against. The engine compares it across
+     * consecutive failing iterations to end a loop that has stopped committing anything,
+     * instead of spending the rest of its budget re-learning that. Absent when unreadable.
+     */
+    headSha?: string
   }
   /**
    * Coding mode (multi-repo): the PRs opened in the connected services' PEER repos, one per
@@ -1029,71 +1034,6 @@ function parseContextFiles(value: unknown): ContextFileSpec[] {
   return files
 }
 
-/**
- * Sanitize a skill resource's relative path: keep the subdirectory structure (so
- * `templates/report.md` materialises nested) but reject anything that could escape the skill
- * directory — absolute paths, `..` traversal, backslashes, empty/dot segments. Returns undefined
- * for an unsafe path (the resource is then dropped).
- */
-function sanitizeSkillRelPath(value: unknown): string | undefined {
-  if (typeof value !== 'string') return undefined
-  const segments = value.replace(/\\/g, '/').split('/')
-  const clean: string[] = []
-  for (const seg of segments) {
-    if (seg === '' || seg === '.') continue
-    if (seg === '..') return undefined
-    // Same character class as a context-file name, per segment.
-    const c = seg.replace(/[^A-Za-z0-9._-]/g, '')
-    if (!c || c === '.' || c === '..' || c.startsWith('.')) return undefined
-    clean.push(c)
-  }
-  return clean.length ? clean.join('/') : undefined
-}
-
-/**
- * Fallback native-skill directory name when the authored name has no id-safe characters (e.g. a
- * purely non-ASCII skill name). The name is only a path segment / manifest label, so a safe
- * default keeps the skill installable rather than dropping it — which, on the claude-code path,
- * would leave the prompt pointing at a skill that was never installed (a blind run).
- */
-const FALLBACK_SKILL_NAME = 'skill'
-
-/** A skill's own directory name, sanitized to a safe single path segment (undefined if empty). */
-function sanitizeSkillName(value: unknown): string | undefined {
-  if (typeof value !== 'string') return undefined
-  const base = value.replace(/\\/g, '/').split('/').pop() ?? ''
-  const cleaned = base.replace(/[^A-Za-z0-9._-]/g, '')
-  if (!cleaned || cleaned === '.' || cleaned === '..' || cleaned.startsWith('.')) return undefined
-  return cleaned
-}
-
-/** Validate the optional `skill` field, or undefined when absent/malformed. */
-function parseSkillSpec(value: unknown): SkillSpec | undefined {
-  if (typeof value !== 'object' || value === null) return undefined
-  const o = value as Record<string, unknown>
-  const instructions = typeof o.instructions === 'string' ? o.instructions : undefined
-  // No instructions ⇒ there is nothing to run — drop the skill (the prompt still carries the
-  // folded-in directive on the Pi/codex path). An unsafe/empty NAME only affects the install
-  // directory, so fall back to a safe default rather than dropping the whole skill.
-  if (!instructions) return undefined
-  const name = sanitizeSkillName(o.name) ?? FALLBACK_SKILL_NAME
-  const description = typeof o.description === 'string' ? o.description : ''
-  const resources: SkillResourceSpec[] = []
-  if (Array.isArray(o.resources)) {
-    const used = new Set<string>()
-    for (const entry of o.resources) {
-      if (typeof entry !== 'object' || entry === null) continue
-      const e = entry as Record<string, unknown>
-      const relPath = sanitizeSkillRelPath(e.relPath)
-      if (!relPath || used.has(relPath)) continue
-      if (typeof e.content !== 'string') continue
-      used.add(relPath)
-      resources.push({ relPath, content: e.content })
-    }
-  }
-  return { name, description, instructions, resources }
-}
-
 /** Parse the explore-mode infra stand-up spec, or undefined when absent/unrecognised. */
 function parseAgentInfraSpec(value: unknown): AgentInfraSpec | undefined {
   if (typeof value !== 'object' || value === null) return undefined
@@ -1251,7 +1191,20 @@ export interface InlineResult {
   text: string
   /** `length` when the model hit its output cap (the reviewer rejects a truncated doc). */
   finishReason?: 'stop' | 'length'
-  usage?: { inputTokens: number; outputTokens: number }
+  /**
+   * The job's token usage with the input side split into its three ORTHOGONAL classes:
+   * `inputTokens` is FRESH input only, so the total input is
+   * `inputTokens + cacheReadTokens + cacheWriteTokens`. Folded from the per-call metrics below,
+   * which is the only channel that knows the split; a CLI that streamed none falls back to the
+   * coarse total with both cache classes 0 — honest, since on that shape nothing is known to
+   * have been cached.
+   */
+  usage?: {
+    inputTokens: number
+    cacheReadTokens: number
+    cacheWriteTokens: number
+    outputTokens: number
+  }
   /** Per-model-call telemetry lifted from the CLI stream (recorded into `llm_call_metrics`). */
   callMetrics?: HarnessCallMetric[]
   /** A structured failure marks a job-level failure even on a clean HTTP exit (see JobResultBase). */
@@ -1322,7 +1275,8 @@ export function parseAgentJob(input: unknown): AgentJob {
     bootstrap: parseAgentBootstrapSpec(o.bootstrap),
     contextFiles: parseContextFiles(o.contextFiles),
     packageRegistries: parsePackageRegistries(o.packageRegistries),
-    skill: parseSkillSpec(o.skill),
+    skills: parseSkillSpecs(o.skills),
+    mcpServers: parseMcpServerSpecs(o.mcpServers),
     testSecrets: parseTestSecrets(o.testSecrets),
     guardLimits: parseGuardLimits(o.guardLimits),
     validation: parseValidationSpec(o.validation),
@@ -1361,7 +1315,8 @@ interface ParsedAgentJobParts {
   bootstrap: ReturnType<typeof parseAgentBootstrapSpec>
   contextFiles: ReturnType<typeof parseContextFiles>
   packageRegistries: ReturnType<typeof parsePackageRegistries>
-  skill: ReturnType<typeof parseSkillSpec>
+  skills: ReturnType<typeof parseSkillSpecs>
+  mcpServers: ReturnType<typeof parseMcpServerSpecs>
   testSecrets: ReturnType<typeof parseTestSecrets>
   guardLimits: ReturnType<typeof parseGuardLimits>
   validation: ReturnType<typeof parseValidationSpec>
@@ -1415,7 +1370,8 @@ function assembleAgentJob(
     bootstrap,
     contextFiles,
     packageRegistries,
-    skill,
+    skills,
+    mcpServers,
     testSecrets,
     guardLimits,
     validation,
@@ -1439,7 +1395,8 @@ function assembleAgentJob(
     ...(output ? { output } : {}),
     ...(contextFiles.length ? { contextFiles } : {}),
     ...(packageRegistries.length ? { packageRegistries } : {}),
-    ...(skill ? { skill } : {}),
+    ...(skills ? { skills } : {}),
+    ...(mcpServers ? { mcpServers } : {}),
     ...(testSecrets.length ? { testSecrets } : {}),
     ...(infra ? { infra } : {}),
     ...(pr ? { pr } : {}),

package/src/pi-workspace.ts

@@ -1,7 +1,8 @@
 import { mkdir, mkdtemp, readdir, rm, stat } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import type { RepoSpec, SkillSpec } from './job.js'
+import type { RepoSpec } from './job.js'
+import type { McpServerSpec, SkillSpec } from './agent-capabilities.js'
 import { readEffortReport } from './effort.js'
 import { log } from './logger.js'
 import {
@@ -206,12 +207,20 @@ export interface AgentRunSpec {
    */
   contextFiles?: ContextFileInfo[]
   /**
-   * A repo-sourced Claude Skill to make available for this run (slice 2). Installed HARNESS-AWARE:
-   * the claude-code runner writes it natively into the config dir's `skills/`; for Pi/codex the
-   * resource files are materialised under `.cat-context/skill/` (their prompt already carries the
-   * folded-in instructions). Absent ⇒ no skill.
+   * The skills to make available for this run — a `skill` step's picked skill and/or the playbooks
+   * the running agent kind declares. Installed HARNESS-AWARE: the claude-code runner writes them
+   * natively into the config dir's `skills/`; for Pi/codex the resource files are materialised
+   * under `.cat-context/skill/<name>/` (their prompt already carries the folded-in instructions).
+   * Absent ⇒ no skills.
    */
-  skill?: SkillSpec
+  skills?: SkillSpec[]
+  /**
+   * Tool servers (MCP) to wire into the agent CLI. Served by the subscription harnesses only —
+   * Pi has no MCP client, and the BACKEND is what decides that (it drops an unservable server and
+   * tells the agent so), which is why this path simply forwards whatever it is given rather than
+   * re-deciding. Absent ⇒ the CLI's built-in tools only.
+   */
+  mcpServers?: McpServerSpec[]
   /**
    * Enable proxy-backed web search: point the rpiv-web-tools SearXNG provider at the
    * backend's search proxy (`${proxyBaseUrl}/web-search`) with the session token as
@@ -268,14 +277,14 @@ export async function runAgentInWorkspace(
   // harness paths; kept out of the agent's commits via a local git exclude entry.
   const contextFiles = spec.contextFiles ?? []
   await materializeContextFiles(spec.dir, contextFiles)
-  // Repo-sourced skill (slice 2): claude-code installs it natively into its ISOLATED config dir,
-  // so it reads from there. Everything else reads the checkout, so materialise the skill's
-  // resources under `.cat-context/skill/` (its instructions are folded into the prompt by the
-  // backend) — Pi, codex, and AMBIENT claude-code, which has no isolated config dir to install
-  // into (the runner refuses to write a repo's skill into the developer's own `~/.claude`; see
-  // `runClaudeCode`). A resource-free skill is a no-op here.
-  if (spec.skill && !installsSkillNatively(spec)) {
-    await materializeSkillResources(spec.dir, spec.skill)
+  // Skills: claude-code installs them natively into its ISOLATED config dir, so it reads from
+  // there. Everything else reads the checkout, so materialise each skill's resources under
+  // `.cat-context/skill/<name>/` (their instructions are folded into the prompt by the backend) —
+  // Pi, codex, and AMBIENT claude-code, which has no isolated config dir to install into (the
+  // runner refuses to write a skill into the developer's own `~/.claude`; see `runClaudeCode`).
+  // Resource-free skills are a no-op here.
+  if (spec.skills?.length && !installsSkillNatively(spec)) {
+    await materializeSkillResources(spec.dir, spec.skills)
   }
 
   // Subscription harnesses (Claude Code / Codex) authenticate with the leased
@@ -296,7 +305,8 @@ export async function runAgentInWorkspace(
       ...(spec.subscriptionToken ? { subscriptionToken: spec.subscriptionToken } : {}),
       subscriptionBaseUrl: spec.subscriptionBaseUrl,
       ...(spec.ambientAuth ? { ambientAuth: true } : {}),
-      ...(spec.skill ? { skill: spec.skill } : {}),
+      ...(spec.skills?.length ? { skills: spec.skills } : {}),
+      ...(spec.mcpServers?.length ? { mcpServers: spec.mcpServers } : {}),
       ...(opts.agentEnv ? { extraEnv: opts.agentEnv } : {}),
       signal: opts.signal,
       // Run the SAME no-progress guard Pi gets (previously claude-code/codex had none): env
@@ -367,12 +377,12 @@ export async function runAgentInWorkspace(
 }
 
 /**
- * Whether the claude-code runner will install this run's repo-sourced skill natively (into the
- * CLI's config dir) rather than the caller materialising it into the checkout. True ONLY for a
+ * Whether the claude-code runner will install this run's skills natively (into the CLI's config
+ * dir) rather than the caller materialising them into the checkout. True ONLY for a
  * leased-credential claude-code run, which gets a throwaway per-run config home. An AMBIENT run
- * uses the developer's own `~/.claude`, which the runner will not write a repo's skill into —
- * it would outlive the run in their personal setup, and two concurrent jobs carrying same-named
- * skills from different repos would overwrite each other's.
+ * uses the developer's own `~/.claude`, which the runner will not write a skill into — it would
+ * outlive the run in their personal setup, and two concurrent jobs carrying same-named skills
+ * would overwrite each other's.
  */
 export function installsSkillNatively(
   spec: Pick<AgentRunSpec, 'harness' | 'ambientAuth'>,

package/src/pi.ts

@@ -253,29 +253,37 @@ export async function materializeContextFiles(
   }
 }
 
-/** Subdirectory of {@link CONTEXT_DIR} where a repo-sourced skill's resources are materialised. */
+/** Subdirectory of {@link CONTEXT_DIR} where a skill's resources are materialised, per skill. */
 export const SKILL_CONTEXT_SUBDIR = 'skill'
 
 /**
- * Materialise a repo-sourced skill's RESOURCE files under `.cat-context/skill/` in the checkout
- * (repo-sourced Claude Skills, slice 2) — the path for every run that does NOT get a native
- * install: Pi, codex, and ambient claude-code (no isolated `CLAUDE_CONFIG_DIR` to install into).
- * Their agents read the checkout, and the skill's instructions are folded into their prompt by the
- * backend (`renderSkillForHarness`, which keys off ambient auth as well as the harness). Resource sub-paths were sanitized at the job boundary (no traversal), so nested
- * dirs are created as needed. Kept out of the agent's commits via the same `.cat-context/` git
- * exclude entry. A skill with no resource bodies is a no-op.
+ * Materialise the run's skills' RESOURCE files under `.cat-context/skill/<name>/` in the checkout
+ * — the path for every run that does NOT get a native install: Pi, codex, and ambient claude-code
+ * (no isolated `CLAUDE_CONFIG_DIR` to install into). Their agents read the checkout, and the
+ * skills' instructions are folded into their prompt by the backend (`renderSkillsForHarness`,
+ * which keys off ambient auth as well as the harness).
+ *
+ * Each skill gets its OWN subdirectory: several skills can apply to one run (a step's pick plus
+ * the kind's declared playbooks), and a flat directory would let two skills' `templates/report.md`
+ * overwrite each other — silently handing the agent the wrong template. The names were sanitized
+ * to a single safe path segment at the job boundary, as were the resource sub-paths (no
+ * traversal), so nested dirs are created as needed. Kept out of the agent's commits via the same
+ * `.cat-context/` git exclude entry. Skills with no resource bodies are a no-op.
  */
 export async function materializeSkillResources(
   cwd: string,
-  skill: { resources: { relPath: string; content: string }[] },
+  skills: { name: string; resources: { relPath: string; content: string }[] }[],
 ): Promise<void> {
-  if (!skill.resources.length) return
-  const dir = join(cwd, CONTEXT_DIR, SKILL_CONTEXT_SUBDIR)
-  await mkdir(dir, { recursive: true })
-  for (const r of skill.resources) {
-    const dest = join(dir, r.relPath)
-    await mkdir(dirname(dest), { recursive: true })
-    await writeFile(dest, r.content, 'utf8')
+  const withResources = skills.filter((s) => s.resources.length)
+  if (!withResources.length) return
+  for (const skill of withResources) {
+    const dir = join(cwd, CONTEXT_DIR, SKILL_CONTEXT_SUBDIR, skill.name)
+    await mkdir(dir, { recursive: true })
+    for (const r of skill.resources) {
+      const dest = join(dir, r.relPath)
+      await mkdir(dirname(dest), { recursive: true })
+      await writeFile(dest, r.content, 'utf8')
+    }
   }
   const gitRoot = await findGitRoot(cwd)
   if (!gitRoot) return
@@ -509,8 +517,21 @@ export interface HarnessCallMetric {
   responseText: string
   /** The reasoning/thinking trace, as a plain string (`''` when none). */
   reasoningText: string
+  /**
+   * FRESH (uncached) input tokens: exclusive of BOTH cache classes below, so the three
+   * are orthogonal and additive. Every producer normalises to this — reading the already
+   * exclusive field where the vendor reports the classes apart (Anthropic), subtracting
+   * the cached share where the vendor reports an inclusive prompt count (Codex/OpenAI).
+   */
   inputTokens: number
-  cachedInputTokens: number
+  /** Input tokens served from the vendor's prompt cache (~0.1× base input). */
+  cacheReadTokens: number
+  /**
+   * Input tokens written INTO the vendor's cache (1.25–2× base input — dearer than fresh),
+   * kept apart from the reads so a loop that keeps re-writing the prefix is distinguishable
+   * from one riding a warm cache. 0 where the CLI reports no separate write class.
+   */
+  cacheWriteTokens: number
   outputTokens: number
   /** The provider finish/stop reason when the CLI reports one (else null). */
   finishReason: string | null

package/src/subagents.ts

@@ -236,7 +236,16 @@ export function startSubagentWatcher(root: string, opts: SubagentWatcherOptions)
     if (event.type !== 'assistant' || !isObject(event.message)) return
     const message = event.message as Record<string, unknown>
     const u = claudeCallUsage(message.usage)
-    if (u.inputTokens === 0 && u.outputTokens === 0) return
+    // Every input class counts towards "did this turn report usage at all": a turn riding a
+    // warm cache legitimately reports 0 fresh input, and skipping it would drop precisely the
+    // cache-heavy calls this telemetry exists to weigh.
+    if (
+      u.inputTokens === 0 &&
+      u.cacheReadTokens === 0 &&
+      u.cacheWriteTokens === 0 &&
+      u.outputTokens === 0
+    )
+      return
     const content = Array.isArray(message.content) ? message.content : []
     const { text, reasoning } = claudeAssistantContent(content)
     publishCallMetric(
@@ -254,13 +263,18 @@ export function startSubagentWatcher(root: string, opts: SubagentWatcherOptions)
         responseText: redactBody(text, secrets),
         reasoningText: redactBody(reasoning, secrets),
         inputTokens: u.inputTokens,
-        cachedInputTokens: u.cachedInputTokens,
+        cacheReadTokens: u.cacheReadTokens,
+        cacheWriteTokens: u.cacheWriteTokens,
         outputTokens: u.outputTokens,
         finishReason: typeof message.stop_reason === 'string' ? message.stop_reason : null,
       },
       opts.onCallMetric,
     )
-    usage.inputTokens += u.inputTokens
+    // The run-level `usage` is the COARSE rotation-window weight, which counts every billed
+    // input bucket — unlike the per-call metric above, whose `inputTokens` is fresh-only. Sum
+    // all three classes back together here or a cache-heavy subagent looks nearly free to the
+    // rotation.
+    usage.inputTokens += u.inputTokens + u.cacheReadTokens + u.cacheWriteTokens
     usage.outputTokens += u.outputTokens
   }