@cat-factory/executor-harness 1.66.0 → 1.70.0

package/src/pi.ts

@@ -33,6 +33,66 @@ import {
  */
 export const PI_MAX_OUTPUT_TOKENS = 32_768
 
+/**
+ * Longest phase label the backend keeps. Mirrors kernel's `MAX_PHASE_CHARS`; see
+ * {@link normalizeProxyPhase} for why this is a copy rather than an import.
+ */
+const MAX_PHASE_CHARS = 32
+
+/**
+ * Normalise a phase label to what the backend will actually store: trimmed, lowercased,
+ * `[a-z0-9-]` only, bounded. `''` when the label is not a phase at all.
+ *
+ * A deliberate COPY of kernel's `normalizeCallPhase` — the container image is built from `src/`
+ * plus typescript alone, so the harness can carry no runtime dependency on a workspace package
+ * (the same constraint that forced `src/host-markdown.ts`). A copy that can drift is worse than
+ * no copy: if the harness rejected a label the backend would have accepted, the call would take
+ * the plain path and land unattributed, and if it accepted one the backend rejects it would
+ * spend a request on a segment destined for `''`. `test/llm-phase.conformity.test.ts` pins the
+ * two to identical verdicts over a corpus, so the alphabet can only be changed in both.
+ */
+export function normalizeProxyPhase(phase: string | undefined): string {
+  if (typeof phase !== 'string') return ''
+  const trimmed = phase.trim().toLowerCase()
+  if (!trimmed || trimmed.length > MAX_PHASE_CHARS) return ''
+  return /^[a-z0-9-]+$/.test(trimmed) ? trimmed : ''
+}
+
+/**
+ * Point Pi's provider at the phase-tagged completions path for the pass about to run, so the
+ * backend can stamp WHICH slice of the run spent each call (the agent's own loop vs a pre-PR
+ * validation repair round vs a reproduction-proof repair round) — see
+ * `docs/initiatives/token-burn-instrumentation.md`. The harness drives those loops, so it is
+ * the only component that knows; reconstructing the boundary downstream from wall-clock
+ * timestamps is exactly the brittle inference this avoids.
+ *
+ * A URL segment because the harness does not make these requests: Pi does, from a config whose
+ * only per-run knobs are the base URL and the token — there is no per-request header to set.
+ *
+ * `supported` is the BACKEND's declaration that it serves the phase-tagged route, carried on the
+ * job body exactly as `webSearch` carries "point the search tool at my `/web-search`". Without it
+ * this function would encode a routing shape the receiving backend may not have: a runner pool
+ * pins its OWN harness image (`RunnerPoolManifest`), and `LOCAL_HARNESS_IMAGE` overrides the
+ * recommended pin outright, so "the image and the backend are a matched set" holds for the
+ * Cloudflare deployment and nowhere else. An image ahead of its backend would 404 EVERY model
+ * call — a dead run, not degraded telemetry. Absent/false ⇒ the plain path, and the calls land
+ * in the backend's unattributed slice.
+ *
+ * Pure so the join is unit-testable without spawning anything.
+ */
+export function phasedProxyBaseUrl(
+  proxyBaseUrl: string,
+  phase: string | undefined,
+  supported: boolean | undefined,
+): string {
+  if (!supported) return proxyBaseUrl
+  // A label the backend would discard would be sent only to be thrown away, so send the plain
+  // path instead — the call is then honestly unattributed rather than attributed to nothing.
+  const normalized = normalizeProxyPhase(phase)
+  if (!normalized) return proxyBaseUrl
+  return `${proxyBaseUrl.replace(/\/+$/, '')}/phase/${normalized}`
+}
+
 /** Write the Pi provider config that routes all model calls through the proxy. */
 export async function writePiModelsConfig(opts: {
   model: string
@@ -253,29 +313,37 @@ export async function materializeContextFiles(
   }
 }
 
-/** Subdirectory of {@link CONTEXT_DIR} where a repo-sourced skill's resources are materialised. */
+/** Subdirectory of {@link CONTEXT_DIR} where a skill's resources are materialised, per skill. */
 export const SKILL_CONTEXT_SUBDIR = 'skill'
 
 /**
- * Materialise a repo-sourced skill's RESOURCE files under `.cat-context/skill/` in the checkout
- * (repo-sourced Claude Skills, slice 2) — the path for every run that does NOT get a native
- * install: Pi, codex, and ambient claude-code (no isolated `CLAUDE_CONFIG_DIR` to install into).
- * Their agents read the checkout, and the skill's instructions are folded into their prompt by the
- * backend (`renderSkillForHarness`, which keys off ambient auth as well as the harness). Resource sub-paths were sanitized at the job boundary (no traversal), so nested
- * dirs are created as needed. Kept out of the agent's commits via the same `.cat-context/` git
- * exclude entry. A skill with no resource bodies is a no-op.
+ * Materialise the run's skills' RESOURCE files under `.cat-context/skill/<name>/` in the checkout
+ * — the path for every run that does NOT get a native install: Pi, codex, and ambient claude-code
+ * (no isolated `CLAUDE_CONFIG_DIR` to install into). Their agents read the checkout, and the
+ * skills' instructions are folded into their prompt by the backend (`renderSkillsForHarness`,
+ * which keys off ambient auth as well as the harness).
+ *
+ * Each skill gets its OWN subdirectory: several skills can apply to one run (a step's pick plus
+ * the kind's declared playbooks), and a flat directory would let two skills' `templates/report.md`
+ * overwrite each other — silently handing the agent the wrong template. The names were sanitized
+ * to a single safe path segment at the job boundary, as were the resource sub-paths (no
+ * traversal), so nested dirs are created as needed. Kept out of the agent's commits via the same
+ * `.cat-context/` git exclude entry. Skills with no resource bodies are a no-op.
  */
 export async function materializeSkillResources(
   cwd: string,
-  skill: { resources: { relPath: string; content: string }[] },
+  skills: { name: string; resources: { relPath: string; content: string }[] }[],
 ): Promise<void> {
-  if (!skill.resources.length) return
-  const dir = join(cwd, CONTEXT_DIR, SKILL_CONTEXT_SUBDIR)
-  await mkdir(dir, { recursive: true })
-  for (const r of skill.resources) {
-    const dest = join(dir, r.relPath)
-    await mkdir(dirname(dest), { recursive: true })
-    await writeFile(dest, r.content, 'utf8')
+  const withResources = skills.filter((s) => s.resources.length)
+  if (!withResources.length) return
+  for (const skill of withResources) {
+    const dir = join(cwd, CONTEXT_DIR, SKILL_CONTEXT_SUBDIR, skill.name)
+    await mkdir(dir, { recursive: true })
+    for (const r of skill.resources) {
+      const dest = join(dir, r.relPath)
+      await mkdir(dirname(dest), { recursive: true })
+      await writeFile(dest, r.content, 'utf8')
+    }
   }
   const gitRoot = await findGitRoot(cwd)
   if (!gitRoot) return
@@ -540,6 +608,17 @@ export interface HarnessCallMetric {
    * falls back to the array index, which is what it always used before streaming existed.
    */
   seq?: number
+  /**
+   * The run PHASE that spent this call (`agent` / `validation-repair` / `reproduction-repair` /
+   * …), stamped by the job registry from the same marker the handlers set as they enter each
+   * phase — so the phase axis on `llm_call_metrics` comes from the component that owns the
+   * boundary rather than from a downstream guess
+   * (`docs/initiatives/token-burn-instrumentation.md`).
+   *
+   * Stamped on the SAME object as {@link seq}, so the live drain and the terminal result can
+   * never disagree about which phase billed a call.
+   */
+  phase?: string
 }
 
 /**

package/src/runner.ts

@@ -66,6 +66,15 @@ export interface RunOptions {
    * per-phase wall-clock is logged on completion. Free-form; unknown phases just show verbatim.
    */
   onPhase?: (phase: string) => void
+  /**
+   * The phase most recently marked via {@link onPhase} — the read side of the same marker, for
+   * work that has to TELL the backend which phase it is in rather than merely record it. Today
+   * that is the Pi path, whose calls are metered server-side by the LLM proxy: the harness tags
+   * the proxy URL with this so a repair round's spend is attributable
+   * (`docs/initiatives/token-burn-instrumentation.md`). Absent ⇒ no phase is carried and those
+   * calls land in the backend's unattributed slice.
+   */
+  currentPhase?: () => string
   /** A per-job child logger carrying the run's correlation fields (jobId, repo, branch, …). */
   log?: Logger
   /**
@@ -468,9 +477,17 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
           // instance for its terminal result, so both channels carry the same `seq` and the
           // backend mints one stable row id per call.
           call.seq = entry.callMetricSeq++
+          // …and the phase the job is in RIGHT NOW, which is what spent the call: the handlers
+          // mark `validation-repair` / `reproduction-repair` around each repair pass, so a
+          // looped run's telemetry says which loop the tokens went to instead of filing every
+          // turn under one undifferentiated "agent"
+          // (`docs/initiatives/token-burn-instrumentation.md`). Stamped at EMIT time, not at
+          // drain time: a poll can land long after the phase moved on.
+          call.phase = phase
           entry.callMetricBuffer.push(call)
         },
         onPhase: (next) => markPhase(next),
+        currentPhase: () => phase,
         log: jobLog,
       })
       markPhase('done')

package/src/structured-output.ts

@@ -1,6 +1,6 @@
 import { redact, redactSecrets, secretsToRedact } from './redact.js'
 import { log } from './logger.js'
-import { PI_MAX_OUTPUT_TOKENS } from './pi.js'
+import { PI_MAX_OUTPUT_TOKENS, phasedProxyBaseUrl } from './pi.js'
 
 // A reusable abstraction for the "agent returns a structured JSON document as its
 // final assistant message" pattern (requirements, blueprint, merger — and any future
@@ -52,10 +52,23 @@ export interface StructuredOutputSpec<T> {
   parse: (text: string) => T | null
 }
 
+/**
+ * The run phase a structured-output repair call is billed to. A constant, not a `currentPhase`
+ * read: this call is made by the harness itself (the agent has already finished and left text
+ * that won't parse), so it belongs to no pass the registry marks.
+ */
+const STRUCTURED_REPAIR_PHASE = 'structured-repair'
+
 /** Runtime wiring to reach the LLM proxy for the repair call. */
 export interface ProxyAccess {
   /** Pi-harness proxy base URL; absent for subscription harnesses (no proxy repair). */
   proxyBaseUrl?: string
+  /**
+   * The backend serves the phase-tagged completions route, so the repair call can be attributed
+   * to {@link STRUCTURED_REPAIR_PHASE} rather than piling into the unattributed slice
+   * (see {@link HarnessAuthFields.proxyPhasePath}).
+   */
+  proxyPhasePath?: boolean
   /** Pi-harness proxy session token; absent for subscription harnesses. */
   sessionToken?: string
   model: string
@@ -266,7 +279,16 @@ async function callRepair<T>(
   if (!access.proxyBaseUrl || !access.sessionToken) {
     throw new Error('structured-output repair requires the LLM proxy (Pi harness)')
   }
-  const url = `${access.proxyBaseUrl.replace(/\/+$/, '')}/chat/completions`
+  // A repair round is its own slice of the run's burn, not part of the agent's loop that
+  // produced the unparseable text — and unlike the phases the registry marks, this call is made
+  // by the HARNESS itself, so its phase is a constant rather than a read of `currentPhase`
+  // (docs/initiatives/token-burn-instrumentation.md).
+  const repairBaseUrl = phasedProxyBaseUrl(
+    access.proxyBaseUrl,
+    STRUCTURED_REPAIR_PHASE,
+    access.proxyPhasePath,
+  )
+  const url = `${repairBaseUrl.replace(/\/+$/, '')}/chat/completions`
   const messages = [
     { role: 'system', content: REPAIR_SYSTEM },
     {