@cat-factory/executor-harness 1.31.12 → 1.34.0

package/dist/package-registries.js

@@ -0,0 +1,51 @@
+import { chmod, rm, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { registerKnownSecrets } from './redact.js';
+// Private package-registry auth for the checkout's installs (npm private orgs,
+// GitHub Packages). The job's allowlisted entries are rendered into the USER
+// `~/.npmrc` — read by npm, pnpm and yarn v1 alike, and inherited by every child
+// process (the agent's own shell installs and the frontend-infra stand-up's) — so
+// the token never rides argv or the checkout. Written per job; a job with NO
+// entries removes any stale file, because warm-pool containers are reused across
+// jobs and must not leak a prior workspace's token.
+/** Where the per-job npm auth lands (the user npmrc, outside any checkout). */
+export function npmrcPath() {
+    return join(homedir(), '.npmrc');
+}
+/**
+ * Render the job's registry entries as npmrc lines: each scope routed to its
+ * registry, plus one `_authToken` credential line per distinct host.
+ */
+export function renderNpmrc(entries) {
+    const lines = [];
+    const hosts = new Map();
+    for (const entry of entries) {
+        for (const scope of entry.scopes) {
+            lines.push(`${scope}:registry=https://${entry.host}/`);
+        }
+        // Last entry wins per host — entries for the same host carry the same vendor
+        // token in practice (the backend stores one token per entry).
+        hosts.set(entry.host, entry.token);
+    }
+    for (const [host, token] of hosts) {
+        lines.push(`//${host}/:_authToken=${token}`);
+    }
+    return `${lines.join('\n')}\n`;
+}
+/**
+ * Write (or clear) the per-job `~/.npmrc` before the agent runs. Tokens are
+ * registered for output redaction so a token echoed in an npm error never reaches
+ * logs or stored output.
+ */
+export async function configurePackageRegistries(entries) {
+    const path = npmrcPath();
+    if (!entries || entries.length === 0) {
+        await rm(path, { force: true });
+        return;
+    }
+    registerKnownSecrets(entries.map((entry) => entry.token));
+    await writeFile(path, renderNpmrc(entries), { mode: 0o600 });
+    // writeFile's mode only applies on create — tighten an existing file too.
+    await chmod(path, 0o600);
+}

package/dist/redact.js

@@ -28,12 +28,25 @@ const MIN_HARVEST_LEN = 12;
 // `DB_ACCESS_KEY`/`api_key` are covered; `auth` is deliberately excluded so it can't
 // clobber a git `Author:` line. The value is the first whitespace-delimited run.
 const CREDENTIAL_ASSIGNMENT = /\b([A-Za-z0-9_]*(?:password|passwd|pwd|secret|token|key|credential)[A-Za-z0-9_]*\s*[:=]\s*)\S+/gi;
+// Known-secret values registered per JOB (e.g. the job's private-registry tokens),
+// scrubbed on EVERY redaction — including the pattern-only `redactSecrets` call sites
+// that carry no per-call secret list. Accumulating across jobs on a reused container
+// is safe: redaction only ever widens.
+const REGISTERED_SECRETS = new Set();
+/** Register known secret values to scrub on every subsequent redaction. */
+export function registerKnownSecrets(values) {
+    for (const value of values) {
+        if (value && value.length >= MIN_REDACT_LEN)
+            REGISTERED_SECRETS.add(value);
+    }
+}
 /**
  * Strip credentials out of any string before it is logged or stored. Applies the
  * pattern rules (URL userinfo `https://user:pass@host`, `x-access-token:<token>`, bare
  * `ghs_`/`ghp_`/`gho_`/`github_pat_` shapes, and credential-named `KEY=value` / `KEY:
- * value` assignments) and then scrubs every supplied known-secret value. Idempotent —
- * safe to call on already-redacted text.
+ * value` assignments) and then scrubs every supplied known-secret value plus the
+ * module-registered ones ({@link registerKnownSecrets}). Idempotent — safe to call on
+ * already-redacted text.
  */
 export function redact(input, knownSecrets = []) {
     let out = input
@@ -41,14 +54,14 @@ export function redact(input, knownSecrets = []) {
         .replace(/x-access-token:[^@\s]+/gi, 'x-access-token:***')
         .replace(/\b(gh[pso]_|github_pat_)[A-Za-z0-9_]+/g, '$1***')
         .replace(CREDENTIAL_ASSIGNMENT, '$1***');
-    for (const secret of knownSecrets) {
+    for (const secret of [...knownSecrets, ...REGISTERED_SECRETS]) {
         // Guard against scrubbing trivially-short values that would mangle output.
         if (secret.length >= MIN_REDACT_LEN)
             out = out.split(secret).join('***');
     }
     return out;
 }
-/** Pattern-only redaction (no known values). Kept for callers without a secret list. */
+/** Pattern + registered-value redaction. Kept for callers without a per-call secret list. */
 export function redactSecrets(input) {
     return redact(input);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cat-factory/executor-harness",
-  "version": "1.31.12",
+  "version": "1.34.0",
   "description": "Container payload: a thin TypeScript wrapper that runs the Pi coding agent against a cloned repo and opens a PR. Runs in the Cloudflare Container (and, in local native mode, as a host process); carries no secrets.",
   "repository": {
     "type": "git",
@@ -26,8 +26,8 @@
     "hono": "^4.12.27",
     "typescript": "^6.0.3",
     "vitest": "^4.1.9",
-    "@cat-factory/spend": "0.10.73",
-    "@cat-factory/server": "0.67.0"
+    "@cat-factory/server": "0.69.0",
+    "@cat-factory/spend": "0.10.77"
   },
   "scripts": {
     "build": "tsc -p tsconfig.json",

package/src/agent-runner.ts

@@ -2,7 +2,7 @@ import { spawn } from 'node:child_process'
 import { mkdtemp, rm, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import type { PiRunOutcome, PiRunStats, TodoProgress } from './pi.js'
+import type { HarnessCallMetric, PiRunOutcome, PiRunStats, TodoProgress } from './pi.js'
 import { killChildProcess, spawnDetached } from './process.js'
 import { redact, secretsToRedact } from './redact.js'
 
@@ -64,6 +64,29 @@ function isObject(value: unknown): value is Record<string, unknown> {
   return typeof value === 'object' && value !== null
 }
 
+/** Scrub any leased-credential occurrences from a telemetry body (no-op when none). */
+function redactBody(text: string, secrets: string[]): string {
+  return secrets.length ? redact(text, secrets) : text
+}
+
+/**
+ * Fallback token attribution: if a CLI reported a cumulative total but no per-turn
+ * usage (so every captured call has zero tokens), pin the whole total onto the LAST
+ * call rather than dropping it — the run's tokens are still accounted, just not split
+ * per turn. A no-op when the calls already carry per-turn tokens.
+ */
+function attributeCumulativeUsage(
+  calls: HarnessCallMetric[],
+  usage: { inputTokens: number; outputTokens: number } | undefined,
+): void {
+  if (!usage || calls.length === 0) return
+  const anyTokens = calls.some((c) => c.inputTokens > 0 || c.outputTokens > 0)
+  if (anyTokens) return
+  const last = calls[calls.length - 1]!
+  last.inputTokens = usage.inputTokens
+  last.outputTokens = usage.outputTokens
+}
+
 /**
  * Drive one CLI subprocess to completion, streaming LF-framed JSONL from stdout
  * through `onEvent`. Mirrors `runPi`'s lifecycle: prompt over stdin (out-of-band,
@@ -184,25 +207,59 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
   let summary = ''
   let usage: { inputTokens: number; outputTokens: number } | undefined
 
+  // Reconstruct the full per-call request/response bodies for telemetry from the
+  // stream. `--output-format stream-json --verbose` emits each turn as a near-verbatim
+  // Anthropic Messages envelope, so `assistant` events carry the complete response
+  // (text + tool_use blocks + usage), and `user` events carry the tool_result blocks
+  // fed back — together the growing prompt transcript. We seed it with the two inputs
+  // the harness supplies (they never appear in the stream): the system + first user
+  // message. Bodies are credential-scrubbed (they can echo the leased token).
+  const secrets = opts.subscriptionToken ? secretsToRedact(opts.subscriptionToken) : []
+  const messages: Array<{ role: string; content: unknown }> = [
+    { role: 'system', content: opts.systemPrompt },
+    { role: 'user', content: opts.userPrompt },
+  ]
+  const calls: HarnessCallMetric[] = []
+
   const onEvent = (event: Record<string, unknown>): void => {
     const type = event.type
     if (type === 'assistant' && isObject(event.message)) {
-      const content = (event.message as Record<string, unknown>).content
-      if (Array.isArray(content)) {
-        for (const block of content) {
-          if (!isObject(block)) continue
-          if (block.type === 'text' && typeof block.text === 'string') {
-            stats.assistantChars += block.text.length
-          }
-          if (block.type === 'tool_use') {
-            stats.toolCalls += 1
-            if (block.name === 'TodoWrite' && opts.onProgress) {
-              const progress = todosToProgress((block.input as Record<string, unknown>)?.todos)
-              if (progress) opts.onProgress(progress)
-            }
-          }
+      const message = event.message as Record<string, unknown>
+      const content = Array.isArray(message.content) ? message.content : []
+      const { text, reasoning, toolUses } = claudeAssistantContent(content)
+      stats.assistantChars += text.length
+      stats.toolCalls += toolUses
+      for (const block of content) {
+        if (
+          isObject(block) &&
+          block.type === 'tool_use' &&
+          block.name === 'TodoWrite' &&
+          opts.onProgress
+        ) {
+          const progress = todosToProgress((block.input as Record<string, unknown>)?.todos)
+          if (progress) opts.onProgress(progress)
         }
       }
+      // Record this call BEFORE appending its turn: the prompt is the history that
+      // produced this response. The append-only array keeps each call's prompt a strict
+      // prefix of the next, so the backend's telemetry chain delta-compresses cleanly.
+      const u = claudeCallUsage(message.usage)
+      calls.push({
+        ...(typeof message.model === 'string' ? { model: message.model } : {}),
+        promptText: redactBody(JSON.stringify(messages), secrets),
+        messageCount: messages.length,
+        responseText: redactBody(text, secrets),
+        reasoningText: redactBody(reasoning, secrets),
+        inputTokens: u.inputTokens,
+        cachedInputTokens: u.cachedInputTokens,
+        outputTokens: u.outputTokens,
+        finishReason: typeof message.stop_reason === 'string' ? message.stop_reason : null,
+      })
+      messages.push({ role: 'assistant', content })
+    } else if (type === 'user' && isObject(event.message)) {
+      // tool_result blocks the harness fed back to the model — part of the next prompt.
+      const content = (event.message as Record<string, unknown>).content
+      if (Array.isArray(content)) messages.push({ role: 'tool', content })
     } else if (type === 'result') {
       if (typeof event.result === 'string') summary = event.result
       usage = claudeUsage(event.usage) ?? usage
@@ -282,7 +339,14 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
       onEvent,
     )
 
-    return { summary, stats, stderrTail, ...(usage ? { usage } : {}) }
+    attributeCumulativeUsage(calls, usage)
+    return {
+      summary,
+      stats,
+      stderrTail,
+      ...(usage ? { usage } : {}),
+      ...(calls.length ? { callMetrics: calls } : {}),
+    }
   } finally {
     // Never leave the config dir (and any cached credential) on disk past the run.
     if (configHome) await rm(configHome, { recursive: true, force: true }).catch(() => {})
@@ -322,6 +386,44 @@ function claudeUsage(raw: unknown): { inputTokens: number; outputTokens: number
   return { inputTokens: input, outputTokens: output }
 }
 
+/** Pull the text + reasoning out of a Claude `assistant` message's content blocks. */
+function claudeAssistantContent(content: unknown[]): {
+  text: string
+  reasoning: string
+  toolUses: number
+} {
+  let text = ''
+  let reasoning = ''
+  let toolUses = 0
+  for (const block of content) {
+    if (!isObject(block)) continue
+    if (block.type === 'text' && typeof block.text === 'string') text += block.text
+    else if (block.type === 'thinking' && typeof block.thinking === 'string')
+      reasoning += block.thinking
+    else if (block.type === 'tool_use') toolUses += 1
+  }
+  return { text, reasoning, toolUses }
+}
+
+/**
+ * Per-CALL token usage off a Claude `assistant` message's `usage` (this turn only, not
+ * the cumulative `result` total). `inputTokens` counts every billed input bucket (fresh
+ * + both cache buckets); `cachedInputTokens` is the cache share, surfaced separately.
+ */
+function claudeCallUsage(raw: unknown): {
+  inputTokens: number
+  cachedInputTokens: number
+  outputTokens: number
+} {
+  if (!isObject(raw)) return { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 }
+  const cached = numberOf(raw.cache_read_input_tokens) + numberOf(raw.cache_creation_input_tokens)
+  return {
+    inputTokens: numberOf(raw.input_tokens) + cached,
+    cachedInputTokens: cached,
+    outputTokens: numberOf(raw.output_tokens),
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Codex
 // ---------------------------------------------------------------------------
@@ -366,13 +468,33 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
     await writeFile(join(codexHome, 'config.toml'), 'cli_auth_credentials_store = "file"\n', 'utf8')
   }
 
+  // Codex has no system-prompt flag, so fold the composed role + best-practice
+  // context into the prompt itself (Claude Code instead rides --append-system-prompt).
+  const prompt = opts.systemPrompt
+    ? `${opts.systemPrompt}\n\n---\n\n${opts.userPrompt}`
+    : opts.userPrompt
+
+  // Codex's `exec --json` is far thinner than Claude Code's stream: it surfaces only
+  // flat assistant text and (on `token_count` events) the per-turn `last_token_usage`
+  // plus a cumulative total. It never exposes the request transcript or structured
+  // tool/command bodies, so the captured prompt is just the folded input — the response
+  // text + per-turn tokens are faithful; the request side is best-effort by design.
+  const secrets = opts.subscriptionToken ? secretsToRedact(opts.subscriptionToken) : []
+  const messages: Array<{ role: string; content: unknown }> = [{ role: 'user', content: prompt }]
+  const calls: HarnessCallMetric[] = []
+  let pendingText = ''
+
   const onEvent = (event: Record<string, unknown>): void => {
     const type = typeof event.type === 'string' ? event.type : ''
-    if (type.includes('agent_message') || type === 'item.completed') {
+    if (
+      type.includes('agent_message') ||
+      (type === 'item.completed' && isCodexMessageItem(event))
+    ) {
       const text = extractText(event)
       if (text) {
         stats.assistantChars += text.length
         summary = text
+        pendingText = text
       }
     }
     if (type.includes('tool') || type.includes('command') || type.includes('exec')) {
@@ -382,14 +504,26 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
     if (progress && opts.onProgress) opts.onProgress(progress)
     const turnUsage = codexUsage(event)
     if (turnUsage) usage = turnUsage
+    // A `token_count` event closes a model turn: pair its per-turn usage with the
+    // assistant text seen since the previous turn as one telemetry call.
+    const perTurn = codexLastTurnUsage(event)
+    if (perTurn) {
+      calls.push({
+        model: opts.model,
+        promptText: redactBody(JSON.stringify(messages), secrets),
+        messageCount: messages.length,
+        responseText: redactBody(pendingText, secrets),
+        reasoningText: '',
+        inputTokens: perTurn.inputTokens,
+        cachedInputTokens: perTurn.cachedInputTokens,
+        outputTokens: perTurn.outputTokens,
+        finishReason: null,
+      })
+      if (pendingText) messages.push({ role: 'assistant', content: pendingText })
+      pendingText = ''
+    }
   }
 
-  // Codex has no system-prompt flag, so fold the composed role + best-practice
-  // context into the prompt itself (Claude Code instead rides --append-system-prompt).
-  const prompt = opts.systemPrompt
-    ? `${opts.systemPrompt}\n\n---\n\n${opts.userPrompt}`
-    : opts.userPrompt
-
   try {
     const { stderrTail } = await streamCli(
       'codex',
@@ -411,13 +545,53 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
       onEvent,
     )
 
-    return { summary, stats, stderrTail, ...(usage ? { usage } : {}) }
+    // Fallback for a CLI/version that never emits per-turn `last_token_usage`: record a
+    // single call from the cumulative total + final text so the run is still observable.
+    if (calls.length === 0 && (usage || summary)) {
+      calls.push({
+        model: opts.model,
+        promptText: redactBody(JSON.stringify(messages), secrets),
+        messageCount: messages.length,
+        responseText: redactBody(summary, secrets),
+        reasoningText: '',
+        inputTokens: usage?.inputTokens ?? 0,
+        cachedInputTokens: 0,
+        outputTokens: usage?.outputTokens ?? 0,
+        finishReason: null,
+      })
+    }
+    return {
+      summary,
+      stats,
+      stderrTail,
+      ...(usage ? { usage } : {}),
+      ...(calls.length ? { callMetrics: calls } : {}),
+    }
   } finally {
     // Never leave the decrypted credential on disk past the run.
     if (codexHome) await rm(codexHome, { recursive: true, force: true }).catch(() => {})
   }
 }
 
+/**
+ * Whether a Codex `item.completed` event carries the model's ASSISTANT text (as
+ * opposed to a command/exec/tool/reasoning item, which also carry a `text` field —
+ * their command output or thinking — and must NOT be captured as the turn's response).
+ * A message item's kind contains `message` (`agent_message`/`assistant_message`); an
+ * item with no kind is treated as a message so older/simple shapes don't regress.
+ */
+function isCodexMessageItem(event: Record<string, unknown>): boolean {
+  const item = isObject(event.item) ? (event.item as Record<string, unknown>) : undefined
+  if (!item) return false
+  const kind =
+    typeof item.item_type === 'string'
+      ? item.item_type
+      : typeof item.type === 'string'
+        ? item.type
+        : ''
+  return kind === '' || /message/i.test(kind)
+}
+
 /** Best-effort: pull a textual message out of a Codex event. */
 function extractText(event: Record<string, unknown>): string | undefined {
   if (typeof event.message === 'string') return event.message
@@ -456,6 +630,8 @@ function codexPlanProgress(event: Record<string, unknown>): TodoProgress | undef
  * other shapes put it on `usage` / `info.usage` directly. We read the cumulative
  * total when present so the caller can simply overwrite (not sum) — summing
  * cumulative totals across events would multiply-count. Checked most-likely first.
+ * `input_tokens` is the TOTAL prompt count (OpenAI semantics: `cached_input_tokens`
+ * is a subset already inside it), so it is NOT summed with the cached share.
  */
 function codexUsage(
   event: Record<string, unknown>,
@@ -467,12 +643,36 @@ function codexUsage(
     (isObject(event.usage) ? event.usage : undefined) ??
     (info && isObject(info.usage) ? info.usage : undefined)
   if (!isObject(raw)) return undefined
-  const input = numberOf(raw.input_tokens) + numberOf(raw.cached_input_tokens)
+  const input = numberOf(raw.input_tokens)
   const output = numberOf(raw.output_tokens)
   if (input === 0 && output === 0) return undefined
   return { inputTokens: input, outputTokens: output }
 }
 
+/**
+ * Per-TURN Codex token usage off a `token_count` event's `info.last_token_usage` (the
+ * delta for the turn just completed, as opposed to `codexUsage`'s cumulative total).
+ * `input_tokens` is the total prompt count for the turn and already INCLUDES the cached
+ * share (OpenAI semantics), so `cachedInputTokens` is surfaced as the subset it is —
+ * NOT added on top (adding it would double-count every cached token).
+ */
+function codexLastTurnUsage(event: Record<string, unknown>):
+  | {
+      inputTokens: number
+      cachedInputTokens: number
+      outputTokens: number
+    }
+  | undefined {
+  const info = isObject(event.info) ? (event.info as Record<string, unknown>) : undefined
+  const raw = info && isObject(info.last_token_usage) ? info.last_token_usage : undefined
+  if (!isObject(raw)) return undefined
+  const input = numberOf(raw.input_tokens)
+  const cached = numberOf(raw.cached_input_tokens)
+  const output = numberOf(raw.output_tokens)
+  if (input === 0 && output === 0) return undefined
+  return { inputTokens: input, cachedInputTokens: cached, outputTokens: output }
+}
+
 function numberOf(value: unknown): number {
   return typeof value === 'number' && Number.isFinite(value) ? value : 0
 }