@cat-factory/executor-harness 1.50.16 → 1.52.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -35,6 +35,19 @@ are surfaced as `progress` while a job runs. The exact request/response shapes
35
35
  cat-factory sends are documented in
36
36
  [`docs/runner-pool-integration.md`](../../docs/runner-pool-integration.md).
37
37
 
38
+ `GET /jobs/{id}` is also the harness's observability channel: `spans`, `followUps`
39
+ and `callMetrics` are **drain-on-read** — each poll returns what accumulated since
40
+ the previous one and clears the buffer. That is deliberate. A job that dies before
41
+ it can return a terminal result (an evicted container, an OOM-killed process) has
42
+ still reported the tool spans it ran and the model calls it paid for. Each drained
43
+ `callMetrics` entry carries a job-scoped `seq`, and the terminal result repeats the
44
+ complete list, so the backend can take both channels without double-counting a call.
45
+
46
+ Because the backend records a call as soon as it drains it (and ignores the terminal
47
+ repeat), a drained call is FINAL. A call whose tokens are still open — a CLI that reports
48
+ only a cumulative total, costed at the end — is withheld from the drain until it is
49
+ complete; see `createCallMetricPublisher` in `src/pi.ts`.
50
+
38
51
  ## What a job does
39
52
 
40
53
  The implementation job (`POST /run`) is the canonical sequence:
@@ -3,6 +3,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
3
3
  import { homedir, tmpdir } from 'node:os';
4
4
  import { dirname, join } from 'node:path';
5
5
  import { claudeAssistantContent, claudeCallUsage, isObject, numberOf, redactBody, } from './claude-stream.js';
6
+ import { createCallMetricPublisher, publishCallMetric, } from './pi.js';
6
7
  import { killChildProcess, spawnDetached } from './process.js';
7
8
  import { redact, secretsToRedact } from './redact.js';
8
9
  import { createSliceTracker, pickProgress, startSubagentWatcher } from './subagents.js';
@@ -219,6 +220,9 @@ export async function runClaudeCode(opts) {
219
220
  { role: 'user', content: opts.userPrompt },
220
221
  ];
221
222
  const calls = [];
223
+ // Streams each call as the CLI yields it, EXCEPT one whose tokens `attributeCumulativeUsage`
224
+ // may still rewrite below (a published call must be final — see the publisher).
225
+ const publisher = createCallMetricPublisher(calls, opts.onCallMetric);
222
226
  // ADR 0026 D2.1 + ADR 0027 Defect B: surface live slice progress from TWO reconciled
223
227
  // sources. The parent's `Task` dispatches + their terminal tool_results DO appear on this
224
228
  // stream (only a subagent's intermediate turns don't), so `sliceTracker` derives per-slice
@@ -256,7 +260,7 @@ export async function runClaudeCode(opts) {
256
260
  // produced this response. The append-only array keeps each call's prompt a strict
257
261
  // prefix of the next, so the backend's telemetry chain delta-compresses cleanly.
258
262
  const u = claudeCallUsage(message.usage);
259
- calls.push({
263
+ publisher.publish({
260
264
  ...(typeof message.model === 'string' ? { model: message.model } : {}),
261
265
  promptText: redactBody(JSON.stringify(messages), secrets),
262
266
  messageCount: messages.length,
@@ -319,21 +323,7 @@ export async function runClaudeCode(opts) {
319
323
  : join(homedir(), '.claude', 'skills');
320
324
  await writeNativeSkill(skillsRoot, opts.skill).catch(() => { });
321
325
  }
322
- // Anthropic itself authenticates with the subscription OAuth token; a
323
- // non-Anthropic Claude-Code vendor (GLM via Z.ai, Kimi via Moonshot, DeepSeek)
324
- // points Claude Code at its Anthropic-compatible endpoint with an auth-token key.
325
- // Ambient mode injects neither — the CLI uses the developer's logged-in `~/.claude`.
326
- const env = opts.ambientAuth
327
- ? {}
328
- : {
329
- CLAUDE_CONFIG_DIR: configHome,
330
- ...(opts.subscriptionBaseUrl
331
- ? {
332
- ANTHROPIC_BASE_URL: opts.subscriptionBaseUrl,
333
- ANTHROPIC_AUTH_TOKEN: opts.subscriptionToken,
334
- }
335
- : { CLAUDE_CODE_OAUTH_TOKEN: opts.subscriptionToken }),
336
- };
326
+ const env = buildClaudeEnv(opts, configHome);
337
327
  // ADR 0026 D3 (path corrected by ADR 0027 Defect A): while the run is live, tail the CLI's
338
328
  // subagent `*.jsonl` transcripts so a parallel-subagent review keeps the inactivity
339
329
  // heartbeat alive (any new bytes ⇒ `onActivity`) and its otherwise-invisible token spend is
@@ -347,6 +337,7 @@ export async function runClaudeCode(opts) {
347
337
  ...(opts.onActivity ? { onActivity: opts.onActivity } : {}),
348
338
  secrets,
349
339
  model: opts.model,
340
+ ...(opts.onCallMetric ? { onCallMetric: opts.onCallMetric } : {}),
350
341
  ...(opts.log ? { log: opts.log } : {}),
351
342
  })
352
343
  : undefined;
@@ -369,38 +360,15 @@ export async function runClaudeCode(opts) {
369
360
  ...appendArgs,
370
361
  ],
371
362
  }, prompt, opts, env, opts.subscriptionToken ? secretsToRedact(opts.subscriptionToken) : [], onEvent);
372
- // The parent's cumulative-usage fallback applies to the PARENT calls only (before the
373
- // subagent calls, which carry their own per-turn tokens, are concatenated).
374
- attributeCumulativeUsage(calls, usage);
375
- // Final drain of any subagent transcript writes that landed after the last poll, then
376
- // fold the subagents' usage + per-call telemetry into the run's outcome — their tokens
377
- // never appear on the parent stream, so this is the only place they are accounted.
378
- await subagents?.stop();
379
- const subUsage = subagents?.usage() ?? { inputTokens: 0, outputTokens: 0 };
380
- const subCalls = subagents?.calls() ?? [];
381
- const mergedCalls = [...calls, ...subCalls];
382
- // INVARIANT (do not "fix" this into a double count): the run total is the parent usage
383
- // PLUS the subagent usage because the two are disjoint sources. The parent `usage` here
384
- // is the terminal `result` event's cumulative, which covers ONLY the parent loop — the
385
- // ADR 0026 incident is itself the proof: a heavily subagent-parallelised review reported
386
- // ~0 tokens, i.e. the parent stream (and its `result` total) never included the subagent
387
- // spend. The subagent tokens live exclusively in the per-session `subagents/*.jsonl`
388
- // transcripts, which the watcher reads and nothing else does; it deliberately EXCLUDES the
389
- // sibling parent session transcript (whose usage `result` already totals), so neither
390
- // `calls` nor `usage` can already contain the subagent spend.
391
- const mergedUsage = usage || subUsage.inputTokens || subUsage.outputTokens
392
- ? {
393
- inputTokens: (usage?.inputTokens ?? 0) + subUsage.inputTokens,
394
- outputTokens: (usage?.outputTokens ?? 0) + subUsage.outputTokens,
395
- }
396
- : undefined;
397
- return {
363
+ return await assembleClaudeOutcome({
398
364
  summary,
399
365
  stats,
400
366
  stderrTail,
401
- ...(mergedUsage ? { usage: mergedUsage } : {}),
402
- ...(mergedCalls.length ? { callMetrics: mergedCalls } : {}),
403
- };
367
+ calls,
368
+ publisher,
369
+ usage,
370
+ subagents,
371
+ });
404
372
  }
405
373
  finally {
406
374
  await subagents?.stop();
@@ -417,6 +385,63 @@ export async function runClaudeCode(opts) {
417
385
  }
418
386
  }
419
387
  }
388
+ /**
389
+ * Build the child-process env for the `claude` CLI: an isolated config home plus subscription
390
+ * auth (Anthropic OAuth token, or an Anthropic-compatible base URL + auth token for a
391
+ * non-Anthropic Claude-Code vendor like GLM/Kimi/DeepSeek), or an empty env in ambient mode
392
+ * (the developer's own logged-in `~/.claude` is used). Extracted from {@link runClaudeCode} to
393
+ * keep its cyclomatic complexity down; behaviour is a straight move of the original expression.
394
+ */
395
+ function buildClaudeEnv(opts, configHome) {
396
+ if (opts.ambientAuth)
397
+ return {};
398
+ return {
399
+ CLAUDE_CONFIG_DIR: configHome,
400
+ ...(opts.subscriptionBaseUrl
401
+ ? {
402
+ ANTHROPIC_BASE_URL: opts.subscriptionBaseUrl,
403
+ ANTHROPIC_AUTH_TOKEN: opts.subscriptionToken,
404
+ }
405
+ : { CLAUDE_CODE_OAUTH_TOKEN: opts.subscriptionToken }),
406
+ };
407
+ }
408
+ /**
409
+ * Merge the parent-loop telemetry with the subagents' out-of-band usage + per-call metrics into
410
+ * the run outcome. INVARIANT (do not "fix" this into a double count): the run total is the parent
411
+ * usage PLUS the subagent usage because the two are disjoint sources — the parent `usage` (the
412
+ * terminal `result` event's cumulative) covers ONLY the parent loop, and the subagent tokens live
413
+ * exclusively in the per-session `subagents/*.jsonl` transcripts the watcher reads. Extracted from
414
+ * {@link runClaudeCode} verbatim to keep its cyclomatic complexity down.
415
+ */
416
+ async function assembleClaudeOutcome(args) {
417
+ const { summary, stats, stderrTail, calls, publisher, usage, subagents } = args;
418
+ // The parent's cumulative-usage fallback applies to the PARENT calls only (before the
419
+ // subagent calls, which carry their own per-turn tokens, are concatenated).
420
+ attributeCumulativeUsage(calls, usage);
421
+ // The withheld calls are final only NOW, so stream them: the completion poll drains them
422
+ // alongside the result, and the backend records the attributed numbers rather than the zeros
423
+ // they carried while the run was in flight.
424
+ publisher.flush();
425
+ // Final drain of any subagent transcript writes that landed after the last poll, then
426
+ // fold the subagents' usage + per-call telemetry into the run's outcome.
427
+ await subagents?.stop();
428
+ const subUsage = subagents?.usage() ?? { inputTokens: 0, outputTokens: 0 };
429
+ const subCalls = subagents?.calls() ?? [];
430
+ const mergedCalls = [...calls, ...subCalls];
431
+ const mergedUsage = usage || subUsage.inputTokens || subUsage.outputTokens
432
+ ? {
433
+ inputTokens: (usage?.inputTokens ?? 0) + subUsage.inputTokens,
434
+ outputTokens: (usage?.outputTokens ?? 0) + subUsage.outputTokens,
435
+ }
436
+ : undefined;
437
+ return {
438
+ summary,
439
+ stats,
440
+ stderrTail,
441
+ ...(mergedUsage ? { usage: mergedUsage } : {}),
442
+ ...(mergedCalls.length ? { callMetrics: mergedCalls } : {}),
443
+ };
444
+ }
420
445
  /** Map Claude Code's `TodoWrite` todos array onto subtask counts. */
421
446
  function todosToProgress(todos) {
422
447
  if (!Array.isArray(todos))
@@ -529,7 +554,7 @@ export async function runCodex(opts) {
529
554
  // assistant text seen since the previous turn as one telemetry call.
530
555
  const perTurn = codexLastTurnUsage(event);
531
556
  if (perTurn) {
532
- calls.push({
557
+ publishCallMetric(calls, {
533
558
  model: opts.model,
534
559
  promptText: redactBody(JSON.stringify(messages), secrets),
535
560
  messageCount: messages.length,
@@ -539,7 +564,7 @@ export async function runCodex(opts) {
539
564
  cachedInputTokens: perTurn.cachedInputTokens,
540
565
  outputTokens: perTurn.outputTokens,
541
566
  finishReason: null,
542
- });
567
+ }, opts.onCallMetric);
543
568
  if (pendingText)
544
569
  messages.push({ role: 'assistant', content: pendingText });
545
570
  pendingText = '';
@@ -563,7 +588,7 @@ export async function runCodex(opts) {
563
588
  // Fallback for a CLI/version that never emits per-turn `last_token_usage`: record a
564
589
  // single call from the cumulative total + final text so the run is still observable.
565
590
  if (calls.length === 0 && (usage || summary)) {
566
- calls.push({
591
+ publishCallMetric(calls, {
567
592
  model: opts.model,
568
593
  promptText: redactBody(JSON.stringify(messages), secrets),
569
594
  messageCount: messages.length,
@@ -573,7 +598,7 @@ export async function runCodex(opts) {
573
598
  cachedInputTokens: 0,
574
599
  outputTokens: usage?.outputTokens ?? 0,
575
600
  finishReason: null,
576
- });
601
+ }, opts.onCallMetric);
577
602
  }
578
603
  return {
579
604
  summary,
package/dist/agent.js CHANGED
@@ -745,14 +745,12 @@ async function runCodingMode(job, opts) {
745
745
  return result;
746
746
  }
747
747
  /**
748
- * The ordinary single-repo coding flow: clone `branch` (or resume `newBranch`), run the agent,
749
- * commit + push to `pushBranch`, and open `pr` when one is set and the run produced changes. A
750
- * no-op is a failure for the implementer (`noChangesIsError` default) and a non-fatal no-op for
751
- * the in-place fixers (and for a seed-only kind like `repro-test`).
748
+ * Assemble the {@link runCodingAgent} spec for the ordinary single-repo coding flow. Extracted
749
+ * from {@link runSingleRepoCoding} so the many optional-field spreads don't inflate that
750
+ * function's cyclomatic complexity; the mapping is a straight field copy off `job`.
752
751
  */
753
- async function runSingleRepoCoding(job, opts) {
754
- const pushBranch = job.pushBranch ?? job.newBranch ?? job.branch;
755
- const { summary, stats, stderrTail, pushed, usage, callMetrics, validation, effortReport } = await runCodingAgent({
752
+ function buildSingleRepoCodingSpec(job, pushBranch) {
753
+ return {
756
754
  kind: 'agent',
757
755
  jobId: job.jobId,
758
756
  repo: job.repo,
@@ -789,7 +787,17 @@ async function runSingleRepoCoding(job, opts) {
789
787
  },
790
788
  }
791
789
  : {}),
792
- }, opts);
790
+ };
791
+ }
792
+ /**
793
+ * The ordinary single-repo coding flow: clone `branch` (or resume `newBranch`), run the agent,
794
+ * commit + push to `pushBranch`, and open `pr` when one is set and the run produced changes. A
795
+ * no-op is a failure for the implementer (`noChangesIsError` default) and a non-fatal no-op for
796
+ * the in-place fixers (and for a seed-only kind like `repro-test`).
797
+ */
798
+ async function runSingleRepoCoding(job, opts) {
799
+ const pushBranch = job.pushBranch ?? job.newBranch ?? job.branch;
800
+ const { summary, stats, stderrTail, pushed, usage, callMetrics, validation, effortReport } = await runCodingAgent(buildSingleRepoCodingSpec(job, pushBranch), opts);
793
801
  // Ralph loop: the harness-computed validation verdict, forwarded onto the coding result as
794
802
  // `ralphVerdict` so the backend's `toRunResult` lifts it onto `AgentRunResult.ralphVerdict`.
795
803
  const ralphVerdict = validation ? { ralphVerdict: validation } : {};
package/dist/job.js CHANGED
@@ -555,6 +555,24 @@ function isReservedEnvName(key) {
555
555
  const lower = key.toLowerCase();
556
556
  return RESERVED_ENV_PREFIXES.some((p) => lower.startsWith(p));
557
557
  }
558
+ /**
559
+ * Collect only string→string entries from a raw `env` bag. A non-string value is dropped so a
560
+ * malformed binding can't inject `[object Object]` (or undefined) as an upstream URL. Reserved
561
+ * names that would break the toolchain or enable injection (PATH, NODE_OPTIONS, LD_PRELOAD, …) are
562
+ * dropped too: they are spread over `process.env` at build time, so a binding named `PATH` would
563
+ * replace it with a URL and the build would no longer find its tools. Extracted from the infra
564
+ * parsers to keep their cyclomatic complexity down.
565
+ */
566
+ function parseInfraEnv(raw) {
567
+ const env = {};
568
+ if (typeof raw === 'object' && raw !== null) {
569
+ for (const [key, val] of Object.entries(raw)) {
570
+ if (key && !isReservedEnvName(key) && typeof val === 'string')
571
+ env[key] = val;
572
+ }
573
+ }
574
+ return env;
575
+ }
558
576
  /** Parse the frontend UI-test infra spec (`kind: 'frontend'`), tolerating missing knobs. */
559
577
  function parseFrontendInfraSpec(o) {
560
578
  const packageManager = o.packageManager === 'pnpm' || o.packageManager === 'npm' || o.packageManager === 'yarn'
@@ -562,18 +580,7 @@ function parseFrontendInfraSpec(o) {
562
580
  : undefined;
563
581
  const serveMode = o.serveMode === 'static' || o.serveMode === 'command' ? o.serveMode : undefined;
564
582
  const envInjection = o.envInjection === 'build' || o.envInjection === 'runtime' ? o.envInjection : undefined;
565
- // Only string→string entries survive; a non-string value is dropped so a malformed
566
- // binding can't inject `[object Object]` (or undefined) as an upstream URL. Reserved names
567
- // that would break the toolchain or enable injection (PATH, NODE_OPTIONS, LD_PRELOAD, …) are
568
- // dropped too: they are spread over `process.env` at build time, so a binding named `PATH`
569
- // would replace it with a URL and the build would no longer find its tools.
570
- const env = {};
571
- if (typeof o.env === 'object' && o.env !== null) {
572
- for (const [key, val] of Object.entries(o.env)) {
573
- if (key && !isReservedEnvName(key) && typeof val === 'string')
574
- env[key] = val;
575
- }
576
- }
583
+ const env = parseInfraEnv(o.env);
577
584
  const servePort = port(o.servePort);
578
585
  const wiremockPort = port(o.wiremockPort);
579
586
  // The app's monorepo subdirectory becomes the install/build/serve cwd, so it goes through the
@@ -732,11 +739,7 @@ function assembleAgentJob(o, mode, agentField, parts) {
732
739
  ghToken: str(o.ghToken, 'ghToken'),
733
740
  repo: parseRepoSpec(repo),
734
741
  branch: str(o.branch, 'branch'),
735
- ...(typeof o.githubApiBase === 'string' ? { githubApiBase: o.githubApiBase } : {}),
736
- ...(typeof o.webToolsGuidance === 'string' ? { webToolsGuidance: o.webToolsGuidance } : {}),
737
- ...(o.webSearch === true ? { webSearch: true } : {}),
738
- ...(o.full === true ? { full: true } : {}),
739
- ...(typeof o.mergeBase === 'string' && o.mergeBase ? { mergeBase: o.mergeBase } : {}),
742
+ ...collectOptionalRequestFields(o),
740
743
  ...(bootstrap ? { bootstrap } : {}),
741
744
  ...(output ? { output } : {}),
742
745
  ...(contextFiles.length ? { contextFiles } : {}),
@@ -744,20 +747,35 @@ function assembleAgentJob(o, mode, agentField, parts) {
744
747
  ...(skill ? { skill } : {}),
745
748
  ...(testSecrets.length ? { testSecrets } : {}),
746
749
  ...(infra ? { infra } : {}),
747
- ...(typeof o.newBranch === 'string' && o.newBranch ? { newBranch: o.newBranch } : {}),
748
- ...(typeof o.pushBranch === 'string' && o.pushBranch ? { pushBranch: o.pushBranch } : {}),
749
- ...(typeof o.commitMessage === 'string' && o.commitMessage
750
- ? { commitMessage: o.commitMessage }
751
- : {}),
752
750
  ...(pr ? { pr } : {}),
753
751
  ...(peerRepos.length ? { peerRepos } : {}),
754
752
  ...(referenceRepos.length ? { referenceRepos } : {}),
755
753
  ...(referenceBranches.length ? { referenceBranches } : {}),
756
754
  ...(reviewPrNumber !== undefined ? { reviewPrNumber } : {}),
755
+ ...(guardLimits ? { guardLimits } : {}),
756
+ ...(validation ? { validation } : {}),
757
+ };
758
+ }
759
+ /**
760
+ * The optional {@link AgentJob} fields read directly off the request `o` (booleans + trimmed
761
+ * strings). Extracted from {@link assembleAgentJob} to keep its cyclomatic complexity down; every
762
+ * key is unique so grouping the conditional spreads is behaviour-neutral (spread order is
763
+ * irrelevant with no colliding keys).
764
+ */
765
+ function collectOptionalRequestFields(o) {
766
+ return {
767
+ ...(typeof o.githubApiBase === 'string' ? { githubApiBase: o.githubApiBase } : {}),
768
+ ...(typeof o.webToolsGuidance === 'string' ? { webToolsGuidance: o.webToolsGuidance } : {}),
769
+ ...(o.webSearch === true ? { webSearch: true } : {}),
770
+ ...(o.full === true ? { full: true } : {}),
771
+ ...(typeof o.mergeBase === 'string' && o.mergeBase ? { mergeBase: o.mergeBase } : {}),
772
+ ...(typeof o.newBranch === 'string' && o.newBranch ? { newBranch: o.newBranch } : {}),
773
+ ...(typeof o.pushBranch === 'string' && o.pushBranch ? { pushBranch: o.pushBranch } : {}),
774
+ ...(typeof o.commitMessage === 'string' && o.commitMessage
775
+ ? { commitMessage: o.commitMessage }
776
+ : {}),
757
777
  ...(o.noChangesIsError === false ? { noChangesIsError: false } : {}),
758
778
  ...(o.persistentCheckout === true ? { persistentCheckout: true } : {}),
759
779
  ...(o.streamFollowUps === true ? { streamFollowUps: true } : {}),
760
- ...(guardLimits ? { guardLimits } : {}),
761
- ...(validation ? { validation } : {}),
762
780
  };
763
781
  }
@@ -147,6 +147,10 @@ export async function runAgentInWorkspace(spec, opts = {}) {
147
147
  signal: opts.signal,
148
148
  onActivity: opts.onActivity,
149
149
  onProgress: opts.onProgress,
150
+ // Stream this run's per-call telemetry to the job's live drain. The subscription
151
+ // harnesses are the only producers of `callMetrics` (Pi's calls are metered by the LLM
152
+ // proxy as they happen), so this is the only path that needs the hook.
153
+ onCallMetric: opts.onCallMetric,
150
154
  ...(opts.log ? { log: opts.log } : {}),
151
155
  });
152
156
  return withEffortReport(spec.dir, subOutcome);
package/dist/pi.js CHANGED
@@ -344,6 +344,65 @@ export async function writeWebToolsConfig(config) {
344
344
  function isObject(value) {
345
345
  return typeof value === 'object' && value !== null;
346
346
  }
347
+ /**
348
+ * Publish one captured model call: append it to the run's list (which becomes the terminal
349
+ * result's `callMetrics`) AND hand the SAME object to the live stream, where the job registry
350
+ * stamps its {@link HarnessCallMetric.seq} and buffers it for the next poll to drain.
351
+ *
352
+ * Every producer goes through here rather than a bare `calls.push`, so the two channels can't
353
+ * drift: a call that reaches the terminal list but never the live stream would be invisible
354
+ * until the job ends, and one that reaches only the live stream would go unrecorded if the
355
+ * poll response were lost.
356
+ *
357
+ * A published call must be FINAL. The backend records it the moment the drain reaches it and
358
+ * IGNORES the terminal repeat (first write wins, so its stored prompt delta stays valid against
359
+ * the chain tip it was written against), which means a field mutated after publishing never
360
+ * reaches the store. A producer whose calls can still change (the cumulative-usage fallback,
361
+ * whose totals arrive with the CLI's terminal `result` event) publishes through
362
+ * {@link createCallMetricPublisher} instead, which withholds exactly those.
363
+ */
364
+ export function publishCallMetric(calls, call, onCallMetric) {
365
+ calls.push(call);
366
+ onCallMetric?.(call);
367
+ }
368
+ /**
369
+ * A {@link publishCallMetric} wrapper for a producer whose per-call tokens may be filled in at
370
+ * the END of the run: a CLI that reports only a cumulative total leaves every turn at zero, and
371
+ * `attributeCumulativeUsage` pins the total onto the last call once the terminal `result` event
372
+ * arrives.
373
+ *
374
+ * Since a published call must be final (the backend stores it on the drain and ignores the
375
+ * terminal repeat), a call the CLI did NOT cost is appended to the list but WITHHELD from the
376
+ * live stream — otherwise it records as a zero-token row and the attributed numbers never land.
377
+ * The withholding window closes the moment any call IS costed: attribution can no longer fire, so
378
+ * everything held is final and released at once, in capture order, and every later call streams
379
+ * immediately whatever its tokens. {@link flush} covers the run that was never costed at all.
380
+ */
381
+ export function createCallMetricPublisher(calls, onCallMetric) {
382
+ const withheld = [];
383
+ let anyCosted = false;
384
+ const flush = () => {
385
+ for (const call of withheld)
386
+ onCallMetric?.(call);
387
+ withheld.length = 0;
388
+ };
389
+ return {
390
+ publish(call) {
391
+ const costed = call.inputTokens > 0 || call.outputTokens > 0;
392
+ if (!costed && !anyCosted) {
393
+ publishCallMetric(calls, call);
394
+ withheld.push(call);
395
+ return;
396
+ }
397
+ if (costed)
398
+ anyCosted = true;
399
+ // Released BEFORE this call so the live sequence stays in capture order.
400
+ flush();
401
+ publishCallMetric(calls, call, onCallMetric);
402
+ },
403
+ flush,
404
+ };
405
+ }
347
406
  /**
348
407
  * Pull the `todo` tool's result `details` out of a Pi `--mode json` event, or
349
408
  * undefined if the event isn't a successful `todo` tool result.
package/dist/runner.js CHANGED
@@ -30,7 +30,7 @@ export function loadRunnerLimits(env = process.env) {
30
30
  };
31
31
  }
32
32
  function toView(entry) {
33
- const { promise: _promise, spanBuffer: _spanBuffer, followUpBuffer: _followUpBuffer, abort: _abort, ...view } = entry;
33
+ const { promise: _promise, spanBuffer: _spanBuffer, followUpBuffer: _followUpBuffer, callMetricBuffer: _callMetricBuffer, callMetricSeq: _callMetricSeq, abort: _abort, ...view } = entry;
34
34
  return { ...view };
35
35
  }
36
36
  /**
@@ -75,15 +75,18 @@ export class JobRegistry {
75
75
  promise: Promise.resolve(),
76
76
  spanBuffer: [],
77
77
  followUpBuffer: [],
78
+ callMetricBuffer: [],
79
+ callMetricSeq: 0,
78
80
  };
79
81
  this.jobs.set(id, entry);
80
82
  entry.promise = this.drive(entry, job);
81
83
  return toView(entry);
82
84
  }
83
85
  /**
84
- * Poll the job — and DRAIN its tool-span buffer (drain-on-read). The GET /jobs/{id}
85
- * handler is the sole caller, so each poll returns the spans accumulated since the
86
- * previous poll and clears them, bounding the harness buffer to one poll interval.
86
+ * Poll the job — and DRAIN its observability buffers (drain-on-read). The GET /jobs/{id}
87
+ * handler is the sole caller, so each poll returns the spans / follow-ups / call metrics
88
+ * accumulated since the previous poll and clears them, bounding the harness buffers to one
89
+ * poll interval.
87
90
  */
88
91
  get(id) {
89
92
  const entry = this.jobs.get(id);
@@ -98,6 +101,10 @@ export class JobRegistry {
98
101
  view.followUps = entry.followUpBuffer;
99
102
  entry.followUpBuffer = [];
100
103
  }
104
+ if (entry.callMetricBuffer.length > 0) {
105
+ view.callMetrics = entry.callMetricBuffer;
106
+ entry.callMetricBuffer = [];
107
+ }
101
108
  return view;
102
109
  }
103
110
  /**
@@ -206,6 +213,13 @@ export class JobRegistry {
206
213
  onFollowUp: (items) => {
207
214
  entry.followUpBuffer.push(...items);
208
215
  },
216
+ onCallMetric: (call) => {
217
+ // Stamp the job-scoped sequence on the metric OBJECT: the handler keeps the same
218
+ // instance for its terminal result, so both channels carry the same `seq` and the
219
+ // backend mints one stable row id per call.
220
+ call.seq = entry.callMetricSeq++;
221
+ entry.callMetricBuffer.push(call);
222
+ },
209
223
  onPhase: (next) => markPhase(next),
210
224
  log: jobLog,
211
225
  });
package/dist/subagents.js CHANGED
@@ -2,6 +2,7 @@ import { readdir, stat } from 'node:fs/promises';
2
2
  import { createReadStream } from 'node:fs';
3
3
  import { basename, join } from 'node:path';
4
4
  import { claudeAssistantContent, claudeCallUsage, isObject, redactBody } from './claude-stream.js';
5
+ import { publishCallMetric } from './pi.js';
5
6
  export function createSliceTracker() {
6
7
  // Insertion-ordered so the progress `items` render in dispatch order.
7
8
  const slices = new Map();
@@ -164,7 +165,7 @@ export function startSubagentWatcher(root, opts) {
164
165
  return;
165
166
  const content = Array.isArray(message.content) ? message.content : [];
166
167
  const { text, reasoning } = claudeAssistantContent(content);
167
- calls.push({
168
+ publishCallMetric(calls, {
168
169
  ...(typeof message.model === 'string'
169
170
  ? { model: message.model }
170
171
  : opts.model
@@ -180,7 +181,7 @@ export function startSubagentWatcher(root, opts) {
180
181
  cachedInputTokens: u.cachedInputTokens,
181
182
  outputTokens: u.outputTokens,
182
183
  finishReason: typeof message.stop_reason === 'string' ? message.stop_reason : null,
183
- });
184
+ }, opts.onCallMetric);
184
185
  usage.inputTokens += u.inputTokens;
185
186
  usage.outputTokens += u.outputTokens;
186
187
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cat-factory/executor-harness",
3
- "version": "1.50.16",
3
+ "version": "1.52.0",
4
4
  "description": "Container payload: a thin TypeScript wrapper that runs the Pi coding agent against a cloned repo and opens a PR. Runs in the Cloudflare Container (and, in local native mode, as a host process); carries no secrets.",
5
5
  "repository": {
6
6
  "type": "git",
@@ -26,8 +26,8 @@
26
26
  "hono": "^4.12.30",
27
27
  "typescript": "7.0.2",
28
28
  "vitest": "^4.1.10",
29
- "@cat-factory/server": "0.142.0",
30
- "@cat-factory/spend": "0.12.74"
29
+ "@cat-factory/server": "0.144.0",
30
+ "@cat-factory/spend": "0.12.77"
31
31
  },
32
32
  "scripts": {
33
33
  "build": "tsc -p tsconfig.json",
@@ -10,7 +10,15 @@ import {
10
10
  redactBody,
11
11
  } from './claude-stream.js'
12
12
  import type { Logger } from './logger.js'
13
- import type { HarnessCallMetric, PiRunOutcome, PiRunStats, TodoProgress } from './pi.js'
13
+ import {
14
+ createCallMetricPublisher,
15
+ publishCallMetric,
16
+ type CallMetricPublisher,
17
+ type HarnessCallMetric,
18
+ type PiRunOutcome,
19
+ type PiRunStats,
20
+ type TodoProgress,
21
+ } from './pi.js'
14
22
  import { killChildProcess, spawnDetached } from './process.js'
15
23
  import { redact, secretsToRedact } from './redact.js'
16
24
  import { createSliceTracker, pickProgress, startSubagentWatcher } from './subagents.js'
@@ -81,6 +89,12 @@ export interface SubscriptionRunOptions {
81
89
  onActivity?: () => void
82
90
  /** Called with the latest subtask counts each time the CLI updates its todo/plan list. */
83
91
  onProgress?: (progress: TodoProgress) => void
92
+ /**
93
+ * Called with each per-call telemetry row as the CLI stream yields it, so the backend can
94
+ * record the run's model calls WHILE it runs instead of only from its terminal result. The
95
+ * same row still rides the result, so a lost poll response costs nothing.
96
+ */
97
+ onCallMetric?: (call: HarnessCallMetric) => void
84
98
  /**
85
99
  * The per-job child logger (jobId/repo/branch correlation). Threaded so the retained
86
100
  * session-transcript path is logged for the run when the isolated config home is torn down.
@@ -324,6 +338,9 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
324
338
  { role: 'user', content: opts.userPrompt },
325
339
  ]
326
340
  const calls: HarnessCallMetric[] = []
341
+ // Streams each call as the CLI yields it, EXCEPT one whose tokens `attributeCumulativeUsage`
342
+ // may still rewrite below (a published call must be final — see the publisher).
343
+ const publisher = createCallMetricPublisher(calls, opts.onCallMetric)
327
344
 
328
345
  // ADR 0026 D2.1 + ADR 0027 Defect B: surface live slice progress from TWO reconciled
329
346
  // sources. The parent's `Task` dispatches + their terminal tool_results DO appear on this
@@ -360,7 +377,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
360
377
  // produced this response. The append-only array keeps each call's prompt a strict
361
378
  // prefix of the next, so the backend's telemetry chain delta-compresses cleanly.
362
379
  const u = claudeCallUsage(message.usage)
363
- calls.push({
380
+ publisher.publish({
364
381
  ...(typeof message.model === 'string' ? { model: message.model } : {}),
365
382
  promptText: redactBody(JSON.stringify(messages), secrets),
366
383
  messageCount: messages.length,
@@ -424,21 +441,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
424
441
  await writeNativeSkill(skillsRoot, opts.skill).catch(() => {})
425
442
  }
426
443
 
427
- // Anthropic itself authenticates with the subscription OAuth token; a
428
- // non-Anthropic Claude-Code vendor (GLM via Z.ai, Kimi via Moonshot, DeepSeek)
429
- // points Claude Code at its Anthropic-compatible endpoint with an auth-token key.
430
- // Ambient mode injects neither — the CLI uses the developer's logged-in `~/.claude`.
431
- const env: Record<string, string> = opts.ambientAuth
432
- ? {}
433
- : {
434
- CLAUDE_CONFIG_DIR: configHome!,
435
- ...(opts.subscriptionBaseUrl
436
- ? {
437
- ANTHROPIC_BASE_URL: opts.subscriptionBaseUrl,
438
- ANTHROPIC_AUTH_TOKEN: opts.subscriptionToken!,
439
- }
440
- : { CLAUDE_CODE_OAUTH_TOKEN: opts.subscriptionToken! }),
441
- }
444
+ const env = buildClaudeEnv(opts, configHome)
442
445
 
443
446
  // ADR 0026 D3 (path corrected by ADR 0027 Defect A): while the run is live, tail the CLI's
444
447
  // subagent `*.jsonl` transcripts so a parallel-subagent review keeps the inactivity
@@ -453,6 +456,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
453
456
  ...(opts.onActivity ? { onActivity: opts.onActivity } : {}),
454
457
  secrets,
455
458
  model: opts.model,
459
+ ...(opts.onCallMetric ? { onCallMetric: opts.onCallMetric } : {}),
456
460
  ...(opts.log ? { log: opts.log } : {}),
457
461
  })
458
462
  : undefined
@@ -484,39 +488,15 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
484
488
  onEvent,
485
489
  )
486
490
 
487
- // The parent's cumulative-usage fallback applies to the PARENT calls only (before the
488
- // subagent calls, which carry their own per-turn tokens, are concatenated).
489
- attributeCumulativeUsage(calls, usage)
490
- // Final drain of any subagent transcript writes that landed after the last poll, then
491
- // fold the subagents' usage + per-call telemetry into the run's outcome — their tokens
492
- // never appear on the parent stream, so this is the only place they are accounted.
493
- await subagents?.stop()
494
- const subUsage = subagents?.usage() ?? { inputTokens: 0, outputTokens: 0 }
495
- const subCalls = subagents?.calls() ?? []
496
- const mergedCalls = [...calls, ...subCalls]
497
- // INVARIANT (do not "fix" this into a double count): the run total is the parent usage
498
- // PLUS the subagent usage because the two are disjoint sources. The parent `usage` here
499
- // is the terminal `result` event's cumulative, which covers ONLY the parent loop — the
500
- // ADR 0026 incident is itself the proof: a heavily subagent-parallelised review reported
501
- // ~0 tokens, i.e. the parent stream (and its `result` total) never included the subagent
502
- // spend. The subagent tokens live exclusively in the per-session `subagents/*.jsonl`
503
- // transcripts, which the watcher reads and nothing else does; it deliberately EXCLUDES the
504
- // sibling parent session transcript (whose usage `result` already totals), so neither
505
- // `calls` nor `usage` can already contain the subagent spend.
506
- const mergedUsage =
507
- usage || subUsage.inputTokens || subUsage.outputTokens
508
- ? {
509
- inputTokens: (usage?.inputTokens ?? 0) + subUsage.inputTokens,
510
- outputTokens: (usage?.outputTokens ?? 0) + subUsage.outputTokens,
511
- }
512
- : undefined
513
- return {
491
+ return await assembleClaudeOutcome({
514
492
  summary,
515
493
  stats,
516
494
  stderrTail,
517
- ...(mergedUsage ? { usage: mergedUsage } : {}),
518
- ...(mergedCalls.length ? { callMetrics: mergedCalls } : {}),
519
- }
495
+ calls,
496
+ publisher,
497
+ usage,
498
+ subagents,
499
+ })
520
500
  } finally {
521
501
  await subagents?.stop()
522
502
  if (configHome) {
@@ -533,6 +513,77 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
533
513
  }
534
514
  }
535
515
 
516
+ /**
517
+ * Build the child-process env for the `claude` CLI: an isolated config home plus subscription
518
+ * auth (Anthropic OAuth token, or an Anthropic-compatible base URL + auth token for a
519
+ * non-Anthropic Claude-Code vendor like GLM/Kimi/DeepSeek), or an empty env in ambient mode
520
+ * (the developer's own logged-in `~/.claude` is used). Extracted from {@link runClaudeCode} to
521
+ * keep its cyclomatic complexity down; behaviour is a straight move of the original expression.
522
+ */
523
+ function buildClaudeEnv(
524
+ opts: SubscriptionRunOptions,
525
+ configHome: string | undefined,
526
+ ): Record<string, string> {
527
+ if (opts.ambientAuth) return {}
528
+ return {
529
+ CLAUDE_CONFIG_DIR: configHome!,
530
+ ...(opts.subscriptionBaseUrl
531
+ ? {
532
+ ANTHROPIC_BASE_URL: opts.subscriptionBaseUrl,
533
+ ANTHROPIC_AUTH_TOKEN: opts.subscriptionToken!,
534
+ }
535
+ : { CLAUDE_CODE_OAUTH_TOKEN: opts.subscriptionToken! }),
536
+ }
537
+ }
538
+
539
+ /**
540
+ * Merge the parent-loop telemetry with the subagents' out-of-band usage + per-call metrics into
541
+ * the run outcome. INVARIANT (do not "fix" this into a double count): the run total is the parent
542
+ * usage PLUS the subagent usage because the two are disjoint sources — the parent `usage` (the
543
+ * terminal `result` event's cumulative) covers ONLY the parent loop, and the subagent tokens live
544
+ * exclusively in the per-session `subagents/*.jsonl` transcripts the watcher reads. Extracted from
545
+ * {@link runClaudeCode} verbatim to keep its cyclomatic complexity down.
546
+ */
547
+ async function assembleClaudeOutcome(args: {
548
+ summary: string
549
+ stats: PiRunStats
550
+ stderrTail: string
551
+ calls: HarnessCallMetric[]
552
+ /** The live-stream publisher, flushed once attribution has finalised the calls' tokens. */
553
+ publisher: CallMetricPublisher
554
+ usage: { inputTokens: number; outputTokens: number } | undefined
555
+ subagents: ReturnType<typeof startSubagentWatcher> | undefined
556
+ }): Promise<PiRunOutcome> {
557
+ const { summary, stats, stderrTail, calls, publisher, usage, subagents } = args
558
+ // The parent's cumulative-usage fallback applies to the PARENT calls only (before the
559
+ // subagent calls, which carry their own per-turn tokens, are concatenated).
560
+ attributeCumulativeUsage(calls, usage)
561
+ // The withheld calls are final only NOW, so stream them: the completion poll drains them
562
+ // alongside the result, and the backend records the attributed numbers rather than the zeros
563
+ // they carried while the run was in flight.
564
+ publisher.flush()
565
+ // Final drain of any subagent transcript writes that landed after the last poll, then
566
+ // fold the subagents' usage + per-call telemetry into the run's outcome.
567
+ await subagents?.stop()
568
+ const subUsage = subagents?.usage() ?? { inputTokens: 0, outputTokens: 0 }
569
+ const subCalls = subagents?.calls() ?? []
570
+ const mergedCalls = [...calls, ...subCalls]
571
+ const mergedUsage =
572
+ usage || subUsage.inputTokens || subUsage.outputTokens
573
+ ? {
574
+ inputTokens: (usage?.inputTokens ?? 0) + subUsage.inputTokens,
575
+ outputTokens: (usage?.outputTokens ?? 0) + subUsage.outputTokens,
576
+ }
577
+ : undefined
578
+ return {
579
+ summary,
580
+ stats,
581
+ stderrTail,
582
+ ...(mergedUsage ? { usage: mergedUsage } : {}),
583
+ ...(mergedCalls.length ? { callMetrics: mergedCalls } : {}),
584
+ }
585
+ }
586
+
536
587
  /** Map Claude Code's `TodoWrite` todos array onto subtask counts. */
537
588
  function todosToProgress(todos: unknown): TodoProgress | undefined {
538
589
  if (!Array.isArray(todos)) return undefined
@@ -649,17 +700,21 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
649
700
  // assistant text seen since the previous turn as one telemetry call.
650
701
  const perTurn = codexLastTurnUsage(event)
651
702
  if (perTurn) {
652
- calls.push({
653
- model: opts.model,
654
- promptText: redactBody(JSON.stringify(messages), secrets),
655
- messageCount: messages.length,
656
- responseText: redactBody(pendingText, secrets),
657
- reasoningText: '',
658
- inputTokens: perTurn.inputTokens,
659
- cachedInputTokens: perTurn.cachedInputTokens,
660
- outputTokens: perTurn.outputTokens,
661
- finishReason: null,
662
- })
703
+ publishCallMetric(
704
+ calls,
705
+ {
706
+ model: opts.model,
707
+ promptText: redactBody(JSON.stringify(messages), secrets),
708
+ messageCount: messages.length,
709
+ responseText: redactBody(pendingText, secrets),
710
+ reasoningText: '',
711
+ inputTokens: perTurn.inputTokens,
712
+ cachedInputTokens: perTurn.cachedInputTokens,
713
+ outputTokens: perTurn.outputTokens,
714
+ finishReason: null,
715
+ },
716
+ opts.onCallMetric,
717
+ )
663
718
  if (pendingText) messages.push({ role: 'assistant', content: pendingText })
664
719
  pendingText = ''
665
720
  }
@@ -691,17 +746,21 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
691
746
  // Fallback for a CLI/version that never emits per-turn `last_token_usage`: record a
692
747
  // single call from the cumulative total + final text so the run is still observable.
693
748
  if (calls.length === 0 && (usage || summary)) {
694
- calls.push({
695
- model: opts.model,
696
- promptText: redactBody(JSON.stringify(messages), secrets),
697
- messageCount: messages.length,
698
- responseText: redactBody(summary, secrets),
699
- reasoningText: '',
700
- inputTokens: usage?.inputTokens ?? 0,
701
- cachedInputTokens: 0,
702
- outputTokens: usage?.outputTokens ?? 0,
703
- finishReason: null,
704
- })
749
+ publishCallMetric(
750
+ calls,
751
+ {
752
+ model: opts.model,
753
+ promptText: redactBody(JSON.stringify(messages), secrets),
754
+ messageCount: messages.length,
755
+ responseText: redactBody(summary, secrets),
756
+ reasoningText: '',
757
+ inputTokens: usage?.inputTokens ?? 0,
758
+ cachedInputTokens: 0,
759
+ outputTokens: usage?.outputTokens ?? 0,
760
+ finishReason: null,
761
+ },
762
+ opts.onCallMetric,
763
+ )
705
764
  }
706
765
  return {
707
766
  summary,
package/src/agent.ts CHANGED
@@ -903,6 +903,55 @@ async function runCodingMode(job: AgentJob, opts: RunOptions): Promise<AgentResu
903
903
  return result
904
904
  }
905
905
 
906
+ /**
907
+ * Assemble the {@link runCodingAgent} spec for the ordinary single-repo coding flow. Extracted
908
+ * from {@link runSingleRepoCoding} so the many optional-field spreads don't inflate that
909
+ * function's cyclomatic complexity; the mapping is a straight field copy off `job`.
910
+ */
911
+ function buildSingleRepoCodingSpec(
912
+ job: AgentJob,
913
+ pushBranch: string,
914
+ ): Parameters<typeof runCodingAgent>[0] {
915
+ return {
916
+ kind: 'agent',
917
+ jobId: job.jobId,
918
+ repo: job.repo,
919
+ cloneBranch: job.branch,
920
+ ...(job.newBranch ? { newBranch: job.newBranch } : {}),
921
+ pushBranch,
922
+ ghToken: job.ghToken,
923
+ systemPrompt: job.systemPrompt,
924
+ userPrompt: job.userPrompt,
925
+ model: job.model,
926
+ harness: job.harness,
927
+ subscriptionToken: job.subscriptionToken,
928
+ subscriptionBaseUrl: job.subscriptionBaseUrl,
929
+ ambientAuth: job.ambientAuth,
930
+ proxyBaseUrl: job.proxyBaseUrl,
931
+ sessionToken: job.sessionToken,
932
+ commitMessage: job.commitMessage ?? job.pr?.title ?? 'Agent changes',
933
+ webToolsGuidance: job.webToolsGuidance,
934
+ webSearchProxy: job.webSearch,
935
+ guardLimits: job.guardLimits,
936
+ ...(job.persistentCheckout ? { persistentCheckout: true } : {}),
937
+ ...(job.streamFollowUps ? { streamFollowUps: true } : {}),
938
+ ...(job.referenceBranches?.length ? { referenceBranches: job.referenceBranches } : {}),
939
+ // Repo-sourced skill (slice 2): installed harness-aware by runAgentInWorkspace.
940
+ ...(job.skill ? { skill: job.skill } : {}),
941
+ // Ralph loop: run the completion command after the agent commits and report its verdict.
942
+ ...(job.validation
943
+ ? {
944
+ validation: {
945
+ command: job.validation.command,
946
+ ...(job.validation.iteration !== undefined
947
+ ? { iteration: job.validation.iteration }
948
+ : {}),
949
+ },
950
+ }
951
+ : {}),
952
+ }
953
+ }
954
+
906
955
  /**
907
956
  * The ordinary single-repo coding flow: clone `branch` (or resume `newBranch`), run the agent,
908
957
  * commit + push to `pushBranch`, and open `pr` when one is set and the run produced changes. A
@@ -912,47 +961,7 @@ async function runCodingMode(job: AgentJob, opts: RunOptions): Promise<AgentResu
912
961
  async function runSingleRepoCoding(job: AgentJob, opts: RunOptions): Promise<AgentResult> {
913
962
  const pushBranch = job.pushBranch ?? job.newBranch ?? job.branch
914
963
  const { summary, stats, stderrTail, pushed, usage, callMetrics, validation, effortReport } =
915
- await runCodingAgent(
916
- {
917
- kind: 'agent',
918
- jobId: job.jobId,
919
- repo: job.repo,
920
- cloneBranch: job.branch,
921
- ...(job.newBranch ? { newBranch: job.newBranch } : {}),
922
- pushBranch,
923
- ghToken: job.ghToken,
924
- systemPrompt: job.systemPrompt,
925
- userPrompt: job.userPrompt,
926
- model: job.model,
927
- harness: job.harness,
928
- subscriptionToken: job.subscriptionToken,
929
- subscriptionBaseUrl: job.subscriptionBaseUrl,
930
- ambientAuth: job.ambientAuth,
931
- proxyBaseUrl: job.proxyBaseUrl,
932
- sessionToken: job.sessionToken,
933
- commitMessage: job.commitMessage ?? job.pr?.title ?? 'Agent changes',
934
- webToolsGuidance: job.webToolsGuidance,
935
- webSearchProxy: job.webSearch,
936
- guardLimits: job.guardLimits,
937
- ...(job.persistentCheckout ? { persistentCheckout: true } : {}),
938
- ...(job.streamFollowUps ? { streamFollowUps: true } : {}),
939
- ...(job.referenceBranches?.length ? { referenceBranches: job.referenceBranches } : {}),
940
- // Repo-sourced skill (slice 2): installed harness-aware by runAgentInWorkspace.
941
- ...(job.skill ? { skill: job.skill } : {}),
942
- // Ralph loop: run the completion command after the agent commits and report its verdict.
943
- ...(job.validation
944
- ? {
945
- validation: {
946
- command: job.validation.command,
947
- ...(job.validation.iteration !== undefined
948
- ? { iteration: job.validation.iteration }
949
- : {}),
950
- },
951
- }
952
- : {}),
953
- },
954
- opts,
955
- )
964
+ await runCodingAgent(buildSingleRepoCodingSpec(job, pushBranch), opts)
956
965
  // Ralph loop: the harness-computed validation verdict, forwarded onto the coding result as
957
966
  // `ralphVerdict` so the backend's `toRunResult` lifts it onto `AgentRunResult.ralphVerdict`.
958
967
  const ralphVerdict = validation ? { ralphVerdict: validation } : {}
package/src/job.ts CHANGED
@@ -1124,6 +1124,24 @@ function isReservedEnvName(key: string): boolean {
1124
1124
  return RESERVED_ENV_PREFIXES.some((p) => lower.startsWith(p))
1125
1125
  }
1126
1126
 
1127
+ /**
1128
+ * Collect only string→string entries from a raw `env` bag. A non-string value is dropped so a
1129
+ * malformed binding can't inject `[object Object]` (or undefined) as an upstream URL. Reserved
1130
+ * names that would break the toolchain or enable injection (PATH, NODE_OPTIONS, LD_PRELOAD, …) are
1131
+ * dropped too: they are spread over `process.env` at build time, so a binding named `PATH` would
1132
+ * replace it with a URL and the build would no longer find its tools. Extracted from the infra
1133
+ * parsers to keep their cyclomatic complexity down.
1134
+ */
1135
+ function parseInfraEnv(raw: unknown): Record<string, string> {
1136
+ const env: Record<string, string> = {}
1137
+ if (typeof raw === 'object' && raw !== null) {
1138
+ for (const [key, val] of Object.entries(raw as Record<string, unknown>)) {
1139
+ if (key && !isReservedEnvName(key) && typeof val === 'string') env[key] = val
1140
+ }
1141
+ }
1142
+ return env
1143
+ }
1144
+
1127
1145
  /** Parse the frontend UI-test infra spec (`kind: 'frontend'`), tolerating missing knobs. */
1128
1146
  function parseFrontendInfraSpec(o: Record<string, unknown>): FrontendInfraSpec {
1129
1147
  const packageManager =
@@ -1133,17 +1151,7 @@ function parseFrontendInfraSpec(o: Record<string, unknown>): FrontendInfraSpec {
1133
1151
  const serveMode = o.serveMode === 'static' || o.serveMode === 'command' ? o.serveMode : undefined
1134
1152
  const envInjection =
1135
1153
  o.envInjection === 'build' || o.envInjection === 'runtime' ? o.envInjection : undefined
1136
- // Only string→string entries survive; a non-string value is dropped so a malformed
1137
- // binding can't inject `[object Object]` (or undefined) as an upstream URL. Reserved names
1138
- // that would break the toolchain or enable injection (PATH, NODE_OPTIONS, LD_PRELOAD, …) are
1139
- // dropped too: they are spread over `process.env` at build time, so a binding named `PATH`
1140
- // would replace it with a URL and the build would no longer find its tools.
1141
- const env: Record<string, string> = {}
1142
- if (typeof o.env === 'object' && o.env !== null) {
1143
- for (const [key, val] of Object.entries(o.env as Record<string, unknown>)) {
1144
- if (key && !isReservedEnvName(key) && typeof val === 'string') env[key] = val
1145
- }
1146
- }
1154
+ const env = parseInfraEnv(o.env)
1147
1155
  const servePort = port(o.servePort)
1148
1156
  const wiremockPort = port(o.wiremockPort)
1149
1157
  // The app's monorepo subdirectory becomes the install/build/serve cwd, so it goes through the
@@ -1376,11 +1384,7 @@ function assembleAgentJob(
1376
1384
  ghToken: str(o.ghToken, 'ghToken'),
1377
1385
  repo: parseRepoSpec(repo),
1378
1386
  branch: str(o.branch, 'branch'),
1379
- ...(typeof o.githubApiBase === 'string' ? { githubApiBase: o.githubApiBase } : {}),
1380
- ...(typeof o.webToolsGuidance === 'string' ? { webToolsGuidance: o.webToolsGuidance } : {}),
1381
- ...(o.webSearch === true ? { webSearch: true } : {}),
1382
- ...(o.full === true ? { full: true } : {}),
1383
- ...(typeof o.mergeBase === 'string' && o.mergeBase ? { mergeBase: o.mergeBase } : {}),
1387
+ ...collectOptionalRequestFields(o),
1384
1388
  ...(bootstrap ? { bootstrap } : {}),
1385
1389
  ...(output ? { output } : {}),
1386
1390
  ...(contextFiles.length ? { contextFiles } : {}),
@@ -1388,20 +1392,36 @@ function assembleAgentJob(
1388
1392
  ...(skill ? { skill } : {}),
1389
1393
  ...(testSecrets.length ? { testSecrets } : {}),
1390
1394
  ...(infra ? { infra } : {}),
1391
- ...(typeof o.newBranch === 'string' && o.newBranch ? { newBranch: o.newBranch } : {}),
1392
- ...(typeof o.pushBranch === 'string' && o.pushBranch ? { pushBranch: o.pushBranch } : {}),
1393
- ...(typeof o.commitMessage === 'string' && o.commitMessage
1394
- ? { commitMessage: o.commitMessage }
1395
- : {}),
1396
1395
  ...(pr ? { pr } : {}),
1397
1396
  ...(peerRepos.length ? { peerRepos } : {}),
1398
1397
  ...(referenceRepos.length ? { referenceRepos } : {}),
1399
1398
  ...(referenceBranches.length ? { referenceBranches } : {}),
1400
1399
  ...(reviewPrNumber !== undefined ? { reviewPrNumber } : {}),
1400
+ ...(guardLimits ? { guardLimits } : {}),
1401
+ ...(validation ? { validation } : {}),
1402
+ }
1403
+ }
1404
+
1405
+ /**
1406
+ * The optional {@link AgentJob} fields read directly off the request `o` (booleans + trimmed
1407
+ * strings). Extracted from {@link assembleAgentJob} to keep its cyclomatic complexity down; every
1408
+ * key is unique so grouping the conditional spreads is behaviour-neutral (spread order is
1409
+ * irrelevant with no colliding keys).
1410
+ */
1411
+ function collectOptionalRequestFields(o: Record<string, unknown>): Partial<AgentJob> {
1412
+ return {
1413
+ ...(typeof o.githubApiBase === 'string' ? { githubApiBase: o.githubApiBase } : {}),
1414
+ ...(typeof o.webToolsGuidance === 'string' ? { webToolsGuidance: o.webToolsGuidance } : {}),
1415
+ ...(o.webSearch === true ? { webSearch: true } : {}),
1416
+ ...(o.full === true ? { full: true } : {}),
1417
+ ...(typeof o.mergeBase === 'string' && o.mergeBase ? { mergeBase: o.mergeBase } : {}),
1418
+ ...(typeof o.newBranch === 'string' && o.newBranch ? { newBranch: o.newBranch } : {}),
1419
+ ...(typeof o.pushBranch === 'string' && o.pushBranch ? { pushBranch: o.pushBranch } : {}),
1420
+ ...(typeof o.commitMessage === 'string' && o.commitMessage
1421
+ ? { commitMessage: o.commitMessage }
1422
+ : {}),
1401
1423
  ...(o.noChangesIsError === false ? { noChangesIsError: false } : {}),
1402
1424
  ...(o.persistentCheckout === true ? { persistentCheckout: true } : {}),
1403
1425
  ...(o.streamFollowUps === true ? { streamFollowUps: true } : {}),
1404
- ...(guardLimits ? { guardLimits } : {}),
1405
- ...(validation ? { validation } : {}),
1406
1426
  }
1407
1427
  }
@@ -271,6 +271,10 @@ export async function runAgentInWorkspace(
271
271
  signal: opts.signal,
272
272
  onActivity: opts.onActivity,
273
273
  onProgress: opts.onProgress,
274
+ // Stream this run's per-call telemetry to the job's live drain. The subscription
275
+ // harnesses are the only producers of `callMetrics` (Pi's calls are metered by the LLM
276
+ // proxy as they happen), so this is the only path that needs the hook.
277
+ onCallMetric: opts.onCallMetric,
274
278
  ...(opts.log ? { log: opts.log } : {}),
275
279
  })
276
280
  return withEffortReport(spec.dir, subOutcome)
package/src/pi.ts CHANGED
@@ -505,6 +505,93 @@ export interface HarnessCallMetric {
505
505
  outputTokens: number
506
506
  /** The provider finish/stop reason when the CLI reports one (else null). */
507
507
  finishReason: string | null
508
+ /**
509
+ * This call's position in the JOB's telemetry sequence, stamped by the job registry the
510
+ * moment the call is emitted (see `RunOptions.onCallMetric`). It is what makes a call's
511
+ * recorded row id stable across the two channels that carry it: the live drain (per poll,
512
+ * so a run's telemetry is inspectable WHILE it runs) and the terminal result (the complete
513
+ * list, so a transport that doesn't drain still records everything). Both channels hold the
514
+ * SAME metric objects, so both mint the same `<jobId>-hc-<seq>` row id and the backend's
515
+ * second write of an already-recorded call is a no-op instead of a duplicate row.
516
+ *
517
+ * Absent only when a producer built a metric without emitting it live; the recorder then
518
+ * falls back to the array index, which is what it always used before streaming existed.
519
+ */
520
+ seq?: number
521
+ }
522
+
523
+ /**
524
+ * Publish one captured model call: append it to the run's list (which becomes the terminal
525
+ * result's `callMetrics`) AND hand the SAME object to the live stream, where the job registry
526
+ * stamps its {@link HarnessCallMetric.seq} and buffers it for the next poll to drain.
527
+ *
528
+ * Every producer goes through here rather than a bare `calls.push`, so the two channels can't
529
+ * drift: a call that reaches the terminal list but never the live stream would be invisible
530
+ * until the job ends, and one that reaches only the live stream would go unrecorded if the
531
+ * poll response were lost.
532
+ *
533
+ * A published call must be FINAL. The backend records it the moment the drain reaches it and
534
+ * IGNORES the terminal repeat (first write wins, so its stored prompt delta stays valid against
535
+ * the chain tip it was written against), which means a field mutated after publishing never
536
+ * reaches the store. A producer whose calls can still change (the cumulative-usage fallback,
537
+ * whose totals arrive with the CLI's terminal `result` event) publishes through
538
+ * {@link createCallMetricPublisher} instead, which withholds exactly those.
539
+ */
540
+ export function publishCallMetric(
541
+ calls: HarnessCallMetric[],
542
+ call: HarnessCallMetric,
543
+ onCallMetric?: (call: HarnessCallMetric) => void,
544
+ ): void {
545
+ calls.push(call)
546
+ onCallMetric?.(call)
547
+ }
548
+
549
+ /** Appends captured calls to a run's list, streaming each one as soon as it is final. */
550
+ export interface CallMetricPublisher {
551
+ /** Append a captured call, streaming it now unless its tokens can still be rewritten. */
552
+ publish(call: HarnessCallMetric): void
553
+ /** Stream whatever is still withheld. Call once the run's totals are attributed. */
554
+ flush(): void
555
+ }
556
+
557
+ /**
558
+ * A {@link publishCallMetric} wrapper for a producer whose per-call tokens may be filled in at
559
+ * the END of the run: a CLI that reports only a cumulative total leaves every turn at zero, and
560
+ * `attributeCumulativeUsage` pins the total onto the last call once the terminal `result` event
561
+ * arrives.
562
+ *
563
+ * Since a published call must be final (the backend stores it on the drain and ignores the
564
+ * terminal repeat), a call the CLI did NOT cost is appended to the list but WITHHELD from the
565
+ * live stream — otherwise it records as a zero-token row and the attributed numbers never land.
566
+ * The withholding window closes the moment any call IS costed: attribution can no longer fire, so
567
+ * everything held is final and released at once, in capture order, and every later call streams
568
+ * immediately whatever its tokens. {@link flush} covers the run that was never costed at all.
569
+ */
570
+ export function createCallMetricPublisher(
571
+ calls: HarnessCallMetric[],
572
+ onCallMetric?: (call: HarnessCallMetric) => void,
573
+ ): CallMetricPublisher {
574
+ const withheld: HarnessCallMetric[] = []
575
+ let anyCosted = false
576
+ const flush = (): void => {
577
+ for (const call of withheld) onCallMetric?.(call)
578
+ withheld.length = 0
579
+ }
580
+ return {
581
+ publish(call) {
582
+ const costed = call.inputTokens > 0 || call.outputTokens > 0
583
+ if (!costed && !anyCosted) {
584
+ publishCallMetric(calls, call)
585
+ withheld.push(call)
586
+ return
587
+ }
588
+ if (costed) anyCosted = true
589
+ // Released BEFORE this call so the live sequence stays in capture order.
590
+ flush()
591
+ publishCallMetric(calls, call, onCallMetric)
592
+ },
593
+ flush,
594
+ }
508
595
  }
509
596
 
510
597
  /** Pi's assistant summary plus {@link PiRunStats} describing what it did. */
package/src/runner.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { redactSecrets } from './redact.js'
2
2
  import type { FollowUpLine } from './follow-ups.js'
3
- import type { TodoProgress, ToolSpan } from './pi.js'
3
+ import type { HarnessCallMetric, TodoProgress, ToolSpan } from './pi.js'
4
4
  import { log, type Logger } from './logger.js'
5
5
  import {
6
6
  type FailureCause,
@@ -30,6 +30,19 @@ export interface RunOptions {
30
30
  onSpan?: (span: ToolSpan) => void
31
31
  /** Receives the forward-looking follow-up / question items the Coder streamed since the last poll. */
32
32
  onFollowUp?: (items: FollowUpLine[]) => void
33
+ /**
34
+ * Receives each per-call telemetry row the moment the agent's CLI stream yields it, so a
35
+ * run's model calls reach `llm_call_metrics` WHILE it runs rather than only in its terminal
36
+ * result. The registry stamps the call's job-scoped {@link HarnessCallMetric.seq} and buffers
37
+ * it for the next poll to drain.
38
+ *
39
+ * Call this for every metric you also put on the result — the SAME object, not a copy: the
40
+ * stamped `seq` is what lets the backend recognise the terminal write of an already-recorded
41
+ * call and skip it. A run that dies mid-flight (the container is evicted, the harness process
42
+ * is OOM-killed) never produces a terminal result, so without this its entire token spend and
43
+ * every prompt/response body are lost — exactly the run an operator most needs to inspect.
44
+ */
45
+ onCallMetric?: (call: HarnessCallMetric) => void
33
46
  /**
34
47
  * Mark the coarse lifecycle phase the handler has entered (`clone` / `agent` / `push` / …).
35
48
  * Drives the stuck-run breadcrumb: an inactivity kill reports WHICH phase was hung, and the
@@ -114,6 +127,17 @@ export interface JobView<TResult extends JobResultBase = JobResultBase> {
114
127
  * surfaces the first one (and only on a follow-ups-enabled coding run).
115
128
  */
116
129
  followUps?: FollowUpLine[]
130
+ /**
131
+ * Per-model-call telemetry the agent's CLI stream yielded SINCE THE LAST POLL
132
+ * (drain-on-read, exactly like {@link spans}). The backend records these into
133
+ * `llm_call_metrics` as they arrive, so a run's token spend and prompt/response bodies are
134
+ * queryable while it is still running — and survive it dying before it can produce a
135
+ * terminal result. Each carries a job-scoped `seq` so the terminal
136
+ * {@link JobResultBase} list can re-offer the same calls without duplicating rows.
137
+ * Absent until the agent's first model call (and on the proxy-metered Pi harness, whose
138
+ * calls the LLM proxy meters directly).
139
+ */
140
+ callMetrics?: HarnessCallMetric[]
117
141
  /**
118
142
  * ADR 0026 D4: set when the cold-start watchdog fired — the job produced NO activity
119
143
  * within {@link RunnerLimits.coldStartMs} of starting, a likely onboarding/auth wedge.
@@ -136,6 +160,13 @@ interface JobEntry<TResult extends JobResultBase> extends JobView<TResult> {
136
160
  spanBuffer: ToolSpan[]
137
161
  /** Follow-up items buffered since the last drain (see {@link JobView.followUps}). */
138
162
  followUpBuffer: FollowUpLine[]
163
+ /** Call telemetry buffered since the last drain (see {@link JobView.callMetrics}). */
164
+ callMetricBuffer: HarnessCallMetric[]
165
+ /**
166
+ * Next job-scoped {@link HarnessCallMetric.seq} to stamp. Monotonic for the life of the job
167
+ * (never reset by a drain), so a call's row id stays unique across every poll window.
168
+ */
169
+ callMetricSeq: number
139
170
  /** Abort the in-flight run (see {@link JobRegistry.abortAll}); set while running only. */
140
171
  abort?: (reason: string) => void
141
172
  }
@@ -192,6 +223,8 @@ function toView<TResult extends JobResultBase>(entry: JobEntry<TResult>): JobVie
192
223
  promise: _promise,
193
224
  spanBuffer: _spanBuffer,
194
225
  followUpBuffer: _followUpBuffer,
226
+ callMetricBuffer: _callMetricBuffer,
227
+ callMetricSeq: _callMetricSeq,
195
228
  abort: _abort,
196
229
  ...view
197
230
  } = entry
@@ -237,6 +270,8 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
237
270
  promise: Promise.resolve(),
238
271
  spanBuffer: [],
239
272
  followUpBuffer: [],
273
+ callMetricBuffer: [],
274
+ callMetricSeq: 0,
240
275
  }
241
276
  this.jobs.set(id, entry)
242
277
  entry.promise = this.drive(entry, job)
@@ -244,9 +279,10 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
244
279
  }
245
280
 
246
281
  /**
247
- * Poll the job — and DRAIN its tool-span buffer (drain-on-read). The GET /jobs/{id}
248
- * handler is the sole caller, so each poll returns the spans accumulated since the
249
- * previous poll and clears them, bounding the harness buffer to one poll interval.
282
+ * Poll the job — and DRAIN its observability buffers (drain-on-read). The GET /jobs/{id}
283
+ * handler is the sole caller, so each poll returns the spans / follow-ups / call metrics
284
+ * accumulated since the previous poll and clears them, bounding the harness buffers to one
285
+ * poll interval.
250
286
  */
251
287
  get(id: string): JobView<TResult> | undefined {
252
288
  const entry = this.jobs.get(id)
@@ -260,6 +296,10 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
260
296
  view.followUps = entry.followUpBuffer
261
297
  entry.followUpBuffer = []
262
298
  }
299
+ if (entry.callMetricBuffer.length > 0) {
300
+ view.callMetrics = entry.callMetricBuffer
301
+ entry.callMetricBuffer = []
302
+ }
263
303
  return view
264
304
  }
265
305
 
@@ -374,6 +414,13 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
374
414
  onFollowUp: (items) => {
375
415
  entry.followUpBuffer.push(...items)
376
416
  },
417
+ onCallMetric: (call) => {
418
+ // Stamp the job-scoped sequence on the metric OBJECT: the handler keeps the same
419
+ // instance for its terminal result, so both channels carry the same `seq` and the
420
+ // backend mints one stable row id per call.
421
+ call.seq = entry.callMetricSeq++
422
+ entry.callMetricBuffer.push(call)
423
+ },
377
424
  onPhase: (next) => markPhase(next),
378
425
  log: jobLog,
379
426
  })
package/src/subagents.ts CHANGED
@@ -3,7 +3,7 @@ import { createReadStream, type Dirent } from 'node:fs'
3
3
  import { basename, join } from 'node:path'
4
4
  import { claudeAssistantContent, claudeCallUsage, isObject, redactBody } from './claude-stream.js'
5
5
  import type { Logger } from './logger.js'
6
- import type { HarnessCallMetric, TodoProgress } from './pi.js'
6
+ import { publishCallMetric, type HarnessCallMetric, type TodoProgress } from './pi.js'
7
7
 
8
8
  // ADR 0026 D2.1 + D3, corrected by ADR 0027. When the Claude Code CLI reviews a large PR
9
9
  // it fans the work out across parallel `Task` subagents. Two things then go dark to the
@@ -155,6 +155,13 @@ export interface SubagentWatcherOptions {
155
155
  secrets?: string[]
156
156
  /** Fallback model id stamped on a subagent call whose transcript omits one. */
157
157
  model?: string
158
+ /**
159
+ * Streams each lifted subagent call to the live telemetry drain (the run's `RunOptions`
160
+ * hook). Subagent work is exactly where a long review spends most of its tokens, and it is
161
+ * the phase the parent stream goes quiet for — so without this a run killed mid-fan-out
162
+ * reports nothing at all.
163
+ */
164
+ onCallMetric?: (call: HarnessCallMetric) => void
158
165
  /** Poll cadence (ms); overridable for tests. */
159
166
  intervalMs?: number
160
167
  log?: Logger
@@ -244,23 +251,27 @@ export function startSubagentWatcher(root: string, opts: SubagentWatcherOptions)
244
251
  if (u.inputTokens === 0 && u.outputTokens === 0) return
245
252
  const content = Array.isArray(message.content) ? message.content : []
246
253
  const { text, reasoning } = claudeAssistantContent(content)
247
- calls.push({
248
- ...(typeof message.model === 'string'
249
- ? { model: message.model }
250
- : opts.model
251
- ? { model: opts.model }
252
- : {}),
253
- // The subagent's own transcript isn't a re-sendable prompt chain, so we don't
254
- // reconstruct the request side (kept empty); the response + tokens are faithful.
255
- promptText: '',
256
- messageCount: 0,
257
- responseText: redactBody(text, secrets),
258
- reasoningText: redactBody(reasoning, secrets),
259
- inputTokens: u.inputTokens,
260
- cachedInputTokens: u.cachedInputTokens,
261
- outputTokens: u.outputTokens,
262
- finishReason: typeof message.stop_reason === 'string' ? message.stop_reason : null,
263
- })
254
+ publishCallMetric(
255
+ calls,
256
+ {
257
+ ...(typeof message.model === 'string'
258
+ ? { model: message.model }
259
+ : opts.model
260
+ ? { model: opts.model }
261
+ : {}),
262
+ // The subagent's own transcript isn't a re-sendable prompt chain, so we don't
263
+ // reconstruct the request side (kept empty); the response + tokens are faithful.
264
+ promptText: '',
265
+ messageCount: 0,
266
+ responseText: redactBody(text, secrets),
267
+ reasoningText: redactBody(reasoning, secrets),
268
+ inputTokens: u.inputTokens,
269
+ cachedInputTokens: u.cachedInputTokens,
270
+ outputTokens: u.outputTokens,
271
+ finishReason: typeof message.stop_reason === 'string' ? message.stop_reason : null,
272
+ },
273
+ opts.onCallMetric,
274
+ )
264
275
  usage.inputTokens += u.inputTokens
265
276
  usage.outputTokens += u.outputTokens
266
277
  }