@cat-factory/executor-harness 1.50.16 → 1.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/agent-runner.js +74 -49
- package/dist/agent.js +16 -8
- package/dist/job.js +42 -24
- package/dist/pi-workspace.js +4 -0
- package/dist/pi.js +59 -0
- package/dist/runner.js +18 -4
- package/dist/subagents.js +3 -2
- package/package.json +3 -3
- package/src/agent-runner.ts +128 -69
- package/src/agent.ts +50 -41
- package/src/job.ts +43 -23
- package/src/pi-workspace.ts +4 -0
- package/src/pi.ts +87 -0
- package/src/runner.ts +51 -4
- package/src/subagents.ts +29 -18
package/README.md
CHANGED
|
@@ -35,6 +35,19 @@ are surfaced as `progress` while a job runs. The exact request/response shapes
|
|
|
35
35
|
cat-factory sends are documented in
|
|
36
36
|
[`docs/runner-pool-integration.md`](../../docs/runner-pool-integration.md).
|
|
37
37
|
|
|
38
|
+
`GET /jobs/{id}` is also the harness's observability channel: `spans`, `followUps`
|
|
39
|
+
and `callMetrics` are **drain-on-read** — each poll returns what accumulated since
|
|
40
|
+
the previous one and clears the buffer. That is deliberate. A job that dies before
|
|
41
|
+
it can return a terminal result (an evicted container, an OOM-killed process) has
|
|
42
|
+
still reported the tool spans it ran and the model calls it paid for. Each drained
|
|
43
|
+
`callMetrics` entry carries a job-scoped `seq`, and the terminal result repeats the
|
|
44
|
+
complete list, so the backend can take both channels without double-counting a call.
|
|
45
|
+
|
|
46
|
+
Because the backend records a call as soon as it drains it (and ignores the terminal
|
|
47
|
+
repeat), a drained call is FINAL. A call whose tokens are still open — a CLI that reports
|
|
48
|
+
only a cumulative total, costed at the end — is withheld from the drain until it is
|
|
49
|
+
complete; see `createCallMetricPublisher` in `src/pi.ts`.
|
|
50
|
+
|
|
38
51
|
## What a job does
|
|
39
52
|
|
|
40
53
|
The implementation job (`POST /run`) is the canonical sequence:
|
package/dist/agent-runner.js
CHANGED
|
@@ -3,6 +3,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
|
3
3
|
import { homedir, tmpdir } from 'node:os';
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
5
|
import { claudeAssistantContent, claudeCallUsage, isObject, numberOf, redactBody, } from './claude-stream.js';
|
|
6
|
+
import { createCallMetricPublisher, publishCallMetric, } from './pi.js';
|
|
6
7
|
import { killChildProcess, spawnDetached } from './process.js';
|
|
7
8
|
import { redact, secretsToRedact } from './redact.js';
|
|
8
9
|
import { createSliceTracker, pickProgress, startSubagentWatcher } from './subagents.js';
|
|
@@ -219,6 +220,9 @@ export async function runClaudeCode(opts) {
|
|
|
219
220
|
{ role: 'user', content: opts.userPrompt },
|
|
220
221
|
];
|
|
221
222
|
const calls = [];
|
|
223
|
+
// Streams each call as the CLI yields it, EXCEPT one whose tokens `attributeCumulativeUsage`
|
|
224
|
+
// may still rewrite below (a published call must be final — see the publisher).
|
|
225
|
+
const publisher = createCallMetricPublisher(calls, opts.onCallMetric);
|
|
222
226
|
// ADR 0026 D2.1 + ADR 0027 Defect B: surface live slice progress from TWO reconciled
|
|
223
227
|
// sources. The parent's `Task` dispatches + their terminal tool_results DO appear on this
|
|
224
228
|
// stream (only a subagent's intermediate turns don't), so `sliceTracker` derives per-slice
|
|
@@ -256,7 +260,7 @@ export async function runClaudeCode(opts) {
|
|
|
256
260
|
// produced this response. The append-only array keeps each call's prompt a strict
|
|
257
261
|
// prefix of the next, so the backend's telemetry chain delta-compresses cleanly.
|
|
258
262
|
const u = claudeCallUsage(message.usage);
|
|
259
|
-
|
|
263
|
+
publisher.publish({
|
|
260
264
|
...(typeof message.model === 'string' ? { model: message.model } : {}),
|
|
261
265
|
promptText: redactBody(JSON.stringify(messages), secrets),
|
|
262
266
|
messageCount: messages.length,
|
|
@@ -319,21 +323,7 @@ export async function runClaudeCode(opts) {
|
|
|
319
323
|
: join(homedir(), '.claude', 'skills');
|
|
320
324
|
await writeNativeSkill(skillsRoot, opts.skill).catch(() => { });
|
|
321
325
|
}
|
|
322
|
-
|
|
323
|
-
// non-Anthropic Claude-Code vendor (GLM via Z.ai, Kimi via Moonshot, DeepSeek)
|
|
324
|
-
// points Claude Code at its Anthropic-compatible endpoint with an auth-token key.
|
|
325
|
-
// Ambient mode injects neither — the CLI uses the developer's logged-in `~/.claude`.
|
|
326
|
-
const env = opts.ambientAuth
|
|
327
|
-
? {}
|
|
328
|
-
: {
|
|
329
|
-
CLAUDE_CONFIG_DIR: configHome,
|
|
330
|
-
...(opts.subscriptionBaseUrl
|
|
331
|
-
? {
|
|
332
|
-
ANTHROPIC_BASE_URL: opts.subscriptionBaseUrl,
|
|
333
|
-
ANTHROPIC_AUTH_TOKEN: opts.subscriptionToken,
|
|
334
|
-
}
|
|
335
|
-
: { CLAUDE_CODE_OAUTH_TOKEN: opts.subscriptionToken }),
|
|
336
|
-
};
|
|
326
|
+
const env = buildClaudeEnv(opts, configHome);
|
|
337
327
|
// ADR 0026 D3 (path corrected by ADR 0027 Defect A): while the run is live, tail the CLI's
|
|
338
328
|
// subagent `*.jsonl` transcripts so a parallel-subagent review keeps the inactivity
|
|
339
329
|
// heartbeat alive (any new bytes ⇒ `onActivity`) and its otherwise-invisible token spend is
|
|
@@ -347,6 +337,7 @@ export async function runClaudeCode(opts) {
|
|
|
347
337
|
...(opts.onActivity ? { onActivity: opts.onActivity } : {}),
|
|
348
338
|
secrets,
|
|
349
339
|
model: opts.model,
|
|
340
|
+
...(opts.onCallMetric ? { onCallMetric: opts.onCallMetric } : {}),
|
|
350
341
|
...(opts.log ? { log: opts.log } : {}),
|
|
351
342
|
})
|
|
352
343
|
: undefined;
|
|
@@ -369,38 +360,15 @@ export async function runClaudeCode(opts) {
|
|
|
369
360
|
...appendArgs,
|
|
370
361
|
],
|
|
371
362
|
}, prompt, opts, env, opts.subscriptionToken ? secretsToRedact(opts.subscriptionToken) : [], onEvent);
|
|
372
|
-
|
|
373
|
-
// subagent calls, which carry their own per-turn tokens, are concatenated).
|
|
374
|
-
attributeCumulativeUsage(calls, usage);
|
|
375
|
-
// Final drain of any subagent transcript writes that landed after the last poll, then
|
|
376
|
-
// fold the subagents' usage + per-call telemetry into the run's outcome — their tokens
|
|
377
|
-
// never appear on the parent stream, so this is the only place they are accounted.
|
|
378
|
-
await subagents?.stop();
|
|
379
|
-
const subUsage = subagents?.usage() ?? { inputTokens: 0, outputTokens: 0 };
|
|
380
|
-
const subCalls = subagents?.calls() ?? [];
|
|
381
|
-
const mergedCalls = [...calls, ...subCalls];
|
|
382
|
-
// INVARIANT (do not "fix" this into a double count): the run total is the parent usage
|
|
383
|
-
// PLUS the subagent usage because the two are disjoint sources. The parent `usage` here
|
|
384
|
-
// is the terminal `result` event's cumulative, which covers ONLY the parent loop — the
|
|
385
|
-
// ADR 0026 incident is itself the proof: a heavily subagent-parallelised review reported
|
|
386
|
-
// ~0 tokens, i.e. the parent stream (and its `result` total) never included the subagent
|
|
387
|
-
// spend. The subagent tokens live exclusively in the per-session `subagents/*.jsonl`
|
|
388
|
-
// transcripts, which the watcher reads and nothing else does; it deliberately EXCLUDES the
|
|
389
|
-
// sibling parent session transcript (whose usage `result` already totals), so neither
|
|
390
|
-
// `calls` nor `usage` can already contain the subagent spend.
|
|
391
|
-
const mergedUsage = usage || subUsage.inputTokens || subUsage.outputTokens
|
|
392
|
-
? {
|
|
393
|
-
inputTokens: (usage?.inputTokens ?? 0) + subUsage.inputTokens,
|
|
394
|
-
outputTokens: (usage?.outputTokens ?? 0) + subUsage.outputTokens,
|
|
395
|
-
}
|
|
396
|
-
: undefined;
|
|
397
|
-
return {
|
|
363
|
+
return await assembleClaudeOutcome({
|
|
398
364
|
summary,
|
|
399
365
|
stats,
|
|
400
366
|
stderrTail,
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
367
|
+
calls,
|
|
368
|
+
publisher,
|
|
369
|
+
usage,
|
|
370
|
+
subagents,
|
|
371
|
+
});
|
|
404
372
|
}
|
|
405
373
|
finally {
|
|
406
374
|
await subagents?.stop();
|
|
@@ -417,6 +385,63 @@ export async function runClaudeCode(opts) {
|
|
|
417
385
|
}
|
|
418
386
|
}
|
|
419
387
|
}
|
|
388
|
+
/**
|
|
389
|
+
* Build the child-process env for the `claude` CLI: an isolated config home plus subscription
|
|
390
|
+
* auth (Anthropic OAuth token, or an Anthropic-compatible base URL + auth token for a
|
|
391
|
+
* non-Anthropic Claude-Code vendor like GLM/Kimi/DeepSeek), or an empty env in ambient mode
|
|
392
|
+
* (the developer's own logged-in `~/.claude` is used). Extracted from {@link runClaudeCode} to
|
|
393
|
+
* keep its cyclomatic complexity down; behaviour is a straight move of the original expression.
|
|
394
|
+
*/
|
|
395
|
+
function buildClaudeEnv(opts, configHome) {
|
|
396
|
+
if (opts.ambientAuth)
|
|
397
|
+
return {};
|
|
398
|
+
return {
|
|
399
|
+
CLAUDE_CONFIG_DIR: configHome,
|
|
400
|
+
...(opts.subscriptionBaseUrl
|
|
401
|
+
? {
|
|
402
|
+
ANTHROPIC_BASE_URL: opts.subscriptionBaseUrl,
|
|
403
|
+
ANTHROPIC_AUTH_TOKEN: opts.subscriptionToken,
|
|
404
|
+
}
|
|
405
|
+
: { CLAUDE_CODE_OAUTH_TOKEN: opts.subscriptionToken }),
|
|
406
|
+
};
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* Merge the parent-loop telemetry with the subagents' out-of-band usage + per-call metrics into
|
|
410
|
+
* the run outcome. INVARIANT (do not "fix" this into a double count): the run total is the parent
|
|
411
|
+
* usage PLUS the subagent usage because the two are disjoint sources — the parent `usage` (the
|
|
412
|
+
* terminal `result` event's cumulative) covers ONLY the parent loop, and the subagent tokens live
|
|
413
|
+
* exclusively in the per-session `subagents/*.jsonl` transcripts the watcher reads. Extracted from
|
|
414
|
+
* {@link runClaudeCode} verbatim to keep its cyclomatic complexity down.
|
|
415
|
+
*/
|
|
416
|
+
async function assembleClaudeOutcome(args) {
|
|
417
|
+
const { summary, stats, stderrTail, calls, publisher, usage, subagents } = args;
|
|
418
|
+
// The parent's cumulative-usage fallback applies to the PARENT calls only (before the
|
|
419
|
+
// subagent calls, which carry their own per-turn tokens, are concatenated).
|
|
420
|
+
attributeCumulativeUsage(calls, usage);
|
|
421
|
+
// The withheld calls are final only NOW, so stream them: the completion poll drains them
|
|
422
|
+
// alongside the result, and the backend records the attributed numbers rather than the zeros
|
|
423
|
+
// they carried while the run was in flight.
|
|
424
|
+
publisher.flush();
|
|
425
|
+
// Final drain of any subagent transcript writes that landed after the last poll, then
|
|
426
|
+
// fold the subagents' usage + per-call telemetry into the run's outcome.
|
|
427
|
+
await subagents?.stop();
|
|
428
|
+
const subUsage = subagents?.usage() ?? { inputTokens: 0, outputTokens: 0 };
|
|
429
|
+
const subCalls = subagents?.calls() ?? [];
|
|
430
|
+
const mergedCalls = [...calls, ...subCalls];
|
|
431
|
+
const mergedUsage = usage || subUsage.inputTokens || subUsage.outputTokens
|
|
432
|
+
? {
|
|
433
|
+
inputTokens: (usage?.inputTokens ?? 0) + subUsage.inputTokens,
|
|
434
|
+
outputTokens: (usage?.outputTokens ?? 0) + subUsage.outputTokens,
|
|
435
|
+
}
|
|
436
|
+
: undefined;
|
|
437
|
+
return {
|
|
438
|
+
summary,
|
|
439
|
+
stats,
|
|
440
|
+
stderrTail,
|
|
441
|
+
...(mergedUsage ? { usage: mergedUsage } : {}),
|
|
442
|
+
...(mergedCalls.length ? { callMetrics: mergedCalls } : {}),
|
|
443
|
+
};
|
|
444
|
+
}
|
|
420
445
|
/** Map Claude Code's `TodoWrite` todos array onto subtask counts. */
|
|
421
446
|
function todosToProgress(todos) {
|
|
422
447
|
if (!Array.isArray(todos))
|
|
@@ -529,7 +554,7 @@ export async function runCodex(opts) {
|
|
|
529
554
|
// assistant text seen since the previous turn as one telemetry call.
|
|
530
555
|
const perTurn = codexLastTurnUsage(event);
|
|
531
556
|
if (perTurn) {
|
|
532
|
-
calls
|
|
557
|
+
publishCallMetric(calls, {
|
|
533
558
|
model: opts.model,
|
|
534
559
|
promptText: redactBody(JSON.stringify(messages), secrets),
|
|
535
560
|
messageCount: messages.length,
|
|
@@ -539,7 +564,7 @@ export async function runCodex(opts) {
|
|
|
539
564
|
cachedInputTokens: perTurn.cachedInputTokens,
|
|
540
565
|
outputTokens: perTurn.outputTokens,
|
|
541
566
|
finishReason: null,
|
|
542
|
-
});
|
|
567
|
+
}, opts.onCallMetric);
|
|
543
568
|
if (pendingText)
|
|
544
569
|
messages.push({ role: 'assistant', content: pendingText });
|
|
545
570
|
pendingText = '';
|
|
@@ -563,7 +588,7 @@ export async function runCodex(opts) {
|
|
|
563
588
|
// Fallback for a CLI/version that never emits per-turn `last_token_usage`: record a
|
|
564
589
|
// single call from the cumulative total + final text so the run is still observable.
|
|
565
590
|
if (calls.length === 0 && (usage || summary)) {
|
|
566
|
-
calls
|
|
591
|
+
publishCallMetric(calls, {
|
|
567
592
|
model: opts.model,
|
|
568
593
|
promptText: redactBody(JSON.stringify(messages), secrets),
|
|
569
594
|
messageCount: messages.length,
|
|
@@ -573,7 +598,7 @@ export async function runCodex(opts) {
|
|
|
573
598
|
cachedInputTokens: 0,
|
|
574
599
|
outputTokens: usage?.outputTokens ?? 0,
|
|
575
600
|
finishReason: null,
|
|
576
|
-
});
|
|
601
|
+
}, opts.onCallMetric);
|
|
577
602
|
}
|
|
578
603
|
return {
|
|
579
604
|
summary,
|
package/dist/agent.js
CHANGED
|
@@ -745,14 +745,12 @@ async function runCodingMode(job, opts) {
|
|
|
745
745
|
return result;
|
|
746
746
|
}
|
|
747
747
|
/**
|
|
748
|
-
*
|
|
749
|
-
*
|
|
750
|
-
*
|
|
751
|
-
* the in-place fixers (and for a seed-only kind like `repro-test`).
|
|
748
|
+
* Assemble the {@link runCodingAgent} spec for the ordinary single-repo coding flow. Extracted
|
|
749
|
+
* from {@link runSingleRepoCoding} so the many optional-field spreads don't inflate that
|
|
750
|
+
* function's cyclomatic complexity; the mapping is a straight field copy off `job`.
|
|
752
751
|
*/
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
const { summary, stats, stderrTail, pushed, usage, callMetrics, validation, effortReport } = await runCodingAgent({
|
|
752
|
+
function buildSingleRepoCodingSpec(job, pushBranch) {
|
|
753
|
+
return {
|
|
756
754
|
kind: 'agent',
|
|
757
755
|
jobId: job.jobId,
|
|
758
756
|
repo: job.repo,
|
|
@@ -789,7 +787,17 @@ async function runSingleRepoCoding(job, opts) {
|
|
|
789
787
|
},
|
|
790
788
|
}
|
|
791
789
|
: {}),
|
|
792
|
-
}
|
|
790
|
+
};
|
|
791
|
+
}
|
|
792
|
+
/**
|
|
793
|
+
* The ordinary single-repo coding flow: clone `branch` (or resume `newBranch`), run the agent,
|
|
794
|
+
* commit + push to `pushBranch`, and open `pr` when one is set and the run produced changes. A
|
|
795
|
+
* no-op is a failure for the implementer (`noChangesIsError` default) and a non-fatal no-op for
|
|
796
|
+
* the in-place fixers (and for a seed-only kind like `repro-test`).
|
|
797
|
+
*/
|
|
798
|
+
async function runSingleRepoCoding(job, opts) {
|
|
799
|
+
const pushBranch = job.pushBranch ?? job.newBranch ?? job.branch;
|
|
800
|
+
const { summary, stats, stderrTail, pushed, usage, callMetrics, validation, effortReport } = await runCodingAgent(buildSingleRepoCodingSpec(job, pushBranch), opts);
|
|
793
801
|
// Ralph loop: the harness-computed validation verdict, forwarded onto the coding result as
|
|
794
802
|
// `ralphVerdict` so the backend's `toRunResult` lifts it onto `AgentRunResult.ralphVerdict`.
|
|
795
803
|
const ralphVerdict = validation ? { ralphVerdict: validation } : {};
|
package/dist/job.js
CHANGED
|
@@ -555,6 +555,24 @@ function isReservedEnvName(key) {
|
|
|
555
555
|
const lower = key.toLowerCase();
|
|
556
556
|
return RESERVED_ENV_PREFIXES.some((p) => lower.startsWith(p));
|
|
557
557
|
}
|
|
558
|
+
/**
|
|
559
|
+
* Collect only string→string entries from a raw `env` bag. A non-string value is dropped so a
|
|
560
|
+
* malformed binding can't inject `[object Object]` (or undefined) as an upstream URL. Reserved
|
|
561
|
+
* names that would break the toolchain or enable injection (PATH, NODE_OPTIONS, LD_PRELOAD, …) are
|
|
562
|
+
* dropped too: they are spread over `process.env` at build time, so a binding named `PATH` would
|
|
563
|
+
* replace it with a URL and the build would no longer find its tools. Extracted from the infra
|
|
564
|
+
* parsers to keep their cyclomatic complexity down.
|
|
565
|
+
*/
|
|
566
|
+
function parseInfraEnv(raw) {
|
|
567
|
+
const env = {};
|
|
568
|
+
if (typeof raw === 'object' && raw !== null) {
|
|
569
|
+
for (const [key, val] of Object.entries(raw)) {
|
|
570
|
+
if (key && !isReservedEnvName(key) && typeof val === 'string')
|
|
571
|
+
env[key] = val;
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
return env;
|
|
575
|
+
}
|
|
558
576
|
/** Parse the frontend UI-test infra spec (`kind: 'frontend'`), tolerating missing knobs. */
|
|
559
577
|
function parseFrontendInfraSpec(o) {
|
|
560
578
|
const packageManager = o.packageManager === 'pnpm' || o.packageManager === 'npm' || o.packageManager === 'yarn'
|
|
@@ -562,18 +580,7 @@ function parseFrontendInfraSpec(o) {
|
|
|
562
580
|
: undefined;
|
|
563
581
|
const serveMode = o.serveMode === 'static' || o.serveMode === 'command' ? o.serveMode : undefined;
|
|
564
582
|
const envInjection = o.envInjection === 'build' || o.envInjection === 'runtime' ? o.envInjection : undefined;
|
|
565
|
-
|
|
566
|
-
// binding can't inject `[object Object]` (or undefined) as an upstream URL. Reserved names
|
|
567
|
-
// that would break the toolchain or enable injection (PATH, NODE_OPTIONS, LD_PRELOAD, …) are
|
|
568
|
-
// dropped too: they are spread over `process.env` at build time, so a binding named `PATH`
|
|
569
|
-
// would replace it with a URL and the build would no longer find its tools.
|
|
570
|
-
const env = {};
|
|
571
|
-
if (typeof o.env === 'object' && o.env !== null) {
|
|
572
|
-
for (const [key, val] of Object.entries(o.env)) {
|
|
573
|
-
if (key && !isReservedEnvName(key) && typeof val === 'string')
|
|
574
|
-
env[key] = val;
|
|
575
|
-
}
|
|
576
|
-
}
|
|
583
|
+
const env = parseInfraEnv(o.env);
|
|
577
584
|
const servePort = port(o.servePort);
|
|
578
585
|
const wiremockPort = port(o.wiremockPort);
|
|
579
586
|
// The app's monorepo subdirectory becomes the install/build/serve cwd, so it goes through the
|
|
@@ -732,11 +739,7 @@ function assembleAgentJob(o, mode, agentField, parts) {
|
|
|
732
739
|
ghToken: str(o.ghToken, 'ghToken'),
|
|
733
740
|
repo: parseRepoSpec(repo),
|
|
734
741
|
branch: str(o.branch, 'branch'),
|
|
735
|
-
...(
|
|
736
|
-
...(typeof o.webToolsGuidance === 'string' ? { webToolsGuidance: o.webToolsGuidance } : {}),
|
|
737
|
-
...(o.webSearch === true ? { webSearch: true } : {}),
|
|
738
|
-
...(o.full === true ? { full: true } : {}),
|
|
739
|
-
...(typeof o.mergeBase === 'string' && o.mergeBase ? { mergeBase: o.mergeBase } : {}),
|
|
742
|
+
...collectOptionalRequestFields(o),
|
|
740
743
|
...(bootstrap ? { bootstrap } : {}),
|
|
741
744
|
...(output ? { output } : {}),
|
|
742
745
|
...(contextFiles.length ? { contextFiles } : {}),
|
|
@@ -744,20 +747,35 @@ function assembleAgentJob(o, mode, agentField, parts) {
|
|
|
744
747
|
...(skill ? { skill } : {}),
|
|
745
748
|
...(testSecrets.length ? { testSecrets } : {}),
|
|
746
749
|
...(infra ? { infra } : {}),
|
|
747
|
-
...(typeof o.newBranch === 'string' && o.newBranch ? { newBranch: o.newBranch } : {}),
|
|
748
|
-
...(typeof o.pushBranch === 'string' && o.pushBranch ? { pushBranch: o.pushBranch } : {}),
|
|
749
|
-
...(typeof o.commitMessage === 'string' && o.commitMessage
|
|
750
|
-
? { commitMessage: o.commitMessage }
|
|
751
|
-
: {}),
|
|
752
750
|
...(pr ? { pr } : {}),
|
|
753
751
|
...(peerRepos.length ? { peerRepos } : {}),
|
|
754
752
|
...(referenceRepos.length ? { referenceRepos } : {}),
|
|
755
753
|
...(referenceBranches.length ? { referenceBranches } : {}),
|
|
756
754
|
...(reviewPrNumber !== undefined ? { reviewPrNumber } : {}),
|
|
755
|
+
...(guardLimits ? { guardLimits } : {}),
|
|
756
|
+
...(validation ? { validation } : {}),
|
|
757
|
+
};
|
|
758
|
+
}
|
|
759
|
+
/**
|
|
760
|
+
* The optional {@link AgentJob} fields read directly off the request `o` (booleans + trimmed
|
|
761
|
+
* strings). Extracted from {@link assembleAgentJob} to keep its cyclomatic complexity down; every
|
|
762
|
+
* key is unique so grouping the conditional spreads is behaviour-neutral (spread order is
|
|
763
|
+
* irrelevant with no colliding keys).
|
|
764
|
+
*/
|
|
765
|
+
function collectOptionalRequestFields(o) {
|
|
766
|
+
return {
|
|
767
|
+
...(typeof o.githubApiBase === 'string' ? { githubApiBase: o.githubApiBase } : {}),
|
|
768
|
+
...(typeof o.webToolsGuidance === 'string' ? { webToolsGuidance: o.webToolsGuidance } : {}),
|
|
769
|
+
...(o.webSearch === true ? { webSearch: true } : {}),
|
|
770
|
+
...(o.full === true ? { full: true } : {}),
|
|
771
|
+
...(typeof o.mergeBase === 'string' && o.mergeBase ? { mergeBase: o.mergeBase } : {}),
|
|
772
|
+
...(typeof o.newBranch === 'string' && o.newBranch ? { newBranch: o.newBranch } : {}),
|
|
773
|
+
...(typeof o.pushBranch === 'string' && o.pushBranch ? { pushBranch: o.pushBranch } : {}),
|
|
774
|
+
...(typeof o.commitMessage === 'string' && o.commitMessage
|
|
775
|
+
? { commitMessage: o.commitMessage }
|
|
776
|
+
: {}),
|
|
757
777
|
...(o.noChangesIsError === false ? { noChangesIsError: false } : {}),
|
|
758
778
|
...(o.persistentCheckout === true ? { persistentCheckout: true } : {}),
|
|
759
779
|
...(o.streamFollowUps === true ? { streamFollowUps: true } : {}),
|
|
760
|
-
...(guardLimits ? { guardLimits } : {}),
|
|
761
|
-
...(validation ? { validation } : {}),
|
|
762
780
|
};
|
|
763
781
|
}
|
package/dist/pi-workspace.js
CHANGED
|
@@ -147,6 +147,10 @@ export async function runAgentInWorkspace(spec, opts = {}) {
|
|
|
147
147
|
signal: opts.signal,
|
|
148
148
|
onActivity: opts.onActivity,
|
|
149
149
|
onProgress: opts.onProgress,
|
|
150
|
+
// Stream this run's per-call telemetry to the job's live drain. The subscription
|
|
151
|
+
// harnesses are the only producers of `callMetrics` (Pi's calls are metered by the LLM
|
|
152
|
+
// proxy as they happen), so this is the only path that needs the hook.
|
|
153
|
+
onCallMetric: opts.onCallMetric,
|
|
150
154
|
...(opts.log ? { log: opts.log } : {}),
|
|
151
155
|
});
|
|
152
156
|
return withEffortReport(spec.dir, subOutcome);
|
package/dist/pi.js
CHANGED
|
@@ -344,6 +344,65 @@ export async function writeWebToolsConfig(config) {
|
|
|
344
344
|
function isObject(value) {
|
|
345
345
|
return typeof value === 'object' && value !== null;
|
|
346
346
|
}
|
|
347
|
+
/**
|
|
348
|
+
* Publish one captured model call: append it to the run's list (which becomes the terminal
|
|
349
|
+
* result's `callMetrics`) AND hand the SAME object to the live stream, where the job registry
|
|
350
|
+
* stamps its {@link HarnessCallMetric.seq} and buffers it for the next poll to drain.
|
|
351
|
+
*
|
|
352
|
+
* Every producer goes through here rather than a bare `calls.push`, so the two channels can't
|
|
353
|
+
* drift: a call that reaches the terminal list but never the live stream would be invisible
|
|
354
|
+
* until the job ends, and one that reaches only the live stream would go unrecorded if the
|
|
355
|
+
* poll response were lost.
|
|
356
|
+
*
|
|
357
|
+
* A published call must be FINAL. The backend records it the moment the drain reaches it and
|
|
358
|
+
* IGNORES the terminal repeat (first write wins, so its stored prompt delta stays valid against
|
|
359
|
+
* the chain tip it was written against), which means a field mutated after publishing never
|
|
360
|
+
* reaches the store. A producer whose calls can still change (the cumulative-usage fallback,
|
|
361
|
+
* whose totals arrive with the CLI's terminal `result` event) publishes through
|
|
362
|
+
* {@link createCallMetricPublisher} instead, which withholds exactly those.
|
|
363
|
+
*/
|
|
364
|
+
export function publishCallMetric(calls, call, onCallMetric) {
|
|
365
|
+
calls.push(call);
|
|
366
|
+
onCallMetric?.(call);
|
|
367
|
+
}
|
|
368
|
+
/**
|
|
369
|
+
* A {@link publishCallMetric} wrapper for a producer whose per-call tokens may be filled in at
|
|
370
|
+
* the END of the run: a CLI that reports only a cumulative total leaves every turn at zero, and
|
|
371
|
+
* `attributeCumulativeUsage` pins the total onto the last call once the terminal `result` event
|
|
372
|
+
* arrives.
|
|
373
|
+
*
|
|
374
|
+
* Since a published call must be final (the backend stores it on the drain and ignores the
|
|
375
|
+
* terminal repeat), a call the CLI did NOT cost is appended to the list but WITHHELD from the
|
|
376
|
+
* live stream — otherwise it records as a zero-token row and the attributed numbers never land.
|
|
377
|
+
* The withholding window closes the moment any call IS costed: attribution can no longer fire, so
|
|
378
|
+
* everything held is final and released at once, in capture order, and every later call streams
|
|
379
|
+
* immediately whatever its tokens. {@link flush} covers the run that was never costed at all.
|
|
380
|
+
*/
|
|
381
|
+
export function createCallMetricPublisher(calls, onCallMetric) {
|
|
382
|
+
const withheld = [];
|
|
383
|
+
let anyCosted = false;
|
|
384
|
+
const flush = () => {
|
|
385
|
+
for (const call of withheld)
|
|
386
|
+
onCallMetric?.(call);
|
|
387
|
+
withheld.length = 0;
|
|
388
|
+
};
|
|
389
|
+
return {
|
|
390
|
+
publish(call) {
|
|
391
|
+
const costed = call.inputTokens > 0 || call.outputTokens > 0;
|
|
392
|
+
if (!costed && !anyCosted) {
|
|
393
|
+
publishCallMetric(calls, call);
|
|
394
|
+
withheld.push(call);
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
if (costed)
|
|
398
|
+
anyCosted = true;
|
|
399
|
+
// Released BEFORE this call so the live sequence stays in capture order.
|
|
400
|
+
flush();
|
|
401
|
+
publishCallMetric(calls, call, onCallMetric);
|
|
402
|
+
},
|
|
403
|
+
flush,
|
|
404
|
+
};
|
|
405
|
+
}
|
|
347
406
|
/**
|
|
348
407
|
* Pull the `todo` tool's result `details` out of a Pi `--mode json` event, or
|
|
349
408
|
* undefined if the event isn't a successful `todo` tool result.
|
package/dist/runner.js
CHANGED
|
@@ -30,7 +30,7 @@ export function loadRunnerLimits(env = process.env) {
|
|
|
30
30
|
};
|
|
31
31
|
}
|
|
32
32
|
function toView(entry) {
|
|
33
|
-
const { promise: _promise, spanBuffer: _spanBuffer, followUpBuffer: _followUpBuffer, abort: _abort, ...view } = entry;
|
|
33
|
+
const { promise: _promise, spanBuffer: _spanBuffer, followUpBuffer: _followUpBuffer, callMetricBuffer: _callMetricBuffer, callMetricSeq: _callMetricSeq, abort: _abort, ...view } = entry;
|
|
34
34
|
return { ...view };
|
|
35
35
|
}
|
|
36
36
|
/**
|
|
@@ -75,15 +75,18 @@ export class JobRegistry {
|
|
|
75
75
|
promise: Promise.resolve(),
|
|
76
76
|
spanBuffer: [],
|
|
77
77
|
followUpBuffer: [],
|
|
78
|
+
callMetricBuffer: [],
|
|
79
|
+
callMetricSeq: 0,
|
|
78
80
|
};
|
|
79
81
|
this.jobs.set(id, entry);
|
|
80
82
|
entry.promise = this.drive(entry, job);
|
|
81
83
|
return toView(entry);
|
|
82
84
|
}
|
|
83
85
|
/**
|
|
84
|
-
* Poll the job — and DRAIN its
|
|
85
|
-
* handler is the sole caller, so each poll returns the spans
|
|
86
|
-
* previous poll and clears them, bounding the harness
|
|
86
|
+
* Poll the job — and DRAIN its observability buffers (drain-on-read). The GET /jobs/{id}
|
|
87
|
+
* handler is the sole caller, so each poll returns the spans / follow-ups / call metrics
|
|
88
|
+
* accumulated since the previous poll and clears them, bounding the harness buffers to one
|
|
89
|
+
* poll interval.
|
|
87
90
|
*/
|
|
88
91
|
get(id) {
|
|
89
92
|
const entry = this.jobs.get(id);
|
|
@@ -98,6 +101,10 @@ export class JobRegistry {
|
|
|
98
101
|
view.followUps = entry.followUpBuffer;
|
|
99
102
|
entry.followUpBuffer = [];
|
|
100
103
|
}
|
|
104
|
+
if (entry.callMetricBuffer.length > 0) {
|
|
105
|
+
view.callMetrics = entry.callMetricBuffer;
|
|
106
|
+
entry.callMetricBuffer = [];
|
|
107
|
+
}
|
|
101
108
|
return view;
|
|
102
109
|
}
|
|
103
110
|
/**
|
|
@@ -206,6 +213,13 @@ export class JobRegistry {
|
|
|
206
213
|
onFollowUp: (items) => {
|
|
207
214
|
entry.followUpBuffer.push(...items);
|
|
208
215
|
},
|
|
216
|
+
onCallMetric: (call) => {
|
|
217
|
+
// Stamp the job-scoped sequence on the metric OBJECT: the handler keeps the same
|
|
218
|
+
// instance for its terminal result, so both channels carry the same `seq` and the
|
|
219
|
+
// backend mints one stable row id per call.
|
|
220
|
+
call.seq = entry.callMetricSeq++;
|
|
221
|
+
entry.callMetricBuffer.push(call);
|
|
222
|
+
},
|
|
209
223
|
onPhase: (next) => markPhase(next),
|
|
210
224
|
log: jobLog,
|
|
211
225
|
});
|
package/dist/subagents.js
CHANGED
|
@@ -2,6 +2,7 @@ import { readdir, stat } from 'node:fs/promises';
|
|
|
2
2
|
import { createReadStream } from 'node:fs';
|
|
3
3
|
import { basename, join } from 'node:path';
|
|
4
4
|
import { claudeAssistantContent, claudeCallUsage, isObject, redactBody } from './claude-stream.js';
|
|
5
|
+
import { publishCallMetric } from './pi.js';
|
|
5
6
|
export function createSliceTracker() {
|
|
6
7
|
// Insertion-ordered so the progress `items` render in dispatch order.
|
|
7
8
|
const slices = new Map();
|
|
@@ -164,7 +165,7 @@ export function startSubagentWatcher(root, opts) {
|
|
|
164
165
|
return;
|
|
165
166
|
const content = Array.isArray(message.content) ? message.content : [];
|
|
166
167
|
const { text, reasoning } = claudeAssistantContent(content);
|
|
167
|
-
calls
|
|
168
|
+
publishCallMetric(calls, {
|
|
168
169
|
...(typeof message.model === 'string'
|
|
169
170
|
? { model: message.model }
|
|
170
171
|
: opts.model
|
|
@@ -180,7 +181,7 @@ export function startSubagentWatcher(root, opts) {
|
|
|
180
181
|
cachedInputTokens: u.cachedInputTokens,
|
|
181
182
|
outputTokens: u.outputTokens,
|
|
182
183
|
finishReason: typeof message.stop_reason === 'string' ? message.stop_reason : null,
|
|
183
|
-
});
|
|
184
|
+
}, opts.onCallMetric);
|
|
184
185
|
usage.inputTokens += u.inputTokens;
|
|
185
186
|
usage.outputTokens += u.outputTokens;
|
|
186
187
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cat-factory/executor-harness",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.52.0",
|
|
4
4
|
"description": "Container payload: a thin TypeScript wrapper that runs the Pi coding agent against a cloned repo and opens a PR. Runs in the Cloudflare Container (and, in local native mode, as a host process); carries no secrets.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -26,8 +26,8 @@
|
|
|
26
26
|
"hono": "^4.12.30",
|
|
27
27
|
"typescript": "7.0.2",
|
|
28
28
|
"vitest": "^4.1.10",
|
|
29
|
-
"@cat-factory/server": "0.
|
|
30
|
-
"@cat-factory/spend": "0.12.
|
|
29
|
+
"@cat-factory/server": "0.144.0",
|
|
30
|
+
"@cat-factory/spend": "0.12.77"
|
|
31
31
|
},
|
|
32
32
|
"scripts": {
|
|
33
33
|
"build": "tsc -p tsconfig.json",
|
package/src/agent-runner.ts
CHANGED
|
@@ -10,7 +10,15 @@ import {
|
|
|
10
10
|
redactBody,
|
|
11
11
|
} from './claude-stream.js'
|
|
12
12
|
import type { Logger } from './logger.js'
|
|
13
|
-
import
|
|
13
|
+
import {
|
|
14
|
+
createCallMetricPublisher,
|
|
15
|
+
publishCallMetric,
|
|
16
|
+
type CallMetricPublisher,
|
|
17
|
+
type HarnessCallMetric,
|
|
18
|
+
type PiRunOutcome,
|
|
19
|
+
type PiRunStats,
|
|
20
|
+
type TodoProgress,
|
|
21
|
+
} from './pi.js'
|
|
14
22
|
import { killChildProcess, spawnDetached } from './process.js'
|
|
15
23
|
import { redact, secretsToRedact } from './redact.js'
|
|
16
24
|
import { createSliceTracker, pickProgress, startSubagentWatcher } from './subagents.js'
|
|
@@ -81,6 +89,12 @@ export interface SubscriptionRunOptions {
|
|
|
81
89
|
onActivity?: () => void
|
|
82
90
|
/** Called with the latest subtask counts each time the CLI updates its todo/plan list. */
|
|
83
91
|
onProgress?: (progress: TodoProgress) => void
|
|
92
|
+
/**
|
|
93
|
+
* Called with each per-call telemetry row as the CLI stream yields it, so the backend can
|
|
94
|
+
* record the run's model calls WHILE it runs instead of only from its terminal result. The
|
|
95
|
+
* same row still rides the result, so a lost poll response costs nothing.
|
|
96
|
+
*/
|
|
97
|
+
onCallMetric?: (call: HarnessCallMetric) => void
|
|
84
98
|
/**
|
|
85
99
|
* The per-job child logger (jobId/repo/branch correlation). Threaded so the retained
|
|
86
100
|
* session-transcript path is logged for the run when the isolated config home is torn down.
|
|
@@ -324,6 +338,9 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
324
338
|
{ role: 'user', content: opts.userPrompt },
|
|
325
339
|
]
|
|
326
340
|
const calls: HarnessCallMetric[] = []
|
|
341
|
+
// Streams each call as the CLI yields it, EXCEPT one whose tokens `attributeCumulativeUsage`
|
|
342
|
+
// may still rewrite below (a published call must be final — see the publisher).
|
|
343
|
+
const publisher = createCallMetricPublisher(calls, opts.onCallMetric)
|
|
327
344
|
|
|
328
345
|
// ADR 0026 D2.1 + ADR 0027 Defect B: surface live slice progress from TWO reconciled
|
|
329
346
|
// sources. The parent's `Task` dispatches + their terminal tool_results DO appear on this
|
|
@@ -360,7 +377,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
360
377
|
// produced this response. The append-only array keeps each call's prompt a strict
|
|
361
378
|
// prefix of the next, so the backend's telemetry chain delta-compresses cleanly.
|
|
362
379
|
const u = claudeCallUsage(message.usage)
|
|
363
|
-
|
|
380
|
+
publisher.publish({
|
|
364
381
|
...(typeof message.model === 'string' ? { model: message.model } : {}),
|
|
365
382
|
promptText: redactBody(JSON.stringify(messages), secrets),
|
|
366
383
|
messageCount: messages.length,
|
|
@@ -424,21 +441,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
424
441
|
await writeNativeSkill(skillsRoot, opts.skill).catch(() => {})
|
|
425
442
|
}
|
|
426
443
|
|
|
427
|
-
|
|
428
|
-
// non-Anthropic Claude-Code vendor (GLM via Z.ai, Kimi via Moonshot, DeepSeek)
|
|
429
|
-
// points Claude Code at its Anthropic-compatible endpoint with an auth-token key.
|
|
430
|
-
// Ambient mode injects neither — the CLI uses the developer's logged-in `~/.claude`.
|
|
431
|
-
const env: Record<string, string> = opts.ambientAuth
|
|
432
|
-
? {}
|
|
433
|
-
: {
|
|
434
|
-
CLAUDE_CONFIG_DIR: configHome!,
|
|
435
|
-
...(opts.subscriptionBaseUrl
|
|
436
|
-
? {
|
|
437
|
-
ANTHROPIC_BASE_URL: opts.subscriptionBaseUrl,
|
|
438
|
-
ANTHROPIC_AUTH_TOKEN: opts.subscriptionToken!,
|
|
439
|
-
}
|
|
440
|
-
: { CLAUDE_CODE_OAUTH_TOKEN: opts.subscriptionToken! }),
|
|
441
|
-
}
|
|
444
|
+
const env = buildClaudeEnv(opts, configHome)
|
|
442
445
|
|
|
443
446
|
// ADR 0026 D3 (path corrected by ADR 0027 Defect A): while the run is live, tail the CLI's
|
|
444
447
|
// subagent `*.jsonl` transcripts so a parallel-subagent review keeps the inactivity
|
|
@@ -453,6 +456,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
453
456
|
...(opts.onActivity ? { onActivity: opts.onActivity } : {}),
|
|
454
457
|
secrets,
|
|
455
458
|
model: opts.model,
|
|
459
|
+
...(opts.onCallMetric ? { onCallMetric: opts.onCallMetric } : {}),
|
|
456
460
|
...(opts.log ? { log: opts.log } : {}),
|
|
457
461
|
})
|
|
458
462
|
: undefined
|
|
@@ -484,39 +488,15 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
484
488
|
onEvent,
|
|
485
489
|
)
|
|
486
490
|
|
|
487
|
-
|
|
488
|
-
// subagent calls, which carry their own per-turn tokens, are concatenated).
|
|
489
|
-
attributeCumulativeUsage(calls, usage)
|
|
490
|
-
// Final drain of any subagent transcript writes that landed after the last poll, then
|
|
491
|
-
// fold the subagents' usage + per-call telemetry into the run's outcome — their tokens
|
|
492
|
-
// never appear on the parent stream, so this is the only place they are accounted.
|
|
493
|
-
await subagents?.stop()
|
|
494
|
-
const subUsage = subagents?.usage() ?? { inputTokens: 0, outputTokens: 0 }
|
|
495
|
-
const subCalls = subagents?.calls() ?? []
|
|
496
|
-
const mergedCalls = [...calls, ...subCalls]
|
|
497
|
-
// INVARIANT (do not "fix" this into a double count): the run total is the parent usage
|
|
498
|
-
// PLUS the subagent usage because the two are disjoint sources. The parent `usage` here
|
|
499
|
-
// is the terminal `result` event's cumulative, which covers ONLY the parent loop — the
|
|
500
|
-
// ADR 0026 incident is itself the proof: a heavily subagent-parallelised review reported
|
|
501
|
-
// ~0 tokens, i.e. the parent stream (and its `result` total) never included the subagent
|
|
502
|
-
// spend. The subagent tokens live exclusively in the per-session `subagents/*.jsonl`
|
|
503
|
-
// transcripts, which the watcher reads and nothing else does; it deliberately EXCLUDES the
|
|
504
|
-
// sibling parent session transcript (whose usage `result` already totals), so neither
|
|
505
|
-
// `calls` nor `usage` can already contain the subagent spend.
|
|
506
|
-
const mergedUsage =
|
|
507
|
-
usage || subUsage.inputTokens || subUsage.outputTokens
|
|
508
|
-
? {
|
|
509
|
-
inputTokens: (usage?.inputTokens ?? 0) + subUsage.inputTokens,
|
|
510
|
-
outputTokens: (usage?.outputTokens ?? 0) + subUsage.outputTokens,
|
|
511
|
-
}
|
|
512
|
-
: undefined
|
|
513
|
-
return {
|
|
491
|
+
return await assembleClaudeOutcome({
|
|
514
492
|
summary,
|
|
515
493
|
stats,
|
|
516
494
|
stderrTail,
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
495
|
+
calls,
|
|
496
|
+
publisher,
|
|
497
|
+
usage,
|
|
498
|
+
subagents,
|
|
499
|
+
})
|
|
520
500
|
} finally {
|
|
521
501
|
await subagents?.stop()
|
|
522
502
|
if (configHome) {
|
|
@@ -533,6 +513,77 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
533
513
|
}
|
|
534
514
|
}
|
|
535
515
|
|
|
516
|
+
/**
|
|
517
|
+
* Build the child-process env for the `claude` CLI: an isolated config home plus subscription
|
|
518
|
+
* auth (Anthropic OAuth token, or an Anthropic-compatible base URL + auth token for a
|
|
519
|
+
* non-Anthropic Claude-Code vendor like GLM/Kimi/DeepSeek), or an empty env in ambient mode
|
|
520
|
+
* (the developer's own logged-in `~/.claude` is used). Extracted from {@link runClaudeCode} to
|
|
521
|
+
* keep its cyclomatic complexity down; behaviour is a straight move of the original expression.
|
|
522
|
+
*/
|
|
523
|
+
function buildClaudeEnv(
|
|
524
|
+
opts: SubscriptionRunOptions,
|
|
525
|
+
configHome: string | undefined,
|
|
526
|
+
): Record<string, string> {
|
|
527
|
+
if (opts.ambientAuth) return {}
|
|
528
|
+
return {
|
|
529
|
+
CLAUDE_CONFIG_DIR: configHome!,
|
|
530
|
+
...(opts.subscriptionBaseUrl
|
|
531
|
+
? {
|
|
532
|
+
ANTHROPIC_BASE_URL: opts.subscriptionBaseUrl,
|
|
533
|
+
ANTHROPIC_AUTH_TOKEN: opts.subscriptionToken!,
|
|
534
|
+
}
|
|
535
|
+
: { CLAUDE_CODE_OAUTH_TOKEN: opts.subscriptionToken! }),
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/**
|
|
540
|
+
* Merge the parent-loop telemetry with the subagents' out-of-band usage + per-call metrics into
|
|
541
|
+
* the run outcome. INVARIANT (do not "fix" this into a double count): the run total is the parent
|
|
542
|
+
* usage PLUS the subagent usage because the two are disjoint sources — the parent `usage` (the
|
|
543
|
+
* terminal `result` event's cumulative) covers ONLY the parent loop, and the subagent tokens live
|
|
544
|
+
* exclusively in the per-session `subagents/*.jsonl` transcripts the watcher reads. Extracted from
|
|
545
|
+
* {@link runClaudeCode} verbatim to keep its cyclomatic complexity down.
|
|
546
|
+
*/
|
|
547
|
+
async function assembleClaudeOutcome(args: {
|
|
548
|
+
summary: string
|
|
549
|
+
stats: PiRunStats
|
|
550
|
+
stderrTail: string
|
|
551
|
+
calls: HarnessCallMetric[]
|
|
552
|
+
/** The live-stream publisher, flushed once attribution has finalised the calls' tokens. */
|
|
553
|
+
publisher: CallMetricPublisher
|
|
554
|
+
usage: { inputTokens: number; outputTokens: number } | undefined
|
|
555
|
+
subagents: ReturnType<typeof startSubagentWatcher> | undefined
|
|
556
|
+
}): Promise<PiRunOutcome> {
|
|
557
|
+
const { summary, stats, stderrTail, calls, publisher, usage, subagents } = args
|
|
558
|
+
// The parent's cumulative-usage fallback applies to the PARENT calls only (before the
|
|
559
|
+
// subagent calls, which carry their own per-turn tokens, are concatenated).
|
|
560
|
+
attributeCumulativeUsage(calls, usage)
|
|
561
|
+
// The withheld calls are final only NOW, so stream them: the completion poll drains them
|
|
562
|
+
// alongside the result, and the backend records the attributed numbers rather than the zeros
|
|
563
|
+
// they carried while the run was in flight.
|
|
564
|
+
publisher.flush()
|
|
565
|
+
// Final drain of any subagent transcript writes that landed after the last poll, then
|
|
566
|
+
// fold the subagents' usage + per-call telemetry into the run's outcome.
|
|
567
|
+
await subagents?.stop()
|
|
568
|
+
const subUsage = subagents?.usage() ?? { inputTokens: 0, outputTokens: 0 }
|
|
569
|
+
const subCalls = subagents?.calls() ?? []
|
|
570
|
+
const mergedCalls = [...calls, ...subCalls]
|
|
571
|
+
const mergedUsage =
|
|
572
|
+
usage || subUsage.inputTokens || subUsage.outputTokens
|
|
573
|
+
? {
|
|
574
|
+
inputTokens: (usage?.inputTokens ?? 0) + subUsage.inputTokens,
|
|
575
|
+
outputTokens: (usage?.outputTokens ?? 0) + subUsage.outputTokens,
|
|
576
|
+
}
|
|
577
|
+
: undefined
|
|
578
|
+
return {
|
|
579
|
+
summary,
|
|
580
|
+
stats,
|
|
581
|
+
stderrTail,
|
|
582
|
+
...(mergedUsage ? { usage: mergedUsage } : {}),
|
|
583
|
+
...(mergedCalls.length ? { callMetrics: mergedCalls } : {}),
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
|
|
536
587
|
/** Map Claude Code's `TodoWrite` todos array onto subtask counts. */
|
|
537
588
|
function todosToProgress(todos: unknown): TodoProgress | undefined {
|
|
538
589
|
if (!Array.isArray(todos)) return undefined
|
|
@@ -649,17 +700,21 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
649
700
|
// assistant text seen since the previous turn as one telemetry call.
|
|
650
701
|
const perTurn = codexLastTurnUsage(event)
|
|
651
702
|
if (perTurn) {
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
703
|
+
publishCallMetric(
|
|
704
|
+
calls,
|
|
705
|
+
{
|
|
706
|
+
model: opts.model,
|
|
707
|
+
promptText: redactBody(JSON.stringify(messages), secrets),
|
|
708
|
+
messageCount: messages.length,
|
|
709
|
+
responseText: redactBody(pendingText, secrets),
|
|
710
|
+
reasoningText: '',
|
|
711
|
+
inputTokens: perTurn.inputTokens,
|
|
712
|
+
cachedInputTokens: perTurn.cachedInputTokens,
|
|
713
|
+
outputTokens: perTurn.outputTokens,
|
|
714
|
+
finishReason: null,
|
|
715
|
+
},
|
|
716
|
+
opts.onCallMetric,
|
|
717
|
+
)
|
|
663
718
|
if (pendingText) messages.push({ role: 'assistant', content: pendingText })
|
|
664
719
|
pendingText = ''
|
|
665
720
|
}
|
|
@@ -691,17 +746,21 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
691
746
|
// Fallback for a CLI/version that never emits per-turn `last_token_usage`: record a
|
|
692
747
|
// single call from the cumulative total + final text so the run is still observable.
|
|
693
748
|
if (calls.length === 0 && (usage || summary)) {
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
749
|
+
publishCallMetric(
|
|
750
|
+
calls,
|
|
751
|
+
{
|
|
752
|
+
model: opts.model,
|
|
753
|
+
promptText: redactBody(JSON.stringify(messages), secrets),
|
|
754
|
+
messageCount: messages.length,
|
|
755
|
+
responseText: redactBody(summary, secrets),
|
|
756
|
+
reasoningText: '',
|
|
757
|
+
inputTokens: usage?.inputTokens ?? 0,
|
|
758
|
+
cachedInputTokens: 0,
|
|
759
|
+
outputTokens: usage?.outputTokens ?? 0,
|
|
760
|
+
finishReason: null,
|
|
761
|
+
},
|
|
762
|
+
opts.onCallMetric,
|
|
763
|
+
)
|
|
705
764
|
}
|
|
706
765
|
return {
|
|
707
766
|
summary,
|
package/src/agent.ts
CHANGED
|
@@ -903,6 +903,55 @@ async function runCodingMode(job: AgentJob, opts: RunOptions): Promise<AgentResu
|
|
|
903
903
|
return result
|
|
904
904
|
}
|
|
905
905
|
|
|
906
|
+
/**
|
|
907
|
+
* Assemble the {@link runCodingAgent} spec for the ordinary single-repo coding flow. Extracted
|
|
908
|
+
* from {@link runSingleRepoCoding} so the many optional-field spreads don't inflate that
|
|
909
|
+
* function's cyclomatic complexity; the mapping is a straight field copy off `job`.
|
|
910
|
+
*/
|
|
911
|
+
function buildSingleRepoCodingSpec(
|
|
912
|
+
job: AgentJob,
|
|
913
|
+
pushBranch: string,
|
|
914
|
+
): Parameters<typeof runCodingAgent>[0] {
|
|
915
|
+
return {
|
|
916
|
+
kind: 'agent',
|
|
917
|
+
jobId: job.jobId,
|
|
918
|
+
repo: job.repo,
|
|
919
|
+
cloneBranch: job.branch,
|
|
920
|
+
...(job.newBranch ? { newBranch: job.newBranch } : {}),
|
|
921
|
+
pushBranch,
|
|
922
|
+
ghToken: job.ghToken,
|
|
923
|
+
systemPrompt: job.systemPrompt,
|
|
924
|
+
userPrompt: job.userPrompt,
|
|
925
|
+
model: job.model,
|
|
926
|
+
harness: job.harness,
|
|
927
|
+
subscriptionToken: job.subscriptionToken,
|
|
928
|
+
subscriptionBaseUrl: job.subscriptionBaseUrl,
|
|
929
|
+
ambientAuth: job.ambientAuth,
|
|
930
|
+
proxyBaseUrl: job.proxyBaseUrl,
|
|
931
|
+
sessionToken: job.sessionToken,
|
|
932
|
+
commitMessage: job.commitMessage ?? job.pr?.title ?? 'Agent changes',
|
|
933
|
+
webToolsGuidance: job.webToolsGuidance,
|
|
934
|
+
webSearchProxy: job.webSearch,
|
|
935
|
+
guardLimits: job.guardLimits,
|
|
936
|
+
...(job.persistentCheckout ? { persistentCheckout: true } : {}),
|
|
937
|
+
...(job.streamFollowUps ? { streamFollowUps: true } : {}),
|
|
938
|
+
...(job.referenceBranches?.length ? { referenceBranches: job.referenceBranches } : {}),
|
|
939
|
+
// Repo-sourced skill (slice 2): installed harness-aware by runAgentInWorkspace.
|
|
940
|
+
...(job.skill ? { skill: job.skill } : {}),
|
|
941
|
+
// Ralph loop: run the completion command after the agent commits and report its verdict.
|
|
942
|
+
...(job.validation
|
|
943
|
+
? {
|
|
944
|
+
validation: {
|
|
945
|
+
command: job.validation.command,
|
|
946
|
+
...(job.validation.iteration !== undefined
|
|
947
|
+
? { iteration: job.validation.iteration }
|
|
948
|
+
: {}),
|
|
949
|
+
},
|
|
950
|
+
}
|
|
951
|
+
: {}),
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
|
|
906
955
|
/**
|
|
907
956
|
* The ordinary single-repo coding flow: clone `branch` (or resume `newBranch`), run the agent,
|
|
908
957
|
* commit + push to `pushBranch`, and open `pr` when one is set and the run produced changes. A
|
|
@@ -912,47 +961,7 @@ async function runCodingMode(job: AgentJob, opts: RunOptions): Promise<AgentResu
|
|
|
912
961
|
async function runSingleRepoCoding(job: AgentJob, opts: RunOptions): Promise<AgentResult> {
|
|
913
962
|
const pushBranch = job.pushBranch ?? job.newBranch ?? job.branch
|
|
914
963
|
const { summary, stats, stderrTail, pushed, usage, callMetrics, validation, effortReport } =
|
|
915
|
-
await runCodingAgent(
|
|
916
|
-
{
|
|
917
|
-
kind: 'agent',
|
|
918
|
-
jobId: job.jobId,
|
|
919
|
-
repo: job.repo,
|
|
920
|
-
cloneBranch: job.branch,
|
|
921
|
-
...(job.newBranch ? { newBranch: job.newBranch } : {}),
|
|
922
|
-
pushBranch,
|
|
923
|
-
ghToken: job.ghToken,
|
|
924
|
-
systemPrompt: job.systemPrompt,
|
|
925
|
-
userPrompt: job.userPrompt,
|
|
926
|
-
model: job.model,
|
|
927
|
-
harness: job.harness,
|
|
928
|
-
subscriptionToken: job.subscriptionToken,
|
|
929
|
-
subscriptionBaseUrl: job.subscriptionBaseUrl,
|
|
930
|
-
ambientAuth: job.ambientAuth,
|
|
931
|
-
proxyBaseUrl: job.proxyBaseUrl,
|
|
932
|
-
sessionToken: job.sessionToken,
|
|
933
|
-
commitMessage: job.commitMessage ?? job.pr?.title ?? 'Agent changes',
|
|
934
|
-
webToolsGuidance: job.webToolsGuidance,
|
|
935
|
-
webSearchProxy: job.webSearch,
|
|
936
|
-
guardLimits: job.guardLimits,
|
|
937
|
-
...(job.persistentCheckout ? { persistentCheckout: true } : {}),
|
|
938
|
-
...(job.streamFollowUps ? { streamFollowUps: true } : {}),
|
|
939
|
-
...(job.referenceBranches?.length ? { referenceBranches: job.referenceBranches } : {}),
|
|
940
|
-
// Repo-sourced skill (slice 2): installed harness-aware by runAgentInWorkspace.
|
|
941
|
-
...(job.skill ? { skill: job.skill } : {}),
|
|
942
|
-
// Ralph loop: run the completion command after the agent commits and report its verdict.
|
|
943
|
-
...(job.validation
|
|
944
|
-
? {
|
|
945
|
-
validation: {
|
|
946
|
-
command: job.validation.command,
|
|
947
|
-
...(job.validation.iteration !== undefined
|
|
948
|
-
? { iteration: job.validation.iteration }
|
|
949
|
-
: {}),
|
|
950
|
-
},
|
|
951
|
-
}
|
|
952
|
-
: {}),
|
|
953
|
-
},
|
|
954
|
-
opts,
|
|
955
|
-
)
|
|
964
|
+
await runCodingAgent(buildSingleRepoCodingSpec(job, pushBranch), opts)
|
|
956
965
|
// Ralph loop: the harness-computed validation verdict, forwarded onto the coding result as
|
|
957
966
|
// `ralphVerdict` so the backend's `toRunResult` lifts it onto `AgentRunResult.ralphVerdict`.
|
|
958
967
|
const ralphVerdict = validation ? { ralphVerdict: validation } : {}
|
package/src/job.ts
CHANGED
|
@@ -1124,6 +1124,24 @@ function isReservedEnvName(key: string): boolean {
|
|
|
1124
1124
|
return RESERVED_ENV_PREFIXES.some((p) => lower.startsWith(p))
|
|
1125
1125
|
}
|
|
1126
1126
|
|
|
1127
|
+
/**
|
|
1128
|
+
* Collect only string→string entries from a raw `env` bag. A non-string value is dropped so a
|
|
1129
|
+
* malformed binding can't inject `[object Object]` (or undefined) as an upstream URL. Reserved
|
|
1130
|
+
* names that would break the toolchain or enable injection (PATH, NODE_OPTIONS, LD_PRELOAD, …) are
|
|
1131
|
+
* dropped too: they are spread over `process.env` at build time, so a binding named `PATH` would
|
|
1132
|
+
* replace it with a URL and the build would no longer find its tools. Extracted from the infra
|
|
1133
|
+
* parsers to keep their cyclomatic complexity down.
|
|
1134
|
+
*/
|
|
1135
|
+
function parseInfraEnv(raw: unknown): Record<string, string> {
|
|
1136
|
+
const env: Record<string, string> = {}
|
|
1137
|
+
if (typeof raw === 'object' && raw !== null) {
|
|
1138
|
+
for (const [key, val] of Object.entries(raw as Record<string, unknown>)) {
|
|
1139
|
+
if (key && !isReservedEnvName(key) && typeof val === 'string') env[key] = val
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
return env
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1127
1145
|
/** Parse the frontend UI-test infra spec (`kind: 'frontend'`), tolerating missing knobs. */
|
|
1128
1146
|
function parseFrontendInfraSpec(o: Record<string, unknown>): FrontendInfraSpec {
|
|
1129
1147
|
const packageManager =
|
|
@@ -1133,17 +1151,7 @@ function parseFrontendInfraSpec(o: Record<string, unknown>): FrontendInfraSpec {
|
|
|
1133
1151
|
const serveMode = o.serveMode === 'static' || o.serveMode === 'command' ? o.serveMode : undefined
|
|
1134
1152
|
const envInjection =
|
|
1135
1153
|
o.envInjection === 'build' || o.envInjection === 'runtime' ? o.envInjection : undefined
|
|
1136
|
-
|
|
1137
|
-
// binding can't inject `[object Object]` (or undefined) as an upstream URL. Reserved names
|
|
1138
|
-
// that would break the toolchain or enable injection (PATH, NODE_OPTIONS, LD_PRELOAD, …) are
|
|
1139
|
-
// dropped too: they are spread over `process.env` at build time, so a binding named `PATH`
|
|
1140
|
-
// would replace it with a URL and the build would no longer find its tools.
|
|
1141
|
-
const env: Record<string, string> = {}
|
|
1142
|
-
if (typeof o.env === 'object' && o.env !== null) {
|
|
1143
|
-
for (const [key, val] of Object.entries(o.env as Record<string, unknown>)) {
|
|
1144
|
-
if (key && !isReservedEnvName(key) && typeof val === 'string') env[key] = val
|
|
1145
|
-
}
|
|
1146
|
-
}
|
|
1154
|
+
const env = parseInfraEnv(o.env)
|
|
1147
1155
|
const servePort = port(o.servePort)
|
|
1148
1156
|
const wiremockPort = port(o.wiremockPort)
|
|
1149
1157
|
// The app's monorepo subdirectory becomes the install/build/serve cwd, so it goes through the
|
|
@@ -1376,11 +1384,7 @@ function assembleAgentJob(
|
|
|
1376
1384
|
ghToken: str(o.ghToken, 'ghToken'),
|
|
1377
1385
|
repo: parseRepoSpec(repo),
|
|
1378
1386
|
branch: str(o.branch, 'branch'),
|
|
1379
|
-
...(
|
|
1380
|
-
...(typeof o.webToolsGuidance === 'string' ? { webToolsGuidance: o.webToolsGuidance } : {}),
|
|
1381
|
-
...(o.webSearch === true ? { webSearch: true } : {}),
|
|
1382
|
-
...(o.full === true ? { full: true } : {}),
|
|
1383
|
-
...(typeof o.mergeBase === 'string' && o.mergeBase ? { mergeBase: o.mergeBase } : {}),
|
|
1387
|
+
...collectOptionalRequestFields(o),
|
|
1384
1388
|
...(bootstrap ? { bootstrap } : {}),
|
|
1385
1389
|
...(output ? { output } : {}),
|
|
1386
1390
|
...(contextFiles.length ? { contextFiles } : {}),
|
|
@@ -1388,20 +1392,36 @@ function assembleAgentJob(
|
|
|
1388
1392
|
...(skill ? { skill } : {}),
|
|
1389
1393
|
...(testSecrets.length ? { testSecrets } : {}),
|
|
1390
1394
|
...(infra ? { infra } : {}),
|
|
1391
|
-
...(typeof o.newBranch === 'string' && o.newBranch ? { newBranch: o.newBranch } : {}),
|
|
1392
|
-
...(typeof o.pushBranch === 'string' && o.pushBranch ? { pushBranch: o.pushBranch } : {}),
|
|
1393
|
-
...(typeof o.commitMessage === 'string' && o.commitMessage
|
|
1394
|
-
? { commitMessage: o.commitMessage }
|
|
1395
|
-
: {}),
|
|
1396
1395
|
...(pr ? { pr } : {}),
|
|
1397
1396
|
...(peerRepos.length ? { peerRepos } : {}),
|
|
1398
1397
|
...(referenceRepos.length ? { referenceRepos } : {}),
|
|
1399
1398
|
...(referenceBranches.length ? { referenceBranches } : {}),
|
|
1400
1399
|
...(reviewPrNumber !== undefined ? { reviewPrNumber } : {}),
|
|
1400
|
+
...(guardLimits ? { guardLimits } : {}),
|
|
1401
|
+
...(validation ? { validation } : {}),
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
/**
|
|
1406
|
+
* The optional {@link AgentJob} fields read directly off the request `o` (booleans + trimmed
|
|
1407
|
+
* strings). Extracted from {@link assembleAgentJob} to keep its cyclomatic complexity down; every
|
|
1408
|
+
* key is unique so grouping the conditional spreads is behaviour-neutral (spread order is
|
|
1409
|
+
* irrelevant with no colliding keys).
|
|
1410
|
+
*/
|
|
1411
|
+
function collectOptionalRequestFields(o: Record<string, unknown>): Partial<AgentJob> {
|
|
1412
|
+
return {
|
|
1413
|
+
...(typeof o.githubApiBase === 'string' ? { githubApiBase: o.githubApiBase } : {}),
|
|
1414
|
+
...(typeof o.webToolsGuidance === 'string' ? { webToolsGuidance: o.webToolsGuidance } : {}),
|
|
1415
|
+
...(o.webSearch === true ? { webSearch: true } : {}),
|
|
1416
|
+
...(o.full === true ? { full: true } : {}),
|
|
1417
|
+
...(typeof o.mergeBase === 'string' && o.mergeBase ? { mergeBase: o.mergeBase } : {}),
|
|
1418
|
+
...(typeof o.newBranch === 'string' && o.newBranch ? { newBranch: o.newBranch } : {}),
|
|
1419
|
+
...(typeof o.pushBranch === 'string' && o.pushBranch ? { pushBranch: o.pushBranch } : {}),
|
|
1420
|
+
...(typeof o.commitMessage === 'string' && o.commitMessage
|
|
1421
|
+
? { commitMessage: o.commitMessage }
|
|
1422
|
+
: {}),
|
|
1401
1423
|
...(o.noChangesIsError === false ? { noChangesIsError: false } : {}),
|
|
1402
1424
|
...(o.persistentCheckout === true ? { persistentCheckout: true } : {}),
|
|
1403
1425
|
...(o.streamFollowUps === true ? { streamFollowUps: true } : {}),
|
|
1404
|
-
...(guardLimits ? { guardLimits } : {}),
|
|
1405
|
-
...(validation ? { validation } : {}),
|
|
1406
1426
|
}
|
|
1407
1427
|
}
|
package/src/pi-workspace.ts
CHANGED
|
@@ -271,6 +271,10 @@ export async function runAgentInWorkspace(
|
|
|
271
271
|
signal: opts.signal,
|
|
272
272
|
onActivity: opts.onActivity,
|
|
273
273
|
onProgress: opts.onProgress,
|
|
274
|
+
// Stream this run's per-call telemetry to the job's live drain. The subscription
|
|
275
|
+
// harnesses are the only producers of `callMetrics` (Pi's calls are metered by the LLM
|
|
276
|
+
// proxy as they happen), so this is the only path that needs the hook.
|
|
277
|
+
onCallMetric: opts.onCallMetric,
|
|
274
278
|
...(opts.log ? { log: opts.log } : {}),
|
|
275
279
|
})
|
|
276
280
|
return withEffortReport(spec.dir, subOutcome)
|
package/src/pi.ts
CHANGED
|
@@ -505,6 +505,93 @@ export interface HarnessCallMetric {
|
|
|
505
505
|
outputTokens: number
|
|
506
506
|
/** The provider finish/stop reason when the CLI reports one (else null). */
|
|
507
507
|
finishReason: string | null
|
|
508
|
+
/**
|
|
509
|
+
* This call's position in the JOB's telemetry sequence, stamped by the job registry the
|
|
510
|
+
* moment the call is emitted (see `RunOptions.onCallMetric`). It is what makes a call's
|
|
511
|
+
* recorded row id stable across the two channels that carry it: the live drain (per poll,
|
|
512
|
+
* so a run's telemetry is inspectable WHILE it runs) and the terminal result (the complete
|
|
513
|
+
* list, so a transport that doesn't drain still records everything). Both channels hold the
|
|
514
|
+
* SAME metric objects, so both mint the same `<jobId>-hc-<seq>` row id and the backend's
|
|
515
|
+
* second write of an already-recorded call is a no-op instead of a duplicate row.
|
|
516
|
+
*
|
|
517
|
+
* Absent only when a producer built a metric without emitting it live; the recorder then
|
|
518
|
+
* falls back to the array index, which is what it always used before streaming existed.
|
|
519
|
+
*/
|
|
520
|
+
seq?: number
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/**
|
|
524
|
+
* Publish one captured model call: append it to the run's list (which becomes the terminal
|
|
525
|
+
* result's `callMetrics`) AND hand the SAME object to the live stream, where the job registry
|
|
526
|
+
* stamps its {@link HarnessCallMetric.seq} and buffers it for the next poll to drain.
|
|
527
|
+
*
|
|
528
|
+
* Every producer goes through here rather than a bare `calls.push`, so the two channels can't
|
|
529
|
+
* drift: a call that reaches the terminal list but never the live stream would be invisible
|
|
530
|
+
* until the job ends, and one that reaches only the live stream would go unrecorded if the
|
|
531
|
+
* poll response were lost.
|
|
532
|
+
*
|
|
533
|
+
* A published call must be FINAL. The backend records it the moment the drain reaches it and
|
|
534
|
+
* IGNORES the terminal repeat (first write wins, so its stored prompt delta stays valid against
|
|
535
|
+
* the chain tip it was written against), which means a field mutated after publishing never
|
|
536
|
+
* reaches the store. A producer whose calls can still change (the cumulative-usage fallback,
|
|
537
|
+
* whose totals arrive with the CLI's terminal `result` event) publishes through
|
|
538
|
+
* {@link createCallMetricPublisher} instead, which withholds exactly those.
|
|
539
|
+
*/
|
|
540
|
+
export function publishCallMetric(
|
|
541
|
+
calls: HarnessCallMetric[],
|
|
542
|
+
call: HarnessCallMetric,
|
|
543
|
+
onCallMetric?: (call: HarnessCallMetric) => void,
|
|
544
|
+
): void {
|
|
545
|
+
calls.push(call)
|
|
546
|
+
onCallMetric?.(call)
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
/** Appends captured calls to a run's list, streaming each one as soon as it is final. */
|
|
550
|
+
export interface CallMetricPublisher {
|
|
551
|
+
/** Append a captured call, streaming it now unless its tokens can still be rewritten. */
|
|
552
|
+
publish(call: HarnessCallMetric): void
|
|
553
|
+
/** Stream whatever is still withheld. Call once the run's totals are attributed. */
|
|
554
|
+
flush(): void
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
/**
|
|
558
|
+
* A {@link publishCallMetric} wrapper for a producer whose per-call tokens may be filled in at
|
|
559
|
+
* the END of the run: a CLI that reports only a cumulative total leaves every turn at zero, and
|
|
560
|
+
* `attributeCumulativeUsage` pins the total onto the last call once the terminal `result` event
|
|
561
|
+
* arrives.
|
|
562
|
+
*
|
|
563
|
+
* Since a published call must be final (the backend stores it on the drain and ignores the
|
|
564
|
+
* terminal repeat), a call the CLI did NOT cost is appended to the list but WITHHELD from the
|
|
565
|
+
* live stream — otherwise it records as a zero-token row and the attributed numbers never land.
|
|
566
|
+
* The withholding window closes the moment any call IS costed: attribution can no longer fire, so
|
|
567
|
+
* everything held is final and released at once, in capture order, and every later call streams
|
|
568
|
+
* immediately whatever its tokens. {@link flush} covers the run that was never costed at all.
|
|
569
|
+
*/
|
|
570
|
+
export function createCallMetricPublisher(
|
|
571
|
+
calls: HarnessCallMetric[],
|
|
572
|
+
onCallMetric?: (call: HarnessCallMetric) => void,
|
|
573
|
+
): CallMetricPublisher {
|
|
574
|
+
const withheld: HarnessCallMetric[] = []
|
|
575
|
+
let anyCosted = false
|
|
576
|
+
const flush = (): void => {
|
|
577
|
+
for (const call of withheld) onCallMetric?.(call)
|
|
578
|
+
withheld.length = 0
|
|
579
|
+
}
|
|
580
|
+
return {
|
|
581
|
+
publish(call) {
|
|
582
|
+
const costed = call.inputTokens > 0 || call.outputTokens > 0
|
|
583
|
+
if (!costed && !anyCosted) {
|
|
584
|
+
publishCallMetric(calls, call)
|
|
585
|
+
withheld.push(call)
|
|
586
|
+
return
|
|
587
|
+
}
|
|
588
|
+
if (costed) anyCosted = true
|
|
589
|
+
// Released BEFORE this call so the live sequence stays in capture order.
|
|
590
|
+
flush()
|
|
591
|
+
publishCallMetric(calls, call, onCallMetric)
|
|
592
|
+
},
|
|
593
|
+
flush,
|
|
594
|
+
}
|
|
508
595
|
}
|
|
509
596
|
|
|
510
597
|
/** Pi's assistant summary plus {@link PiRunStats} describing what it did. */
|
package/src/runner.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { redactSecrets } from './redact.js'
|
|
2
2
|
import type { FollowUpLine } from './follow-ups.js'
|
|
3
|
-
import type { TodoProgress, ToolSpan } from './pi.js'
|
|
3
|
+
import type { HarnessCallMetric, TodoProgress, ToolSpan } from './pi.js'
|
|
4
4
|
import { log, type Logger } from './logger.js'
|
|
5
5
|
import {
|
|
6
6
|
type FailureCause,
|
|
@@ -30,6 +30,19 @@ export interface RunOptions {
|
|
|
30
30
|
onSpan?: (span: ToolSpan) => void
|
|
31
31
|
/** Receives the forward-looking follow-up / question items the Coder streamed since the last poll. */
|
|
32
32
|
onFollowUp?: (items: FollowUpLine[]) => void
|
|
33
|
+
/**
|
|
34
|
+
* Receives each per-call telemetry row the moment the agent's CLI stream yields it, so a
|
|
35
|
+
* run's model calls reach `llm_call_metrics` WHILE it runs rather than only in its terminal
|
|
36
|
+
* result. The registry stamps the call's job-scoped {@link HarnessCallMetric.seq} and buffers
|
|
37
|
+
* it for the next poll to drain.
|
|
38
|
+
*
|
|
39
|
+
* Call this for every metric you also put on the result — the SAME object, not a copy: the
|
|
40
|
+
* stamped `seq` is what lets the backend recognise the terminal write of an already-recorded
|
|
41
|
+
* call and skip it. A run that dies mid-flight (the container is evicted, the harness process
|
|
42
|
+
* is OOM-killed) never produces a terminal result, so without this its entire token spend and
|
|
43
|
+
* every prompt/response body are lost — exactly the run an operator most needs to inspect.
|
|
44
|
+
*/
|
|
45
|
+
onCallMetric?: (call: HarnessCallMetric) => void
|
|
33
46
|
/**
|
|
34
47
|
* Mark the coarse lifecycle phase the handler has entered (`clone` / `agent` / `push` / …).
|
|
35
48
|
* Drives the stuck-run breadcrumb: an inactivity kill reports WHICH phase was hung, and the
|
|
@@ -114,6 +127,17 @@ export interface JobView<TResult extends JobResultBase = JobResultBase> {
|
|
|
114
127
|
* surfaces the first one (and only on a follow-ups-enabled coding run).
|
|
115
128
|
*/
|
|
116
129
|
followUps?: FollowUpLine[]
|
|
130
|
+
/**
|
|
131
|
+
* Per-model-call telemetry the agent's CLI stream yielded SINCE THE LAST POLL
|
|
132
|
+
* (drain-on-read, exactly like {@link spans}). The backend records these into
|
|
133
|
+
* `llm_call_metrics` as they arrive, so a run's token spend and prompt/response bodies are
|
|
134
|
+
* queryable while it is still running — and survive it dying before it can produce a
|
|
135
|
+
* terminal result. Each carries a job-scoped `seq` so the terminal
|
|
136
|
+
* {@link JobResultBase} list can re-offer the same calls without duplicating rows.
|
|
137
|
+
* Absent until the agent's first model call (and on the proxy-metered Pi harness, whose
|
|
138
|
+
* calls the LLM proxy meters directly).
|
|
139
|
+
*/
|
|
140
|
+
callMetrics?: HarnessCallMetric[]
|
|
117
141
|
/**
|
|
118
142
|
* ADR 0026 D4: set when the cold-start watchdog fired — the job produced NO activity
|
|
119
143
|
* within {@link RunnerLimits.coldStartMs} of starting, a likely onboarding/auth wedge.
|
|
@@ -136,6 +160,13 @@ interface JobEntry<TResult extends JobResultBase> extends JobView<TResult> {
|
|
|
136
160
|
spanBuffer: ToolSpan[]
|
|
137
161
|
/** Follow-up items buffered since the last drain (see {@link JobView.followUps}). */
|
|
138
162
|
followUpBuffer: FollowUpLine[]
|
|
163
|
+
/** Call telemetry buffered since the last drain (see {@link JobView.callMetrics}). */
|
|
164
|
+
callMetricBuffer: HarnessCallMetric[]
|
|
165
|
+
/**
|
|
166
|
+
* Next job-scoped {@link HarnessCallMetric.seq} to stamp. Monotonic for the life of the job
|
|
167
|
+
* (never reset by a drain), so a call's row id stays unique across every poll window.
|
|
168
|
+
*/
|
|
169
|
+
callMetricSeq: number
|
|
139
170
|
/** Abort the in-flight run (see {@link JobRegistry.abortAll}); set while running only. */
|
|
140
171
|
abort?: (reason: string) => void
|
|
141
172
|
}
|
|
@@ -192,6 +223,8 @@ function toView<TResult extends JobResultBase>(entry: JobEntry<TResult>): JobVie
|
|
|
192
223
|
promise: _promise,
|
|
193
224
|
spanBuffer: _spanBuffer,
|
|
194
225
|
followUpBuffer: _followUpBuffer,
|
|
226
|
+
callMetricBuffer: _callMetricBuffer,
|
|
227
|
+
callMetricSeq: _callMetricSeq,
|
|
195
228
|
abort: _abort,
|
|
196
229
|
...view
|
|
197
230
|
} = entry
|
|
@@ -237,6 +270,8 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
|
|
|
237
270
|
promise: Promise.resolve(),
|
|
238
271
|
spanBuffer: [],
|
|
239
272
|
followUpBuffer: [],
|
|
273
|
+
callMetricBuffer: [],
|
|
274
|
+
callMetricSeq: 0,
|
|
240
275
|
}
|
|
241
276
|
this.jobs.set(id, entry)
|
|
242
277
|
entry.promise = this.drive(entry, job)
|
|
@@ -244,9 +279,10 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
|
|
|
244
279
|
}
|
|
245
280
|
|
|
246
281
|
/**
|
|
247
|
-
* Poll the job — and DRAIN its
|
|
248
|
-
* handler is the sole caller, so each poll returns the spans
|
|
249
|
-
* previous poll and clears them, bounding the harness
|
|
282
|
+
* Poll the job — and DRAIN its observability buffers (drain-on-read). The GET /jobs/{id}
|
|
283
|
+
* handler is the sole caller, so each poll returns the spans / follow-ups / call metrics
|
|
284
|
+
* accumulated since the previous poll and clears them, bounding the harness buffers to one
|
|
285
|
+
* poll interval.
|
|
250
286
|
*/
|
|
251
287
|
get(id: string): JobView<TResult> | undefined {
|
|
252
288
|
const entry = this.jobs.get(id)
|
|
@@ -260,6 +296,10 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
|
|
|
260
296
|
view.followUps = entry.followUpBuffer
|
|
261
297
|
entry.followUpBuffer = []
|
|
262
298
|
}
|
|
299
|
+
if (entry.callMetricBuffer.length > 0) {
|
|
300
|
+
view.callMetrics = entry.callMetricBuffer
|
|
301
|
+
entry.callMetricBuffer = []
|
|
302
|
+
}
|
|
263
303
|
return view
|
|
264
304
|
}
|
|
265
305
|
|
|
@@ -374,6 +414,13 @@ export class JobRegistry<TJob = unknown, TResult extends JobResultBase = JobResu
|
|
|
374
414
|
onFollowUp: (items) => {
|
|
375
415
|
entry.followUpBuffer.push(...items)
|
|
376
416
|
},
|
|
417
|
+
onCallMetric: (call) => {
|
|
418
|
+
// Stamp the job-scoped sequence on the metric OBJECT: the handler keeps the same
|
|
419
|
+
// instance for its terminal result, so both channels carry the same `seq` and the
|
|
420
|
+
// backend mints one stable row id per call.
|
|
421
|
+
call.seq = entry.callMetricSeq++
|
|
422
|
+
entry.callMetricBuffer.push(call)
|
|
423
|
+
},
|
|
377
424
|
onPhase: (next) => markPhase(next),
|
|
378
425
|
log: jobLog,
|
|
379
426
|
})
|
package/src/subagents.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { createReadStream, type Dirent } from 'node:fs'
|
|
|
3
3
|
import { basename, join } from 'node:path'
|
|
4
4
|
import { claudeAssistantContent, claudeCallUsage, isObject, redactBody } from './claude-stream.js'
|
|
5
5
|
import type { Logger } from './logger.js'
|
|
6
|
-
import type
|
|
6
|
+
import { publishCallMetric, type HarnessCallMetric, type TodoProgress } from './pi.js'
|
|
7
7
|
|
|
8
8
|
// ADR 0026 D2.1 + D3, corrected by ADR 0027. When the Claude Code CLI reviews a large PR
|
|
9
9
|
// it fans the work out across parallel `Task` subagents. Two things then go dark to the
|
|
@@ -155,6 +155,13 @@ export interface SubagentWatcherOptions {
|
|
|
155
155
|
secrets?: string[]
|
|
156
156
|
/** Fallback model id stamped on a subagent call whose transcript omits one. */
|
|
157
157
|
model?: string
|
|
158
|
+
/**
|
|
159
|
+
* Streams each lifted subagent call to the live telemetry drain (the run's `RunOptions`
|
|
160
|
+
* hook). Subagent work is exactly where a long review spends most of its tokens, and it is
|
|
161
|
+
* the phase the parent stream goes quiet for — so without this a run killed mid-fan-out
|
|
162
|
+
* reports nothing at all.
|
|
163
|
+
*/
|
|
164
|
+
onCallMetric?: (call: HarnessCallMetric) => void
|
|
158
165
|
/** Poll cadence (ms); overridable for tests. */
|
|
159
166
|
intervalMs?: number
|
|
160
167
|
log?: Logger
|
|
@@ -244,23 +251,27 @@ export function startSubagentWatcher(root: string, opts: SubagentWatcherOptions)
|
|
|
244
251
|
if (u.inputTokens === 0 && u.outputTokens === 0) return
|
|
245
252
|
const content = Array.isArray(message.content) ? message.content : []
|
|
246
253
|
const { text, reasoning } = claudeAssistantContent(content)
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
? { model:
|
|
252
|
-
:
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
254
|
+
publishCallMetric(
|
|
255
|
+
calls,
|
|
256
|
+
{
|
|
257
|
+
...(typeof message.model === 'string'
|
|
258
|
+
? { model: message.model }
|
|
259
|
+
: opts.model
|
|
260
|
+
? { model: opts.model }
|
|
261
|
+
: {}),
|
|
262
|
+
// The subagent's own transcript isn't a re-sendable prompt chain, so we don't
|
|
263
|
+
// reconstruct the request side (kept empty); the response + tokens are faithful.
|
|
264
|
+
promptText: '',
|
|
265
|
+
messageCount: 0,
|
|
266
|
+
responseText: redactBody(text, secrets),
|
|
267
|
+
reasoningText: redactBody(reasoning, secrets),
|
|
268
|
+
inputTokens: u.inputTokens,
|
|
269
|
+
cachedInputTokens: u.cachedInputTokens,
|
|
270
|
+
outputTokens: u.outputTokens,
|
|
271
|
+
finishReason: typeof message.stop_reason === 'string' ? message.stop_reason : null,
|
|
272
|
+
},
|
|
273
|
+
opts.onCallMetric,
|
|
274
|
+
)
|
|
264
275
|
usage.inputTokens += u.inputTokens
|
|
265
276
|
usage.outputTokens += u.outputTokens
|
|
266
277
|
}
|