@cat-factory/executor-harness 1.90.0 → 1.92.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-runner.d.ts +8 -1
- package/dist/agent-runner.js +27 -0
- package/dist/pi-workspace.js +3 -0
- package/dist/pi.d.ts +26 -5
- package/dist/pi.js +18 -13
- package/dist/tool-trajectory.d.ts +78 -0
- package/dist/tool-trajectory.js +194 -0
- package/package.json +6 -6
- package/src/agent-runner.ts +48 -0
- package/src/coding-agent.ts +27 -31
- package/src/pi-workspace.ts +3 -0
- package/src/pi.ts +53 -18
- package/src/tool-trajectory.ts +239 -0
package/dist/agent-runner.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Logger } from './logger.js';
|
|
2
|
-
import { type HarnessCallMetric, type PiRunOutcome, type TodoProgress } from './pi.js';
|
|
2
|
+
import { type HarnessCallMetric, type PiRunOutcome, type TodoProgress, type ToolSpan } from './pi.js';
|
|
3
3
|
import { type McpServerSpec, type SkillSpec } from './agent-capabilities.js';
|
|
4
4
|
import { type ProgressGuardLimits } from './progress-guard.js';
|
|
5
5
|
import { type SliceReview } from './subagents.js';
|
|
@@ -72,6 +72,13 @@ export interface SubscriptionRunOptions {
|
|
|
72
72
|
onActivity?: () => void;
|
|
73
73
|
/** Called with the latest subtask counts each time the CLI updates its todo/plan list. */
|
|
74
74
|
onProgress?: (progress: TodoProgress) => void;
|
|
75
|
+
/**
|
|
76
|
+
* Called once per completed tool call with a {@link ToolSpan}: the run's TRAJECTORY. The CLI's
|
|
77
|
+
* tool loop is internal to the CLI and never touches our proxy, so its own event stream is the
|
|
78
|
+
* only place these exist — without this hook a subscription-harness run's account of what it
|
|
79
|
+
* DID dies with the container.
|
|
80
|
+
*/
|
|
81
|
+
onSpan?: (span: ToolSpan) => void;
|
|
75
82
|
/**
|
|
76
83
|
* Called with the FULL set of per-slice reviews each time one lands, so the backend can persist
|
|
77
84
|
* a parallel review's completed work as it happens instead of only from the terminal result.
|
package/dist/agent-runner.js
CHANGED
|
@@ -4,6 +4,7 @@ import { tmpdir } from 'node:os';
|
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
5
|
import { claudeAssistantContent, isObject, numberOf, redactBody } from './claude-stream.js';
|
|
6
6
|
import { createClaudeRunTelemetry, subagentDispatchId } from './claude-call-aggregator.js';
|
|
7
|
+
import { ToolCallTracker, recordClaudeToolResults, } from './tool-trajectory.js';
|
|
7
8
|
import { createCallMetricPublisher, publishCallMetric, } from './pi.js';
|
|
8
9
|
import { claudeAllowedToolPatterns, codexMcpConfigToml, mcpServerSecretValues, writeClaudeMcpConfig, } from './agent-capabilities.js';
|
|
9
10
|
import { ProgressGuard } from './progress-guard.js';
|
|
@@ -372,6 +373,27 @@ function createClaudeProgressGuard(opts) {
|
|
|
372
373
|
reason: () => guardReason,
|
|
373
374
|
};
|
|
374
375
|
}
|
|
376
|
+
/**
|
|
377
|
+
* The run's TRAJECTORY, on the claude-code stream: each `tool_use` block paired with the
|
|
378
|
+
* `tool_result` that answers it on the following user turn, numbered and captured (scrubbed +
|
|
379
|
+
* capped). The CLI's stream is the only place this loop is visible at all — its tool calls never
|
|
380
|
+
* touch our proxy — so without this a subscription-harness run's account of what it DID dies with
|
|
381
|
+
* the container.
|
|
382
|
+
*
|
|
383
|
+
* Both halves are no-ops when the caller wants no spans, so a driver that only needs the run's
|
|
384
|
+
* output never pays to serialise a body nothing will read. Split out of {@link runClaudeCode} for
|
|
385
|
+
* the per-function line budget, like {@link createClaudeProgressGuard}.
|
|
386
|
+
*/
|
|
387
|
+
function createClaudeToolTrajectory(opts, secrets) {
|
|
388
|
+
if (!opts.onSpan)
|
|
389
|
+
return { onToolUse: () => { }, onToolResults: () => { } };
|
|
390
|
+
const onSpan = opts.onSpan;
|
|
391
|
+
const tracker = new ToolCallTracker(secrets);
|
|
392
|
+
return {
|
|
393
|
+
onToolUse: (id, name, input) => tracker.started(id, name, input),
|
|
394
|
+
onToolResults: (content) => recordClaudeToolResults(tracker, content, (call) => onSpan({ ...call, bodies: 'stored' })),
|
|
395
|
+
};
|
|
396
|
+
}
|
|
375
397
|
export async function runClaudeCode(opts) {
|
|
376
398
|
const stats = { toolCalls: 0, assistantChars: 0 };
|
|
377
399
|
let summary = '';
|
|
@@ -459,6 +481,7 @@ export async function runClaudeCode(opts) {
|
|
|
459
481
|
// diagnostic. Disabled when the caller supplies no limits (only the external watchdog bounds it).
|
|
460
482
|
const progressGuard = createClaudeProgressGuard(opts);
|
|
461
483
|
const { rememberTool, feedGuard, guardAbort } = progressGuard;
|
|
484
|
+
const trajectory = createClaudeToolTrajectory(opts, secrets);
|
|
462
485
|
const onEvent = (event, meta) => {
|
|
463
486
|
const type = event.type;
|
|
464
487
|
// A subagent's turns ride the parent's stdout tagged with the dispatch that spawned them;
|
|
@@ -481,6 +504,7 @@ export async function runClaudeCode(opts) {
|
|
|
481
504
|
// `is_error` its `tool_result` carries on the next `user` turn.
|
|
482
505
|
if (typeof block.id === 'string' && typeof block.name === 'string') {
|
|
483
506
|
rememberTool(block.id, block.name);
|
|
507
|
+
trajectory.onToolUse(block.id, block.name, block.input);
|
|
484
508
|
}
|
|
485
509
|
if (block.name === 'TodoWrite') {
|
|
486
510
|
const progress = todosToProgress(block.input?.todos);
|
|
@@ -506,6 +530,9 @@ export async function runClaudeCode(opts) {
|
|
|
506
530
|
// would kill nothing and only convert a clean exit into a spurious failure.
|
|
507
531
|
if (!meta?.final)
|
|
508
532
|
feedGuard(content);
|
|
533
|
+
// The trajectory's other half — fed on the FINAL flush too, unlike the guard, since a
|
|
534
|
+
// CLI that has exited is exactly when the last calls' results matter.
|
|
535
|
+
trajectory.onToolResults(content);
|
|
509
536
|
telemetry.onToolResult(dispatchId, content);
|
|
510
537
|
}
|
|
511
538
|
}
|
package/dist/pi-workspace.js
CHANGED
|
@@ -182,6 +182,9 @@ export async function runAgentInWorkspace(spec, opts = {}) {
|
|
|
182
182
|
expectsEdits: spec.expectsEdits ?? true,
|
|
183
183
|
onActivity: opts.onActivity,
|
|
184
184
|
onProgress: opts.onProgress,
|
|
185
|
+
// The run's tool-call trajectory, the same hook the Pi path feeds — so a subscription run
|
|
186
|
+
// and a proxied one produce the same evidence rather than one of them producing none.
|
|
187
|
+
onSpan: opts.onSpan,
|
|
185
188
|
// Per-slice review capture, so a parallel review's finished slices are persisted as they
|
|
186
189
|
// land rather than only in the terminal output. Only the subscription runners fan work out
|
|
187
190
|
// across subagents, so this is the only path that can produce it.
|
package/dist/pi.d.ts
CHANGED
|
@@ -187,18 +187,39 @@ export interface TodoProgress {
|
|
|
187
187
|
items?: TodoItem[];
|
|
188
188
|
}
|
|
189
189
|
/**
|
|
190
|
-
* One tool invocation in
|
|
191
|
-
*
|
|
192
|
-
*
|
|
193
|
-
*
|
|
190
|
+
* One tool invocation in an agent's loop, captured for the run's TRAJECTORY: the ordered
|
|
191
|
+
* account of what the agent did, drained by the backend on its existing job poll and
|
|
192
|
+
* both persisted and emitted as a child span under the run trace.
|
|
193
|
+
*
|
|
194
|
+
* It carries the call's arguments and result (scrubbed and capped at capture — see
|
|
195
|
+
* `tool-trajectory.ts`), because the question asked of a finished run is which command
|
|
196
|
+
* ran against what, not how long a tool named `bash` took. Whether those bodies are
|
|
197
|
+
* RETAINED is the backend's decision, taken against the deployment switch and the
|
|
198
|
+
* workspace's opt-out; the harness's job is to capture them bounded and scrubbed.
|
|
194
199
|
*/
|
|
195
200
|
export interface ToolSpan {
|
|
196
201
|
tool: string;
|
|
197
|
-
/**
|
|
202
|
+
/**
|
|
203
|
+
* The call's 0-based ordinal within this job. Two calls routinely land in the same
|
|
204
|
+
* millisecond, so this is the only thing that orders the trajectory — and it is what
|
|
205
|
+
* makes the backend's stored row id deterministic, so a replayed poll re-records
|
|
206
|
+
* instead of duplicating.
|
|
207
|
+
*/
|
|
208
|
+
seq: number;
|
|
209
|
+
/** Epoch ms the tool call started (the previous call's end when no start was seen). */
|
|
198
210
|
startedAt: number;
|
|
199
211
|
/** Epoch ms the tool call ended (when its `tool_execution_end` event arrived). */
|
|
200
212
|
endedAt: number;
|
|
201
213
|
ok: boolean;
|
|
214
|
+
/** Whether the bodies below were captured at all — always `'stored'` from this harness. */
|
|
215
|
+
bodies: 'stored' | 'withheld';
|
|
216
|
+
/** The call's arguments, serialised, scrubbed and capped. `''` when it took none. */
|
|
217
|
+
args: string;
|
|
218
|
+
/** What the tool returned, scrubbed and capped. `''` when it returned nothing. */
|
|
219
|
+
result: string;
|
|
220
|
+
/** Characters the cap dropped from {@link args} / {@link result}; 0 when nothing was cut. */
|
|
221
|
+
argsDropped: number;
|
|
222
|
+
resultDropped: number;
|
|
202
223
|
}
|
|
203
224
|
/**
|
|
204
225
|
* What the agent actually did this run, independent of any file changes. Used to
|
package/dist/pi.js
CHANGED
|
@@ -4,10 +4,11 @@ import { homedir } from 'node:os';
|
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
5
|
import { killChildProcess, spawnDetached } from './process.js';
|
|
6
6
|
import { pathExists } from './fs-utils.js';
|
|
7
|
-
import { redactSecrets } from './redact.js';
|
|
7
|
+
import { redactSecrets, secretsToRedact } from './redact.js';
|
|
8
8
|
import { HarnessFailure } from './failure.js';
|
|
9
9
|
import { log } from './logger.js';
|
|
10
10
|
import { ProgressGuard, progressGuardLimitsFromEnv, toolCallSignal, } from './progress-guard.js';
|
|
11
|
+
import { ToolCallTracker, readToolCallId, toolCallResult, toolCallStart, } from './tool-trajectory.js';
|
|
11
12
|
// Drives the Pi coding-agent CLI. Pi is pointed at the Worker's OpenAI-compatible
|
|
12
13
|
// proxy via a custom provider in ~/.pi/agent/models.json, authenticated with the
|
|
13
14
|
// per-job session token (interpolated from $PI_PROXY_TOKEN) — so no provider key
|
|
@@ -606,10 +607,17 @@ export function runPi(opts) {
|
|
|
606
607
|
let malformedLines = 0;
|
|
607
608
|
let observerErrors = 0;
|
|
608
609
|
const guard = new ProgressGuard(opts.guardLimits ?? progressGuardLimitsFromEnv(), opts.expectsEdits ?? true);
|
|
609
|
-
//
|
|
610
|
-
//
|
|
611
|
-
//
|
|
612
|
-
|
|
610
|
+
// Pairs each tool call's start with its result, numbers the pairs and captures the two
|
|
611
|
+
// bodies (scrubbed + capped). A call whose start Pi never emitted still gets an entry,
|
|
612
|
+
// timed from the previous call's end — see `ToolCallTracker`.
|
|
613
|
+
//
|
|
614
|
+
// The known-secret list is DERIVED from the token this function itself hands the child
|
|
615
|
+
// (`PI_PROXY_TOKEN` / `SEARXNG_API_KEY`) rather than taken as a parameter: the bodies
|
|
616
|
+
// travel to a store and to external trace sinks, and a caller that forgets to pass the
|
|
617
|
+
// list produces bodies scrubbed of credential SHAPES only, with no signal that the
|
|
618
|
+
// narrower rule ever ran. Deriving it here means the one place that knows the child's
|
|
619
|
+
// credentials is the place that scrubs them.
|
|
620
|
+
const tools = new ToolCallTracker(secretsToRedact(opts.sessionToken));
|
|
613
621
|
// SIGTERM first, then SIGKILL if Pi ignores it. Shared by the watchdog abort
|
|
614
622
|
// and the no-progress guard; the `close` handler turns it into a rejection.
|
|
615
623
|
const killChild = () => killChildProcess(child);
|
|
@@ -648,22 +656,19 @@ export function runPi(opts) {
|
|
|
648
656
|
opts.onProgress(progress);
|
|
649
657
|
}
|
|
650
658
|
if (opts.onSpan) {
|
|
659
|
+
const start = toolCallStart(event);
|
|
660
|
+
if (start)
|
|
661
|
+
tools.started(start.id, start.name, start.args);
|
|
651
662
|
const signal = toolCallSignal(event);
|
|
652
663
|
if (signal && signal.name) {
|
|
653
|
-
const
|
|
664
|
+
const call = tools.finished(readToolCallId(event), signal.name, toolCallResult(event), signal.isError);
|
|
654
665
|
try {
|
|
655
|
-
opts.onSpan({
|
|
656
|
-
tool: signal.name,
|
|
657
|
-
startedAt: toolBoundary,
|
|
658
|
-
endedAt,
|
|
659
|
-
ok: !signal.isError,
|
|
660
|
-
});
|
|
666
|
+
opts.onSpan({ ...call, bodies: 'stored' });
|
|
661
667
|
}
|
|
662
668
|
catch {
|
|
663
669
|
// A faulty observer must never break the run.
|
|
664
670
|
observerErrors++;
|
|
665
671
|
}
|
|
666
|
-
toolBoundary = endedAt;
|
|
667
672
|
}
|
|
668
673
|
}
|
|
669
674
|
if (runGuard && !guardReason && !aborted) {
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/** Cap on a captured argument blob. Generous for a command line, far below a file body. */
|
|
2
|
+
export declare const MAX_TOOL_ARGS_CHARS: number;
|
|
3
|
+
/** Cap on a captured result. Larger than the args cap: a result is where the bytes actually are. */
|
|
4
|
+
export declare const MAX_TOOL_RESULT_CHARS: number;
|
|
5
|
+
/** A captured body plus what the cap dropped from it. */
|
|
6
|
+
export interface CapturedToolBody {
|
|
7
|
+
text: string;
|
|
8
|
+
dropped: number;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Serialise, scrub and cap one tool body.
|
|
12
|
+
*
|
|
13
|
+
* A non-string value is JSON-serialised, and a value that cannot be (a cycle, a `BigInt`, a
|
|
14
|
+
* throwing getter) is NAMED as unserialisable rather than dropped to `''`: an empty body means
|
|
15
|
+
* "the call carried none", and a capture failure is a different fact.
|
|
16
|
+
*/
|
|
17
|
+
export declare function captureToolBody(value: unknown, max: number, secrets: readonly string[]): CapturedToolBody;
|
|
18
|
+
/** What {@link ToolCallTracker} emits per completed call — the fields a `ToolSpan` needs. */
|
|
19
|
+
export interface TrackedToolCall {
|
|
20
|
+
tool: string;
|
|
21
|
+
seq: number;
|
|
22
|
+
startedAt: number;
|
|
23
|
+
endedAt: number;
|
|
24
|
+
ok: boolean;
|
|
25
|
+
args: string;
|
|
26
|
+
result: string;
|
|
27
|
+
argsDropped: number;
|
|
28
|
+
resultDropped: number;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Pairs each tool call's START with its RESULT and numbers the pairs, so the two agent CLIs feed
|
|
32
|
+
* one trajectory shape from two very different streams (Pi's flat `tool_execution_*` events, the
|
|
33
|
+
* claude-code stream's `tool_use` / `tool_result` content blocks).
|
|
34
|
+
*
|
|
35
|
+
* Correlation is BY ID where the stream supplies one, and by tool name otherwise, because that is
|
|
36
|
+
* the difference between the two producers: claude-code's blocks always carry a `tool_use_id`, and
|
|
37
|
+
* a parallel batch of calls is routine there, while Pi's stream is sequential. A result the tracker
|
|
38
|
+
* cannot pair with a start is still EMITTED (with no args and the previous call's end as its
|
|
39
|
+
* start): losing a step of the trajectory to a schema tweak would be worse than an entry that says
|
|
40
|
+
* less than its neighbours, and the `seq` it takes keeps every later entry's ordinal honest.
|
|
41
|
+
*/
|
|
42
|
+
export declare class ToolCallTracker {
|
|
43
|
+
private readonly secrets;
|
|
44
|
+
private seq;
|
|
45
|
+
private readonly byId;
|
|
46
|
+
private readonly byTool;
|
|
47
|
+
/**
|
|
48
|
+
* Start boundary for a call whose own start was never seen: the previous call's end, or the run
|
|
49
|
+
* start. Approximate but contiguous, which is the property the trace tree needs.
|
|
50
|
+
*/
|
|
51
|
+
private boundary;
|
|
52
|
+
constructor(secrets?: readonly string[], now?: number);
|
|
53
|
+
/** Record that a call began, with the arguments the agent supplied. */
|
|
54
|
+
started(id: string | undefined, tool: string, args: unknown, at?: number): void;
|
|
55
|
+
/** Record that a call finished, and return the completed entry. */
|
|
56
|
+
finished(id: string | undefined, tool: string, result: unknown, isError: boolean, at?: number): TrackedToolCall;
|
|
57
|
+
private take;
|
|
58
|
+
}
|
|
59
|
+
/** Read a tool-call id off a stream event, whatever the producer calls it. */
|
|
60
|
+
export declare function readToolCallId(event: Record<string, unknown>): string | undefined;
|
|
61
|
+
/** A tool call BEGINNING, read off a Pi `--mode json` event, or undefined if it isn't one. */
|
|
62
|
+
export declare function toolCallStart(event: Record<string, unknown>): {
|
|
63
|
+
id?: string;
|
|
64
|
+
name: string;
|
|
65
|
+
args: unknown;
|
|
66
|
+
} | undefined;
|
|
67
|
+
/** The RESULT payload of a Pi `tool_execution_end` event, unwrapped from its envelope. */
|
|
68
|
+
export declare function toolCallResult(event: Record<string, unknown>): unknown;
|
|
69
|
+
/**
|
|
70
|
+
* Feed one claude-code `user` turn's content blocks to the tracker, emitting an entry per
|
|
71
|
+
* `tool_result`.
|
|
72
|
+
*
|
|
73
|
+
* The CLI answers each `tool_use` with a `tool_result` carrying the same `tool_use_id`, so the
|
|
74
|
+
* pairing is exact even when the model fired a batch of calls in parallel — which it routinely
|
|
75
|
+
* does, and which is why the trajectory here is ordered by the ordinal the tracker stamps rather
|
|
76
|
+
* than by the turn the results arrived on.
|
|
77
|
+
*/
|
|
78
|
+
export declare function recordClaudeToolResults(tracker: ToolCallTracker, content: readonly unknown[], emit: (call: TrackedToolCall) => void): void;
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { redact } from './redact.js';
|
|
2
|
+
// The TRAJECTORY capture: what the agent DID, one entry per tool call, in the order it made them.
|
|
3
|
+
//
|
|
4
|
+
// The harness has always buffered a compact span per tool call (name + timing + ok) for the run's
|
|
5
|
+
// trace. That is enough to draw a tree and not enough to answer the question anyone actually asks
|
|
6
|
+
// of a finished run — WHICH command, against WHAT, and what came back. The evidence standard for a
|
|
7
|
+
// merged PR is "how, not just the diff", and a span saying `bash` ran for 300ms is not evidence of
|
|
8
|
+
// anything. So each entry now carries the call's arguments and result, captured here because this
|
|
9
|
+
// is the only process that ever sees them: an agent CLI's tool loop is internal to the CLI, and
|
|
10
|
+
// the container is gone the moment the job settles.
|
|
11
|
+
//
|
|
12
|
+
// Two properties keep that affordable and safe:
|
|
13
|
+
//
|
|
14
|
+
// - **Bounded at capture.** Each body is capped (a build log is routinely megabytes) and the entry
|
|
15
|
+
// STATES what the cap dropped, so a reader can tell a short command from the head of a long one.
|
|
16
|
+
// The caps are what keep the drain buffer and the poll response small.
|
|
17
|
+
// - **Scrubbed at capture.** A tool's arguments and output routinely echo an env var, a clone URL
|
|
18
|
+
// or the leased subscription token, and these travel to a store and to external trace sinks.
|
|
19
|
+
//
|
|
20
|
+
// The RETENTION decision is the backend's, not this module's: entries carry `bodies: 'stored'` and
|
|
21
|
+
// the backend's double gate (`LLM_RECORD_PROMPTS` + the workspace's `storeAgentContext`) decides
|
|
22
|
+
// whether to keep them, exactly as it already does for the prompt bodies the call-metric
|
|
23
|
+
// reconstruction assembles here. Deciding it twice would mean the container had to be told the
|
|
24
|
+
// workspace's settings, and an image one release behind its backend would then be deciding it with
|
|
25
|
+
// stale ones.
|
|
26
|
+
/** Cap on a captured argument blob. Generous for a command line, far below a file body. */
|
|
27
|
+
export const MAX_TOOL_ARGS_CHARS = 2 * 1024;
|
|
28
|
+
/** Cap on a captured result. Larger than the args cap: a result is where the bytes actually are. */
|
|
29
|
+
export const MAX_TOOL_RESULT_CHARS = 4 * 1024;
|
|
30
|
+
const EMPTY = { text: '', dropped: 0 };
|
|
31
|
+
/**
|
|
32
|
+
* Serialise, scrub and cap one tool body.
|
|
33
|
+
*
|
|
34
|
+
* A non-string value is JSON-serialised, and a value that cannot be (a cycle, a `BigInt`, a
|
|
35
|
+
* throwing getter) is NAMED as unserialisable rather than dropped to `''`: an empty body means
|
|
36
|
+
* "the call carried none", and a capture failure is a different fact.
|
|
37
|
+
*/
|
|
38
|
+
export function captureToolBody(value, max, secrets) {
|
|
39
|
+
if (value === undefined || value === null)
|
|
40
|
+
return EMPTY;
|
|
41
|
+
let text;
|
|
42
|
+
if (typeof value === 'string') {
|
|
43
|
+
text = value;
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
try {
|
|
47
|
+
text = JSON.stringify(value) ?? '';
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return { text: '[unserialisable]', dropped: 0 };
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (text === '')
|
|
54
|
+
return EMPTY;
|
|
55
|
+
const scrubbed = redact(text, secrets);
|
|
56
|
+
if (scrubbed.length <= max)
|
|
57
|
+
return { text: scrubbed, dropped: 0 };
|
|
58
|
+
return { text: scrubbed.slice(0, max), dropped: scrubbed.length - max };
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Pairs each tool call's START with its RESULT and numbers the pairs, so the two agent CLIs feed
|
|
62
|
+
* one trajectory shape from two very different streams (Pi's flat `tool_execution_*` events, the
|
|
63
|
+
* claude-code stream's `tool_use` / `tool_result` content blocks).
|
|
64
|
+
*
|
|
65
|
+
* Correlation is BY ID where the stream supplies one, and by tool name otherwise, because that is
|
|
66
|
+
* the difference between the two producers: claude-code's blocks always carry a `tool_use_id`, and
|
|
67
|
+
* a parallel batch of calls is routine there, while Pi's stream is sequential. A result the tracker
|
|
68
|
+
* cannot pair with a start is still EMITTED (with no args and the previous call's end as its
|
|
69
|
+
* start): losing a step of the trajectory to a schema tweak would be worse than an entry that says
|
|
70
|
+
* less than its neighbours, and the `seq` it takes keeps every later entry's ordinal honest.
|
|
71
|
+
*/
|
|
72
|
+
export class ToolCallTracker {
|
|
73
|
+
secrets;
|
|
74
|
+
seq = 0;
|
|
75
|
+
byId = new Map();
|
|
76
|
+
byTool = new Map();
|
|
77
|
+
/**
|
|
78
|
+
* Start boundary for a call whose own start was never seen: the previous call's end, or the run
|
|
79
|
+
* start. Approximate but contiguous, which is the property the trace tree needs.
|
|
80
|
+
*/
|
|
81
|
+
boundary;
|
|
82
|
+
constructor(secrets = [], now = Date.now()) {
|
|
83
|
+
this.secrets = secrets;
|
|
84
|
+
this.boundary = now;
|
|
85
|
+
}
|
|
86
|
+
/** Record that a call began, with the arguments the agent supplied. */
|
|
87
|
+
started(id, tool, args, at = Date.now()) {
|
|
88
|
+
const pending = {
|
|
89
|
+
tool,
|
|
90
|
+
startedAt: at,
|
|
91
|
+
args: captureToolBody(args, MAX_TOOL_ARGS_CHARS, this.secrets),
|
|
92
|
+
};
|
|
93
|
+
if (id) {
|
|
94
|
+
this.byId.set(id, pending);
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
const queue = this.byTool.get(tool) ?? [];
|
|
98
|
+
queue.push(pending);
|
|
99
|
+
this.byTool.set(tool, queue);
|
|
100
|
+
}
|
|
101
|
+
/** Record that a call finished, and return the completed entry. */
|
|
102
|
+
finished(id, tool, result, isError, at = Date.now()) {
|
|
103
|
+
const pending = this.take(id, tool);
|
|
104
|
+
const captured = captureToolBody(result, MAX_TOOL_RESULT_CHARS, this.secrets);
|
|
105
|
+
const call = {
|
|
106
|
+
tool: pending?.tool ?? tool,
|
|
107
|
+
seq: this.seq++,
|
|
108
|
+
startedAt: pending?.startedAt ?? this.boundary,
|
|
109
|
+
endedAt: at,
|
|
110
|
+
ok: !isError,
|
|
111
|
+
args: pending?.args.text ?? '',
|
|
112
|
+
argsDropped: pending?.args.dropped ?? 0,
|
|
113
|
+
result: captured.text,
|
|
114
|
+
resultDropped: captured.dropped,
|
|
115
|
+
};
|
|
116
|
+
this.boundary = at;
|
|
117
|
+
return call;
|
|
118
|
+
}
|
|
119
|
+
take(id, tool) {
|
|
120
|
+
if (id) {
|
|
121
|
+
const byId = this.byId.get(id);
|
|
122
|
+
if (byId) {
|
|
123
|
+
this.byId.delete(id);
|
|
124
|
+
return byId;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
const queue = this.byTool.get(tool);
|
|
128
|
+
// FIFO, so a sequential stream pairs each result with the call that opened first. A stream
|
|
129
|
+
// that interleaves un-idded calls of one tool would mis-pair here; none of the two producers
|
|
130
|
+
// does, and the alternative (refusing to pair at all) loses the args on every ordinary run.
|
|
131
|
+
const pending = queue?.shift();
|
|
132
|
+
if (queue && queue.length === 0)
|
|
133
|
+
this.byTool.delete(tool);
|
|
134
|
+
return pending;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
/** Read a tool-call id off a stream event, whatever the producer calls it. */
|
|
138
|
+
export function readToolCallId(event) {
|
|
139
|
+
for (const key of ['toolCallId', 'tool_call_id', 'callId', 'id']) {
|
|
140
|
+
const value = event[key];
|
|
141
|
+
if (typeof value === 'string' && value !== '')
|
|
142
|
+
return value;
|
|
143
|
+
}
|
|
144
|
+
return undefined;
|
|
145
|
+
}
|
|
146
|
+
/** A tool call BEGINNING, read off a Pi `--mode json` event, or undefined if it isn't one. */
|
|
147
|
+
export function toolCallStart(event) {
|
|
148
|
+
if (event.type !== 'tool_execution_start' && event.type !== 'tool_call')
|
|
149
|
+
return undefined;
|
|
150
|
+
const name = typeof event.toolName === 'string' ? event.toolName : '';
|
|
151
|
+
if (!name)
|
|
152
|
+
return undefined;
|
|
153
|
+
// Read every spelling the stream might use rather than pinning one: Pi's event schema has been
|
|
154
|
+
// renamed under us before (see `todoResultDetails`, which reads three shapes of one result), and
|
|
155
|
+
// a rename here costs the ARGUMENTS of every call, silently.
|
|
156
|
+
const args = event.args ?? event.arguments ?? event.input ?? event.parameters;
|
|
157
|
+
const id = readToolCallId(event);
|
|
158
|
+
return { name, args, ...(id ? { id } : {}) };
|
|
159
|
+
}
|
|
160
|
+
/** The RESULT payload of a Pi `tool_execution_end` event, unwrapped from its envelope. */
|
|
161
|
+
export function toolCallResult(event) {
|
|
162
|
+
const result = event.result;
|
|
163
|
+
if (result && typeof result === 'object') {
|
|
164
|
+
const inner = result;
|
|
165
|
+
// `details` is the structured payload Pi's own extensions return (the todo tool's shape), and
|
|
166
|
+
// `content`/`output` the free-text ones. Falling back to the whole envelope keeps a shape none
|
|
167
|
+
// of these match readable rather than empty.
|
|
168
|
+
return inner.details ?? inner.content ?? inner.output ?? result;
|
|
169
|
+
}
|
|
170
|
+
return result ?? event.output ?? event.content;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Feed one claude-code `user` turn's content blocks to the tracker, emitting an entry per
|
|
174
|
+
* `tool_result`.
|
|
175
|
+
*
|
|
176
|
+
* The CLI answers each `tool_use` with a `tool_result` carrying the same `tool_use_id`, so the
|
|
177
|
+
* pairing is exact even when the model fired a batch of calls in parallel — which it routinely
|
|
178
|
+
* does, and which is why the trajectory here is ordered by the ordinal the tracker stamps rather
|
|
179
|
+
* than by the turn the results arrived on.
|
|
180
|
+
*/
|
|
181
|
+
export function recordClaudeToolResults(tracker, content, emit) {
|
|
182
|
+
for (const block of content) {
|
|
183
|
+
if (!block || typeof block !== 'object')
|
|
184
|
+
continue;
|
|
185
|
+
const record = block;
|
|
186
|
+
if (record.type !== 'tool_result')
|
|
187
|
+
continue;
|
|
188
|
+
const id = typeof record.tool_use_id === 'string' ? record.tool_use_id : undefined;
|
|
189
|
+
// The block carries no tool NAME (only the id the assistant turn named), so an unpaired
|
|
190
|
+
// result falls back to a stated placeholder rather than an empty string that would render
|
|
191
|
+
// as a nameless step.
|
|
192
|
+
emit(tracker.finished(id, 'unknown', record.content, record.is_error === true));
|
|
193
|
+
}
|
|
194
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cat-factory/executor-harness",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.92.2",
|
|
4
4
|
"description": "Container payload: a thin TypeScript wrapper that runs the Pi coding agent against a cloned repo and opens a PR. Runs in the Cloudflare Container (and, in local native mode, as a host process); carries no secrets.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -25,14 +25,14 @@
|
|
|
25
25
|
"access": "public"
|
|
26
26
|
},
|
|
27
27
|
"devDependencies": {
|
|
28
|
-
"@hono/node-server": "^2.0
|
|
28
|
+
"@hono/node-server": "^2.1.0",
|
|
29
29
|
"@types/node": "^26.1.2",
|
|
30
|
-
"hono": "^4.
|
|
30
|
+
"hono": "^4.13.0",
|
|
31
31
|
"typescript": "7.0.2",
|
|
32
32
|
"vitest": "^4.1.10",
|
|
33
|
-
"@cat-factory/kernel": "0.
|
|
34
|
-
"@cat-factory/server": "0.
|
|
35
|
-
"@cat-factory/spend": "0.
|
|
33
|
+
"@cat-factory/kernel": "0.242.0",
|
|
34
|
+
"@cat-factory/server": "0.222.0",
|
|
35
|
+
"@cat-factory/spend": "0.15.6"
|
|
36
36
|
},
|
|
37
37
|
"scripts": {
|
|
38
38
|
"build": "tsc -p tsconfig.json",
|
package/src/agent-runner.ts
CHANGED
|
@@ -4,6 +4,11 @@ import { tmpdir } from 'node:os'
|
|
|
4
4
|
import { dirname, join } from 'node:path'
|
|
5
5
|
import { claudeAssistantContent, isObject, numberOf, redactBody } from './claude-stream.js'
|
|
6
6
|
import { createClaudeRunTelemetry, subagentDispatchId } from './claude-call-aggregator.js'
|
|
7
|
+
import {
|
|
8
|
+
ToolCallTracker,
|
|
9
|
+
type TrackedToolCall,
|
|
10
|
+
recordClaudeToolResults,
|
|
11
|
+
} from './tool-trajectory.js'
|
|
7
12
|
import type { Logger } from './logger.js'
|
|
8
13
|
import {
|
|
9
14
|
createCallMetricPublisher,
|
|
@@ -13,6 +18,7 @@ import {
|
|
|
13
18
|
type PiRunOutcome,
|
|
14
19
|
type PiRunStats,
|
|
15
20
|
type TodoProgress,
|
|
21
|
+
type ToolSpan,
|
|
16
22
|
} from './pi.js'
|
|
17
23
|
import {
|
|
18
24
|
claudeAllowedToolPatterns,
|
|
@@ -123,6 +129,13 @@ export interface SubscriptionRunOptions {
|
|
|
123
129
|
onActivity?: () => void
|
|
124
130
|
/** Called with the latest subtask counts each time the CLI updates its todo/plan list. */
|
|
125
131
|
onProgress?: (progress: TodoProgress) => void
|
|
132
|
+
/**
|
|
133
|
+
* Called once per completed tool call with a {@link ToolSpan}: the run's TRAJECTORY. The CLI's
|
|
134
|
+
* tool loop is internal to the CLI and never touches our proxy, so its own event stream is the
|
|
135
|
+
* only place these exist — without this hook a subscription-harness run's account of what it
|
|
136
|
+
* DID dies with the container.
|
|
137
|
+
*/
|
|
138
|
+
onSpan?: (span: ToolSpan) => void
|
|
126
139
|
/**
|
|
127
140
|
* Called with the FULL set of per-slice reviews each time one lands, so the backend can persist
|
|
128
141
|
* a parallel review's completed work as it happens instead of only from the terminal result.
|
|
@@ -538,6 +551,36 @@ function createClaudeProgressGuard(opts: SubscriptionRunOptions): {
|
|
|
538
551
|
}
|
|
539
552
|
}
|
|
540
553
|
|
|
554
|
+
/**
|
|
555
|
+
* The run's TRAJECTORY, on the claude-code stream: each `tool_use` block paired with the
|
|
556
|
+
* `tool_result` that answers it on the following user turn, numbered and captured (scrubbed +
|
|
557
|
+
* capped). The CLI's stream is the only place this loop is visible at all — its tool calls never
|
|
558
|
+
* touch our proxy — so without this a subscription-harness run's account of what it DID dies with
|
|
559
|
+
* the container.
|
|
560
|
+
*
|
|
561
|
+
* Both halves are no-ops when the caller wants no spans, so a driver that only needs the run's
|
|
562
|
+
* output never pays to serialise a body nothing will read. Split out of {@link runClaudeCode} for
|
|
563
|
+
* the per-function line budget, like {@link createClaudeProgressGuard}.
|
|
564
|
+
*/
|
|
565
|
+
function createClaudeToolTrajectory(
|
|
566
|
+
opts: SubscriptionRunOptions,
|
|
567
|
+
secrets: readonly string[],
|
|
568
|
+
): {
|
|
569
|
+
onToolUse: (id: string, name: string, input: unknown) => void
|
|
570
|
+
onToolResults: (content: unknown[]) => void
|
|
571
|
+
} {
|
|
572
|
+
if (!opts.onSpan) return { onToolUse: () => {}, onToolResults: () => {} }
|
|
573
|
+
const onSpan = opts.onSpan
|
|
574
|
+
const tracker = new ToolCallTracker(secrets)
|
|
575
|
+
return {
|
|
576
|
+
onToolUse: (id, name, input) => tracker.started(id, name, input),
|
|
577
|
+
onToolResults: (content) =>
|
|
578
|
+
recordClaudeToolResults(tracker, content, (call: TrackedToolCall) =>
|
|
579
|
+
onSpan({ ...call, bodies: 'stored' }),
|
|
580
|
+
),
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
541
584
|
export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRunOutcome> {
|
|
542
585
|
const stats: PiRunStats = { toolCalls: 0, assistantChars: 0 }
|
|
543
586
|
let summary = ''
|
|
@@ -628,6 +671,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
628
671
|
// diagnostic. Disabled when the caller supplies no limits (only the external watchdog bounds it).
|
|
629
672
|
const progressGuard = createClaudeProgressGuard(opts)
|
|
630
673
|
const { rememberTool, feedGuard, guardAbort } = progressGuard
|
|
674
|
+
const trajectory = createClaudeToolTrajectory(opts, secrets)
|
|
631
675
|
|
|
632
676
|
const onEvent = (event: Record<string, unknown>, meta?: { final?: boolean }): void => {
|
|
633
677
|
const type = event.type
|
|
@@ -650,6 +694,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
650
694
|
// `is_error` its `tool_result` carries on the next `user` turn.
|
|
651
695
|
if (typeof block.id === 'string' && typeof block.name === 'string') {
|
|
652
696
|
rememberTool(block.id, block.name)
|
|
697
|
+
trajectory.onToolUse(block.id, block.name, block.input)
|
|
653
698
|
}
|
|
654
699
|
if (block.name === 'TodoWrite') {
|
|
655
700
|
const progress = todosToProgress((block.input as Record<string, unknown>)?.todos)
|
|
@@ -672,6 +717,9 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
672
717
|
// Not on the at-close flush: the CLI has already exited, so tripping the guard there
|
|
673
718
|
// would kill nothing and only convert a clean exit into a spurious failure.
|
|
674
719
|
if (!meta?.final) feedGuard(content)
|
|
720
|
+
// The trajectory's other half — fed on the FINAL flush too, unlike the guard, since a
|
|
721
|
+
// CLI that has exited is exactly when the last calls' results matter.
|
|
722
|
+
trajectory.onToolResults(content)
|
|
675
723
|
telemetry.onToolResult(dispatchId, content)
|
|
676
724
|
}
|
|
677
725
|
} else if (type === 'result') {
|
package/src/coding-agent.ts
CHANGED
|
@@ -1101,40 +1101,36 @@ export async function runMultiRepoCoding(
|
|
|
1101
1101
|
baseSha: '',
|
|
1102
1102
|
resumed: false,
|
|
1103
1103
|
},
|
|
1104
|
-
...peers.map(
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
}),
|
|
1120
|
-
),
|
|
1104
|
+
...peers.map((peer): RepoLeg => ({
|
|
1105
|
+
repo: peer.repo,
|
|
1106
|
+
dirName: claimDir(peer.repo),
|
|
1107
|
+
dir: '',
|
|
1108
|
+
cloneBranch: peer.repo.baseBranch,
|
|
1109
|
+
// Coding peers always carry `newBranch` (the backend sets the shared work branch);
|
|
1110
|
+
// fall back to the primary's for the type (read-only peers never reach this path).
|
|
1111
|
+
workBranch: peer.newBranch ?? primaryWorkBranch,
|
|
1112
|
+
ghToken: peer.ghToken ?? job.ghToken,
|
|
1113
|
+
...(peer.pr ? { pr: peer.pr } : {}),
|
|
1114
|
+
...(peer.frameId ? { frameId: peer.frameId } : {}),
|
|
1115
|
+
primary: false,
|
|
1116
|
+
baseSha: '',
|
|
1117
|
+
resumed: false,
|
|
1118
|
+
})),
|
|
1121
1119
|
// Read-only reference repos (doc-writer): cloned as siblings the agent reads but never writes.
|
|
1122
1120
|
// `workBranch` is set to the base only to satisfy the type — a read-only leg never branches or
|
|
1123
1121
|
// pushes (guarded by `readOnly` in both the clone and push phases below).
|
|
1124
|
-
...references.map(
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
}),
|
|
1137
|
-
),
|
|
1122
|
+
...references.map((reference): RepoLeg => ({
|
|
1123
|
+
repo: reference.repo,
|
|
1124
|
+
dirName: claimDir(reference.repo),
|
|
1125
|
+
dir: '',
|
|
1126
|
+
cloneBranch: reference.repo.baseBranch,
|
|
1127
|
+
workBranch: reference.repo.baseBranch,
|
|
1128
|
+
ghToken: reference.ghToken ?? job.ghToken,
|
|
1129
|
+
primary: false,
|
|
1130
|
+
readOnly: true,
|
|
1131
|
+
baseSha: '',
|
|
1132
|
+
resumed: false,
|
|
1133
|
+
})),
|
|
1138
1134
|
]
|
|
1139
1135
|
|
|
1140
1136
|
return withWorkspace('multi', async (root) => {
|
package/src/pi-workspace.ts
CHANGED
|
@@ -325,6 +325,9 @@ export async function runAgentInWorkspace(
|
|
|
325
325
|
expectsEdits: spec.expectsEdits ?? true,
|
|
326
326
|
onActivity: opts.onActivity,
|
|
327
327
|
onProgress: opts.onProgress,
|
|
328
|
+
// The run's tool-call trajectory, the same hook the Pi path feeds — so a subscription run
|
|
329
|
+
// and a proxied one produce the same evidence rather than one of them producing none.
|
|
330
|
+
onSpan: opts.onSpan,
|
|
328
331
|
// Per-slice review capture, so a parallel review's finished slices are persisted as they
|
|
329
332
|
// land rather than only in the terminal output. Only the subscription runners fan work out
|
|
330
333
|
// across subagents, so this is the only path that can produce it.
|
package/src/pi.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { homedir } from 'node:os'
|
|
|
4
4
|
import { dirname, join } from 'node:path'
|
|
5
5
|
import { killChildProcess, spawnDetached } from './process.js'
|
|
6
6
|
import { pathExists } from './fs-utils.js'
|
|
7
|
-
import { redactSecrets } from './redact.js'
|
|
7
|
+
import { redactSecrets, secretsToRedact } from './redact.js'
|
|
8
8
|
import { HarnessFailure } from './failure.js'
|
|
9
9
|
import { log } from './logger.js'
|
|
10
10
|
import type { EffortReport } from './effort.js'
|
|
@@ -14,6 +14,12 @@ import {
|
|
|
14
14
|
toolCallSignal,
|
|
15
15
|
type ProgressGuardLimits,
|
|
16
16
|
} from './progress-guard.js'
|
|
17
|
+
import {
|
|
18
|
+
ToolCallTracker,
|
|
19
|
+
readToolCallId,
|
|
20
|
+
toolCallResult,
|
|
21
|
+
toolCallStart,
|
|
22
|
+
} from './tool-trajectory.js'
|
|
17
23
|
|
|
18
24
|
// Drives the Pi coding-agent CLI. Pi is pointed at the Worker's OpenAI-compatible
|
|
19
25
|
// proxy via a custom provider in ~/.pi/agent/models.json, authenticated with the
|
|
@@ -501,18 +507,39 @@ export interface TodoProgress {
|
|
|
501
507
|
}
|
|
502
508
|
|
|
503
509
|
/**
|
|
504
|
-
* One tool invocation in
|
|
505
|
-
*
|
|
506
|
-
*
|
|
507
|
-
*
|
|
510
|
+
* One tool invocation in an agent's loop, captured for the run's TRAJECTORY: the ordered
|
|
511
|
+
* account of what the agent did, drained by the backend on its existing job poll and
|
|
512
|
+
* both persisted and emitted as a child span under the run trace.
|
|
513
|
+
*
|
|
514
|
+
* It carries the call's arguments and result (scrubbed and capped at capture — see
|
|
515
|
+
* `tool-trajectory.ts`), because the question asked of a finished run is which command
|
|
516
|
+
* ran against what, not how long a tool named `bash` took. Whether those bodies are
|
|
517
|
+
* RETAINED is the backend's decision, taken against the deployment switch and the
|
|
518
|
+
* workspace's opt-out; the harness's job is to capture them bounded and scrubbed.
|
|
508
519
|
*/
|
|
509
520
|
export interface ToolSpan {
|
|
510
521
|
tool: string
|
|
511
|
-
/**
|
|
522
|
+
/**
|
|
523
|
+
* The call's 0-based ordinal within this job. Two calls routinely land in the same
|
|
524
|
+
* millisecond, so this is the only thing that orders the trajectory — and it is what
|
|
525
|
+
* makes the backend's stored row id deterministic, so a replayed poll re-records
|
|
526
|
+
* instead of duplicating.
|
|
527
|
+
*/
|
|
528
|
+
seq: number
|
|
529
|
+
/** Epoch ms the tool call started (the previous call's end when no start was seen). */
|
|
512
530
|
startedAt: number
|
|
513
531
|
/** Epoch ms the tool call ended (when its `tool_execution_end` event arrived). */
|
|
514
532
|
endedAt: number
|
|
515
533
|
ok: boolean
|
|
534
|
+
/** Whether the bodies below were captured at all — always `'stored'` from this harness. */
|
|
535
|
+
bodies: 'stored' | 'withheld'
|
|
536
|
+
/** The call's arguments, serialised, scrubbed and capped. `''` when it took none. */
|
|
537
|
+
args: string
|
|
538
|
+
/** What the tool returned, scrubbed and capped. `''` when it returned nothing. */
|
|
539
|
+
result: string
|
|
540
|
+
/** Characters the cap dropped from {@link args} / {@link result}; 0 when nothing was cut. */
|
|
541
|
+
argsDropped: number
|
|
542
|
+
resultDropped: number
|
|
516
543
|
}
|
|
517
544
|
|
|
518
545
|
function isObject(value: unknown): value is Record<string, unknown> {
|
|
@@ -918,10 +945,17 @@ export function runPi(opts: {
|
|
|
918
945
|
opts.guardLimits ?? progressGuardLimitsFromEnv(),
|
|
919
946
|
opts.expectsEdits ?? true,
|
|
920
947
|
)
|
|
921
|
-
//
|
|
922
|
-
//
|
|
923
|
-
//
|
|
924
|
-
|
|
948
|
+
// Pairs each tool call's start with its result, numbers the pairs and captures the two
|
|
949
|
+
// bodies (scrubbed + capped). A call whose start Pi never emitted still gets an entry,
|
|
950
|
+
// timed from the previous call's end — see `ToolCallTracker`.
|
|
951
|
+
//
|
|
952
|
+
// The known-secret list is DERIVED from the token this function itself hands the child
|
|
953
|
+
// (`PI_PROXY_TOKEN` / `SEARXNG_API_KEY`) rather than taken as a parameter: the bodies
|
|
954
|
+
// travel to a store and to external trace sinks, and a caller that forgets to pass the
|
|
955
|
+
// list produces bodies scrubbed of credential SHAPES only, with no signal that the
|
|
956
|
+
// narrower rule ever ran. Deriving it here means the one place that knows the child's
|
|
957
|
+
// credentials is the place that scrubs them.
|
|
958
|
+
const tools = new ToolCallTracker(secretsToRedact(opts.sessionToken))
|
|
925
959
|
|
|
926
960
|
// SIGTERM first, then SIGKILL if Pi ignores it. Shared by the watchdog abort
|
|
927
961
|
// and the no-progress guard; the `close` handler turns it into a rejection.
|
|
@@ -958,21 +992,22 @@ export function runPi(opts: {
|
|
|
958
992
|
if (progress) opts.onProgress(progress)
|
|
959
993
|
}
|
|
960
994
|
if (opts.onSpan) {
|
|
995
|
+
const start = toolCallStart(event)
|
|
996
|
+
if (start) tools.started(start.id, start.name, start.args)
|
|
961
997
|
const signal = toolCallSignal(event)
|
|
962
998
|
if (signal && signal.name) {
|
|
963
|
-
const
|
|
999
|
+
const call = tools.finished(
|
|
1000
|
+
readToolCallId(event),
|
|
1001
|
+
signal.name,
|
|
1002
|
+
toolCallResult(event),
|
|
1003
|
+
signal.isError,
|
|
1004
|
+
)
|
|
964
1005
|
try {
|
|
965
|
-
opts.onSpan({
|
|
966
|
-
tool: signal.name,
|
|
967
|
-
startedAt: toolBoundary,
|
|
968
|
-
endedAt,
|
|
969
|
-
ok: !signal.isError,
|
|
970
|
-
})
|
|
1006
|
+
opts.onSpan({ ...call, bodies: 'stored' })
|
|
971
1007
|
} catch {
|
|
972
1008
|
// A faulty observer must never break the run.
|
|
973
1009
|
observerErrors++
|
|
974
1010
|
}
|
|
975
|
-
toolBoundary = endedAt
|
|
976
1011
|
}
|
|
977
1012
|
}
|
|
978
1013
|
if (runGuard && !guardReason && !aborted) {
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
import { redact } from './redact.js'
|
|
2
|
+
|
|
3
|
+
// The TRAJECTORY capture: what the agent DID, one entry per tool call, in the order it made them.
|
|
4
|
+
//
|
|
5
|
+
// The harness has always buffered a compact span per tool call (name + timing + ok) for the run's
|
|
6
|
+
// trace. That is enough to draw a tree and not enough to answer the question anyone actually asks
|
|
7
|
+
// of a finished run — WHICH command, against WHAT, and what came back. The evidence standard for a
|
|
8
|
+
// merged PR is "how, not just the diff", and a span saying `bash` ran for 300ms is not evidence of
|
|
9
|
+
// anything. So each entry now carries the call's arguments and result, captured here because this
|
|
10
|
+
// is the only process that ever sees them: an agent CLI's tool loop is internal to the CLI, and
|
|
11
|
+
// the container is gone the moment the job settles.
|
|
12
|
+
//
|
|
13
|
+
// Two properties keep that affordable and safe:
|
|
14
|
+
//
|
|
15
|
+
// - **Bounded at capture.** Each body is capped (a build log is routinely megabytes) and the entry
|
|
16
|
+
// STATES what the cap dropped, so a reader can tell a short command from the head of a long one.
|
|
17
|
+
// The caps are what keep the drain buffer and the poll response small.
|
|
18
|
+
// - **Scrubbed at capture.** A tool's arguments and output routinely echo an env var, a clone URL
|
|
19
|
+
// or the leased subscription token, and these travel to a store and to external trace sinks.
|
|
20
|
+
//
|
|
21
|
+
// The RETENTION decision is the backend's, not this module's: entries carry `bodies: 'stored'` and
|
|
22
|
+
// the backend's double gate (`LLM_RECORD_PROMPTS` + the workspace's `storeAgentContext`) decides
|
|
23
|
+
// whether to keep them, exactly as it already does for the prompt bodies the call-metric
|
|
24
|
+
// reconstruction assembles here. Deciding it twice would mean the container had to be told the
|
|
25
|
+
// workspace's settings, and an image one release behind its backend would then be deciding it with
|
|
26
|
+
// stale ones.
|
|
27
|
+
|
|
28
|
+
/** Cap on a captured argument blob. Generous for a command line, far below a file body. */
|
|
29
|
+
export const MAX_TOOL_ARGS_CHARS = 2 * 1024
|
|
30
|
+
/** Cap on a captured result. Larger than the args cap: a result is where the bytes actually are. */
|
|
31
|
+
export const MAX_TOOL_RESULT_CHARS = 4 * 1024
|
|
32
|
+
|
|
33
|
+
/** A captured body plus what the cap dropped from it. */
|
|
34
|
+
export interface CapturedToolBody {
|
|
35
|
+
text: string
|
|
36
|
+
dropped: number
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const EMPTY: CapturedToolBody = { text: '', dropped: 0 }
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Serialise, scrub and cap one tool body.
|
|
43
|
+
*
|
|
44
|
+
* A non-string value is JSON-serialised, and a value that cannot be (a cycle, a `BigInt`, a
|
|
45
|
+
* throwing getter) is NAMED as unserialisable rather than dropped to `''`: an empty body means
|
|
46
|
+
* "the call carried none", and a capture failure is a different fact.
|
|
47
|
+
*/
|
|
48
|
+
export function captureToolBody(
|
|
49
|
+
value: unknown,
|
|
50
|
+
max: number,
|
|
51
|
+
secrets: readonly string[],
|
|
52
|
+
): CapturedToolBody {
|
|
53
|
+
if (value === undefined || value === null) return EMPTY
|
|
54
|
+
let text: string
|
|
55
|
+
if (typeof value === 'string') {
|
|
56
|
+
text = value
|
|
57
|
+
} else {
|
|
58
|
+
try {
|
|
59
|
+
text = JSON.stringify(value) ?? ''
|
|
60
|
+
} catch {
|
|
61
|
+
return { text: '[unserialisable]', dropped: 0 }
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (text === '') return EMPTY
|
|
65
|
+
const scrubbed = redact(text, secrets)
|
|
66
|
+
if (scrubbed.length <= max) return { text: scrubbed, dropped: 0 }
|
|
67
|
+
return { text: scrubbed.slice(0, max), dropped: scrubbed.length - max }
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** One tool call the tracker is holding open, between its start and its result. */
|
|
71
|
+
interface PendingCall {
|
|
72
|
+
tool: string
|
|
73
|
+
startedAt: number
|
|
74
|
+
args: CapturedToolBody
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** What {@link ToolCallTracker} emits per completed call — the fields a `ToolSpan` needs. */
|
|
78
|
+
export interface TrackedToolCall {
|
|
79
|
+
tool: string
|
|
80
|
+
seq: number
|
|
81
|
+
startedAt: number
|
|
82
|
+
endedAt: number
|
|
83
|
+
ok: boolean
|
|
84
|
+
args: string
|
|
85
|
+
result: string
|
|
86
|
+
argsDropped: number
|
|
87
|
+
resultDropped: number
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Pairs each tool call's START with its RESULT and numbers the pairs, so the two agent CLIs feed
|
|
92
|
+
* one trajectory shape from two very different streams (Pi's flat `tool_execution_*` events, the
|
|
93
|
+
* claude-code stream's `tool_use` / `tool_result` content blocks).
|
|
94
|
+
*
|
|
95
|
+
* Correlation is BY ID where the stream supplies one, and by tool name otherwise, because that is
|
|
96
|
+
* the difference between the two producers: claude-code's blocks always carry a `tool_use_id`, and
|
|
97
|
+
* a parallel batch of calls is routine there, while Pi's stream is sequential. A result the tracker
|
|
98
|
+
* cannot pair with a start is still EMITTED (with no args and the previous call's end as its
|
|
99
|
+
* start): losing a step of the trajectory to a schema tweak would be worse than an entry that says
|
|
100
|
+
* less than its neighbours, and the `seq` it takes keeps every later entry's ordinal honest.
|
|
101
|
+
*/
|
|
102
|
+
export class ToolCallTracker {
|
|
103
|
+
private seq = 0
|
|
104
|
+
private readonly byId = new Map<string, PendingCall>()
|
|
105
|
+
private readonly byTool = new Map<string, PendingCall[]>()
|
|
106
|
+
/**
|
|
107
|
+
* Start boundary for a call whose own start was never seen: the previous call's end, or the run
|
|
108
|
+
* start. Approximate but contiguous, which is the property the trace tree needs.
|
|
109
|
+
*/
|
|
110
|
+
private boundary: number
|
|
111
|
+
|
|
112
|
+
constructor(
|
|
113
|
+
private readonly secrets: readonly string[] = [],
|
|
114
|
+
now: number = Date.now(),
|
|
115
|
+
) {
|
|
116
|
+
this.boundary = now
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Record that a call began, with the arguments the agent supplied. */
|
|
120
|
+
started(id: string | undefined, tool: string, args: unknown, at: number = Date.now()): void {
|
|
121
|
+
const pending: PendingCall = {
|
|
122
|
+
tool,
|
|
123
|
+
startedAt: at,
|
|
124
|
+
args: captureToolBody(args, MAX_TOOL_ARGS_CHARS, this.secrets),
|
|
125
|
+
}
|
|
126
|
+
if (id) {
|
|
127
|
+
this.byId.set(id, pending)
|
|
128
|
+
return
|
|
129
|
+
}
|
|
130
|
+
const queue = this.byTool.get(tool) ?? []
|
|
131
|
+
queue.push(pending)
|
|
132
|
+
this.byTool.set(tool, queue)
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Record that a call finished, and return the completed entry. */
|
|
136
|
+
finished(
|
|
137
|
+
id: string | undefined,
|
|
138
|
+
tool: string,
|
|
139
|
+
result: unknown,
|
|
140
|
+
isError: boolean,
|
|
141
|
+
at: number = Date.now(),
|
|
142
|
+
): TrackedToolCall {
|
|
143
|
+
const pending = this.take(id, tool)
|
|
144
|
+
const captured = captureToolBody(result, MAX_TOOL_RESULT_CHARS, this.secrets)
|
|
145
|
+
const call: TrackedToolCall = {
|
|
146
|
+
tool: pending?.tool ?? tool,
|
|
147
|
+
seq: this.seq++,
|
|
148
|
+
startedAt: pending?.startedAt ?? this.boundary,
|
|
149
|
+
endedAt: at,
|
|
150
|
+
ok: !isError,
|
|
151
|
+
args: pending?.args.text ?? '',
|
|
152
|
+
argsDropped: pending?.args.dropped ?? 0,
|
|
153
|
+
result: captured.text,
|
|
154
|
+
resultDropped: captured.dropped,
|
|
155
|
+
}
|
|
156
|
+
this.boundary = at
|
|
157
|
+
return call
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
private take(id: string | undefined, tool: string): PendingCall | undefined {
|
|
161
|
+
if (id) {
|
|
162
|
+
const byId = this.byId.get(id)
|
|
163
|
+
if (byId) {
|
|
164
|
+
this.byId.delete(id)
|
|
165
|
+
return byId
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
const queue = this.byTool.get(tool)
|
|
169
|
+
// FIFO, so a sequential stream pairs each result with the call that opened first. A stream
|
|
170
|
+
// that interleaves un-idded calls of one tool would mis-pair here; none of the two producers
|
|
171
|
+
// does, and the alternative (refusing to pair at all) loses the args on every ordinary run.
|
|
172
|
+
const pending = queue?.shift()
|
|
173
|
+
if (queue && queue.length === 0) this.byTool.delete(tool)
|
|
174
|
+
return pending
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Read a tool-call id off a stream event, whatever the producer calls it. */
|
|
179
|
+
export function readToolCallId(event: Record<string, unknown>): string | undefined {
|
|
180
|
+
for (const key of ['toolCallId', 'tool_call_id', 'callId', 'id']) {
|
|
181
|
+
const value = event[key]
|
|
182
|
+
if (typeof value === 'string' && value !== '') return value
|
|
183
|
+
}
|
|
184
|
+
return undefined
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** A tool call BEGINNING, read off a Pi `--mode json` event, or undefined if it isn't one. */
|
|
188
|
+
export function toolCallStart(
|
|
189
|
+
event: Record<string, unknown>,
|
|
190
|
+
): { id?: string; name: string; args: unknown } | undefined {
|
|
191
|
+
if (event.type !== 'tool_execution_start' && event.type !== 'tool_call') return undefined
|
|
192
|
+
const name = typeof event.toolName === 'string' ? event.toolName : ''
|
|
193
|
+
if (!name) return undefined
|
|
194
|
+
// Read every spelling the stream might use rather than pinning one: Pi's event schema has been
|
|
195
|
+
// renamed under us before (see `todoResultDetails`, which reads three shapes of one result), and
|
|
196
|
+
// a rename here costs the ARGUMENTS of every call, silently.
|
|
197
|
+
const args = event.args ?? event.arguments ?? event.input ?? event.parameters
|
|
198
|
+
const id = readToolCallId(event)
|
|
199
|
+
return { name, args, ...(id ? { id } : {}) }
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** The RESULT payload of a Pi `tool_execution_end` event, unwrapped from its envelope. */
|
|
203
|
+
export function toolCallResult(event: Record<string, unknown>): unknown {
|
|
204
|
+
const result = event.result
|
|
205
|
+
if (result && typeof result === 'object') {
|
|
206
|
+
const inner = result as Record<string, unknown>
|
|
207
|
+
// `details` is the structured payload Pi's own extensions return (the todo tool's shape), and
|
|
208
|
+
// `content`/`output` the free-text ones. Falling back to the whole envelope keeps a shape none
|
|
209
|
+
// of these match readable rather than empty.
|
|
210
|
+
return inner.details ?? inner.content ?? inner.output ?? result
|
|
211
|
+
}
|
|
212
|
+
return result ?? event.output ?? event.content
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Feed one claude-code `user` turn's content blocks to the tracker, emitting an entry per
|
|
217
|
+
* `tool_result`.
|
|
218
|
+
*
|
|
219
|
+
* The CLI answers each `tool_use` with a `tool_result` carrying the same `tool_use_id`, so the
|
|
220
|
+
* pairing is exact even when the model fired a batch of calls in parallel — which it routinely
|
|
221
|
+
* does, and which is why the trajectory here is ordered by the ordinal the tracker stamps rather
|
|
222
|
+
* than by the turn the results arrived on.
|
|
223
|
+
*/
|
|
224
|
+
export function recordClaudeToolResults(
|
|
225
|
+
tracker: ToolCallTracker,
|
|
226
|
+
content: readonly unknown[],
|
|
227
|
+
emit: (call: TrackedToolCall) => void,
|
|
228
|
+
): void {
|
|
229
|
+
for (const block of content) {
|
|
230
|
+
if (!block || typeof block !== 'object') continue
|
|
231
|
+
const record = block as Record<string, unknown>
|
|
232
|
+
if (record.type !== 'tool_result') continue
|
|
233
|
+
const id = typeof record.tool_use_id === 'string' ? record.tool_use_id : undefined
|
|
234
|
+
// The block carries no tool NAME (only the id the assistant turn named), so an unpaired
|
|
235
|
+
// result falls back to a stated placeholder rather than an empty string that would render
|
|
236
|
+
// as a nameless step.
|
|
237
|
+
emit(tracker.finished(id, 'unknown', record.content, record.is_error === true))
|
|
238
|
+
}
|
|
239
|
+
}
|