@cat-factory/executor-harness 1.88.0 → 1.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -49
- package/dist/agent-capabilities.d.ts +20 -3
- package/dist/agent-capabilities.js +32 -4
- package/dist/agent-runner.d.ts +8 -1
- package/dist/agent-runner.js +27 -0
- package/dist/job.d.ts +2 -0
- package/dist/job.js +6 -0
- package/dist/pi-workspace.js +3 -0
- package/dist/pi.d.ts +26 -5
- package/dist/pi.js +18 -13
- package/dist/progress-guard.d.ts +29 -0
- package/dist/progress-guard.js +82 -8
- package/dist/tool-trajectory.d.ts +78 -0
- package/dist/tool-trajectory.js +194 -0
- package/package.json +4 -4
- package/src/agent-capabilities.ts +34 -4
- package/src/agent-runner.ts +48 -0
- package/src/job.ts +6 -0
- package/src/pi-workspace.ts +3 -0
- package/src/pi.ts +53 -18
- package/src/progress-guard.ts +129 -10
- package/src/tool-trajectory.ts +239 -0
package/dist/pi.js
CHANGED
|
@@ -4,10 +4,11 @@ import { homedir } from 'node:os';
|
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
5
|
import { killChildProcess, spawnDetached } from './process.js';
|
|
6
6
|
import { pathExists } from './fs-utils.js';
|
|
7
|
-
import { redactSecrets } from './redact.js';
|
|
7
|
+
import { redactSecrets, secretsToRedact } from './redact.js';
|
|
8
8
|
import { HarnessFailure } from './failure.js';
|
|
9
9
|
import { log } from './logger.js';
|
|
10
10
|
import { ProgressGuard, progressGuardLimitsFromEnv, toolCallSignal, } from './progress-guard.js';
|
|
11
|
+
import { ToolCallTracker, readToolCallId, toolCallResult, toolCallStart, } from './tool-trajectory.js';
|
|
11
12
|
// Drives the Pi coding-agent CLI. Pi is pointed at the Worker's OpenAI-compatible
|
|
12
13
|
// proxy via a custom provider in ~/.pi/agent/models.json, authenticated with the
|
|
13
14
|
// per-job session token (interpolated from $PI_PROXY_TOKEN) — so no provider key
|
|
@@ -606,10 +607,17 @@ export function runPi(opts) {
|
|
|
606
607
|
let malformedLines = 0;
|
|
607
608
|
let observerErrors = 0;
|
|
608
609
|
const guard = new ProgressGuard(opts.guardLimits ?? progressGuardLimitsFromEnv(), opts.expectsEdits ?? true);
|
|
609
|
-
//
|
|
610
|
-
//
|
|
611
|
-
//
|
|
612
|
-
|
|
610
|
+
// Pairs each tool call's start with its result, numbers the pairs and captures the two
|
|
611
|
+
// bodies (scrubbed + capped). A call whose start Pi never emitted still gets an entry,
|
|
612
|
+
// timed from the previous call's end — see `ToolCallTracker`.
|
|
613
|
+
//
|
|
614
|
+
// The known-secret list is DERIVED from the token this function itself hands the child
|
|
615
|
+
// (`PI_PROXY_TOKEN` / `SEARXNG_API_KEY`) rather than taken as a parameter: the bodies
|
|
616
|
+
// travel to a store and to external trace sinks, and a caller that forgets to pass the
|
|
617
|
+
// list produces bodies scrubbed of credential SHAPES only, with no signal that the
|
|
618
|
+
// narrower rule ever ran. Deriving it here means the one place that knows the child's
|
|
619
|
+
// credentials is the place that scrubs them.
|
|
620
|
+
const tools = new ToolCallTracker(secretsToRedact(opts.sessionToken));
|
|
613
621
|
// SIGTERM first, then SIGKILL if Pi ignores it. Shared by the watchdog abort
|
|
614
622
|
// and the no-progress guard; the `close` handler turns it into a rejection.
|
|
615
623
|
const killChild = () => killChildProcess(child);
|
|
@@ -648,22 +656,19 @@ export function runPi(opts) {
|
|
|
648
656
|
opts.onProgress(progress);
|
|
649
657
|
}
|
|
650
658
|
if (opts.onSpan) {
|
|
659
|
+
const start = toolCallStart(event);
|
|
660
|
+
if (start)
|
|
661
|
+
tools.started(start.id, start.name, start.args);
|
|
651
662
|
const signal = toolCallSignal(event);
|
|
652
663
|
if (signal && signal.name) {
|
|
653
|
-
const
|
|
664
|
+
const call = tools.finished(readToolCallId(event), signal.name, toolCallResult(event), signal.isError);
|
|
654
665
|
try {
|
|
655
|
-
opts.onSpan({
|
|
656
|
-
tool: signal.name,
|
|
657
|
-
startedAt: toolBoundary,
|
|
658
|
-
endedAt,
|
|
659
|
-
ok: !signal.isError,
|
|
660
|
-
});
|
|
666
|
+
opts.onSpan({ ...call, bodies: 'stored' });
|
|
661
667
|
}
|
|
662
668
|
catch {
|
|
663
669
|
// A faulty observer must never break the run.
|
|
664
670
|
observerErrors++;
|
|
665
671
|
}
|
|
666
|
-
toolBoundary = endedAt;
|
|
667
672
|
}
|
|
668
673
|
}
|
|
669
674
|
if (runGuard && !guardReason && !aborted) {
|
package/dist/progress-guard.d.ts
CHANGED
|
@@ -35,11 +35,38 @@ export interface ProgressGuardLimits {
|
|
|
35
35
|
* without it.
|
|
36
36
|
*/
|
|
37
37
|
maxConsecutiveWebCalls?: number;
|
|
38
|
+
/**
|
|
39
|
+
* Abort after this many consecutive MCP tool-server calls (`mcp__*`) with no other
|
|
40
|
+
* tool call in between: the tool-server analogue of `maxConsecutiveWebCalls`, and
|
|
41
|
+
* present for the same reason. An `mcp__*` call is exempt from the no-edit bound (see
|
|
42
|
+
* `isMcpToolCall`), so without a streak of its own a run could query a tool server
|
|
43
|
+
* indefinitely without tripping any guard. Any non-MCP tool call resets the streak.
|
|
44
|
+
* Optional: defaults to {@link DEFAULT_PROGRESS_GUARD_LIMITS}.
|
|
45
|
+
*/
|
|
46
|
+
maxConsecutiveMcpCalls?: number;
|
|
47
|
+
/**
|
|
48
|
+
* Abort after this many consecutive calls that are EXEMPT from the no-edit bound
|
|
49
|
+
* (planning, read-only exploration, subagent dispatch, `mcp__*`) with no action call
|
|
50
|
+
* in between. The backstop that makes each individual exemption mean "not counted"
|
|
51
|
+
* rather than "unbounded": every per-family streak above resets on any call outside
|
|
52
|
+
* its own family, so a run alternating `web_search` with `mcp__issues__search` (or
|
|
53
|
+
* with `read`) trips none of them and, having never made an action call, never
|
|
54
|
+
* reaches `maxToolCallsWithoutEdit` either. Only the job's wall-clock ceiling
|
|
55
|
+
* bounded that.
|
|
56
|
+
*
|
|
57
|
+
* Deliberately far above every family cap, because it is not a research bound and
|
|
58
|
+
* must not become one: reading a hundred files before the first edit is legitimate
|
|
59
|
+
* work-up, and any `bash`/edit/action call resets the streak. Optional: defaults to
|
|
60
|
+
* {@link DEFAULT_PROGRESS_GUARD_LIMITS}.
|
|
61
|
+
*/
|
|
62
|
+
maxConsecutiveNonActionCalls?: number;
|
|
38
63
|
}
|
|
39
64
|
export declare const DEFAULT_PROGRESS_GUARD_LIMITS: {
|
|
40
65
|
maxToolCallsWithoutEdit: number;
|
|
41
66
|
maxConsecutiveErrors: number;
|
|
42
67
|
maxConsecutiveWebCalls: number;
|
|
68
|
+
maxConsecutiveMcpCalls: number;
|
|
69
|
+
maxConsecutiveNonActionCalls: number;
|
|
43
70
|
};
|
|
44
71
|
/** Read {@link ProgressGuardLimits} from the environment, falling back to the defaults. */
|
|
45
72
|
export declare function progressGuardLimitsFromEnv(env?: NodeJS.ProcessEnv): ProgressGuardLimits;
|
|
@@ -69,6 +96,8 @@ export declare class ProgressGuard {
|
|
|
69
96
|
private edits;
|
|
70
97
|
private consecutiveErrors;
|
|
71
98
|
private consecutiveWebCalls;
|
|
99
|
+
private consecutiveMcpCalls;
|
|
100
|
+
private consecutiveNonActionCalls;
|
|
72
101
|
constructor(limits: ProgressGuardLimits,
|
|
73
102
|
/** When false (assess-only runs like the merger), the no-edit bound is skipped. */
|
|
74
103
|
expectsEdits?: boolean);
|
package/dist/progress-guard.js
CHANGED
|
@@ -28,6 +28,16 @@ export const DEFAULT_PROGRESS_GUARD_LIMITS = {
|
|
|
28
28
|
// A genuine research burst is a handful of searches; an uninterrupted run of this
|
|
29
29
|
// many web calls (with no read/edit/bash between) is a search loop, not progress.
|
|
30
30
|
maxConsecutiveWebCalls: 25,
|
|
31
|
+
// Looser than the web cap: a tool server is usually the agent's route to the SYSTEM OF
|
|
32
|
+
// RECORD (the issue tracker, the advisory database, the design source), and reading a
|
|
33
|
+
// list and then each of its items is a normal opening move, not a rabbit-hole. A run
|
|
34
|
+
// that makes this many in a row with no read, edit or bash between is looping.
|
|
35
|
+
maxConsecutiveMcpCalls: 40,
|
|
36
|
+
// Well clear of every family cap above, and of any plausible read-up: a run that makes
|
|
37
|
+
// this many exempt calls with not one action call between them has stopped converging,
|
|
38
|
+
// whatever mix of reads, searches and lookups it is cycling through. Sized as a
|
|
39
|
+
// backstop rather than a judgement, because the families are where judgement belongs.
|
|
40
|
+
maxConsecutiveNonActionCalls: 200,
|
|
31
41
|
};
|
|
32
42
|
// Tool names that mutate files, so a call to one clears the no-edit suspicion. Kept
|
|
33
43
|
// broad on purpose: different models/extensions name the same capability differently
|
|
@@ -98,6 +108,28 @@ const EXPLORATION_TOOLS = new Set([
|
|
|
98
108
|
// call between) can be caught as a search loop — see `maxConsecutiveWebCalls`. Covers both
|
|
99
109
|
// Pi's `web_search`/`web_fetch` and Claude Code's `WebSearch`/`WebFetch`.
|
|
100
110
|
const WEB_TOOLS = new Set(['web_search', 'web_fetch', 'websearch', 'webfetch']);
|
|
111
|
+
// A call to a tool server (MCP). Every MCP client names these `mcp__<server>__<tool>`, and
|
|
112
|
+
// the prefix is the ONLY thing the harness can know about them: what a given server's tools
|
|
113
|
+
// do is a backend registration this image has never seen, so the guard classifies by shape.
|
|
114
|
+
//
|
|
115
|
+
// Matched, rather than enumerated in EXPLORATION_TOOLS, because the set is open: it is
|
|
116
|
+
// whatever tool servers the running kind was wired with. A prefix test is also why this is a
|
|
117
|
+
// function, `name.startsWith` on the already-lower-cased name, so `MCP__Issues__search`
|
|
118
|
+
// classifies the same as `mcp__issues__search`.
|
|
119
|
+
function isMcpToolCall(loweredName) {
|
|
120
|
+
return loweredName.startsWith('mcp__');
|
|
121
|
+
}
|
|
122
|
+
// Whether a call is EXEMPT from the no-edit bound: planning and bookkeeping, read-only
|
|
123
|
+
// exploration, a subagent dispatch, or a tool-server call. One predicate rather than the
|
|
124
|
+
// four tests inlined at the branch, because the combined non-action streak and the no-edit
|
|
125
|
+
// exemption must be the SAME set: a family exempted in one place and missed in the other is
|
|
126
|
+
// either an unbounded loop or a run killed for a call the bound says it may make.
|
|
127
|
+
function isNonActionToolCall(loweredName) {
|
|
128
|
+
return (PLANNING_TOOLS.has(loweredName) ||
|
|
129
|
+
EXPLORATION_TOOLS.has(loweredName) ||
|
|
130
|
+
SUBAGENT_DISPATCH_TOOLS.has(loweredName) ||
|
|
131
|
+
isMcpToolCall(loweredName));
|
|
132
|
+
}
|
|
101
133
|
/** Read {@link ProgressGuardLimits} from the environment, falling back to the defaults. */
|
|
102
134
|
export function progressGuardLimitsFromEnv(env = process.env) {
|
|
103
135
|
const num = (raw, fallback) => {
|
|
@@ -108,6 +140,8 @@ export function progressGuardLimitsFromEnv(env = process.env) {
|
|
|
108
140
|
maxToolCallsWithoutEdit: num(env.JOB_MAX_TOOLCALLS_WITHOUT_EDIT, DEFAULT_PROGRESS_GUARD_LIMITS.maxToolCallsWithoutEdit),
|
|
109
141
|
maxConsecutiveErrors: num(env.JOB_MAX_CONSECUTIVE_TOOL_ERRORS, DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveErrors),
|
|
110
142
|
maxConsecutiveWebCalls: num(env.JOB_MAX_CONSECUTIVE_WEB_CALLS, DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveWebCalls),
|
|
143
|
+
maxConsecutiveMcpCalls: num(env.JOB_MAX_CONSECUTIVE_MCP_CALLS, DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveMcpCalls),
|
|
144
|
+
maxConsecutiveNonActionCalls: num(env.JOB_MAX_CONSECUTIVE_NON_ACTION_CALLS, DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveNonActionCalls),
|
|
111
145
|
};
|
|
112
146
|
}
|
|
113
147
|
/**
|
|
@@ -127,9 +161,12 @@ export function mergeGuardLimits(base, overrides) {
|
|
|
127
161
|
return {
|
|
128
162
|
maxToolCallsWithoutEdit: loosen(base.maxToolCallsWithoutEdit, overrides.maxToolCallsWithoutEdit),
|
|
129
163
|
maxConsecutiveErrors: loosen(base.maxConsecutiveErrors, overrides.maxConsecutiveErrors),
|
|
130
|
-
//
|
|
131
|
-
//
|
|
164
|
+
// The streak knobs are optional on the interface (callers may omit them), so fall back
|
|
165
|
+
// to the default before loosening: it keeps `loosen`'s base a concrete number.
|
|
132
166
|
maxConsecutiveWebCalls: loosen(base.maxConsecutiveWebCalls ?? DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveWebCalls, overrides.maxConsecutiveWebCalls),
|
|
167
|
+
maxConsecutiveMcpCalls: loosen(base.maxConsecutiveMcpCalls ?? DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveMcpCalls, overrides.maxConsecutiveMcpCalls),
|
|
168
|
+
maxConsecutiveNonActionCalls: loosen(base.maxConsecutiveNonActionCalls ??
|
|
169
|
+
DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveNonActionCalls, overrides.maxConsecutiveNonActionCalls),
|
|
133
170
|
};
|
|
134
171
|
}
|
|
135
172
|
/**
|
|
@@ -146,6 +183,8 @@ export class ProgressGuard {
|
|
|
146
183
|
edits = 0;
|
|
147
184
|
consecutiveErrors = 0;
|
|
148
185
|
consecutiveWebCalls = 0;
|
|
186
|
+
consecutiveMcpCalls = 0;
|
|
187
|
+
consecutiveNonActionCalls = 0;
|
|
149
188
|
constructor(limits,
|
|
150
189
|
/** When false (assess-only runs like the merger), the no-edit bound is skipped. */
|
|
151
190
|
expectsEdits = true) {
|
|
@@ -189,14 +228,49 @@ export class ProgressGuard {
|
|
|
189
228
|
else {
|
|
190
229
|
this.consecutiveWebCalls = 0;
|
|
191
230
|
}
|
|
192
|
-
//
|
|
193
|
-
// no-edit bound
|
|
194
|
-
//
|
|
195
|
-
if (
|
|
196
|
-
|
|
197
|
-
|
|
231
|
+
// Tool-server (MCP) calls: bounded as their own streak for exactly the reason the web
|
|
232
|
+
// streak exists. They are exempt from the no-edit bound below, and an exemption with no
|
|
233
|
+
// counter-bound is a loop the guard cannot see. Any non-MCP call resets it.
|
|
234
|
+
if (isMcpToolCall(name)) {
|
|
235
|
+
this.consecutiveMcpCalls++;
|
|
236
|
+
const mcpCap = this.limits.maxConsecutiveMcpCalls ?? DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveMcpCalls;
|
|
237
|
+
if (this.consecutiveMcpCalls >= mcpCap) {
|
|
238
|
+
return (`no progress: ${this.consecutiveMcpCalls} consecutive tool-server (MCP) calls without ` +
|
|
239
|
+
`any other action. The agent is stuck querying its tools instead of doing the work. ` +
|
|
240
|
+
`Aborting.`);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
else {
|
|
244
|
+
this.consecutiveMcpCalls = 0;
|
|
245
|
+
}
|
|
246
|
+
// Planning, read-only exploration, subagent-dispatch and tool-server calls don't count
|
|
247
|
+
// toward the no-edit bound (see `isNonActionToolCall`): only "action" calls without an
|
|
248
|
+
// edit do.
|
|
249
|
+
//
|
|
250
|
+
// An `mcp__*` call is exempt for the same reason a `read` is: the bound targets the
|
|
251
|
+
// credential rabbit-hole (endless `bash` probing with nothing implemented), and reaching
|
|
252
|
+
// a registered tool server is the platform TELLING the agent to look something up
|
|
253
|
+
// ("prefer them over guessing"). Counting them would abort an edits-expected kind for
|
|
254
|
+
// consulting the issue tracker the deployment wired for it, punishing the run for
|
|
255
|
+
// following its own prompt. They are neutral rather than edit-satisfying, exactly like a
|
|
256
|
+
// subagent dispatch: a read-only lookup must not clear the suspicion the bound holds.
|
|
257
|
+
//
|
|
258
|
+
// The exempt calls carry ONE streak of their own, and it is what keeps every exemption
|
|
259
|
+
// above from adding up to an unbounded run: each per-family cap resets on any call
|
|
260
|
+
// outside its family, so alternating two exempt families trips neither, and a run that
|
|
261
|
+
// never makes an action call never reaches the no-edit bound either.
|
|
262
|
+
if (isNonActionToolCall(name)) {
|
|
263
|
+
this.consecutiveNonActionCalls++;
|
|
264
|
+
const nonActionCap = this.limits.maxConsecutiveNonActionCalls ??
|
|
265
|
+
DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveNonActionCalls;
|
|
266
|
+
if (this.consecutiveNonActionCalls >= nonActionCap) {
|
|
267
|
+
return (`no progress: ${this.consecutiveNonActionCalls} consecutive read-only calls (searching, ` +
|
|
268
|
+
`reading, tool-server lookups, subagent dispatches) with no action call between them. ` +
|
|
269
|
+
`The agent is cycling through research instead of doing the work. Aborting.`);
|
|
270
|
+
}
|
|
198
271
|
return null;
|
|
199
272
|
}
|
|
273
|
+
this.consecutiveNonActionCalls = 0;
|
|
200
274
|
this.toolCalls++;
|
|
201
275
|
if (FILE_EDIT_TOOLS.has(name))
|
|
202
276
|
this.edits++;
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/** Cap on a captured argument blob. Generous for a command line, far below a file body. */
|
|
2
|
+
export declare const MAX_TOOL_ARGS_CHARS: number;
|
|
3
|
+
/** Cap on a captured result. Larger than the args cap: a result is where the bytes actually are. */
|
|
4
|
+
export declare const MAX_TOOL_RESULT_CHARS: number;
|
|
5
|
+
/** A captured body plus what the cap dropped from it. */
|
|
6
|
+
export interface CapturedToolBody {
|
|
7
|
+
text: string;
|
|
8
|
+
dropped: number;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Serialise, scrub and cap one tool body.
|
|
12
|
+
*
|
|
13
|
+
* A non-string value is JSON-serialised, and a value that cannot be (a cycle, a `BigInt`, a
|
|
14
|
+
* throwing getter) is NAMED as unserialisable rather than dropped to `''`: an empty body means
|
|
15
|
+
* "the call carried none", and a capture failure is a different fact.
|
|
16
|
+
*/
|
|
17
|
+
export declare function captureToolBody(value: unknown, max: number, secrets: readonly string[]): CapturedToolBody;
|
|
18
|
+
/** What {@link ToolCallTracker} emits per completed call — the fields a `ToolSpan` needs. */
|
|
19
|
+
export interface TrackedToolCall {
|
|
20
|
+
tool: string;
|
|
21
|
+
seq: number;
|
|
22
|
+
startedAt: number;
|
|
23
|
+
endedAt: number;
|
|
24
|
+
ok: boolean;
|
|
25
|
+
args: string;
|
|
26
|
+
result: string;
|
|
27
|
+
argsDropped: number;
|
|
28
|
+
resultDropped: number;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Pairs each tool call's START with its RESULT and numbers the pairs, so the two agent CLIs feed
|
|
32
|
+
* one trajectory shape from two very different streams (Pi's flat `tool_execution_*` events, the
|
|
33
|
+
* claude-code stream's `tool_use` / `tool_result` content blocks).
|
|
34
|
+
*
|
|
35
|
+
* Correlation is BY ID where the stream supplies one, and by tool name otherwise, because that is
|
|
36
|
+
* the difference between the two producers: claude-code's blocks always carry a `tool_use_id`, and
|
|
37
|
+
* a parallel batch of calls is routine there, while Pi's stream is sequential. A result the tracker
|
|
38
|
+
* cannot pair with a start is still EMITTED (with no args and the previous call's end as its
|
|
39
|
+
* start): losing a step of the trajectory to a schema tweak would be worse than an entry that says
|
|
40
|
+
* less than its neighbours, and the `seq` it takes keeps every later entry's ordinal honest.
|
|
41
|
+
*/
|
|
42
|
+
export declare class ToolCallTracker {
|
|
43
|
+
private readonly secrets;
|
|
44
|
+
private seq;
|
|
45
|
+
private readonly byId;
|
|
46
|
+
private readonly byTool;
|
|
47
|
+
/**
|
|
48
|
+
* Start boundary for a call whose own start was never seen: the previous call's end, or the run
|
|
49
|
+
* start. Approximate but contiguous, which is the property the trace tree needs.
|
|
50
|
+
*/
|
|
51
|
+
private boundary;
|
|
52
|
+
constructor(secrets?: readonly string[], now?: number);
|
|
53
|
+
/** Record that a call began, with the arguments the agent supplied. */
|
|
54
|
+
started(id: string | undefined, tool: string, args: unknown, at?: number): void;
|
|
55
|
+
/** Record that a call finished, and return the completed entry. */
|
|
56
|
+
finished(id: string | undefined, tool: string, result: unknown, isError: boolean, at?: number): TrackedToolCall;
|
|
57
|
+
private take;
|
|
58
|
+
}
|
|
59
|
+
/** Read a tool-call id off a stream event, whatever the producer calls it. */
|
|
60
|
+
export declare function readToolCallId(event: Record<string, unknown>): string | undefined;
|
|
61
|
+
/** A tool call BEGINNING, read off a Pi `--mode json` event, or undefined if it isn't one. */
|
|
62
|
+
export declare function toolCallStart(event: Record<string, unknown>): {
|
|
63
|
+
id?: string;
|
|
64
|
+
name: string;
|
|
65
|
+
args: unknown;
|
|
66
|
+
} | undefined;
|
|
67
|
+
/** The RESULT payload of a Pi `tool_execution_end` event, unwrapped from its envelope. */
|
|
68
|
+
export declare function toolCallResult(event: Record<string, unknown>): unknown;
|
|
69
|
+
/**
|
|
70
|
+
* Feed one claude-code `user` turn's content blocks to the tracker, emitting an entry per
|
|
71
|
+
* `tool_result`.
|
|
72
|
+
*
|
|
73
|
+
* The CLI answers each `tool_use` with a `tool_result` carrying the same `tool_use_id`, so the
|
|
74
|
+
* pairing is exact even when the model fired a batch of calls in parallel — which it routinely
|
|
75
|
+
* does, and which is why the trajectory here is ordered by the ordinal the tracker stamps rather
|
|
76
|
+
* than by the turn the results arrived on.
|
|
77
|
+
*/
|
|
78
|
+
export declare function recordClaudeToolResults(tracker: ToolCallTracker, content: readonly unknown[], emit: (call: TrackedToolCall) => void): void;
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { redact } from './redact.js';
|
|
2
|
+
// The TRAJECTORY capture: what the agent DID, one entry per tool call, in the order it made them.
|
|
3
|
+
//
|
|
4
|
+
// The harness has always buffered a compact span per tool call (name + timing + ok) for the run's
|
|
5
|
+
// trace. That is enough to draw a tree and not enough to answer the question anyone actually asks
|
|
6
|
+
// of a finished run — WHICH command, against WHAT, and what came back. The evidence standard for a
|
|
7
|
+
// merged PR is "how, not just the diff", and a span saying `bash` ran for 300ms is not evidence of
|
|
8
|
+
// anything. So each entry now carries the call's arguments and result, captured here because this
|
|
9
|
+
// is the only process that ever sees them: an agent CLI's tool loop is internal to the CLI, and
|
|
10
|
+
// the container is gone the moment the job settles.
|
|
11
|
+
//
|
|
12
|
+
// Two properties keep that affordable and safe:
|
|
13
|
+
//
|
|
14
|
+
// - **Bounded at capture.** Each body is capped (a build log is routinely megabytes) and the entry
|
|
15
|
+
// STATES what the cap dropped, so a reader can tell a short command from the head of a long one.
|
|
16
|
+
// The caps are what keep the drain buffer and the poll response small.
|
|
17
|
+
// - **Scrubbed at capture.** A tool's arguments and output routinely echo an env var, a clone URL
|
|
18
|
+
// or the leased subscription token, and these travel to a store and to external trace sinks.
|
|
19
|
+
//
|
|
20
|
+
// The RETENTION decision is the backend's, not this module's: entries carry `bodies: 'stored'` and
|
|
21
|
+
// the backend's double gate (`LLM_RECORD_PROMPTS` + the workspace's `storeAgentContext`) decides
|
|
22
|
+
// whether to keep them, exactly as it already does for the prompt bodies the call-metric
|
|
23
|
+
// reconstruction assembles here. Deciding it twice would mean the container had to be told the
|
|
24
|
+
// workspace's settings, and an image one release behind its backend would then be deciding it with
|
|
25
|
+
// stale ones.
|
|
26
|
+
/** Cap on a captured argument blob. Generous for a command line, far below a file body. */
|
|
27
|
+
export const MAX_TOOL_ARGS_CHARS = 2 * 1024;
|
|
28
|
+
/** Cap on a captured result. Larger than the args cap: a result is where the bytes actually are. */
|
|
29
|
+
export const MAX_TOOL_RESULT_CHARS = 4 * 1024;
|
|
30
|
+
const EMPTY = { text: '', dropped: 0 };
|
|
31
|
+
/**
|
|
32
|
+
* Serialise, scrub and cap one tool body.
|
|
33
|
+
*
|
|
34
|
+
* A non-string value is JSON-serialised, and a value that cannot be (a cycle, a `BigInt`, a
|
|
35
|
+
* throwing getter) is NAMED as unserialisable rather than dropped to `''`: an empty body means
|
|
36
|
+
* "the call carried none", and a capture failure is a different fact.
|
|
37
|
+
*/
|
|
38
|
+
export function captureToolBody(value, max, secrets) {
|
|
39
|
+
if (value === undefined || value === null)
|
|
40
|
+
return EMPTY;
|
|
41
|
+
let text;
|
|
42
|
+
if (typeof value === 'string') {
|
|
43
|
+
text = value;
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
try {
|
|
47
|
+
text = JSON.stringify(value) ?? '';
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return { text: '[unserialisable]', dropped: 0 };
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (text === '')
|
|
54
|
+
return EMPTY;
|
|
55
|
+
const scrubbed = redact(text, secrets);
|
|
56
|
+
if (scrubbed.length <= max)
|
|
57
|
+
return { text: scrubbed, dropped: 0 };
|
|
58
|
+
return { text: scrubbed.slice(0, max), dropped: scrubbed.length - max };
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Pairs each tool call's START with its RESULT and numbers the pairs, so the two agent CLIs feed
|
|
62
|
+
* one trajectory shape from two very different streams (Pi's flat `tool_execution_*` events, the
|
|
63
|
+
* claude-code stream's `tool_use` / `tool_result` content blocks).
|
|
64
|
+
*
|
|
65
|
+
* Correlation is BY ID where the stream supplies one, and by tool name otherwise, because that is
|
|
66
|
+
* the difference between the two producers: claude-code's blocks always carry a `tool_use_id`, and
|
|
67
|
+
* a parallel batch of calls is routine there, while Pi's stream is sequential. A result the tracker
|
|
68
|
+
* cannot pair with a start is still EMITTED (with no args and the previous call's end as its
|
|
69
|
+
* start): losing a step of the trajectory to a schema tweak would be worse than an entry that says
|
|
70
|
+
* less than its neighbours, and the `seq` it takes keeps every later entry's ordinal honest.
|
|
71
|
+
*/
|
|
72
|
+
export class ToolCallTracker {
|
|
73
|
+
secrets;
|
|
74
|
+
seq = 0;
|
|
75
|
+
byId = new Map();
|
|
76
|
+
byTool = new Map();
|
|
77
|
+
/**
|
|
78
|
+
* Start boundary for a call whose own start was never seen: the previous call's end, or the run
|
|
79
|
+
* start. Approximate but contiguous, which is the property the trace tree needs.
|
|
80
|
+
*/
|
|
81
|
+
boundary;
|
|
82
|
+
constructor(secrets = [], now = Date.now()) {
|
|
83
|
+
this.secrets = secrets;
|
|
84
|
+
this.boundary = now;
|
|
85
|
+
}
|
|
86
|
+
/** Record that a call began, with the arguments the agent supplied. */
|
|
87
|
+
started(id, tool, args, at = Date.now()) {
|
|
88
|
+
const pending = {
|
|
89
|
+
tool,
|
|
90
|
+
startedAt: at,
|
|
91
|
+
args: captureToolBody(args, MAX_TOOL_ARGS_CHARS, this.secrets),
|
|
92
|
+
};
|
|
93
|
+
if (id) {
|
|
94
|
+
this.byId.set(id, pending);
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
const queue = this.byTool.get(tool) ?? [];
|
|
98
|
+
queue.push(pending);
|
|
99
|
+
this.byTool.set(tool, queue);
|
|
100
|
+
}
|
|
101
|
+
/** Record that a call finished, and return the completed entry. */
|
|
102
|
+
finished(id, tool, result, isError, at = Date.now()) {
|
|
103
|
+
const pending = this.take(id, tool);
|
|
104
|
+
const captured = captureToolBody(result, MAX_TOOL_RESULT_CHARS, this.secrets);
|
|
105
|
+
const call = {
|
|
106
|
+
tool: pending?.tool ?? tool,
|
|
107
|
+
seq: this.seq++,
|
|
108
|
+
startedAt: pending?.startedAt ?? this.boundary,
|
|
109
|
+
endedAt: at,
|
|
110
|
+
ok: !isError,
|
|
111
|
+
args: pending?.args.text ?? '',
|
|
112
|
+
argsDropped: pending?.args.dropped ?? 0,
|
|
113
|
+
result: captured.text,
|
|
114
|
+
resultDropped: captured.dropped,
|
|
115
|
+
};
|
|
116
|
+
this.boundary = at;
|
|
117
|
+
return call;
|
|
118
|
+
}
|
|
119
|
+
take(id, tool) {
|
|
120
|
+
if (id) {
|
|
121
|
+
const byId = this.byId.get(id);
|
|
122
|
+
if (byId) {
|
|
123
|
+
this.byId.delete(id);
|
|
124
|
+
return byId;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
const queue = this.byTool.get(tool);
|
|
128
|
+
// FIFO, so a sequential stream pairs each result with the call that opened first. A stream
|
|
129
|
+
// that interleaves un-idded calls of one tool would mis-pair here; none of the two producers
|
|
130
|
+
// does, and the alternative (refusing to pair at all) loses the args on every ordinary run.
|
|
131
|
+
const pending = queue?.shift();
|
|
132
|
+
if (queue && queue.length === 0)
|
|
133
|
+
this.byTool.delete(tool);
|
|
134
|
+
return pending;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
/** Read a tool-call id off a stream event, whatever the producer calls it. */
|
|
138
|
+
export function readToolCallId(event) {
|
|
139
|
+
for (const key of ['toolCallId', 'tool_call_id', 'callId', 'id']) {
|
|
140
|
+
const value = event[key];
|
|
141
|
+
if (typeof value === 'string' && value !== '')
|
|
142
|
+
return value;
|
|
143
|
+
}
|
|
144
|
+
return undefined;
|
|
145
|
+
}
|
|
146
|
+
/** A tool call BEGINNING, read off a Pi `--mode json` event, or undefined if it isn't one. */
|
|
147
|
+
export function toolCallStart(event) {
|
|
148
|
+
if (event.type !== 'tool_execution_start' && event.type !== 'tool_call')
|
|
149
|
+
return undefined;
|
|
150
|
+
const name = typeof event.toolName === 'string' ? event.toolName : '';
|
|
151
|
+
if (!name)
|
|
152
|
+
return undefined;
|
|
153
|
+
// Read every spelling the stream might use rather than pinning one: Pi's event schema has been
|
|
154
|
+
// renamed under us before (see `todoResultDetails`, which reads three shapes of one result), and
|
|
155
|
+
// a rename here costs the ARGUMENTS of every call, silently.
|
|
156
|
+
const args = event.args ?? event.arguments ?? event.input ?? event.parameters;
|
|
157
|
+
const id = readToolCallId(event);
|
|
158
|
+
return { name, args, ...(id ? { id } : {}) };
|
|
159
|
+
}
|
|
160
|
+
/** The RESULT payload of a Pi `tool_execution_end` event, unwrapped from its envelope. */
|
|
161
|
+
export function toolCallResult(event) {
|
|
162
|
+
const result = event.result;
|
|
163
|
+
if (result && typeof result === 'object') {
|
|
164
|
+
const inner = result;
|
|
165
|
+
// `details` is the structured payload Pi's own extensions return (the todo tool's shape), and
|
|
166
|
+
// `content`/`output` the free-text ones. Falling back to the whole envelope keeps a shape none
|
|
167
|
+
// of these match readable rather than empty.
|
|
168
|
+
return inner.details ?? inner.content ?? inner.output ?? result;
|
|
169
|
+
}
|
|
170
|
+
return result ?? event.output ?? event.content;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Feed one claude-code `user` turn's content blocks to the tracker, emitting an entry per
|
|
174
|
+
* `tool_result`.
|
|
175
|
+
*
|
|
176
|
+
* The CLI answers each `tool_use` with a `tool_result` carrying the same `tool_use_id`, so the
|
|
177
|
+
* pairing is exact even when the model fired a batch of calls in parallel — which it routinely
|
|
178
|
+
* does, and which is why the trajectory here is ordered by the ordinal the tracker stamps rather
|
|
179
|
+
* than by the turn the results arrived on.
|
|
180
|
+
*/
|
|
181
|
+
export function recordClaudeToolResults(tracker, content, emit) {
|
|
182
|
+
for (const block of content) {
|
|
183
|
+
if (!block || typeof block !== 'object')
|
|
184
|
+
continue;
|
|
185
|
+
const record = block;
|
|
186
|
+
if (record.type !== 'tool_result')
|
|
187
|
+
continue;
|
|
188
|
+
const id = typeof record.tool_use_id === 'string' ? record.tool_use_id : undefined;
|
|
189
|
+
// The block carries no tool NAME (only the id the assistant turn named), so an unpaired
|
|
190
|
+
// result falls back to a stated placeholder rather than an empty string that would render
|
|
191
|
+
// as a nameless step.
|
|
192
|
+
emit(tracker.finished(id, 'unknown', record.content, record.is_error === true));
|
|
193
|
+
}
|
|
194
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cat-factory/executor-harness",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.92.0",
|
|
4
4
|
"description": "Container payload: a thin TypeScript wrapper that runs the Pi coding agent against a cloned repo and opens a PR. Runs in the Cloudflare Container (and, in local native mode, as a host process); carries no secrets.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -30,9 +30,9 @@
|
|
|
30
30
|
"hono": "^4.12.33",
|
|
31
31
|
"typescript": "7.0.2",
|
|
32
32
|
"vitest": "^4.1.10",
|
|
33
|
-
"@cat-factory/kernel": "0.
|
|
34
|
-
"@cat-factory/server": "0.
|
|
35
|
-
"@cat-factory/spend": "0.
|
|
33
|
+
"@cat-factory/kernel": "0.236.0",
|
|
34
|
+
"@cat-factory/server": "0.215.0",
|
|
35
|
+
"@cat-factory/spend": "0.14.14"
|
|
36
36
|
},
|
|
37
37
|
"scripts": {
|
|
38
38
|
"build": "tsc -p tsconfig.json",
|
|
@@ -178,6 +178,20 @@ function sanitizeServerId(value: unknown): string | undefined {
|
|
|
178
178
|
return MCP_SERVER_ID_PATTERN.test(value) ? value : undefined
|
|
179
179
|
}
|
|
180
180
|
|
|
181
|
+
/**
|
|
182
|
+
* A tool name an `allowedTools` entry may name. Kept byte-identical to kernel's
|
|
183
|
+
* `MCP_TOOL_NAME_PATTERN` for the same reason {@link MCP_SERVER_ID_PATTERN} is a copy, and pinned
|
|
184
|
+
* against it by `test/agent-capabilities.conformity.test.ts`.
|
|
185
|
+
*
|
|
186
|
+
* The comma is the reason the rule exists on THIS side of the boundary too:
|
|
187
|
+
* {@link claudeAllowedToolPatterns} builds the list that the runner joins into one
|
|
188
|
+
* `--allowedTools` argument with commas, so an entry carrying one splits into two patterns of which
|
|
189
|
+
* the second matches no tool the CLI has. Dropped rather than passed through, because the entries
|
|
190
|
+
* that survive are what narrows the session: a bad one would silently take the run's whole MCP
|
|
191
|
+
* surface with it.
|
|
192
|
+
*/
|
|
193
|
+
export const MCP_TOOL_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/
|
|
194
|
+
|
|
181
195
|
/**
|
|
182
196
|
* Whether an HTTP tool server's URL may be started. Mirrors kernel's `isAllowedMcpHttpUrl` (see
|
|
183
197
|
* {@link MCP_SERVER_ID_PATTERN} for why it is a copy, and the conformity suite that pins it):
|
|
@@ -219,13 +233,25 @@ function parseStringArray(value: unknown): string[] | undefined {
|
|
|
219
233
|
return out.length ? out : undefined
|
|
220
234
|
}
|
|
221
235
|
|
|
236
|
+
/**
|
|
237
|
+
* The `allowedTools` list: string entries that are single tool NAMES (see
|
|
238
|
+
* {@link MCP_TOOL_NAME_PATTERN}). Undefined when nothing survives, which is the same answer as an
|
|
239
|
+
* absent field (every tool the server exposes) and the right one: the alternative is a list whose
|
|
240
|
+
* only surviving entries are the platform's own built-in tool names, i.e. a run narrowed to no MCP
|
|
241
|
+
* tools at all. The backend refuses these at registration; this is the boundary check.
|
|
242
|
+
*/
|
|
243
|
+
function parseAllowedTools(value: unknown): string[] | undefined {
|
|
244
|
+
const names = parseStringArray(value)?.filter((name) => MCP_TOOL_NAME_PATTERN.test(name))
|
|
245
|
+
return names?.length ? names : undefined
|
|
246
|
+
}
|
|
247
|
+
|
|
222
248
|
/** Validate one `mcpServers` entry, or undefined when malformed for its transport. */
|
|
223
249
|
function parseMcpServerSpec(value: unknown): McpServerSpec | undefined {
|
|
224
250
|
if (typeof value !== 'object' || value === null) return undefined
|
|
225
251
|
const o = value as Record<string, unknown>
|
|
226
252
|
const id = sanitizeServerId(o.id)
|
|
227
253
|
if (!id) return undefined
|
|
228
|
-
const allowedTools =
|
|
254
|
+
const allowedTools = parseAllowedTools(o.allowedTools)
|
|
229
255
|
const secretKeys = parseStringArray(o.secretKeys)
|
|
230
256
|
if (o.transport === 'http') {
|
|
231
257
|
// https anywhere, plain http only on loopback: the CLI would happily be pointed at a
|
|
@@ -370,9 +396,13 @@ function tomlString(value: string): string {
|
|
|
370
396
|
|
|
371
397
|
/**
|
|
372
398
|
* The `[mcp_servers.<id>]` TOML block Codex reads from its `CODEX_HOME/config.toml`. Codex's MCP
|
|
373
|
-
* client is stdio-only, so an `http` server is skipped here
|
|
374
|
-
*
|
|
375
|
-
*
|
|
399
|
+
* client is stdio-only, so an `http` server is skipped here.
|
|
400
|
+
*
|
|
401
|
+
* The skip is now a BACKSTOP rather than the decision: the backend knows which transports each
|
|
402
|
+
* harness reaches (`MCP_HARNESS_TRANSPORTS`) and drops an `http` server from a Codex dispatch under
|
|
403
|
+
* its own `transport_unsupported` reason, so the prompt states the gap instead of advertising a tool
|
|
404
|
+
* this writer then silently omitted. It stays because a body that reached the container by any other
|
|
405
|
+
* route must still produce a valid config rather than a malformed one.
|
|
376
406
|
*/
|
|
377
407
|
export function codexMcpConfigToml(servers: McpServerSpec[]): string {
|
|
378
408
|
const blocks: string[] = []
|