@cat-factory/executor-harness 1.96.0 → 1.98.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/dist/agent-runner.d.ts +14 -1
- package/dist/agent-runner.js +84 -28
- package/dist/bootstrap-mode.js +1 -0
- package/dist/coding-agent.d.ts +2 -1
- package/dist/embed.d.ts +2 -1
- package/dist/embed.js +2 -1
- package/dist/failure.d.ts +19 -1
- package/dist/failure.js +40 -0
- package/dist/git.d.ts +6 -0
- package/dist/git.js +16 -9
- package/dist/inline.d.ts +6 -0
- package/dist/inline.js +6 -0
- package/dist/job.d.ts +2 -1
- package/dist/jsonl-stream.d.ts +70 -0
- package/dist/jsonl-stream.js +149 -0
- package/dist/pi-reduction.d.ts +136 -0
- package/dist/pi-reduction.js +303 -0
- package/dist/pi-workspace.d.ts +2 -1
- package/dist/pi-workspace.js +6 -1
- package/dist/pi.d.ts +8 -81
- package/dist/pi.js +124 -310
- package/dist/runner.d.ts +31 -0
- package/dist/runner.js +50 -3
- package/dist/structured-output.js +2 -1
- package/dist/tool-silence.d.ts +74 -0
- package/dist/tool-silence.js +99 -0
- package/package.json +4 -4
- package/src/agent-runner.ts +100 -30
- package/src/agent.ts +1 -1
- package/src/bootstrap-mode.ts +2 -1
- package/src/coding-agent.ts +2 -1
- package/src/embed.ts +8 -5
- package/src/failure.ts +36 -9
- package/src/git.ts +17 -9
- package/src/inline.ts +6 -0
- package/src/job.ts +2 -1
- package/src/jsonl-stream.ts +149 -0
- package/src/pi-reduction.ts +359 -0
- package/src/pi-workspace.ts +7 -3
- package/src/pi.ts +144 -349
- package/src/runner.ts +91 -4
- package/src/structured-output.ts +2 -1
- package/src/tool-silence.ts +125 -0
package/dist/runner.d.ts
CHANGED
|
@@ -5,6 +5,7 @@ import type { SliceReview } from './subagents.js';
|
|
|
5
5
|
import type { ObservedMcpServer } from './agent-capabilities.js';
|
|
6
6
|
import type { HarnessCallMetric, TodoProgress, ToolSpan } from './pi.js';
|
|
7
7
|
import { type Logger } from './logger.js';
|
|
8
|
+
import { type ToolProgressWindow } from './tool-silence.js';
|
|
8
9
|
import { type FailureCause } from './failure.js';
|
|
9
10
|
/** Non-secret correlation fields a job carries on every log line (jobId, repo, branch, …). */
|
|
10
11
|
type LogFields = Record<string, unknown>;
|
|
@@ -16,6 +17,22 @@ export interface RunOptions {
|
|
|
16
17
|
onProgress?: (progress: TodoProgress) => void;
|
|
17
18
|
/** Receives one compact {@link ToolSpan} per completed tool call (observability). */
|
|
18
19
|
onSpan?: (span: ToolSpan) => void;
|
|
20
|
+
/**
|
|
21
|
+
* Opens the tool-silence window (stuck-run audit F13) for ONE agent stream, returning the
|
|
22
|
+
* handle that stream beats on every completed tool call and closes when it ends.
|
|
23
|
+
*
|
|
24
|
+
* Called by the agent-CLI runners themselves rather than by the phase marker, because the
|
|
25
|
+
* window is only meaningful while something able to RESET it is running and only the runner
|
|
26
|
+
* knows whether its CLI reports completed tool calls at all. A caller that runs no tool loop
|
|
27
|
+
* (the inline one-shot completion) simply does not forward this, which is a statement, not an
|
|
28
|
+
* omission: the run stays bounded by the inactivity and max-duration watchdogs, and a window
|
|
29
|
+
* nothing could ever beat would only be able to expire.
|
|
30
|
+
*
|
|
31
|
+
* Absent ⇒ no window is opened and this watchdog is silent for that work. It fails toward NOT
|
|
32
|
+
* killing on purpose: this audit exists as much to stop recovery machinery ending healthy runs
|
|
33
|
+
* as to bound wedged ones, and the wall-clock cap is underneath either way.
|
|
34
|
+
*/
|
|
35
|
+
beginToolWindow?: () => ToolProgressWindow;
|
|
19
36
|
/** Receives the forward-looking follow-up / question items the Coder streamed since the last poll. */
|
|
20
37
|
onFollowUp?: (items: FollowUpLine[]) => void;
|
|
21
38
|
/**
|
|
@@ -246,6 +263,20 @@ export interface RunnerLimits {
|
|
|
246
263
|
* progress, which counts as activity). Set to 0 to disable.
|
|
247
264
|
*/
|
|
248
265
|
coldStartMs: number;
|
|
266
|
+
/**
|
|
267
|
+
* Stuck-run audit F13: force-fail the job if a running agent stream completes no tool call for
|
|
268
|
+
* this long. The gap the other two watchdogs structurally cannot see — a model that keeps
|
|
269
|
+
* talking (or thinking out loud) resets the inactivity timer on every chunk while completing
|
|
270
|
+
* nothing, so the only remaining bound was the full wall-clock cap and the engine's
|
|
271
|
+
* ~70-minute poll budget behind it.
|
|
272
|
+
*
|
|
273
|
+
* Armed only for as long as a stream that REPORTS completed tool calls is running (see
|
|
274
|
+
* {@link RunOptions.beginToolWindow}), so the activity-silent stretches — clone, dependency
|
|
275
|
+
* install, push, a validation loop's check commands — are outside it by construction: they
|
|
276
|
+
* legitimately complete no tool calls, and they are bounded by their own per-command timeouts.
|
|
277
|
+
* Set to 0 to disable.
|
|
278
|
+
*/
|
|
279
|
+
toolSilenceMs: number;
|
|
249
280
|
}
|
|
250
281
|
export declare function loadRunnerLimits(env?: NodeJS.ProcessEnv): RunnerLimits;
|
|
251
282
|
/**
|
package/dist/runner.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { redactSecrets } from './redact.js';
|
|
2
2
|
import { log } from './logger.js';
|
|
3
|
-
import {
|
|
3
|
+
import { ToolSilenceWatchdog } from './tool-silence.js';
|
|
4
|
+
import { failureCauseOf, inactivityAbortMessage, maxDurationAbortMessage, toolSilenceAbortMessage, } from './failure.js';
|
|
4
5
|
function intEnv(value, fallback) {
|
|
5
6
|
const n = value ? Number(value) : NaN;
|
|
6
7
|
return Number.isFinite(n) && n > 0 ? n : fallback;
|
|
@@ -13,22 +14,43 @@ function intEnvAllowZero(value, fallback) {
|
|
|
13
14
|
return Number.isFinite(n) && n >= 0 ? n : fallback;
|
|
14
15
|
}
|
|
15
16
|
export function loadRunnerLimits(env = process.env) {
|
|
17
|
+
const maxDurationMs = intEnv(env.JOB_MAX_DURATION_MS, 60 * 60_000);
|
|
18
|
+
const inactivityMs = intEnv(env.JOB_INACTIVITY_MS, 10 * 60_000);
|
|
16
19
|
return {
|
|
17
20
|
// 60 minutes: generous headroom for serious multi-file coding tasks while
|
|
18
21
|
// still bounding a runaway container.
|
|
19
|
-
maxDurationMs
|
|
22
|
+
maxDurationMs,
|
|
20
23
|
// 10 minutes of zero output is treated as hung (a single long LLM/tool call
|
|
21
24
|
// is far shorter; Pi streams events as it works). The per-git command ceiling
|
|
22
25
|
// (`GIT_TIMEOUT_MS` in git.ts) is DERIVED from this value — a fixed margin below
|
|
23
26
|
// it — so a slow clone/push (which emits no activity events) always times out
|
|
24
27
|
// with git's own clear reason rather than this watchdog's "likely hung" message,
|
|
25
28
|
// for any configured window. See the invariant note in git.ts.
|
|
26
|
-
inactivityMs
|
|
29
|
+
inactivityMs,
|
|
27
30
|
// 2 minutes: comfortably longer than a warm agent's time-to-first-token yet far
|
|
28
31
|
// under the 10-minute inactivity kill, so a truly output-less start is flagged early.
|
|
29
32
|
coldStartMs: intEnvAllowZero(env.JOB_COLD_START_MS, 2 * 60_000),
|
|
33
|
+
toolSilenceMs: intEnvAllowZero(env.JOB_TOOL_SILENCE_MS, toolSilenceDefault(maxDurationMs, inactivityMs)),
|
|
30
34
|
};
|
|
31
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* The tool-silence window when the operator has not set one: HALF the configured wall-clock cap,
|
|
38
|
+
* DERIVED rather than a constant so lowering `JOB_MAX_DURATION_MS` tightens it too (a fixed
|
|
39
|
+
* 30 minutes would sit past the whole budget of a deployment that runs 20-minute jobs, i.e. be
|
|
40
|
+
* silently disabled).
|
|
41
|
+
*
|
|
42
|
+
* Floored at the inactivity window so the default never races the gone-quiet diagnostic more
|
|
43
|
+
* often than it has to. The floor is a sizing choice, NOT the thing that keeps the two watchdogs
|
|
44
|
+
* apart: they anchor on different events (the last completed tool call vs the last byte of
|
|
45
|
+
* output), so the tool-silence anchor is always the earlier of the two and equal windows would
|
|
46
|
+
* still have it firing first. What actually keeps this watchdog off a hang is the expiry test in
|
|
47
|
+
* {@link ToolSilenceWatchdog}, which fires only when output arrived DURING the window that
|
|
48
|
+
* elapsed — and which also holds for an operator who sets `JOB_TOOL_SILENCE_MS` below
|
|
49
|
+
* `JOB_INACTIVITY_MS`, where no default-side clamp applies at all.
|
|
50
|
+
*/
|
|
51
|
+
function toolSilenceDefault(maxDurationMs, inactivityMs) {
|
|
52
|
+
return Math.max(Math.round(maxDurationMs / 2), inactivityMs);
|
|
53
|
+
}
|
|
32
54
|
function toView(entry) {
|
|
33
55
|
const { promise: _promise, spanBuffer: _spanBuffer, followUpBuffer: _followUpBuffer, callMetricBuffer: _callMetricBuffer, callMetricSeq: _callMetricSeq, abort: _abort, ...view } = entry;
|
|
34
56
|
return { ...view };
|
|
@@ -224,6 +246,20 @@ export class JobRegistry {
|
|
|
224
246
|
// spoken yet" test and, on a failure, the difference between a run that died mid-work and one
|
|
225
247
|
// that never got going at all.
|
|
226
248
|
let lastActivityAt;
|
|
249
|
+
// Stuck-run audit F13: the third watchdog, and the only one that can see a model which keeps
|
|
250
|
+
// TALKING while completing nothing — its output resets the inactivity timer on every chunk,
|
|
251
|
+
// and it is nowhere near the wall-clock cap. It is armed by the agent stream itself rather
|
|
252
|
+
// than by the phase marker (see `RunOptions.beginToolWindow`) and reads `lastActivityAt` at
|
|
253
|
+
// expiry, which is what keeps the gone-quiet case with the inactivity watchdog that owns it.
|
|
254
|
+
const toolSilence = new ToolSilenceWatchdog({
|
|
255
|
+
windowMs: this.limits.toolSilenceMs,
|
|
256
|
+
lastActivityAt: () => lastActivityAt,
|
|
257
|
+
onExpired: () => {
|
|
258
|
+
// First watchdog to fire wins the reason (see `resetInactivity` above).
|
|
259
|
+
killReason ??= 'no-tool-progress';
|
|
260
|
+
controller.abort(new Error('no tool progress'));
|
|
261
|
+
},
|
|
262
|
+
});
|
|
227
263
|
// ADR 0026 D4: a one-shot cold-start watchdog. If the job produces no activity within
|
|
228
264
|
// `coldStartMs`, record a structured diagnostic (a likely onboarding/auth wedge) so it
|
|
229
265
|
// is legible early — it does NOT abort the run (the inactivity watchdog still owns
|
|
@@ -261,6 +297,7 @@ export class JobRegistry {
|
|
|
261
297
|
entry.spanBuffer.push(span);
|
|
262
298
|
lastTool = { name: span.tool, at: span.endedAt };
|
|
263
299
|
},
|
|
300
|
+
beginToolWindow: () => toolSilence.open(),
|
|
264
301
|
onFollowUp: (items) => {
|
|
265
302
|
entry.followUpBuffer.push(...items);
|
|
266
303
|
},
|
|
@@ -338,6 +375,7 @@ export class JobRegistry {
|
|
|
338
375
|
clearTimeout(inactivity);
|
|
339
376
|
clearTimeout(cap);
|
|
340
377
|
clearTimeout(coldStart);
|
|
378
|
+
toolSilence.stop();
|
|
341
379
|
entry.abort = undefined;
|
|
342
380
|
entry.heartbeatAt = Date.now();
|
|
343
381
|
}
|
|
@@ -376,6 +414,15 @@ export class JobRegistry {
|
|
|
376
414
|
detail: redactSecrets(`Phase timings: ${phaseBreakdown || '(none)'}. ${breadcrumb}.${cold}`),
|
|
377
415
|
};
|
|
378
416
|
}
|
|
417
|
+
if (ctx.killReason === 'no-tool-progress') {
|
|
418
|
+
return {
|
|
419
|
+
// The breadcrumb carries the last completed tool, which is the whole diagnostic here:
|
|
420
|
+
// it names what the agent was doing when it stopped doing anything.
|
|
421
|
+
message: redactSecrets(`${toolSilenceAbortMessage(this.limits.toolSilenceMs)} (${breadcrumb})`),
|
|
422
|
+
cause: 'no-tool-progress',
|
|
423
|
+
detail: redactSecrets(`Phase timings: ${phaseBreakdown || '(none)'}. ${breadcrumb}.${cold}`),
|
|
424
|
+
};
|
|
425
|
+
}
|
|
379
426
|
const raw = ctx.error instanceof Error ? ctx.error.message : String(ctx.error);
|
|
380
427
|
// A thrown error tagged with a structured cause (a git op / an upstream API call) keeps
|
|
381
428
|
// it; an untagged throw is a generic agent failure.
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { redact, redactSecrets, secretsToRedact } from './redact.js';
|
|
2
2
|
import { log } from './logger.js';
|
|
3
|
-
import {
|
|
3
|
+
import { phasedProxyBaseUrl } from './pi.js';
|
|
4
|
+
import { PI_MAX_OUTPUT_TOKENS } from './pi-reduction.js';
|
|
4
5
|
// A reusable abstraction for the "agent returns a structured JSON document as its
|
|
5
6
|
// final assistant message" pattern (requirements, blueprint, merger — and any future
|
|
6
7
|
// kind). An agent of this kind emits its result as text, not a tool call, and the
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The live window for ONE tool-reporting agent stream. Opened by the stream, beaten by each
|
|
3
|
+
* completed tool call, closed when the stream ends (cleanly or not) — so the window can never
|
|
4
|
+
* outlive the only thing able to reset it.
|
|
5
|
+
*/
|
|
6
|
+
export interface ToolProgressWindow {
|
|
7
|
+
/** A tool call completed: the only evidence this watchdog accepts as progress. */
|
|
8
|
+
toolCompleted(): void;
|
|
9
|
+
/** The stream ended; the window closes with it. Idempotent. */
|
|
10
|
+
close(): void;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* A window that measures nothing: what the watchdog hands out when disabled, and what a producer
|
|
14
|
+
* substitutes when its caller wired no watchdog at all. Having one means every producer holds a
|
|
15
|
+
* real window and states its tool progress unconditionally, rather than guarding each call site
|
|
16
|
+
* with a `?.` that reads as though beating the window were optional.
|
|
17
|
+
*/
|
|
18
|
+
export declare const NO_TOOL_WINDOW: ToolProgressWindow;
|
|
19
|
+
export interface ToolSilenceDeps {
|
|
20
|
+
/** The window length. `<= 0` disables the watchdog entirely (every window is inert). */
|
|
21
|
+
windowMs: number;
|
|
22
|
+
/**
|
|
23
|
+
* When the run last produced ANY output, or undefined if it never has — the same clock the
|
|
24
|
+
* inactivity watchdog resets on. Read at EXPIRY (see {@link ToolSilenceWatchdog.open}), which
|
|
25
|
+
* is what keeps this watchdog off the gone-quiet case.
|
|
26
|
+
*/
|
|
27
|
+
lastActivityAt: () => number | undefined;
|
|
28
|
+
/** Called when a window expires with the run demonstrably still talking. */
|
|
29
|
+
onExpired: () => void;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Hands out {@link ToolProgressWindow}s and fires `onExpired` for one that goes a full window
|
|
33
|
+
* without a completed tool call while the run keeps producing output.
|
|
34
|
+
*
|
|
35
|
+
* ONE window is open at a time (a job runs one agent stream at a time, and a repair loop runs
|
|
36
|
+
* them in sequence). Opening a second supersedes the first, and a superseded handle's calls are
|
|
37
|
+
* ignored rather than reaching back into the live window — a stale `close()` from a stream that
|
|
38
|
+
* finished after its successor started would otherwise disarm the watchdog for the rest of the job.
|
|
39
|
+
*/
|
|
40
|
+
export declare class ToolSilenceWatchdog {
|
|
41
|
+
private readonly deps;
|
|
42
|
+
private timer;
|
|
43
|
+
/** Identity of the window currently open; every handout takes the next number. */
|
|
44
|
+
private openId;
|
|
45
|
+
/** When the live window was last armed — the start of the span `onExpired` is a verdict about. */
|
|
46
|
+
private armedAt;
|
|
47
|
+
constructor(deps: ToolSilenceDeps);
|
|
48
|
+
/** Open a window for one agent stream. Returns an inert handle when the watchdog is disabled. */
|
|
49
|
+
open(): ToolProgressWindow;
|
|
50
|
+
/** Disarm for good (the job settled). Safe to call with no window open. */
|
|
51
|
+
stop(): void;
|
|
52
|
+
private arm;
|
|
53
|
+
/**
|
|
54
|
+
* A window elapsed with no completed tool call. Fire ONLY if the run was talking through it.
|
|
55
|
+
*
|
|
56
|
+
* `no-tool-progress` claims something specific — output arrived, but nothing got done — and
|
|
57
|
+
* that is only a truthful reading when output actually arrived DURING the window that just
|
|
58
|
+
* expired. A window that passed in total silence is the INACTIVITY watchdog's fact, whose
|
|
59
|
+
* diagnostic ("the container went quiet") is the one an operator can act on; relabelling it as
|
|
60
|
+
* a rabbit-hole would send them looking at the model instead of at the hang.
|
|
61
|
+
*
|
|
62
|
+
* This is a structural guard, not a tie-breaker: the two timers anchor on different events (the
|
|
63
|
+
* last tool call vs the last byte of output), so no arithmetic between the two window LENGTHS
|
|
64
|
+
* can order them. Equal windows put the tool-silence anchor strictly earlier — it fires first —
|
|
65
|
+
* and an operator setting `JOB_TOOL_SILENCE_MS` below `JOB_INACTIVITY_MS` gets that on every
|
|
66
|
+
* quiet run. Deciding from what the expired window actually SAW is independent of both numbers.
|
|
67
|
+
*
|
|
68
|
+
* A deferred window re-arms rather than standing down, so a run that goes quiet and then
|
|
69
|
+
* resumes its monologue is still caught, one full window later. The deferral terminates:
|
|
70
|
+
* either output resumes (the next expiry has output in its window and fires) or it does not
|
|
71
|
+
* (inactivity fires).
|
|
72
|
+
*/
|
|
73
|
+
private expire;
|
|
74
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
// The tool-silence watchdog (stuck-run audit F13): the third bound on a job, and the only one
|
|
2
|
+
// that can see a model which keeps TALKING while completing nothing. Its output resets the
|
|
3
|
+
// inactivity timer on every chunk, and it is nowhere near the wall-clock cap.
|
|
4
|
+
//
|
|
5
|
+
// WHY THIS IS ITS OWN MODULE — the window is only meaningful while something that REPORTS
|
|
6
|
+
// completed tool calls is running, and only that producer knows whether it does. Keying the
|
|
7
|
+
// window on the job's coarse phase label instead looked equivalent and was not: `agent` is a
|
|
8
|
+
// telemetry breadcrumb several call sites mark for several different things (a Codex pass, a
|
|
9
|
+
// tool-less inline completion, the label restored around a repair loop's shell commands), so a
|
|
10
|
+
// phase-armed window spent most of its time armed over work that could not possibly reset it.
|
|
11
|
+
// The producer opens its own window instead, which makes "armed" and "can beat" the same fact by
|
|
12
|
+
// construction rather than by two call sites agreeing.
|
|
13
|
+
/**
|
|
14
|
+
* A window that measures nothing: what the watchdog hands out when disabled, and what a producer
|
|
15
|
+
* substitutes when its caller wired no watchdog at all. Having one means every producer holds a
|
|
16
|
+
* real window and states its tool progress unconditionally, rather than guarding each call site
|
|
17
|
+
* with a `?.` that reads as though beating the window were optional.
|
|
18
|
+
*/
|
|
19
|
+
export const NO_TOOL_WINDOW = { toolCompleted: () => { }, close: () => { } };
|
|
20
|
+
/**
|
|
21
|
+
* Hands out {@link ToolProgressWindow}s and fires `onExpired` for one that goes a full window
|
|
22
|
+
* without a completed tool call while the run keeps producing output.
|
|
23
|
+
*
|
|
24
|
+
* ONE window is open at a time (a job runs one agent stream at a time, and a repair loop runs
|
|
25
|
+
* them in sequence). Opening a second supersedes the first, and a superseded handle's calls are
|
|
26
|
+
* ignored rather than reaching back into the live window — a stale `close()` from a stream that
|
|
27
|
+
* finished after its successor started would otherwise disarm the watchdog for the rest of the job.
|
|
28
|
+
*/
|
|
29
|
+
export class ToolSilenceWatchdog {
|
|
30
|
+
deps;
|
|
31
|
+
timer;
|
|
32
|
+
/** Identity of the window currently open; every handout takes the next number. */
|
|
33
|
+
openId = 0;
|
|
34
|
+
/** When the live window was last armed — the start of the span `onExpired` is a verdict about. */
|
|
35
|
+
armedAt = 0;
|
|
36
|
+
constructor(deps) {
|
|
37
|
+
this.deps = deps;
|
|
38
|
+
}
|
|
39
|
+
/** Open a window for one agent stream. Returns an inert handle when the watchdog is disabled. */
|
|
40
|
+
open() {
|
|
41
|
+
if (this.deps.windowMs <= 0)
|
|
42
|
+
return NO_TOOL_WINDOW;
|
|
43
|
+
const id = ++this.openId;
|
|
44
|
+
this.arm();
|
|
45
|
+
const live = () => this.openId === id;
|
|
46
|
+
return {
|
|
47
|
+
toolCompleted: () => {
|
|
48
|
+
if (live())
|
|
49
|
+
this.arm();
|
|
50
|
+
},
|
|
51
|
+
close: () => {
|
|
52
|
+
if (!live())
|
|
53
|
+
return;
|
|
54
|
+
// Retire the id as well as the timer, so a late `toolCompleted()` from this stream
|
|
55
|
+
// cannot re-arm a window whose producer has already gone.
|
|
56
|
+
this.openId++;
|
|
57
|
+
this.stop();
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
/** Disarm for good (the job settled). Safe to call with no window open. */
|
|
62
|
+
stop() {
|
|
63
|
+
clearTimeout(this.timer);
|
|
64
|
+
this.timer = undefined;
|
|
65
|
+
}
|
|
66
|
+
arm() {
|
|
67
|
+
clearTimeout(this.timer);
|
|
68
|
+
this.armedAt = Date.now();
|
|
69
|
+
this.timer = setTimeout(() => this.expire(), this.deps.windowMs);
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* A window elapsed with no completed tool call. Fire ONLY if the run was talking through it.
|
|
73
|
+
*
|
|
74
|
+
* `no-tool-progress` claims something specific — output arrived, but nothing got done — and
|
|
75
|
+
* that is only a truthful reading when output actually arrived DURING the window that just
|
|
76
|
+
* expired. A window that passed in total silence is the INACTIVITY watchdog's fact, whose
|
|
77
|
+
* diagnostic ("the container went quiet") is the one an operator can act on; relabelling it as
|
|
78
|
+
* a rabbit-hole would send them looking at the model instead of at the hang.
|
|
79
|
+
*
|
|
80
|
+
* This is a structural guard, not a tie-breaker: the two timers anchor on different events (the
|
|
81
|
+
* last tool call vs the last byte of output), so no arithmetic between the two window LENGTHS
|
|
82
|
+
* can order them. Equal windows put the tool-silence anchor strictly earlier — it fires first —
|
|
83
|
+
* and an operator setting `JOB_TOOL_SILENCE_MS` below `JOB_INACTIVITY_MS` gets that on every
|
|
84
|
+
* quiet run. Deciding from what the expired window actually SAW is independent of both numbers.
|
|
85
|
+
*
|
|
86
|
+
* A deferred window re-arms rather than standing down, so a run that goes quiet and then
|
|
87
|
+
* resumes its monologue is still caught, one full window later. The deferral terminates:
|
|
88
|
+
* either output resumes (the next expiry has output in its window and fires) or it does not
|
|
89
|
+
* (inactivity fires).
|
|
90
|
+
*/
|
|
91
|
+
expire() {
|
|
92
|
+
const lastActivityAt = this.deps.lastActivityAt();
|
|
93
|
+
if (lastActivityAt === undefined || lastActivityAt <= this.armedAt) {
|
|
94
|
+
this.arm();
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
this.deps.onExpired();
|
|
98
|
+
}
|
|
99
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cat-factory/executor-harness",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.98.0",
|
|
4
4
|
"description": "Container payload: a thin TypeScript wrapper that runs the Pi coding agent against a cloned repo and opens a PR. Runs in the Cloudflare Container (and, in local native mode, as a host process); carries no secrets.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -30,9 +30,9 @@
|
|
|
30
30
|
"hono": "^4.13.0",
|
|
31
31
|
"typescript": "7.0.2",
|
|
32
32
|
"vitest": "^4.1.10",
|
|
33
|
-
"@cat-factory/kernel": "0.
|
|
34
|
-
"@cat-factory/server": "0.
|
|
35
|
-
"@cat-factory/spend": "0.15.
|
|
33
|
+
"@cat-factory/kernel": "0.267.0",
|
|
34
|
+
"@cat-factory/server": "0.247.0",
|
|
35
|
+
"@cat-factory/spend": "0.15.35"
|
|
36
36
|
},
|
|
37
37
|
"scripts": {
|
|
38
38
|
"build": "tsc -p tsconfig.json",
|
package/src/agent-runner.ts
CHANGED
|
@@ -9,17 +9,18 @@ import {
|
|
|
9
9
|
type TrackedToolCall,
|
|
10
10
|
recordClaudeToolResults,
|
|
11
11
|
} from './tool-trajectory.js'
|
|
12
|
-
import type
|
|
12
|
+
import { log, type Logger } from './logger.js'
|
|
13
|
+
import { NO_TOOL_WINDOW, type ToolProgressWindow } from './tool-silence.js'
|
|
13
14
|
import {
|
|
14
15
|
createCallMetricPublisher,
|
|
15
16
|
publishCallMetric,
|
|
16
17
|
type CallMetricPublisher,
|
|
17
18
|
type HarnessCallMetric,
|
|
18
19
|
type PiRunOutcome,
|
|
19
|
-
type PiRunStats,
|
|
20
20
|
type TodoProgress,
|
|
21
21
|
type ToolSpan,
|
|
22
22
|
} from './pi.js'
|
|
23
|
+
import type { PiRunStats } from './pi-reduction.js'
|
|
23
24
|
import {
|
|
24
25
|
claudeAllowedToolPatterns,
|
|
25
26
|
codexMcpConfigToml,
|
|
@@ -31,6 +32,7 @@ import {
|
|
|
31
32
|
type SkillSpec,
|
|
32
33
|
} from './agent-capabilities.js'
|
|
33
34
|
import { ProgressGuard, type ProgressGuardLimits } from './progress-guard.js'
|
|
35
|
+
import { BoundedTail, JsonlLineReader } from './jsonl-stream.js'
|
|
34
36
|
import { killChildProcess, spawnDetached } from './process.js'
|
|
35
37
|
import { describeProcessExit } from './process-exit.js'
|
|
36
38
|
import { redact, registerKnownSecrets, secretsToRedact } from './redact.js'
|
|
@@ -138,6 +140,18 @@ export interface SubscriptionRunOptions {
|
|
|
138
140
|
* DID dies with the container.
|
|
139
141
|
*/
|
|
140
142
|
onSpan?: (span: ToolSpan) => void
|
|
143
|
+
/**
|
|
144
|
+
* Opens this stream's tool-silence window (see `RunOptions.beginToolWindow`), closed when the
|
|
145
|
+
* CLI exits. Both subscription CLIs report tool activity — claude-code on the `tool_result`
|
|
146
|
+
* turn that answers each call, codex on its tool/command/exec events — so a window either
|
|
147
|
+
* opens is one the run can beat. It is deliberately NOT tied to {@link onSpan}: the trajectory
|
|
148
|
+
* is an observability opt-in, and the codex stream produces none at all while still doing tool
|
|
149
|
+
* work, which a span-keyed window would have read as a run making no progress.
|
|
150
|
+
*
|
|
151
|
+
* A caller with no tool loop (the inline one-shot completion) passes nothing; see the note at
|
|
152
|
+
* `handleInline`.
|
|
153
|
+
*/
|
|
154
|
+
beginToolWindow?: () => ToolProgressWindow
|
|
141
155
|
/**
|
|
142
156
|
* Called with the FULL set of per-slice reviews each time one lands, so the backend can persist
|
|
143
157
|
* a parallel review's completed work as it happens instead of only from the terminal result.
|
|
@@ -227,9 +241,10 @@ function streamCli(
|
|
|
227
241
|
child.stdin.on('error', () => {})
|
|
228
242
|
child.stdin.end(prompt)
|
|
229
243
|
|
|
230
|
-
|
|
244
|
+
// 8 KB is well over the 700 B tail anyone quotes below, and the CLI's stderr is diagnostic
|
|
245
|
+
// noise rather than a product, so a bounded tail is all this ever needed to be.
|
|
246
|
+
const stderr = new BoundedTail(8_000)
|
|
231
247
|
let aborted = false
|
|
232
|
-
let lineBuffer = ''
|
|
233
248
|
|
|
234
249
|
const killChild = (): void => killChildProcess(child)
|
|
235
250
|
|
|
@@ -253,16 +268,9 @@ function streamCli(
|
|
|
253
268
|
}
|
|
254
269
|
}
|
|
255
270
|
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
while (nl !== -1) {
|
|
260
|
-
const line = lineBuffer.slice(0, nl).trim()
|
|
261
|
-
lineBuffer = lineBuffer.slice(nl + 1)
|
|
262
|
-
nl = lineBuffer.indexOf('\n')
|
|
263
|
-
processLine(line)
|
|
264
|
-
}
|
|
265
|
-
}
|
|
271
|
+
// Bounded framing, shared with `runPi`: an unterminated record must not be able to grow
|
|
272
|
+
// until parsing it stalls the loop the watchdogs and poll handlers run on (audit F6).
|
|
273
|
+
const reader = new JsonlLineReader(processLine)
|
|
266
274
|
|
|
267
275
|
const onAbort = (): void => {
|
|
268
276
|
aborted = true
|
|
@@ -272,12 +280,11 @@ function streamCli(
|
|
|
272
280
|
|
|
273
281
|
child.stdout.on('data', (chunk: Buffer) => {
|
|
274
282
|
opts.onActivity?.()
|
|
275
|
-
|
|
283
|
+
reader.push(chunk.toString())
|
|
276
284
|
})
|
|
277
285
|
child.stderr.on('data', (chunk: Buffer) => {
|
|
278
286
|
opts.onActivity?.()
|
|
279
|
-
stderr
|
|
280
|
-
if (stderr.length > 8_000) stderr = stderr.slice(-8_000)
|
|
287
|
+
stderr.push(chunk.toString())
|
|
281
288
|
})
|
|
282
289
|
|
|
283
290
|
child.on('error', (err) => {
|
|
@@ -286,8 +293,19 @@ function streamCli(
|
|
|
286
293
|
})
|
|
287
294
|
child.on('close', (code, signal) => {
|
|
288
295
|
opts.signal?.removeEventListener('abort', onAbort)
|
|
289
|
-
const stderrTail = redact(stderr, secrets).slice(-700)
|
|
290
|
-
|
|
296
|
+
const stderrTail = redact(stderr.toString(), secrets).slice(-700)
|
|
297
|
+
reader.flush()
|
|
298
|
+
// Surface an oversized record the reader refused to buffer ONCE (a count, not per line),
|
|
299
|
+
// for the same reason `runPi` does: a dropped record costs this run its progress, its
|
|
300
|
+
// trajectory and its per-call telemetry for that turn, and a silent loss reads exactly
|
|
301
|
+
// like a CLI that never emitted it. Falls back to the module logger so the report cannot
|
|
302
|
+
// depend on a caller having wired a per-job one.
|
|
303
|
+
if (reader.droppedLines > 0) {
|
|
304
|
+
;(opts.log ?? log).warn('agent CLI: skipped oversized JSONL records', {
|
|
305
|
+
command,
|
|
306
|
+
oversizedLines: reader.droppedLines,
|
|
307
|
+
})
|
|
308
|
+
}
|
|
291
309
|
if (aborted) {
|
|
292
310
|
// Carry the tail on the rejection so a caller that REPLACES this generic message with a
|
|
293
311
|
// more specific cause (the no-progress guard's diagnostic) can still append it — the
|
|
@@ -659,6 +677,25 @@ function createClaudeToolTrajectory(
|
|
|
659
677
|
}
|
|
660
678
|
}
|
|
661
679
|
|
|
680
|
+
/**
|
|
681
|
+
* Open this run's tool-silence window, or the inert one when the caller wired no watchdog. One
|
|
682
|
+
* definition so both runners resolve "is there a watchdog?" identically, and so neither carries
|
|
683
|
+
* the optional-call noise at the point where it should simply have a window.
|
|
684
|
+
*/
|
|
685
|
+
function openToolWindow(opts: SubscriptionRunOptions): ToolProgressWindow {
|
|
686
|
+
return opts.beginToolWindow ? opts.beginToolWindow() : NO_TOOL_WINDOW
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/**
|
|
690
|
+
* Whether a claude-code `user` turn carries a `tool_result` block, i.e. whether a tool call just
|
|
691
|
+
* COMPLETED — the progress the tool-silence watchdog measures. Tested explicitly rather than
|
|
692
|
+
* taken from "the model sent a user turn", which a plain follow-up prompt also is: a watchdog
|
|
693
|
+
* reset handed out for work that did nothing is the same as no watchdog.
|
|
694
|
+
*/
|
|
695
|
+
function carriesToolResult(content: unknown[]): boolean {
|
|
696
|
+
return content.some((block) => isObject(block) && block.type === 'tool_result')
|
|
697
|
+
}
|
|
698
|
+
|
|
662
699
|
export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRunOutcome> {
|
|
663
700
|
const stats: PiRunStats = { toolCalls: 0, assistantChars: 0 }
|
|
664
701
|
let summary = ''
|
|
@@ -739,6 +776,9 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
739
776
|
const progressGuard = createClaudeProgressGuard(opts)
|
|
740
777
|
const { rememberTool, feedGuard, guardAbort } = progressGuard
|
|
741
778
|
const trajectory = createClaudeToolTrajectory(opts, secrets)
|
|
779
|
+
// This stream's tool-silence window; opened just before the CLI starts and closed in the
|
|
780
|
+
// `finally` below, so it can only ever be armed while the CLI it watches is running.
|
|
781
|
+
let toolWindow: ToolProgressWindow = NO_TOOL_WINDOW
|
|
742
782
|
|
|
743
783
|
const onEvent = (event: Record<string, unknown>, meta?: { final?: boolean }): void => {
|
|
744
784
|
const type = event.type
|
|
@@ -776,6 +816,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
776
816
|
// tool_result blocks the harness fed back to the model — part of the next prompt.
|
|
777
817
|
const content = (event.message as Record<string, unknown>).content
|
|
778
818
|
if (Array.isArray(content)) {
|
|
819
|
+
if (carriesToolResult(content)) toolWindow.toolCompleted()
|
|
779
820
|
sliceTracker.onUser(content)
|
|
780
821
|
planTracker.onUser(content)
|
|
781
822
|
emitProgress()
|
|
@@ -824,6 +865,9 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
824
865
|
? AbortSignal.any([opts.signal, guardAbort.signal])
|
|
825
866
|
: guardAbort.signal
|
|
826
867
|
|
|
868
|
+
// Opened around the CLI itself, not around this function: everything above is per-run setup
|
|
869
|
+
// (the config home, the skills, the MCP config) which completes no tool calls by nature.
|
|
870
|
+
toolWindow = openToolWindow(opts)
|
|
827
871
|
try {
|
|
828
872
|
const { stderrTail } = await streamCli(
|
|
829
873
|
{
|
|
@@ -895,6 +939,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
895
939
|
}
|
|
896
940
|
throw withAgentReport(err, terminalReport, secrets)
|
|
897
941
|
} finally {
|
|
942
|
+
toolWindow.close()
|
|
898
943
|
await subagents?.stop()
|
|
899
944
|
await home.dispose()
|
|
900
945
|
}
|
|
@@ -1097,6 +1142,30 @@ function claudeUsage(raw: unknown): { inputTokens: number; outputTokens: number
|
|
|
1097
1142
|
// Codex
|
|
1098
1143
|
// ---------------------------------------------------------------------------
|
|
1099
1144
|
|
|
1145
|
+
/**
|
|
1146
|
+
* The assistant text a codex event carries, or `''`. Two shapes because the CLI changed its
|
|
1147
|
+
* stream between versions and the harness serves both: the flat `agent_message*` events and the
|
|
1148
|
+
* newer `item.completed` envelope around a message item.
|
|
1149
|
+
*/
|
|
1150
|
+
function codexAssistantText(event: Record<string, unknown>, type: string): string {
|
|
1151
|
+
const isMessage =
|
|
1152
|
+
type.includes('agent_message') || (type === 'item.completed' && isCodexMessageItem(event))
|
|
1153
|
+
return (isMessage ? extractText(event) : '') ?? ''
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
/**
|
|
1157
|
+
* Whether a codex event reports tool activity — a substring test because the CLI names these
|
|
1158
|
+
* events differently across versions (`exec_command_end`, `item.*` around a command execution,
|
|
1159
|
+
* `tool_*`) and the harness cares only that SOMETHING ran.
|
|
1160
|
+
*
|
|
1161
|
+
* This is also the tool-silence watchdog's only signal on this stream. Codex exposes no
|
|
1162
|
+
* structured tool bodies, so `runCodex` produces no `ToolSpan` at all, and a window keyed on the
|
|
1163
|
+
* trajectory would have force-failed every codex pass that outran it while the run was working.
|
|
1164
|
+
*/
|
|
1165
|
+
function isCodexToolActivity(type: string): boolean {
|
|
1166
|
+
return type.includes('tool') || type.includes('command') || type.includes('exec')
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1100
1169
|
/**
|
|
1101
1170
|
* Run the Codex CLI headlessly against `opts.cwd`, authenticated with the leased
|
|
1102
1171
|
* ChatGPT `auth.json` bundle written to an isolated CODEX_HOME, talking direct to
|
|
@@ -1158,6 +1227,9 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
1158
1227
|
// context into the prompt itself (Claude Code instead rides --append-system-prompt,
|
|
1159
1228
|
// falling back to this same fold when the prompt overflows argv).
|
|
1160
1229
|
const prompt = foldSystemPrompt(opts.systemPrompt, opts.userPrompt)
|
|
1230
|
+
// This stream's tool-silence window (see the claude runner for the shape); opened just before
|
|
1231
|
+
// the CLI starts and closed in the `finally` below.
|
|
1232
|
+
let toolWindow: ToolProgressWindow = NO_TOOL_WINDOW
|
|
1161
1233
|
|
|
1162
1234
|
// Codex's `exec --json` is far thinner than Claude Code's stream: it surfaces only
|
|
1163
1235
|
// flat assistant text and (on `token_count` events) the per-turn `last_token_usage`
|
|
@@ -1171,19 +1243,15 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
1171
1243
|
|
|
1172
1244
|
const onEvent = (event: Record<string, unknown>): void => {
|
|
1173
1245
|
const type = typeof event.type === 'string' ? event.type : ''
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
if (text) {
|
|
1180
|
-
stats.assistantChars += text.length
|
|
1181
|
-
summary = text
|
|
1182
|
-
pendingText = text
|
|
1183
|
-
}
|
|
1246
|
+
const text = codexAssistantText(event, type)
|
|
1247
|
+
if (text) {
|
|
1248
|
+
stats.assistantChars += text.length
|
|
1249
|
+
summary = text
|
|
1250
|
+
pendingText = text
|
|
1184
1251
|
}
|
|
1185
|
-
if (
|
|
1252
|
+
if (isCodexToolActivity(type)) {
|
|
1186
1253
|
stats.toolCalls += 1
|
|
1254
|
+
toolWindow.toolCompleted()
|
|
1187
1255
|
}
|
|
1188
1256
|
const progress = codexPlanProgress(event)
|
|
1189
1257
|
if (progress && opts.onProgress) opts.onProgress(progress)
|
|
@@ -1214,6 +1282,7 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
1214
1282
|
}
|
|
1215
1283
|
}
|
|
1216
1284
|
|
|
1285
|
+
toolWindow = openToolWindow(opts)
|
|
1217
1286
|
try {
|
|
1218
1287
|
const { stderrTail } = await streamCli(
|
|
1219
1288
|
{
|
|
@@ -1281,6 +1350,7 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
1281
1350
|
// stream, not on stderr — so a bad exit carries the last thing the agent said.
|
|
1282
1351
|
throw withAgentReport(err, summary, secrets)
|
|
1283
1352
|
} finally {
|
|
1353
|
+
toolWindow.close()
|
|
1284
1354
|
if (codexHome) {
|
|
1285
1355
|
// Lift the CLI session transcripts (`sessions/`) out for short-lived retention BEFORE the
|
|
1286
1356
|
// home is deleted — the credential (`auth.json`) lives at the home root, never in
|
package/src/agent.ts
CHANGED
|
@@ -27,7 +27,7 @@ import {
|
|
|
27
27
|
unmergedPaths,
|
|
28
28
|
} from './git.js'
|
|
29
29
|
import { inferVcsProvider, openPullRequest } from './vcs-api.js'
|
|
30
|
-
import type { PiRunStats, RunDiagnostics } from './pi.js'
|
|
30
|
+
import type { PiRunStats, RunDiagnostics } from './pi-reduction.js'
|
|
31
31
|
import { applyPrDescription } from './pr-description.js'
|
|
32
32
|
import {
|
|
33
33
|
makeDirClaimer,
|
package/src/bootstrap-mode.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { opendir } from 'node:fs/promises'
|
|
2
2
|
import { join } from 'node:path'
|
|
3
3
|
import type { AgentJob, AgentResult } from './job.js'
|
|
4
|
-
import type { PiRunStats } from './pi.js'
|
|
4
|
+
import type { PiRunStats } from './pi-reduction.js'
|
|
5
5
|
import type { RunOptions } from './runner.js'
|
|
6
6
|
import {
|
|
7
7
|
NEVER_ACTED_CAUSE,
|
|
@@ -101,6 +101,7 @@ export async function runBootstrap(job: AgentJob, opts: RunOptions): Promise<Age
|
|
|
101
101
|
dir,
|
|
102
102
|
target: boot.target,
|
|
103
103
|
ghToken: job.ghToken,
|
|
104
|
+
signal,
|
|
104
105
|
message: fromScratch
|
|
105
106
|
? 'Bootstrap new repository'
|
|
106
107
|
: `Bootstrap from ${job.repo.owner}/${job.repo.name}`,
|
package/src/coding-agent.ts
CHANGED
|
@@ -30,7 +30,8 @@ import {
|
|
|
30
30
|
} from './git.js'
|
|
31
31
|
import { openPullRequest } from './vcs-api.js'
|
|
32
32
|
import { FOLLOW_UPS_FILENAME, FollowUpTailer } from './follow-ups.js'
|
|
33
|
-
import type { HarnessCallMetric
|
|
33
|
+
import type { HarnessCallMetric } from './pi.js'
|
|
34
|
+
import type { PiRunStats } from './pi-reduction.js'
|
|
34
35
|
import { EFFORT_REPORT_FILE, type EffortReport } from './effort.js'
|
|
35
36
|
import {
|
|
36
37
|
type AgentPrDescription,
|