@cat-factory/executor-harness 1.96.0 → 1.100.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/dist/agent-capabilities.js +25 -11
- package/dist/agent-runner.d.ts +14 -1
- package/dist/agent-runner.js +84 -28
- package/dist/bootstrap-mode.js +1 -0
- package/dist/coding-agent.d.ts +2 -1
- package/dist/embed.d.ts +2 -1
- package/dist/embed.js +2 -1
- package/dist/failure.d.ts +19 -1
- package/dist/failure.js +40 -0
- package/dist/git.d.ts +6 -0
- package/dist/git.js +16 -9
- package/dist/inline.d.ts +6 -0
- package/dist/inline.js +6 -0
- package/dist/job.d.ts +2 -1
- package/dist/jsonl-stream.d.ts +70 -0
- package/dist/jsonl-stream.js +149 -0
- package/dist/pi-reduction.d.ts +136 -0
- package/dist/pi-reduction.js +303 -0
- package/dist/pi-workspace.d.ts +2 -1
- package/dist/pi-workspace.js +6 -1
- package/dist/pi.d.ts +8 -81
- package/dist/pi.js +124 -310
- package/dist/runner.d.ts +31 -0
- package/dist/runner.js +50 -3
- package/dist/structured-output.js +2 -1
- package/dist/tool-silence.d.ts +74 -0
- package/dist/tool-silence.js +99 -0
- package/package.json +4 -4
- package/src/agent-capabilities.ts +22 -13
- package/src/agent-runner.ts +100 -30
- package/src/agent.ts +1 -1
- package/src/bootstrap-mode.ts +2 -1
- package/src/coding-agent.ts +2 -1
- package/src/embed.ts +8 -5
- package/src/failure.ts +36 -9
- package/src/git.ts +17 -9
- package/src/inline.ts +6 -0
- package/src/job.ts +2 -1
- package/src/jsonl-stream.ts +149 -0
- package/src/pi-reduction.ts +359 -0
- package/src/pi-workspace.ts +7 -3
- package/src/pi.ts +144 -349
- package/src/runner.ts +91 -4
- package/src/structured-output.ts +2 -1
- package/src/tool-silence.ts +125 -0
package/src/agent-runner.ts
CHANGED
|
@@ -9,17 +9,18 @@ import {
|
|
|
9
9
|
type TrackedToolCall,
|
|
10
10
|
recordClaudeToolResults,
|
|
11
11
|
} from './tool-trajectory.js'
|
|
12
|
-
import type
|
|
12
|
+
import { log, type Logger } from './logger.js'
|
|
13
|
+
import { NO_TOOL_WINDOW, type ToolProgressWindow } from './tool-silence.js'
|
|
13
14
|
import {
|
|
14
15
|
createCallMetricPublisher,
|
|
15
16
|
publishCallMetric,
|
|
16
17
|
type CallMetricPublisher,
|
|
17
18
|
type HarnessCallMetric,
|
|
18
19
|
type PiRunOutcome,
|
|
19
|
-
type PiRunStats,
|
|
20
20
|
type TodoProgress,
|
|
21
21
|
type ToolSpan,
|
|
22
22
|
} from './pi.js'
|
|
23
|
+
import type { PiRunStats } from './pi-reduction.js'
|
|
23
24
|
import {
|
|
24
25
|
claudeAllowedToolPatterns,
|
|
25
26
|
codexMcpConfigToml,
|
|
@@ -31,6 +32,7 @@ import {
|
|
|
31
32
|
type SkillSpec,
|
|
32
33
|
} from './agent-capabilities.js'
|
|
33
34
|
import { ProgressGuard, type ProgressGuardLimits } from './progress-guard.js'
|
|
35
|
+
import { BoundedTail, JsonlLineReader } from './jsonl-stream.js'
|
|
34
36
|
import { killChildProcess, spawnDetached } from './process.js'
|
|
35
37
|
import { describeProcessExit } from './process-exit.js'
|
|
36
38
|
import { redact, registerKnownSecrets, secretsToRedact } from './redact.js'
|
|
@@ -138,6 +140,18 @@ export interface SubscriptionRunOptions {
|
|
|
138
140
|
* DID dies with the container.
|
|
139
141
|
*/
|
|
140
142
|
onSpan?: (span: ToolSpan) => void
|
|
143
|
+
/**
|
|
144
|
+
* Opens this stream's tool-silence window (see `RunOptions.beginToolWindow`), closed when the
|
|
145
|
+
* CLI exits. Both subscription CLIs report tool activity — claude-code on the `tool_result`
|
|
146
|
+
* turn that answers each call, codex on its tool/command/exec events — so a window either
|
|
147
|
+
* opens is one the run can beat. It is deliberately NOT tied to {@link onSpan}: the trajectory
|
|
148
|
+
* is an observability opt-in, and the codex stream produces none at all while still doing tool
|
|
149
|
+
* work, which a span-keyed window would have read as a run making no progress.
|
|
150
|
+
*
|
|
151
|
+
* A caller with no tool loop (the inline one-shot completion) passes nothing; see the note at
|
|
152
|
+
* `handleInline`.
|
|
153
|
+
*/
|
|
154
|
+
beginToolWindow?: () => ToolProgressWindow
|
|
141
155
|
/**
|
|
142
156
|
* Called with the FULL set of per-slice reviews each time one lands, so the backend can persist
|
|
143
157
|
* a parallel review's completed work as it happens instead of only from the terminal result.
|
|
@@ -227,9 +241,10 @@ function streamCli(
|
|
|
227
241
|
child.stdin.on('error', () => {})
|
|
228
242
|
child.stdin.end(prompt)
|
|
229
243
|
|
|
230
|
-
|
|
244
|
+
// 8 KB is well over the 700 B tail anyone quotes below, and the CLI's stderr is diagnostic
|
|
245
|
+
// noise rather than a product, so a bounded tail is all this ever needed to be.
|
|
246
|
+
const stderr = new BoundedTail(8_000)
|
|
231
247
|
let aborted = false
|
|
232
|
-
let lineBuffer = ''
|
|
233
248
|
|
|
234
249
|
const killChild = (): void => killChildProcess(child)
|
|
235
250
|
|
|
@@ -253,16 +268,9 @@ function streamCli(
|
|
|
253
268
|
}
|
|
254
269
|
}
|
|
255
270
|
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
while (nl !== -1) {
|
|
260
|
-
const line = lineBuffer.slice(0, nl).trim()
|
|
261
|
-
lineBuffer = lineBuffer.slice(nl + 1)
|
|
262
|
-
nl = lineBuffer.indexOf('\n')
|
|
263
|
-
processLine(line)
|
|
264
|
-
}
|
|
265
|
-
}
|
|
271
|
+
// Bounded framing, shared with `runPi`: an unterminated record must not be able to grow
|
|
272
|
+
// until parsing it stalls the loop the watchdogs and poll handlers run on (audit F6).
|
|
273
|
+
const reader = new JsonlLineReader(processLine)
|
|
266
274
|
|
|
267
275
|
const onAbort = (): void => {
|
|
268
276
|
aborted = true
|
|
@@ -272,12 +280,11 @@ function streamCli(
|
|
|
272
280
|
|
|
273
281
|
child.stdout.on('data', (chunk: Buffer) => {
|
|
274
282
|
opts.onActivity?.()
|
|
275
|
-
|
|
283
|
+
reader.push(chunk.toString())
|
|
276
284
|
})
|
|
277
285
|
child.stderr.on('data', (chunk: Buffer) => {
|
|
278
286
|
opts.onActivity?.()
|
|
279
|
-
stderr
|
|
280
|
-
if (stderr.length > 8_000) stderr = stderr.slice(-8_000)
|
|
287
|
+
stderr.push(chunk.toString())
|
|
281
288
|
})
|
|
282
289
|
|
|
283
290
|
child.on('error', (err) => {
|
|
@@ -286,8 +293,19 @@ function streamCli(
|
|
|
286
293
|
})
|
|
287
294
|
child.on('close', (code, signal) => {
|
|
288
295
|
opts.signal?.removeEventListener('abort', onAbort)
|
|
289
|
-
const stderrTail = redact(stderr, secrets).slice(-700)
|
|
290
|
-
|
|
296
|
+
const stderrTail = redact(stderr.toString(), secrets).slice(-700)
|
|
297
|
+
reader.flush()
|
|
298
|
+
// Surface an oversized record the reader refused to buffer ONCE (a count, not per line),
|
|
299
|
+
// for the same reason `runPi` does: a dropped record costs this run its progress, its
|
|
300
|
+
// trajectory and its per-call telemetry for that turn, and a silent loss reads exactly
|
|
301
|
+
// like a CLI that never emitted it. Falls back to the module logger so the report cannot
|
|
302
|
+
// depend on a caller having wired a per-job one.
|
|
303
|
+
if (reader.droppedLines > 0) {
|
|
304
|
+
;(opts.log ?? log).warn('agent CLI: skipped oversized JSONL records', {
|
|
305
|
+
command,
|
|
306
|
+
oversizedLines: reader.droppedLines,
|
|
307
|
+
})
|
|
308
|
+
}
|
|
291
309
|
if (aborted) {
|
|
292
310
|
// Carry the tail on the rejection so a caller that REPLACES this generic message with a
|
|
293
311
|
// more specific cause (the no-progress guard's diagnostic) can still append it — the
|
|
@@ -659,6 +677,25 @@ function createClaudeToolTrajectory(
|
|
|
659
677
|
}
|
|
660
678
|
}
|
|
661
679
|
|
|
680
|
+
/**
|
|
681
|
+
* Open this run's tool-silence window, or the inert one when the caller wired no watchdog. One
|
|
682
|
+
* definition so both runners resolve "is there a watchdog?" identically, and so neither carries
|
|
683
|
+
* the optional-call noise at the point where it should simply have a window.
|
|
684
|
+
*/
|
|
685
|
+
function openToolWindow(opts: SubscriptionRunOptions): ToolProgressWindow {
|
|
686
|
+
return opts.beginToolWindow ? opts.beginToolWindow() : NO_TOOL_WINDOW
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/**
|
|
690
|
+
* Whether a claude-code `user` turn carries a `tool_result` block, i.e. whether a tool call just
|
|
691
|
+
* COMPLETED — the progress the tool-silence watchdog measures. Tested explicitly rather than
|
|
692
|
+
* taken from "the model sent a user turn", which a plain follow-up prompt also is: a watchdog
|
|
693
|
+
* reset handed out for work that did nothing is the same as no watchdog.
|
|
694
|
+
*/
|
|
695
|
+
function carriesToolResult(content: unknown[]): boolean {
|
|
696
|
+
return content.some((block) => isObject(block) && block.type === 'tool_result')
|
|
697
|
+
}
|
|
698
|
+
|
|
662
699
|
export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRunOutcome> {
|
|
663
700
|
const stats: PiRunStats = { toolCalls: 0, assistantChars: 0 }
|
|
664
701
|
let summary = ''
|
|
@@ -739,6 +776,9 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
739
776
|
const progressGuard = createClaudeProgressGuard(opts)
|
|
740
777
|
const { rememberTool, feedGuard, guardAbort } = progressGuard
|
|
741
778
|
const trajectory = createClaudeToolTrajectory(opts, secrets)
|
|
779
|
+
// This stream's tool-silence window; opened just before the CLI starts and closed in the
|
|
780
|
+
// `finally` below, so it can only ever be armed while the CLI it watches is running.
|
|
781
|
+
let toolWindow: ToolProgressWindow = NO_TOOL_WINDOW
|
|
742
782
|
|
|
743
783
|
const onEvent = (event: Record<string, unknown>, meta?: { final?: boolean }): void => {
|
|
744
784
|
const type = event.type
|
|
@@ -776,6 +816,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
776
816
|
// tool_result blocks the harness fed back to the model — part of the next prompt.
|
|
777
817
|
const content = (event.message as Record<string, unknown>).content
|
|
778
818
|
if (Array.isArray(content)) {
|
|
819
|
+
if (carriesToolResult(content)) toolWindow.toolCompleted()
|
|
779
820
|
sliceTracker.onUser(content)
|
|
780
821
|
planTracker.onUser(content)
|
|
781
822
|
emitProgress()
|
|
@@ -824,6 +865,9 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
824
865
|
? AbortSignal.any([opts.signal, guardAbort.signal])
|
|
825
866
|
: guardAbort.signal
|
|
826
867
|
|
|
868
|
+
// Opened around the CLI itself, not around this function: everything above is per-run setup
|
|
869
|
+
// (the config home, the skills, the MCP config) which completes no tool calls by nature.
|
|
870
|
+
toolWindow = openToolWindow(opts)
|
|
827
871
|
try {
|
|
828
872
|
const { stderrTail } = await streamCli(
|
|
829
873
|
{
|
|
@@ -895,6 +939,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
895
939
|
}
|
|
896
940
|
throw withAgentReport(err, terminalReport, secrets)
|
|
897
941
|
} finally {
|
|
942
|
+
toolWindow.close()
|
|
898
943
|
await subagents?.stop()
|
|
899
944
|
await home.dispose()
|
|
900
945
|
}
|
|
@@ -1097,6 +1142,30 @@ function claudeUsage(raw: unknown): { inputTokens: number; outputTokens: number
|
|
|
1097
1142
|
// Codex
|
|
1098
1143
|
// ---------------------------------------------------------------------------
|
|
1099
1144
|
|
|
1145
|
+
/**
|
|
1146
|
+
* The assistant text a codex event carries, or `''`. Two shapes because the CLI changed its
|
|
1147
|
+
* stream between versions and the harness serves both: the flat `agent_message*` events and the
|
|
1148
|
+
* newer `item.completed` envelope around a message item.
|
|
1149
|
+
*/
|
|
1150
|
+
function codexAssistantText(event: Record<string, unknown>, type: string): string {
|
|
1151
|
+
const isMessage =
|
|
1152
|
+
type.includes('agent_message') || (type === 'item.completed' && isCodexMessageItem(event))
|
|
1153
|
+
return (isMessage ? extractText(event) : '') ?? ''
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
/**
|
|
1157
|
+
* Whether a codex event reports tool activity — a substring test because the CLI names these
|
|
1158
|
+
* events differently across versions (`exec_command_end`, `item.*` around a command execution,
|
|
1159
|
+
* `tool_*`) and the harness cares only that SOMETHING ran.
|
|
1160
|
+
*
|
|
1161
|
+
* This is also the tool-silence watchdog's only signal on this stream. Codex exposes no
|
|
1162
|
+
* structured tool bodies, so `runCodex` produces no `ToolSpan` at all, and a window keyed on the
|
|
1163
|
+
* trajectory would have force-failed every codex pass that outran it while the run was working.
|
|
1164
|
+
*/
|
|
1165
|
+
function isCodexToolActivity(type: string): boolean {
|
|
1166
|
+
return type.includes('tool') || type.includes('command') || type.includes('exec')
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1100
1169
|
/**
|
|
1101
1170
|
* Run the Codex CLI headlessly against `opts.cwd`, authenticated with the leased
|
|
1102
1171
|
* ChatGPT `auth.json` bundle written to an isolated CODEX_HOME, talking direct to
|
|
@@ -1158,6 +1227,9 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
1158
1227
|
// context into the prompt itself (Claude Code instead rides --append-system-prompt,
|
|
1159
1228
|
// falling back to this same fold when the prompt overflows argv).
|
|
1160
1229
|
const prompt = foldSystemPrompt(opts.systemPrompt, opts.userPrompt)
|
|
1230
|
+
// This stream's tool-silence window (see the claude runner for the shape); opened just before
|
|
1231
|
+
// the CLI starts and closed in the `finally` below.
|
|
1232
|
+
let toolWindow: ToolProgressWindow = NO_TOOL_WINDOW
|
|
1161
1233
|
|
|
1162
1234
|
// Codex's `exec --json` is far thinner than Claude Code's stream: it surfaces only
|
|
1163
1235
|
// flat assistant text and (on `token_count` events) the per-turn `last_token_usage`
|
|
@@ -1171,19 +1243,15 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
1171
1243
|
|
|
1172
1244
|
const onEvent = (event: Record<string, unknown>): void => {
|
|
1173
1245
|
const type = typeof event.type === 'string' ? event.type : ''
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
if (text) {
|
|
1180
|
-
stats.assistantChars += text.length
|
|
1181
|
-
summary = text
|
|
1182
|
-
pendingText = text
|
|
1183
|
-
}
|
|
1246
|
+
const text = codexAssistantText(event, type)
|
|
1247
|
+
if (text) {
|
|
1248
|
+
stats.assistantChars += text.length
|
|
1249
|
+
summary = text
|
|
1250
|
+
pendingText = text
|
|
1184
1251
|
}
|
|
1185
|
-
if (
|
|
1252
|
+
if (isCodexToolActivity(type)) {
|
|
1186
1253
|
stats.toolCalls += 1
|
|
1254
|
+
toolWindow.toolCompleted()
|
|
1187
1255
|
}
|
|
1188
1256
|
const progress = codexPlanProgress(event)
|
|
1189
1257
|
if (progress && opts.onProgress) opts.onProgress(progress)
|
|
@@ -1214,6 +1282,7 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
1214
1282
|
}
|
|
1215
1283
|
}
|
|
1216
1284
|
|
|
1285
|
+
toolWindow = openToolWindow(opts)
|
|
1217
1286
|
try {
|
|
1218
1287
|
const { stderrTail } = await streamCli(
|
|
1219
1288
|
{
|
|
@@ -1281,6 +1350,7 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
1281
1350
|
// stream, not on stderr — so a bad exit carries the last thing the agent said.
|
|
1282
1351
|
throw withAgentReport(err, summary, secrets)
|
|
1283
1352
|
} finally {
|
|
1353
|
+
toolWindow.close()
|
|
1284
1354
|
if (codexHome) {
|
|
1285
1355
|
// Lift the CLI session transcripts (`sessions/`) out for short-lived retention BEFORE the
|
|
1286
1356
|
// home is deleted — the credential (`auth.json`) lives at the home root, never in
|
package/src/agent.ts
CHANGED
|
@@ -27,7 +27,7 @@ import {
|
|
|
27
27
|
unmergedPaths,
|
|
28
28
|
} from './git.js'
|
|
29
29
|
import { inferVcsProvider, openPullRequest } from './vcs-api.js'
|
|
30
|
-
import type { PiRunStats, RunDiagnostics } from './pi.js'
|
|
30
|
+
import type { PiRunStats, RunDiagnostics } from './pi-reduction.js'
|
|
31
31
|
import { applyPrDescription } from './pr-description.js'
|
|
32
32
|
import {
|
|
33
33
|
makeDirClaimer,
|
package/src/bootstrap-mode.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { opendir } from 'node:fs/promises'
|
|
2
2
|
import { join } from 'node:path'
|
|
3
3
|
import type { AgentJob, AgentResult } from './job.js'
|
|
4
|
-
import type { PiRunStats } from './pi.js'
|
|
4
|
+
import type { PiRunStats } from './pi-reduction.js'
|
|
5
5
|
import type { RunOptions } from './runner.js'
|
|
6
6
|
import {
|
|
7
7
|
NEVER_ACTED_CAUSE,
|
|
@@ -101,6 +101,7 @@ export async function runBootstrap(job: AgentJob, opts: RunOptions): Promise<Age
|
|
|
101
101
|
dir,
|
|
102
102
|
target: boot.target,
|
|
103
103
|
ghToken: job.ghToken,
|
|
104
|
+
signal,
|
|
104
105
|
message: fromScratch
|
|
105
106
|
? 'Bootstrap new repository'
|
|
106
107
|
: `Bootstrap from ${job.repo.owner}/${job.repo.name}`,
|
package/src/coding-agent.ts
CHANGED
|
@@ -30,7 +30,8 @@ import {
|
|
|
30
30
|
} from './git.js'
|
|
31
31
|
import { openPullRequest } from './vcs-api.js'
|
|
32
32
|
import { FOLLOW_UPS_FILENAME, FollowUpTailer } from './follow-ups.js'
|
|
33
|
-
import type { HarnessCallMetric
|
|
33
|
+
import type { HarnessCallMetric } from './pi.js'
|
|
34
|
+
import type { PiRunStats } from './pi-reduction.js'
|
|
34
35
|
import { EFFORT_REPORT_FILE, type EffortReport } from './effort.js'
|
|
35
36
|
import {
|
|
36
37
|
type AgentPrDescription,
|
package/src/embed.ts
CHANGED
|
@@ -6,19 +6,22 @@
|
|
|
6
6
|
// only the reusable primitives are exposed here.
|
|
7
7
|
|
|
8
8
|
export {
|
|
9
|
-
PI_MAX_OUTPUT_TOKENS,
|
|
10
9
|
writePiModelsConfig,
|
|
11
10
|
writeAgentsContext,
|
|
12
11
|
runPi,
|
|
13
|
-
summarizePiRun,
|
|
14
|
-
parsePiOutput,
|
|
15
12
|
parseTodoProgress,
|
|
16
|
-
terminalRunError,
|
|
17
13
|
type PiRunOutcome,
|
|
18
|
-
type PiRunStats,
|
|
19
14
|
type TodoItem,
|
|
20
15
|
type TodoProgress,
|
|
21
16
|
} from './pi.js'
|
|
17
|
+
export {
|
|
18
|
+
PI_MAX_OUTPUT_TOKENS,
|
|
19
|
+
parsePiOutput,
|
|
20
|
+
summarizePiRun,
|
|
21
|
+
terminalRunError,
|
|
22
|
+
type PiRunReduction,
|
|
23
|
+
type PiRunStats,
|
|
24
|
+
} from './pi-reduction.js'
|
|
22
25
|
export {
|
|
23
26
|
DEFAULT_PROGRESS_GUARD_LIMITS,
|
|
24
27
|
progressGuardLimitsFromEnv,
|
package/src/failure.ts
CHANGED
|
@@ -20,6 +20,10 @@
|
|
|
20
20
|
*
|
|
21
21
|
* - `inactivity-timeout` — the inactivity watchdog fired (no agent output for the window).
|
|
22
22
|
* - `max-duration` — the overall wall-clock cap fired.
|
|
23
|
+
* - `no-tool-progress` — the tool-silence watchdog fired: the agent kept TALKING but completed
|
|
24
|
+
* no tool call for the window. Distinct from `inactivity-timeout` on
|
|
25
|
+
* purpose, because the two need different fixes: one says the container
|
|
26
|
+
* went quiet, this one says the model rabbit-holed while streaming.
|
|
23
27
|
* - `agent` — the agent ran but produced an unusable/failed result, or threw.
|
|
24
28
|
* - `git` — a git operation failed (clone/push/merge/PR).
|
|
25
29
|
* - `api` — an upstream API call failed (e.g. the GitHub/GitLab PR/MR REST call).
|
|
@@ -28,15 +32,25 @@
|
|
|
28
32
|
* - `no-usable-output` — the agent finished but returned no usable report / structured output.
|
|
29
33
|
* - `no-changes` — a coding agent finished without producing any change to push.
|
|
30
34
|
*/
|
|
31
|
-
export
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
35
|
+
export const FAILURE_CAUSES = [
|
|
36
|
+
'inactivity-timeout',
|
|
37
|
+
'max-duration',
|
|
38
|
+
'no-tool-progress',
|
|
39
|
+
'agent',
|
|
40
|
+
'git',
|
|
41
|
+
'api',
|
|
42
|
+
'llm-upstream',
|
|
43
|
+
'no-usable-output',
|
|
44
|
+
'no-changes',
|
|
45
|
+
] as const
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* See {@link FAILURE_CAUSES}. Derived from the array rather than declared beside it so the two
|
|
49
|
+
* cannot disagree, and so the list is ENUMERABLE at runtime — which is what lets
|
|
50
|
+
* `failure-cause.conformity.test.ts` check this image's vocabulary against the kernel union that
|
|
51
|
+
* has to classify it (the two are kept in step by hand; the image can carry no workspace dep).
|
|
52
|
+
*/
|
|
53
|
+
export type FailureCause = (typeof FAILURE_CAUSES)[number]
|
|
40
54
|
|
|
41
55
|
/**
|
|
42
56
|
* A thrown failure that carries a structured {@link FailureCause}, so a `git` / `api`
|
|
@@ -77,3 +91,16 @@ export function inactivityAbortMessage(inactivityMs: number): string {
|
|
|
77
91
|
export function maxDurationAbortMessage(maxDurationMs: number): string {
|
|
78
92
|
return `Aborted: exceeded max duration of ${Math.round(maxDurationMs / 1000)}s`
|
|
79
93
|
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* The tool-silence-watchdog abort message. Human-readable only, like its two siblings — the
|
|
97
|
+
* backend reads the structured `no-tool-progress` {@link FailureCause}. Says what it observed
|
|
98
|
+
* (output, but no completed tool call) rather than "hung": the run was demonstrably alive, which
|
|
99
|
+
* is exactly why the inactivity watchdog never fired.
|
|
100
|
+
*/
|
|
101
|
+
export function toolSilenceAbortMessage(toolSilenceMs: number): string {
|
|
102
|
+
return (
|
|
103
|
+
`Aborted: the agent produced output but completed no tool call for ` +
|
|
104
|
+
`${Math.round(toolSilenceMs / 1000)}s`
|
|
105
|
+
)
|
|
106
|
+
}
|
package/src/git.ts
CHANGED
|
@@ -1081,25 +1081,33 @@ export async function pushBranch(
|
|
|
1081
1081
|
* .gitignore and/or license picked on the new-repo page), so a fast-forward is
|
|
1082
1082
|
* impossible. The Worker pre-flights that the target is empty or holds only that
|
|
1083
1083
|
* boilerplate, so overwriting it is safe and intended.
|
|
1084
|
+
*
|
|
1085
|
+
* `signal` is the job watchdog's, and threading it is load-bearing rather than tidy: without
|
|
1086
|
+
* it the six commands below are bounded only by their own per-command timeouts, so an abort
|
|
1087
|
+
* raised during the push phase cannot interrupt them and the job keeps working for up to
|
|
1088
|
+
* ~6 × `GIT_TIMEOUT_MS` past its max-duration kill. Every other git helper here threads it.
|
|
1084
1089
|
*/
|
|
1085
1090
|
export async function reinitAndPush(opts: {
|
|
1086
1091
|
dir: string
|
|
1087
1092
|
target: BootstrapTargetSpec
|
|
1088
1093
|
ghToken: string
|
|
1089
1094
|
message: string
|
|
1095
|
+
signal?: AbortSignal
|
|
1090
1096
|
}): Promise<void> {
|
|
1091
|
-
|
|
1092
|
-
await
|
|
1097
|
+
const { dir, signal } = opts
|
|
1098
|
+
await rm(join(dir, '.git'), { recursive: true, force: true })
|
|
1099
|
+
await git(['init'], { cwd: dir, signal })
|
|
1093
1100
|
// Start the history on the target's default branch (init may default to master).
|
|
1094
|
-
await git(['checkout', '-b', opts.target.defaultBranch], { cwd:
|
|
1095
|
-
await git(['config', 'user.name', GIT_AUTHOR], { cwd:
|
|
1096
|
-
await git(['config', 'user.email', GIT_EMAIL], { cwd:
|
|
1097
|
-
await git(['add', '-A'], { cwd:
|
|
1098
|
-
await git(['commit', '-m', opts.message], { cwd:
|
|
1101
|
+
await git(['checkout', '-b', opts.target.defaultBranch], { cwd: dir, signal })
|
|
1102
|
+
await git(['config', 'user.name', GIT_AUTHOR], { cwd: dir, signal })
|
|
1103
|
+
await git(['config', 'user.email', GIT_EMAIL], { cwd: dir, signal })
|
|
1104
|
+
await git(['add', '-A'], { cwd: dir, signal })
|
|
1105
|
+
await git(['commit', '-m', opts.message], { cwd: dir, signal })
|
|
1099
1106
|
const url = authenticatedCloneUrl(opts.target.cloneUrl)
|
|
1100
|
-
await git(['remote', 'add', 'origin', url], { cwd:
|
|
1107
|
+
await git(['remote', 'add', 'origin', url], { cwd: dir, signal })
|
|
1101
1108
|
await git(['push', '--force', '-u', 'origin', opts.target.defaultBranch], {
|
|
1102
|
-
cwd:
|
|
1109
|
+
cwd: dir,
|
|
1110
|
+
signal,
|
|
1103
1111
|
env: await authEnv(opts.ghToken),
|
|
1104
1112
|
})
|
|
1105
1113
|
}
|
package/src/inline.ts
CHANGED
|
@@ -35,6 +35,12 @@ function deriveFinishReason(calls: HarnessCallMetric[] | undefined): 'stop' | 'l
|
|
|
35
35
|
* directory. The job's watchdog (inactivity + max-duration, see {@link JobRegistry}) bounds
|
|
36
36
|
* it through `opts.signal`; `opts.onActivity` keeps the inactivity timer alive while the CLI
|
|
37
37
|
* streams. The temp cwd is always removed.
|
|
38
|
+
*
|
|
39
|
+
* `opts.beginToolWindow` is deliberately NOT forwarded. An inline completion is a one-shot
|
|
40
|
+
* answer with no checkout and no tool loop, so the tool-silence watchdog would arm a window this
|
|
41
|
+
* run could never beat and could only ever expire — force-failing a healthy completion under a
|
|
42
|
+
* cause ("kept talking, completed no tool call") that misdescribes what it is. The inactivity and
|
|
43
|
+
* max-duration watchdogs still bound it, which is the whole bound this work ever had.
|
|
38
44
|
*/
|
|
39
45
|
export async function handleInline(job: InlineJob, opts: RunOptions): Promise<InlineResult> {
|
|
40
46
|
opts.onPhase?.('agent')
|
package/src/job.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type { HarnessCallMetric
|
|
1
|
+
import type { HarnessCallMetric } from './pi.js'
|
|
2
|
+
import type { PiRunStats } from './pi-reduction.js'
|
|
2
3
|
import type { HarnessKind } from './pi-workspace.js'
|
|
3
4
|
import type { FailureCause } from './failure.js'
|
|
4
5
|
import type { EffortReport } from './effort.js'
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
// BOUNDED reading of an agent CLI's child streams: LF-framing its JSONL stdout, and holding a
|
|
2
|
+
// capped tail of raw output for diagnostics.
|
|
3
|
+
//
|
|
4
|
+
// WHY THIS MODULE EXISTS — the harness's two watchdog timers and its `/health` + `/jobs` poll
|
|
5
|
+
// endpoints share ONE Node event loop with the stream-parsing hot path, so the advertised "a
|
|
6
|
+
// container can never run forever" guarantee only holds while that loop stays live (stuck-run
|
|
7
|
+
// audit F6). Both CLI readers (`runPi`, `runSubscriptionAgent`) had grown the same unbounded
|
|
8
|
+
// framing loop: a record with no terminating newline accumulated without limit, so a runaway
|
|
9
|
+
// producer could drive the buffer until a single `JSON.parse` (or the allocation behind it)
|
|
10
|
+
// stalled the loop past the abort timers and the poll handlers alike. The container then stops
|
|
11
|
+
// answering polls while its own watchdogs never fire — the exact wedge the timers exist to
|
|
12
|
+
// prevent, with only the engine-side poll-failure tolerance and the reaper left underneath.
|
|
13
|
+
//
|
|
14
|
+
// One definition of "how much of a child's output we are willing to hold" therefore serves both
|
|
15
|
+
// harnesses, for the same reason `ProgressGuard` does: two copies of a bound are two bounds.
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Longest single JSONL record either CLI may emit before the reader stops buffering it.
|
|
19
|
+
*
|
|
20
|
+
* Deliberately far above the largest LEGITIMATE record — the terminal `agent_end`, which carries
|
|
21
|
+
* the run's whole message transcript including tool results — because dropping that one costs the
|
|
22
|
+
* run its summary and stats. The cap is not a size policy, it is the ceiling that keeps a
|
|
23
|
+
* runaway producer from growing the buffer until parsing it wedges the event loop, so it only
|
|
24
|
+
* has to be low enough that one parse of it stays well inside the poll cadence.
|
|
25
|
+
*/
|
|
26
|
+
export const MAX_JSONL_LINE_CHARS = 32 * 1024 * 1024
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* A fixed-size tail of a text stream, for output kept ONLY to quote back on a failure.
|
|
30
|
+
*
|
|
31
|
+
* Retaining a whole run's stdout to slice the last 2 KB off it at close is the memory half of
|
|
32
|
+
* F6: a chatty agent's output is unbounded, and the container OOMing is another way for a job to
|
|
33
|
+
* stop answering polls with no watchdog having fired. The tail is trimmed lazily — only once it
|
|
34
|
+
* has grown past twice the bound — so a run that streams thousands of chunks pays an amortized
|
|
35
|
+
* O(1) copy per chunk rather than an O(maxChars) slice on every one of them.
|
|
36
|
+
*/
|
|
37
|
+
export class BoundedTail {
|
|
38
|
+
private text = ''
|
|
39
|
+
private total = 0
|
|
40
|
+
|
|
41
|
+
constructor(private readonly maxChars: number) {}
|
|
42
|
+
|
|
43
|
+
push(chunk: string): void {
|
|
44
|
+
this.text += chunk
|
|
45
|
+
this.total += chunk.length
|
|
46
|
+
if (this.text.length > this.maxChars * 2) this.text = this.text.slice(-this.maxChars)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The last `maxChars` characters seen. */
|
|
50
|
+
toString(): string {
|
|
51
|
+
return this.text.length > this.maxChars ? this.text.slice(-this.maxChars) : this.text
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Everything ever pushed, whether or not it is still retained. */
|
|
55
|
+
get totalChars(): number {
|
|
56
|
+
return this.total
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Characters dropped off the FRONT because the tail is bounded; 0 while everything still fits.
|
|
61
|
+
*
|
|
62
|
+
* A caller that renders the tail to a human owes them this: a bounded tail is the opposite of a
|
|
63
|
+
* prefix, so a reader who assumes one concludes the producer stopped where the text begins.
|
|
64
|
+
* Diagnostic quotes (a stderr tail) need no such note — being a tail is what they are for.
|
|
65
|
+
*/
|
|
66
|
+
get droppedChars(): number {
|
|
67
|
+
return this.total - this.toString().length
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Frames a child's LF-delimited JSONL stdout into complete records, bounding what it will buffer
|
|
73
|
+
* for any one of them.
|
|
74
|
+
*
|
|
75
|
+
* `onLine` is invoked per complete record with `final: false`, and once more from {@link flush}
|
|
76
|
+
* with `final: true` for a trailing record that arrived without its newline (a clean exit can
|
|
77
|
+
* leave the last event unterminated). `final` is what lets a caller deliver the record's
|
|
78
|
+
* progress/telemetry signal while suppressing any decision that would KILL the run: the process
|
|
79
|
+
* has already exited, so a guard tripping on that last buffered record would turn a clean exit
|
|
80
|
+
* into a spurious failure.
|
|
81
|
+
*
|
|
82
|
+
* A record that outgrows {@link MAX_JSONL_LINE_CHARS} is DROPPED, not truncated: a partial JSON
|
|
83
|
+
* document is not a record, and handing the parser half of one would report it as corrupt output
|
|
84
|
+
* rather than as the bound firing. The reader then resynchronises on the next newline, so the
|
|
85
|
+
* oversized record costs its own signal and nothing after it. Callers report {@link droppedLines}
|
|
86
|
+
* at close (never per line) so the loss is diagnosable instead of silent.
|
|
87
|
+
*/
|
|
88
|
+
export class JsonlLineReader {
|
|
89
|
+
private buffer = ''
|
|
90
|
+
/** True while discarding the tail of a record that already blew the cap. */
|
|
91
|
+
private skipping = false
|
|
92
|
+
private dropped = 0
|
|
93
|
+
|
|
94
|
+
constructor(
|
|
95
|
+
private readonly onLine: (line: string, final: boolean) => void,
|
|
96
|
+
private readonly maxLineChars: number = MAX_JSONL_LINE_CHARS,
|
|
97
|
+
) {}
|
|
98
|
+
|
|
99
|
+
/** Feed one stdout chunk, emitting every complete record it finishes. */
|
|
100
|
+
push(text: string): void {
|
|
101
|
+
// Framing scans the incoming CHUNK, never the accumulated buffer. `buffer += chunk` is a
|
|
102
|
+
// cheap rope in V8 and `.length` reads off it in constant time, but ANY search over it
|
|
103
|
+
// flattens the rope — so scanning the buffer once per chunk costs O(record) per chunk, i.e.
|
|
104
|
+
// quadratic in a runaway record, paid on the very event loop this class exists to keep
|
|
105
|
+
// answering polls. Measured, a 32 MB unterminated record cost ~6s of solid blocking that
|
|
106
|
+
// way: the cap bounded the memory and handed back the stall in its place.
|
|
107
|
+
let rest = text
|
|
108
|
+
for (;;) {
|
|
109
|
+
const nl = rest.indexOf('\n')
|
|
110
|
+
if (nl === -1) break
|
|
111
|
+
if (this.skipping) {
|
|
112
|
+
// The newline that ends an oversized record ends the skip with it: everything buffered
|
|
113
|
+
// for that record is already gone, and what follows is a fresh one.
|
|
114
|
+
this.skipping = false
|
|
115
|
+
} else {
|
|
116
|
+
// The only place the buffer is materialised, and only for a record that COMPLETED —
|
|
117
|
+
// which the branch below has already kept under the cap.
|
|
118
|
+
const raw = this.buffer + rest.slice(0, nl)
|
|
119
|
+
this.buffer = ''
|
|
120
|
+
// The cap is on the RECORD, not on the leftover buffer: a record that arrived whole
|
|
121
|
+
// inside one chunk was never buffered across pushes, and dropping it only when it
|
|
122
|
+
// straddles a chunk boundary would make the bound depend on how the OS split the reads.
|
|
123
|
+
if (raw.length > this.maxLineChars) this.dropped++
|
|
124
|
+
else this.onLine(raw.trim(), false)
|
|
125
|
+
}
|
|
126
|
+
rest = rest.slice(nl + 1)
|
|
127
|
+
}
|
|
128
|
+
if (this.skipping) return
|
|
129
|
+
this.buffer += rest
|
|
130
|
+
if (this.buffer.length > this.maxLineChars) {
|
|
131
|
+
this.buffer = ''
|
|
132
|
+
// Count the record once, however many chunks it goes on to spill.
|
|
133
|
+
this.dropped++
|
|
134
|
+
this.skipping = true
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Emit any trailing unterminated record (see the class doc); call once, after the child exits. */
|
|
139
|
+
flush(): void {
|
|
140
|
+
const line = this.buffer.trim()
|
|
141
|
+
this.buffer = ''
|
|
142
|
+
if (line && !this.skipping) this.onLine(line, true)
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Records dropped for exceeding the line cap; 0 on every ordinary run. */
|
|
146
|
+
get droppedLines(): number {
|
|
147
|
+
return this.dropped
|
|
148
|
+
}
|
|
149
|
+
}
|