@cat-factory/executor-harness 1.60.0 → 1.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/dist/agent-runner.js +81 -8
- package/dist/agent.js +9 -3
- package/dist/claude-stream.js +18 -0
- package/dist/coding-agent.js +32 -4
- package/dist/embed.js +2 -1
- package/dist/git.js +0 -319
- package/dist/host-markdown.js +142 -0
- package/dist/pi-workspace.js +35 -2
- package/dist/pi.js +15 -184
- package/dist/pr-description.js +157 -0
- package/dist/progress-guard.js +211 -0
- package/dist/subagents.js +1 -50
- package/dist/vcs-api.js +402 -0
- package/package.json +4 -3
- package/src/agent-runner.ts +88 -8
- package/src/agent.ts +8 -3
- package/src/claude-stream.ts +19 -0
- package/src/coding-agent.ts +45 -3
- package/src/embed.ts +5 -3
- package/src/git.ts +1 -385
- package/src/host-markdown.ts +155 -0
- package/src/pi-workspace.ts +40 -4
- package/src/pi.ts +26 -252
- package/src/pr-description.ts +171 -0
- package/src/progress-guard.ts +285 -0
- package/src/subagents.ts +7 -16
- package/src/vcs-api.ts +512 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// The TEXT BOUNDARY for agent-authored text the harness writes onto a VCS host.
|
|
3
|
+
//
|
|
4
|
+
// A pull-request description is NOT an inert string sink. The host parses it: `#123` becomes
|
|
5
|
+
// an issue link, `@name` notifies a real person, a closing keyword in front of an issue
|
|
6
|
+
// reference CLOSES that issue when the PR merges, and an unbalanced code fence swallows
|
|
7
|
+
// everything rendered after it — including the fenced JSON block the engine later appends as
|
|
8
|
+
// the verification report's machine-readable contract.
|
|
9
|
+
//
|
|
10
|
+
// The agent's reviewer briefing (`pr-description.ts`) is model-authored prose that lands
|
|
11
|
+
// verbatim on that surface, so it crosses this boundary first. "This closes #42" is idiomatic
|
|
12
|
+
// for a briefing to emit and must not close issue 42; "@alice owns the rounding rule" is
|
|
13
|
+
// idiomatic and must not page whoever holds that handle.
|
|
14
|
+
//
|
|
15
|
+
// This is a deliberate COPY of `hostMarkdown` in `@cat-factory/kernel`
|
|
16
|
+
// (`src/shared/host-markdown.logic.ts`), for the same reason `isSafeTestPath` is copied: the
|
|
17
|
+
// container image is built from `src/` plus typescript alone (the Dockerfile cannot resolve a
|
|
18
|
+
// `workspace:*` dependency), so the harness carries no runtime dependency on any package here.
|
|
19
|
+
// `test/host-markdown.conformity.test.ts` pins the two implementations to byte-identical
|
|
20
|
+
// output over a shared corpus, so the copy cannot drift — change one, change the other.
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The host's closing keywords. A PR body carrying one of these in front of an issue reference
|
|
25
|
+
* closes that issue on merge — a side effect the harness must never trigger on the agent's
|
|
26
|
+
* behalf. Same list on GitHub and GitLab.
|
|
27
|
+
*/
|
|
28
|
+
const CLOSING_KEYWORDS =
|
|
29
|
+
'close[sd]?|closing|fix|fixe[sd]|fixing|resolve[sd]?|resolving|implement(?:s|ed)?|implementing'
|
|
30
|
+
|
|
31
|
+
/** An issue/MR URL on either host, in the form a closing keyword can reference. */
|
|
32
|
+
const ISSUE_URL = String.raw`https?://\S+?/(?:issues|-/issues|merge_requests|pull)/\d+`
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Every auto-linking trigger, in ONE alternation.
|
|
36
|
+
*
|
|
37
|
+
* Deliberately a single pass rather than chained `.replace()` calls: each escape EMITS a `#`,
|
|
38
|
+
* so a later rule would re-escape the output of an earlier one (`@` → `@` → `@`).
|
|
39
|
+
* One regex means the replacement text is never rescanned.
|
|
40
|
+
*/
|
|
41
|
+
const AUTO_LINK_TRIGGERS = new RegExp(
|
|
42
|
+
[
|
|
43
|
+
// A closing keyword in front of an issue/MR URL. The URL form survives the character
|
|
44
|
+
// escapes below (nothing in it is a trigger), so the KEYWORD is what gets defused.
|
|
45
|
+
String.raw`(?<keyword>\b(?:${CLOSING_KEYWORDS}))(?=\s*:?\s+${ISSUE_URL})`,
|
|
46
|
+
// `@name` / `@org/team` — a mention notifies a real account.
|
|
47
|
+
String.raw`(?<at>@(?=[A-Za-z0-9]))`,
|
|
48
|
+
// `#123` and `owner/repo#123` — an issue/PR cross-reference.
|
|
49
|
+
String.raw`(?<hash>#(?=\d))`,
|
|
50
|
+
// `!123` — GitLab's merge-request reference.
|
|
51
|
+
String.raw`(?<bang>!(?=\d))`,
|
|
52
|
+
].join('|'),
|
|
53
|
+
'gi',
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Neutralise the host's auto-linking triggers in ONE line of untrusted text, leaving inline
|
|
58
|
+
* code spans alone (the host does not auto-link inside them, so escaping there would only
|
|
59
|
+
* show the reader a literal `#`).
|
|
60
|
+
*
|
|
61
|
+
* The escapes are numeric HTML entities, which render as the original character but are
|
|
62
|
+
* invisible to the reference parser — so the reader sees exactly what the agent wrote while
|
|
63
|
+
* the mention/close side effects are defused.
|
|
64
|
+
*/
|
|
65
|
+
function inertLine(line: string): string {
|
|
66
|
+
return mapOutsideCodeSpans(line, (text) =>
|
|
67
|
+
text.replace(AUTO_LINK_TRIGGERS, (match, ...args) => {
|
|
68
|
+
const groups = args[args.length - 1] as Record<string, string | undefined>
|
|
69
|
+
// Entity-escaping the FIRST character is enough to break the parser's match while
|
|
70
|
+
// rendering identically — which matters most for the keyword, whose remaining letters
|
|
71
|
+
// are ordinary prose the reader should still see.
|
|
72
|
+
return `&#${match.charCodeAt(0)};${groups.keyword ? match.slice(1) : ''}`
|
|
73
|
+
}),
|
|
74
|
+
)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Apply `fn` to the parts of `line` that are NOT inline code spans. Code spans are matched by
|
|
79
|
+
* a backtick run and its matching closer, which is CommonMark's rule and — more to the point
|
|
80
|
+
* — the rule the host renderer applies when deciding where to auto-link.
|
|
81
|
+
*/
|
|
82
|
+
function mapOutsideCodeSpans(line: string, fn: (text: string) => string): string {
|
|
83
|
+
const out: string[] = []
|
|
84
|
+
let index = 0
|
|
85
|
+
const span = /(`+)[\s\S]*?\1/g
|
|
86
|
+
let match: RegExpExecArray | null
|
|
87
|
+
while ((match = span.exec(line)) !== null) {
|
|
88
|
+
out.push(fn(line.slice(index, match.index)), match[0])
|
|
89
|
+
index = match.index + match[0].length
|
|
90
|
+
}
|
|
91
|
+
return out.join('') + fn(line.slice(index))
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** A line that opens or closes a fenced code block, with the fence it uses. */
|
|
95
|
+
function fenceAt(line: string): { char: string; length: number; info: boolean } | null {
|
|
96
|
+
const match = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line)
|
|
97
|
+
if (!match) return null
|
|
98
|
+
const fence = match[1]!
|
|
99
|
+
// A ``` fence's info string may not contain a backtick (CommonMark), which is what stops an
|
|
100
|
+
// inline span from being read as a fence.
|
|
101
|
+
if (fence.startsWith('`') && match[2]!.includes('`')) return null
|
|
102
|
+
return { char: fence[0]!, length: fence.length, info: match[2]!.trim().length > 0 }
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Walk `lines`, tracking fenced-code state, and hand each line to `visit` together with
|
|
107
|
+
* whether it sits INSIDE a fenced block. Returns the fence still open at the end, if any.
|
|
108
|
+
*
|
|
109
|
+
* One shared walker so the three things that care about fences — leaving code untouched,
|
|
110
|
+
* closing what the text left open, and finding the briefing's title heading — can never
|
|
111
|
+
* disagree about where a block starts and ends.
|
|
112
|
+
*/
|
|
113
|
+
export function walkFences(
|
|
114
|
+
lines: readonly string[],
|
|
115
|
+
visit: (line: string, insideFence: boolean) => void,
|
|
116
|
+
): { char: string; length: number } | null {
|
|
117
|
+
let open: { char: string; length: number } | null = null
|
|
118
|
+
for (const line of lines) {
|
|
119
|
+
const fence = fenceAt(line)
|
|
120
|
+
// The fence line itself belongs to the code block, so it is never rewritten.
|
|
121
|
+
visit(line, open !== null || fence !== null)
|
|
122
|
+
if (!fence) continue
|
|
123
|
+
if (!open) open = { char: fence.char, length: fence.length }
|
|
124
|
+
else if (fence.char === open.char && fence.length >= open.length && !fence.info) open = null
|
|
125
|
+
}
|
|
126
|
+
return open
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Render untrusted multi-line markdown safe to send to a host: auto-link triggers defused
|
|
131
|
+
* outside fenced code, and any fence the text leaves open closed again.
|
|
132
|
+
*
|
|
133
|
+
* Unlike kernel's `hostMarkdown.prose` this does NOT cap the length — the caller
|
|
134
|
+
* ({@link import('./pr-description.js')}) applies its own budget with its own visible note
|
|
135
|
+
* BEFORE calling here, so an escape entity can never be sliced in half. With that one
|
|
136
|
+
* difference the output is identical, which the conformity test pins.
|
|
137
|
+
*/
|
|
138
|
+
export function inertMarkdown(text: string): string {
|
|
139
|
+
const normalised = text.replace(/\r\n?/g, '\n')
|
|
140
|
+
const rewritten: string[] = []
|
|
141
|
+
const open = walkFences(normalised.split('\n'), (line, insideFence) => {
|
|
142
|
+
rewritten.push(insideFence ? line : inertLine(line))
|
|
143
|
+
})
|
|
144
|
+
const joined = rewritten.join('\n')
|
|
145
|
+
return open ? `${joined}\n${open.char.repeat(open.length)}` : joined
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Render untrusted text INLINE (a pull-request title): newlines folded to spaces because the
|
|
150
|
+
* surrounding line has its own meaning, and auto-link triggers defused. The caller caps the
|
|
151
|
+
* length first, for the same reason as {@link inertMarkdown}.
|
|
152
|
+
*/
|
|
153
|
+
export function inertInline(text: string): string {
|
|
154
|
+
return inertLine(text.replace(/\s+/g, ' '))
|
|
155
|
+
}
|
package/src/pi-workspace.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { mkdir, mkdtemp, rm } from 'node:fs/promises'
|
|
1
|
+
import { mkdir, mkdtemp, readdir, rm, stat } from 'node:fs/promises'
|
|
2
2
|
import { tmpdir } from 'node:os'
|
|
3
3
|
import { join } from 'node:path'
|
|
4
4
|
import type { RepoSpec, SkillSpec } from './job.js'
|
|
@@ -8,13 +8,10 @@ import {
|
|
|
8
8
|
type ContextFileInfo,
|
|
9
9
|
type PiRunOutcome,
|
|
10
10
|
type PiRunStats,
|
|
11
|
-
type ProgressGuardLimits,
|
|
12
11
|
type RunDiagnostics,
|
|
13
12
|
CONTEXT_DIR,
|
|
14
13
|
materializeContextFiles,
|
|
15
14
|
materializeSkillResources,
|
|
16
|
-
mergeGuardLimits,
|
|
17
|
-
progressGuardLimitsFromEnv,
|
|
18
15
|
runPi,
|
|
19
16
|
webSearchConfigFromEnv,
|
|
20
17
|
webSearchProxyEnv,
|
|
@@ -22,6 +19,11 @@ import {
|
|
|
22
19
|
writePiModelsConfig,
|
|
23
20
|
writeWebToolsConfig,
|
|
24
21
|
} from './pi.js'
|
|
22
|
+
import {
|
|
23
|
+
type ProgressGuardLimits,
|
|
24
|
+
mergeGuardLimits,
|
|
25
|
+
progressGuardLimitsFromEnv,
|
|
26
|
+
} from './progress-guard.js'
|
|
25
27
|
import type { RunOptions } from './runner.js'
|
|
26
28
|
import { type SubscriptionHarness, runSubscriptionHarness } from './agent-runner.js'
|
|
27
29
|
|
|
@@ -226,6 +228,31 @@ export interface AgentRunSpec {
|
|
|
226
228
|
multiRepo?: boolean
|
|
227
229
|
}
|
|
228
230
|
|
|
231
|
+
/**
|
|
232
|
+
* Whether the run's checkout actually ships a `blueprints/` folder — what gates the blueprint
|
|
233
|
+
* orientation note in AGENTS.md (an external repo has none, so the note would be ~10 lines of
|
|
234
|
+
* dead guidance pointing at files that don't exist, re-sent on every turn).
|
|
235
|
+
*
|
|
236
|
+
* A MULTI-REPO run's `dir` is the workspace ROOT with each repo checked out as a sibling under
|
|
237
|
+
* it, so the root itself never holds `blueprints/`: the legs are checked too, and the note is
|
|
238
|
+
* included when ANY leg ships one (it orients the agent to the concept, and the agent finds the
|
|
239
|
+
* per-repo folder from there). Best-effort throughout — any stat/readdir failure simply omits
|
|
240
|
+
* the note rather than failing the dispatch.
|
|
241
|
+
*/
|
|
242
|
+
export async function checkoutHasBlueprints(dir: string, multiRepo: boolean): Promise<boolean> {
|
|
243
|
+
const isBlueprintDir = (path: string): Promise<boolean> =>
|
|
244
|
+
stat(join(path, 'blueprints'))
|
|
245
|
+
.then((s) => s.isDirectory())
|
|
246
|
+
.catch(() => false)
|
|
247
|
+
if (await isBlueprintDir(dir)) return true
|
|
248
|
+
if (!multiRepo) return false
|
|
249
|
+
const legs = await readdir(dir, { withFileTypes: true }).catch(() => [])
|
|
250
|
+
const checks = await Promise.all(
|
|
251
|
+
legs.filter((e) => e.isDirectory()).map((e) => isBlueprintDir(join(dir, e.name))),
|
|
252
|
+
)
|
|
253
|
+
return checks.some(Boolean)
|
|
254
|
+
}
|
|
255
|
+
|
|
229
256
|
/**
|
|
230
257
|
* Write Pi's global agent context (`~/.pi/agent/AGENTS.md`) + provider config,
|
|
231
258
|
* then run Pi once in `spec.dir` and return its summary/stats/stderr. The context
|
|
@@ -272,6 +299,13 @@ export async function runAgentInWorkspace(
|
|
|
272
299
|
...(spec.skill ? { skill: spec.skill } : {}),
|
|
273
300
|
...(opts.agentEnv ? { extraEnv: opts.agentEnv } : {}),
|
|
274
301
|
signal: opts.signal,
|
|
302
|
+
// Run the SAME no-progress guard Pi gets (previously claude-code/codex had none): env
|
|
303
|
+
// defaults merged loosen-only with the kind's tuning + the backend's complexity-scaled
|
|
304
|
+
// no-edit allowance, so a claude-code run that stops making progress is killed early
|
|
305
|
+
// instead of burning the full wall-clock budget. The claude runner consumes it; codex
|
|
306
|
+
// ignores it for now (its stream isn't wired to the guard).
|
|
307
|
+
guardLimits: mergeGuardLimits(progressGuardLimitsFromEnv(), spec.guardLimits),
|
|
308
|
+
expectsEdits: spec.expectsEdits ?? true,
|
|
275
309
|
onActivity: opts.onActivity,
|
|
276
310
|
onProgress: opts.onProgress,
|
|
277
311
|
// Stream this run's per-call telemetry to the job's live drain. The subscription
|
|
@@ -303,11 +337,13 @@ export async function runAgentInWorkspace(
|
|
|
303
337
|
}
|
|
304
338
|
const webSearch = webSearchConfigFromEnv({ ...process.env, ...extraEnv })
|
|
305
339
|
if (webSearch) await writeWebToolsConfig(webSearch)
|
|
340
|
+
const hasBlueprints = await checkoutHasBlueprints(spec.dir, spec.multiRepo === true)
|
|
306
341
|
await writeAgentsContext(spec.systemPrompt, {
|
|
307
342
|
webSearch: Boolean(webSearch),
|
|
308
343
|
guidance: spec.webToolsGuidance,
|
|
309
344
|
serviceDirectory: spec.serviceDirectory,
|
|
310
345
|
contextFiles,
|
|
346
|
+
hasBlueprints,
|
|
311
347
|
...(spec.multiRepo ? { multiRepo: true } : {}),
|
|
312
348
|
})
|
|
313
349
|
await writePiModelsConfig({ model: spec.model, proxyBaseUrl })
|
package/src/pi.ts
CHANGED
|
@@ -8,6 +8,12 @@ import { redactSecrets } from './redact.js'
|
|
|
8
8
|
import { HarnessFailure } from './failure.js'
|
|
9
9
|
import { log } from './logger.js'
|
|
10
10
|
import type { EffortReport } from './effort.js'
|
|
11
|
+
import {
|
|
12
|
+
ProgressGuard,
|
|
13
|
+
progressGuardLimitsFromEnv,
|
|
14
|
+
toolCallSignal,
|
|
15
|
+
type ProgressGuardLimits,
|
|
16
|
+
} from './progress-guard.js'
|
|
11
17
|
|
|
12
18
|
// Drives the Pi coding-agent CLI. Pi is pointed at the Worker's OpenAI-compatible
|
|
13
19
|
// proxy via a custom provider in ~/.pi/agent/models.json, authenticated with the
|
|
@@ -106,21 +112,12 @@ for a module that is directly relevant to your task, when you need its summary a
|
|
|
106
112
|
exact code references. \`blueprints/version.json\` is a tiny manifest for quick
|
|
107
113
|
staleness checks. Treat the blueprint as orientation, not a task list.`
|
|
108
114
|
|
|
109
|
-
//
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
If a \`spec/\` folder exists, it is the specification for this service. It is sharded
|
|
117
|
-
by a module (domain) → feature (group) taxonomy. **Read \`spec/overview.md\` first** —
|
|
118
|
-
it states what MUST be true and indexes the modules and their features (with links).
|
|
119
|
-
Open \`spec/modules/<module>/<feature>.md\` (or its \`.json\` for exact detail) for the
|
|
120
|
-
feature you are working on — it carries that feature's requirements AND the domain
|
|
121
|
-
rules scoped to it. \`spec/features/<module>/<feature>.feature\` are the Gherkin
|
|
122
|
-
acceptance scenarios your work must satisfy — treat them as the source of truth for
|
|
123
|
-
behaviour and tests. Read only the modules/features relevant to your task.`
|
|
115
|
+
// NOTE: the spec-reading guidance is NOT appended here. It is contributed once, backend-side, by
|
|
116
|
+
// the `spec-aware` trait (`SPEC_AWARE_GUIDANCE` in @cat-factory/agents), which lands in the
|
|
117
|
+
// composed system prompt for every spec-aware kind on BOTH harness paths. This harness used to
|
|
118
|
+
// append a near-duplicate block, so a spec-aware Pi run carried the guidance twice; the claude-code
|
|
119
|
+
// path never appended it. Sourcing it solely from the trait dedupes the Pi prompt and makes the two
|
|
120
|
+
// paths consistent. (A non-spec-aware kind is deliberately not told to read the spec.)
|
|
124
121
|
|
|
125
122
|
/**
|
|
126
123
|
* Write the composed system prompt as Pi's GLOBAL agent context
|
|
@@ -144,6 +141,12 @@ export async function writeAgentsContext(
|
|
|
144
141
|
serviceDirectory?: string
|
|
145
142
|
contextFiles?: ContextFileInfo[]
|
|
146
143
|
multiRepo?: boolean
|
|
144
|
+
/**
|
|
145
|
+
* Whether the checkout actually ships a `blueprints/` folder. The blueprint orientation
|
|
146
|
+
* note is only appended when it does — otherwise it is ~10 lines of dead guidance (re-sent
|
|
147
|
+
* every turn) pointing at files that don't exist. Absent/false ⇒ the note is omitted.
|
|
148
|
+
*/
|
|
149
|
+
hasBlueprints?: boolean
|
|
147
150
|
} = {},
|
|
148
151
|
): Promise<void> {
|
|
149
152
|
const dir = join(homedir(), '.pi', 'agent')
|
|
@@ -167,9 +170,14 @@ export async function writeAgentsContext(
|
|
|
167
170
|
// Point the agent at any linked context the backend materialised into the checkout
|
|
168
171
|
// (requirements / RFCs / PRDs / tracker issues) so it reads them on demand.
|
|
169
172
|
const context = contextGuidance(opts.contextFiles ?? [])
|
|
173
|
+
// Only orient the agent to `blueprints/` when the checkout actually has them — otherwise the
|
|
174
|
+
// note is dead weight re-sent on every turn. The spec-reading guidance is NOT appended here
|
|
175
|
+
// (see the note above `writeAgentsContext`): it comes solely from the backend `spec-aware`
|
|
176
|
+
// trait, so a spec-aware run no longer carries it twice.
|
|
177
|
+
const blueprint = opts.hasBlueprints ? BLUEPRINT_GUIDANCE : ''
|
|
170
178
|
await writeFile(
|
|
171
179
|
join(dir, 'AGENTS.md'),
|
|
172
|
-
`${systemPrompt}${
|
|
180
|
+
`${systemPrompt}${blueprint}${TODO_GUIDANCE}${monorepo}${multiRepo}${webTools}${context}`,
|
|
173
181
|
'utf8',
|
|
174
182
|
)
|
|
175
183
|
}
|
|
@@ -727,242 +735,8 @@ export function parseTodoProgress(event: Record<string, unknown>): TodoProgress
|
|
|
727
735
|
|
|
728
736
|
return undefined
|
|
729
737
|
}
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
function toolCallSignal(
|
|
733
|
-
event: Record<string, unknown>,
|
|
734
|
-
): { name: string; isError: boolean } | undefined {
|
|
735
|
-
// `tool_execution_end` is the canonical per-call stream event (statsFromEvents
|
|
736
|
-
// counts the same one), so the guard reads it and nothing else — no double count.
|
|
737
|
-
if (event.type !== 'tool_execution_end') return undefined
|
|
738
|
-
const name = typeof event.toolName === 'string' ? event.toolName : ''
|
|
739
|
-
return { name, isError: event.isError === true }
|
|
740
|
-
}
|
|
741
|
-
|
|
742
|
-
/** Tunable bounds for the {@link ProgressGuard}. */
|
|
743
|
-
export interface ProgressGuardLimits {
|
|
744
|
-
/**
|
|
745
|
-
* Abort once the agent has made this many NON-exploration tool calls without ever
|
|
746
|
-
* using a file-editing tool (see `FILE_EDIT_TOOLS`). The signature of the credential
|
|
747
|
-
* rabbit-hole that motivated this: probing the environment (`bash`/exec) endlessly
|
|
748
|
-
* without implementing anything. Read-only exploration (`read`/`grep`/… — see
|
|
749
|
-
* `EXPLORATION_TOOLS`) and planning (`todo`) do NOT count, so a large task that
|
|
750
|
-
* legitimately reads/searches many files before its first edit is not killed for it.
|
|
751
|
-
* Disabled when `expectsEdits` is false (e.g. the assess-only merger / Blueprinter,
|
|
752
|
-
* which legitimately edit nothing). Note this bound only guards the run UNTIL its
|
|
753
|
-
* first edit: once the agent has edited a file at all, it has demonstrably started
|
|
754
|
-
* the work, so only `maxConsecutiveErrors` guards a later stall.
|
|
755
|
-
*/
|
|
756
|
-
maxToolCallsWithoutEdit: number
|
|
757
|
-
/**
|
|
758
|
-
* Abort after this many consecutive failing tool calls — the agent is stuck
|
|
759
|
-
* retrying an operation that keeps failing rather than making progress.
|
|
760
|
-
*/
|
|
761
|
-
maxConsecutiveErrors: number
|
|
762
|
-
/**
|
|
763
|
-
* Abort after this many consecutive web-search/web-fetch calls with no other tool
|
|
764
|
-
* call in between. Web tools are read-only exploration (they don't count toward the
|
|
765
|
-
* no-edit bound), so without this a model could rabbit-hole on searches indefinitely
|
|
766
|
-
* without ever tripping a guard. Any non-web tool call resets the streak. Optional:
|
|
767
|
-
* defaults to {@link DEFAULT_PROGRESS_GUARD_LIMITS} when a caller builds limits
|
|
768
|
-
* without it.
|
|
769
|
-
*/
|
|
770
|
-
maxConsecutiveWebCalls?: number
|
|
771
|
-
}
|
|
772
|
-
|
|
773
|
-
// `satisfies` (not a type annotation) so each property keeps its concrete `number`
|
|
774
|
-
// type — `maxConsecutiveWebCalls` is optional on the interface (callers may omit it),
|
|
775
|
-
// but the defaults always define it, so consumers reading it off here get a `number`.
|
|
776
|
-
export const DEFAULT_PROGRESS_GUARD_LIMITS = {
|
|
777
|
-
// Counts only non-exploration, non-planning calls (see EXPLORATION_TOOLS), so the
|
|
778
|
-
// ceiling can be generous without risking a false kill on a read-heavy large task.
|
|
779
|
-
maxToolCallsWithoutEdit: 40,
|
|
780
|
-
maxConsecutiveErrors: 12,
|
|
781
|
-
// A genuine research burst is a handful of searches; an uninterrupted run of this
|
|
782
|
-
// many web calls (with no read/edit/bash between) is a search loop, not progress.
|
|
783
|
-
maxConsecutiveWebCalls: 25,
|
|
784
|
-
} satisfies ProgressGuardLimits
|
|
785
|
-
|
|
786
|
-
// Tool names that mutate files, so a call to one clears the no-edit suspicion. Kept
|
|
787
|
-
// broad on purpose: different models/extensions name the same capability differently
|
|
788
|
-
// (`edit`/`write`, but also `apply_patch`/`patch`/`str_replace`/`multiedit`/`create`),
|
|
789
|
-
// and a false "no edits" reading would kill a run that IS making changes. Matched
|
|
790
|
-
// case-insensitively. NOTE: a file written purely via `bash` (e.g. a heredoc) is not
|
|
791
|
-
// recognised here — broaden or move to a working-tree signal if that becomes common.
|
|
792
|
-
const FILE_EDIT_TOOLS = new Set([
|
|
793
|
-
'edit',
|
|
794
|
-
'write',
|
|
795
|
-
'apply_patch',
|
|
796
|
-
'patch',
|
|
797
|
-
'str_replace',
|
|
798
|
-
'multiedit',
|
|
799
|
-
'create',
|
|
800
|
-
])
|
|
801
|
-
|
|
802
|
-
// Planning/bookkeeping tools that are neither file edits nor the environment-probing
|
|
803
|
-
// the no-edit bound targets — the todo list the agent maintains as it works. These do
|
|
804
|
-
// NOT count toward `maxToolCallsWithoutEdit`: a run that diligently updates a long
|
|
805
|
-
// todo list before its first edit (common on a large task) would otherwise be killed
|
|
806
|
-
// for "no edits" purely from planning calls. They still reset the consecutive-error
|
|
807
|
-
// streak (a successful call means the agent isn't wedged). Matched case-insensitively.
|
|
808
|
-
const PLANNING_TOOLS = new Set(['todo'])
|
|
809
|
-
|
|
810
|
-
// Read-only exploration tools: reading/searching the repo is legitimate work-up to an
|
|
811
|
-
// edit, NOT the environment-probing the no-edit bound targets, so they don't count
|
|
812
|
-
// toward `maxToolCallsWithoutEdit` (a large task may read/search dozens of files
|
|
813
|
-
// before its first edit). The bound thus counts only "action" calls — chiefly `bash`
|
|
814
|
-
// (the credential rabbit-hole's vector) — that have yet to produce an edit. Kept broad
|
|
815
|
-
// since models/extensions name the same capability differently. Matched case-insensitively.
|
|
816
|
-
const EXPLORATION_TOOLS = new Set([
|
|
817
|
-
'read',
|
|
818
|
-
'grep',
|
|
819
|
-
'search',
|
|
820
|
-
'glob',
|
|
821
|
-
'ls',
|
|
822
|
-
'list',
|
|
823
|
-
'find',
|
|
824
|
-
'tree',
|
|
825
|
-
'cat',
|
|
826
|
-
'view',
|
|
827
|
-
'head',
|
|
828
|
-
'tail',
|
|
829
|
-
'stat',
|
|
830
|
-
// rpiv-web-tools: querying/reading the web is read-only research up to an edit,
|
|
831
|
-
// not the environment-probing the no-edit bound targets, so it doesn't count.
|
|
832
|
-
'web_search',
|
|
833
|
-
'web_fetch',
|
|
834
|
-
])
|
|
835
|
-
|
|
836
|
-
// The rpiv-web-tools calls, tracked separately so an unbounded run of them (with no
|
|
837
|
-
// other tool call between) can be caught as a search loop — see `maxConsecutiveWebCalls`.
|
|
838
|
-
const WEB_TOOLS = new Set(['web_search', 'web_fetch'])
|
|
839
|
-
|
|
840
|
-
/** Read {@link ProgressGuardLimits} from the environment, falling back to the defaults. */
|
|
841
|
-
export function progressGuardLimitsFromEnv(
|
|
842
|
-
env: NodeJS.ProcessEnv = process.env,
|
|
843
|
-
): ProgressGuardLimits {
|
|
844
|
-
const num = (raw: string | undefined, fallback: number): number => {
|
|
845
|
-
const n = Number(raw)
|
|
846
|
-
return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback
|
|
847
|
-
}
|
|
848
|
-
return {
|
|
849
|
-
maxToolCallsWithoutEdit: num(
|
|
850
|
-
env.JOB_MAX_TOOLCALLS_WITHOUT_EDIT,
|
|
851
|
-
DEFAULT_PROGRESS_GUARD_LIMITS.maxToolCallsWithoutEdit,
|
|
852
|
-
),
|
|
853
|
-
maxConsecutiveErrors: num(
|
|
854
|
-
env.JOB_MAX_CONSECUTIVE_TOOL_ERRORS,
|
|
855
|
-
DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveErrors,
|
|
856
|
-
),
|
|
857
|
-
maxConsecutiveWebCalls: num(
|
|
858
|
-
env.JOB_MAX_CONSECUTIVE_WEB_CALLS,
|
|
859
|
-
DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveWebCalls,
|
|
860
|
-
),
|
|
861
|
-
}
|
|
862
|
-
}
|
|
863
|
-
|
|
864
|
-
/**
|
|
865
|
-
* Apply per-knob overrides onto a base set of guard limits, ENFORCING loosen-only: an
|
|
866
|
-
* override can only RAISE a knob (more headroom), never lower it below the base. A
|
|
867
|
-
* larger value is more lenient for every knob (more no-edit tool calls / errors / web
|
|
868
|
-
* calls tolerated), so each result is `max(base, override)`. This is a hard guarantee,
|
|
869
|
-
* not a convention — a tuning entry (built-in or a custom kind's, which reaches this via
|
|
870
|
-
* an untrusted job body) that supplies a value TIGHTER than the base is clamped back up
|
|
871
|
-
* to the base rather than aborting a legitimately-progressing run. An absent/undefined
|
|
872
|
-
* knob keeps the base value untouched.
|
|
873
|
-
*/
|
|
874
|
-
export function mergeGuardLimits(
|
|
875
|
-
base: ProgressGuardLimits,
|
|
876
|
-
overrides: Partial<ProgressGuardLimits> | undefined,
|
|
877
|
-
): ProgressGuardLimits {
|
|
878
|
-
if (!overrides) return base
|
|
879
|
-
const loosen = (b: number, o: number | undefined): number =>
|
|
880
|
-
typeof o === 'number' ? Math.max(b, o) : b
|
|
881
|
-
return {
|
|
882
|
-
maxToolCallsWithoutEdit: loosen(
|
|
883
|
-
base.maxToolCallsWithoutEdit,
|
|
884
|
-
overrides.maxToolCallsWithoutEdit,
|
|
885
|
-
),
|
|
886
|
-
maxConsecutiveErrors: loosen(base.maxConsecutiveErrors, overrides.maxConsecutiveErrors),
|
|
887
|
-
// `maxConsecutiveWebCalls` is optional on the interface (callers may omit it), so
|
|
888
|
-
// fall back to the default before loosening — keeps `loosen`'s base a concrete number.
|
|
889
|
-
maxConsecutiveWebCalls: loosen(
|
|
890
|
-
base.maxConsecutiveWebCalls ?? DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveWebCalls,
|
|
891
|
-
overrides.maxConsecutiveWebCalls,
|
|
892
|
-
),
|
|
893
|
-
}
|
|
894
|
-
}
|
|
895
|
-
|
|
896
|
-
/**
|
|
897
|
-
* Live anti-rabbithole guard: fed each streamed Pi event, it returns a diagnostic
|
|
898
|
-
* reason the moment a run has plainly stopped making progress, so the harness can
|
|
899
|
-
* kill Pi early instead of letting it burn the whole budget (and then surface a
|
|
900
|
-
* useful failure instead of a generic "no file changes"). Pure and incremental so
|
|
901
|
-
* it can be unit-tested over a fixed event sequence.
|
|
902
|
-
*/
|
|
903
|
-
export class ProgressGuard {
|
|
904
|
-
private toolCalls = 0
|
|
905
|
-
private edits = 0
|
|
906
|
-
private consecutiveErrors = 0
|
|
907
|
-
private consecutiveWebCalls = 0
|
|
908
|
-
|
|
909
|
-
constructor(
|
|
910
|
-
private readonly limits: ProgressGuardLimits,
|
|
911
|
-
/** When false (assess-only runs like the merger), the no-edit bound is skipped. */
|
|
912
|
-
private readonly expectsEdits: boolean = true,
|
|
913
|
-
) {}
|
|
914
|
-
|
|
915
|
-
/** Feed one parsed Pi event; returns a diagnostic reason when the run should abort, else null. */
|
|
916
|
-
observe(event: Record<string, unknown>): string | null {
|
|
917
|
-
const tool = toolCallSignal(event)
|
|
918
|
-
if (!tool) return null
|
|
919
|
-
const name = tool.name.toLowerCase()
|
|
920
|
-
// The error streak tracks ANY tool call (a planning call still proves the agent
|
|
921
|
-
// isn't wedged in a failing-op loop), so it's updated before the planning skip.
|
|
922
|
-
this.consecutiveErrors = tool.isError ? this.consecutiveErrors + 1 : 0
|
|
923
|
-
if (this.consecutiveErrors >= this.limits.maxConsecutiveErrors) {
|
|
924
|
-
return (
|
|
925
|
-
`no progress: ${this.consecutiveErrors} consecutive failing tool calls — the agent is stuck ` +
|
|
926
|
-
`retrying a failing operation rather than making progress. Aborting.`
|
|
927
|
-
)
|
|
928
|
-
}
|
|
929
|
-
|
|
930
|
-
// Web search/fetch loop: web tools are read-only (they don't count toward the
|
|
931
|
-
// no-edit bound), so guard them separately — an uninterrupted streak of them is a
|
|
932
|
-
// research rabbit-hole. Any non-web tool call resets the streak.
|
|
933
|
-
if (WEB_TOOLS.has(name)) {
|
|
934
|
-
this.consecutiveWebCalls++
|
|
935
|
-
const webCap =
|
|
936
|
-
this.limits.maxConsecutiveWebCalls ?? DEFAULT_PROGRESS_GUARD_LIMITS.maxConsecutiveWebCalls
|
|
937
|
-
if (this.consecutiveWebCalls >= webCap) {
|
|
938
|
-
return (
|
|
939
|
-
`no progress: ${this.consecutiveWebCalls} consecutive web search/fetch calls without ` +
|
|
940
|
-
`any other action — the agent is stuck researching instead of doing the work. Aborting.`
|
|
941
|
-
)
|
|
942
|
-
}
|
|
943
|
-
} else {
|
|
944
|
-
this.consecutiveWebCalls = 0
|
|
945
|
-
}
|
|
946
|
-
|
|
947
|
-
// Planning and read-only exploration calls don't count toward the no-edit bound
|
|
948
|
-
// (see PLANNING_TOOLS / EXPLORATION_TOOLS) — only "action" calls without an edit do.
|
|
949
|
-
if (PLANNING_TOOLS.has(name) || EXPLORATION_TOOLS.has(name)) return null
|
|
950
|
-
this.toolCalls++
|
|
951
|
-
if (FILE_EDIT_TOOLS.has(name)) this.edits++
|
|
952
|
-
|
|
953
|
-
if (
|
|
954
|
-
this.expectsEdits &&
|
|
955
|
-
this.edits === 0 &&
|
|
956
|
-
this.toolCalls >= this.limits.maxToolCallsWithoutEdit
|
|
957
|
-
) {
|
|
958
|
-
return (
|
|
959
|
-
`no progress: ${this.toolCalls} tool calls and not one file edit — the agent is exploring or ` +
|
|
960
|
-
`probing the environment without implementing anything. Aborting before it burns the whole run.`
|
|
961
|
-
)
|
|
962
|
-
}
|
|
963
|
-
return null
|
|
964
|
-
}
|
|
965
|
-
}
|
|
738
|
+
// The no-progress guard (its limits, tool vocabulary and the `ProgressGuard` itself) lives in
|
|
739
|
+
// `progress-guard.ts` — it is shared with the claude-code runner, so it is no longer Pi's.
|
|
966
740
|
|
|
967
741
|
/**
|
|
968
742
|
* Run Pi non-interactively against `cwd` and return its assistant summary. Uses
|