@cat-factory/executor-harness 1.96.0 → 1.98.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/dist/agent-runner.d.ts +14 -1
- package/dist/agent-runner.js +84 -28
- package/dist/bootstrap-mode.js +1 -0
- package/dist/coding-agent.d.ts +2 -1
- package/dist/embed.d.ts +2 -1
- package/dist/embed.js +2 -1
- package/dist/failure.d.ts +19 -1
- package/dist/failure.js +40 -0
- package/dist/git.d.ts +6 -0
- package/dist/git.js +16 -9
- package/dist/inline.d.ts +6 -0
- package/dist/inline.js +6 -0
- package/dist/job.d.ts +2 -1
- package/dist/jsonl-stream.d.ts +70 -0
- package/dist/jsonl-stream.js +149 -0
- package/dist/pi-reduction.d.ts +136 -0
- package/dist/pi-reduction.js +303 -0
- package/dist/pi-workspace.d.ts +2 -1
- package/dist/pi-workspace.js +6 -1
- package/dist/pi.d.ts +8 -81
- package/dist/pi.js +124 -310
- package/dist/runner.d.ts +31 -0
- package/dist/runner.js +50 -3
- package/dist/structured-output.js +2 -1
- package/dist/tool-silence.d.ts +74 -0
- package/dist/tool-silence.js +99 -0
- package/package.json +4 -4
- package/src/agent-runner.ts +100 -30
- package/src/agent.ts +1 -1
- package/src/bootstrap-mode.ts +2 -1
- package/src/coding-agent.ts +2 -1
- package/src/embed.ts +8 -5
- package/src/failure.ts +36 -9
- package/src/git.ts +17 -9
- package/src/inline.ts +6 -0
- package/src/job.ts +2 -1
- package/src/jsonl-stream.ts +149 -0
- package/src/pi-reduction.ts +359 -0
- package/src/pi-workspace.ts +7 -3
- package/src/pi.ts +144 -349
- package/src/runner.ts +91 -4
- package/src/structured-output.ts +2 -1
- package/src/tool-silence.ts +125 -0
package/dist/pi.d.ts
CHANGED
|
@@ -1,17 +1,7 @@
|
|
|
1
1
|
import type { EffortReport } from './effort.js';
|
|
2
2
|
import { type ProgressGuardLimits } from './progress-guard.js';
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
* Generous on purpose: a reasoning model (e.g. GLM-5.2) spends tokens on its
|
|
6
|
-
* `<think>` trace before the answer + tool calls, so a tight cap truncates it
|
|
7
|
-
* mid-reasoning and the agent never commits edits. It is a ceiling, not a target
|
|
8
|
-
* — unused output tokens are not billed and Workers AI clamps the request to the
|
|
9
|
-
* model's real max — so erring high is safe. Raised to 32k after a spec-writer run
|
|
10
|
-
* truncated an intermediate tool call at the old 16k cap; the document itself
|
|
11
|
-
* stopped well under it, so this is headroom for larger specs/diffs, with
|
|
12
|
-
* {@link runDiagnostics} flagging the rare case where even 32k is not enough.
|
|
13
|
-
*/
|
|
14
|
-
export declare const PI_MAX_OUTPUT_TOKENS = 32768;
|
|
3
|
+
import { type PiRunStats, type RunDiagnostics } from './pi-reduction.js';
|
|
4
|
+
import { type ToolProgressWindow } from './tool-silence.js';
|
|
15
5
|
/**
|
|
16
6
|
* Normalise a phase label to what the backend will actually store: trimmed, lowercased,
|
|
17
7
|
* `[a-z0-9-]` only, bounded. `''` when the label is not a phase at all.
|
|
@@ -221,37 +211,6 @@ export interface ToolSpan {
|
|
|
221
211
|
argsDropped: number;
|
|
222
212
|
resultDropped: number;
|
|
223
213
|
}
|
|
224
|
-
/**
|
|
225
|
-
* What the agent actually did this run, independent of any file changes. Used to
|
|
226
|
-
* tell a genuine no-op (the agent never reached the model / never acted) apart
|
|
227
|
-
* from a real run, so a bootstrap that produced nothing is failed rather than
|
|
228
|
-
* pushed as an empty repo. `toolCalls === 0 && assistantChars === 0` is the
|
|
229
|
-
* signature of a run where Pi never made a successful model call.
|
|
230
|
-
*/
|
|
231
|
-
export interface PiRunStats {
|
|
232
|
-
/** Tool calls the assistant emitted across the transcript (0 ⇒ it never acted). */
|
|
233
|
-
toolCalls: number;
|
|
234
|
-
/** Total characters of assistant text (0 ⇒ the model produced nothing). */
|
|
235
|
-
assistantChars: number;
|
|
236
|
-
}
|
|
237
|
-
/**
|
|
238
|
-
* Output-quality signals lifted from the agent's transcript, so the harness can fail
|
|
239
|
-
* LOUDLY on a malformed run instead of silently handing a half-baked artifact to the
|
|
240
|
-
* structured-output repair (which would manufacture a doc from garbage — the trap
|
|
241
|
-
* behind the spec-writer ⇄ companion rework loop). Two distinct invalid states, both
|
|
242
|
-
* seen in production from `kimi-k2.7-code`:
|
|
243
|
-
* - a completion that hit the output ceiling (its answer/tool call was cut off), and
|
|
244
|
-
* - a FINAL turn that carried no text at all (an empty `content: []` despite spending
|
|
245
|
-
* output tokens), so there is no answer to parse.
|
|
246
|
-
*/
|
|
247
|
-
export interface RunDiagnostics {
|
|
248
|
-
/** Some completion ended at the output-token ceiling — its content was cut off. */
|
|
249
|
-
truncated: boolean;
|
|
250
|
-
/** The agent's FINAL completion hit the ceiling: its ANSWER (not a mid-run step) was cut off. */
|
|
251
|
-
finalTruncated: boolean;
|
|
252
|
-
/** The agent's final turn carried no text content (e.g. an empty `content: []`). */
|
|
253
|
-
finalAnswerEmpty: boolean;
|
|
254
|
-
}
|
|
255
214
|
/**
|
|
256
215
|
* One model call captured from a subscription harness's CLI event stream, shaped so
|
|
257
216
|
* the backend can record it into the same `llm_call_metrics` telemetry the LLM proxy
|
|
@@ -426,6 +385,12 @@ export declare function runPi(opts: {
|
|
|
426
385
|
* the container payload doesn't pass it, so production behaviour is unchanged.
|
|
427
386
|
*/
|
|
428
387
|
onSpan?: (span: ToolSpan) => void;
|
|
388
|
+
/**
|
|
389
|
+
* Opens this stream's tool-silence window (see `RunOptions.beginToolWindow`), closed when Pi
|
|
390
|
+
* exits. Pi reports every completed tool call, so the window it opens is one this run can
|
|
391
|
+
* always beat; a caller that passes nothing leaves the watchdog silent for the run.
|
|
392
|
+
*/
|
|
393
|
+
beginToolWindow?: () => ToolProgressWindow;
|
|
429
394
|
/**
|
|
430
395
|
* Called with every parsed Pi `--mode json` event, in stream order — the raw
|
|
431
396
|
* observability seam over the run. Used by offline tooling (the smoketest
|
|
@@ -446,16 +411,6 @@ export declare function runPi(opts: {
|
|
|
446
411
|
*/
|
|
447
412
|
extraEnv?: Record<string, string>;
|
|
448
413
|
}): Promise<PiRunOutcome>;
|
|
449
|
-
/**
|
|
450
|
-
* The terminal-failure message when Pi's run ended in a hard error (the model was
|
|
451
|
-
* unreachable / refused, and Pi exhausted its auto-retries), else undefined. Only
|
|
452
|
-
* the FINAL outcome counts: a mid-run hiccup the agent recovered from leaves a clean
|
|
453
|
-
* terminal `agent_end`, so it returns undefined. Scans from the end and decides on
|
|
454
|
-
* the first terminal signal it meets — the trailing `auto_retry_end` (its `success`
|
|
455
|
-
* flag) or the last `agent_end` (its `stopReason`). Pure so it is unit-testable over
|
|
456
|
-
* a fixed event sequence.
|
|
457
|
-
*/
|
|
458
|
-
export declare function terminalRunError(stdout: string): string | undefined;
|
|
459
414
|
/**
|
|
460
415
|
* Classify a terminal run error whose text points at the LLM PROXY rejecting every model call
|
|
461
416
|
* (auth / quota / rate-limit) into an actionable remedy, else undefined. All model traffic goes
|
|
@@ -466,31 +421,3 @@ export declare function terminalRunError(stdout: string): string | undefined;
|
|
|
466
421
|
* `llm-upstream` cause + this remedy. Pure, so it is unit-tested over fixed error strings.
|
|
467
422
|
*/
|
|
468
423
|
export declare function classifyLlmUpstreamError(finalError: string): string | undefined;
|
|
469
|
-
/**
|
|
470
|
-
* Pi's assistant summary plus {@link PiRunStats}, derived from one pass over its
|
|
471
|
-
* output — the canonical close-of-run signal the harness uses both to report the
|
|
472
|
-
* answer and to detect a no-op run (the agent never acted).
|
|
473
|
-
*/
|
|
474
|
-
export declare function summarizePiRun(stdout: string): PiRunOutcome;
|
|
475
|
-
/**
|
|
476
|
-
* Output-quality signals over the canonical `agent_end` transcript: whether any
|
|
477
|
-
* completion hit the output ceiling (its content was cut off), whether the FINAL
|
|
478
|
-
* completion did, and whether that final turn carried no text at all. Pure so it is
|
|
479
|
-
* unit-testable over a fixed event sequence. Defaults to all-false when there is no
|
|
480
|
-
* terminal transcript (a no-op run is already caught by {@link agentNeverActed}).
|
|
481
|
-
*
|
|
482
|
-
* `cap` is the per-completion ceiling Pi requested ({@link PI_MAX_OUTPUT_TOKENS});
|
|
483
|
-
* truncation is detected by an assistant message whose `usage.output` reached it,
|
|
484
|
-
* which is reliable even when the model reports a non-`length` stop reason (Workers
|
|
485
|
-
* AI labelled a cut-off tool call `tool_calls`, not `length`).
|
|
486
|
-
*/
|
|
487
|
-
export declare function diagnosticsFromEvents(events: Record<string, unknown>[], cap?: number): RunDiagnostics;
|
|
488
|
-
/** {@link RunDiagnostics} over Pi's raw `--mode json` stdout (see {@link diagnosticsFromEvents}). */
|
|
489
|
-
export declare function runDiagnostics(stdout: string, cap?: number): RunDiagnostics;
|
|
490
|
-
/**
|
|
491
|
-
* Extract the assistant's final summary from Pi's JSON-lines output. Pi emits a
|
|
492
|
-
* terminal `agent_end` event whose `messages` is the full transcript, so the
|
|
493
|
-
* last assistant message there is the canonical answer. Falls back to scanning
|
|
494
|
-
* `message_end` events, then to a raw tail, so a schema tweak never loses output.
|
|
495
|
-
*/
|
|
496
|
-
export declare function parsePiOutput(stdout: string): string;
|
package/dist/pi.js
CHANGED
|
@@ -9,22 +9,21 @@ import { HarnessFailure } from './failure.js';
|
|
|
9
9
|
import { log } from './logger.js';
|
|
10
10
|
import { ProgressGuard, progressGuardLimitsFromEnv, toolCallSignal, } from './progress-guard.js';
|
|
11
11
|
import { ToolCallTracker, readToolCallId, toolCallResult, toolCallStart, } from './tool-trajectory.js';
|
|
12
|
+
import { BoundedTail, JsonlLineReader } from './jsonl-stream.js';
|
|
13
|
+
import { PI_MAX_OUTPUT_TOKENS, PiRunReducer, isObject, } from './pi-reduction.js';
|
|
14
|
+
import { NO_TOOL_WINDOW } from './tool-silence.js';
|
|
12
15
|
// Drives the Pi coding-agent CLI. Pi is pointed at the Worker's OpenAI-compatible
|
|
13
16
|
// proxy via a custom provider in ~/.pi/agent/models.json, authenticated with the
|
|
14
17
|
// per-job session token (interpolated from $PI_PROXY_TOKEN) — so no provider key
|
|
15
18
|
// ever lives in the image or in Pi's config on disk.
|
|
16
19
|
/**
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* model's real max — so erring high is safe. Raised to 32k after a spec-writer run
|
|
23
|
-
* truncated an intermediate tool call at the old 16k cap; the document itself
|
|
24
|
-
* stopped well under it, so this is headroom for larger specs/diffs, with
|
|
25
|
-
* {@link runDiagnostics} flagging the rare case where even 32k is not enough.
|
|
20
|
+
* How much of Pi's raw stdout/stderr the run holds for diagnostics. Every consumer takes a tail
|
|
21
|
+
* of it (2 KB for the last-resort summary, 1.5 KB for a stderr quote, 500 B for a crash detail),
|
|
22
|
+
* so this is generous headroom over the largest of them rather than a number anything depends
|
|
23
|
+
* on. What it replaces is retaining the WHOLE of a chatty run's output to slice 2 KB off the end
|
|
24
|
+
* (stuck-run audit F6).
|
|
26
25
|
*/
|
|
27
|
-
|
|
26
|
+
const OUTPUT_TAIL_CHARS = 64 * 1024;
|
|
28
27
|
/**
|
|
29
28
|
* Longest phase label the backend keeps. Mirrors kernel's `MAX_PHASE_CHARS`; see
|
|
30
29
|
* {@link normalizeProxyPhase} for why this is a copy rather than an import.
|
|
@@ -405,9 +404,6 @@ export async function writeWebToolsConfig(config) {
|
|
|
405
404
|
await writeFile(path, JSON.stringify({ provider: config.provider }, null, 2), { mode: 0o600 });
|
|
406
405
|
return path;
|
|
407
406
|
}
|
|
408
|
-
function isObject(value) {
|
|
409
|
-
return typeof value === 'object' && value !== null;
|
|
410
|
-
}
|
|
411
407
|
/**
|
|
412
408
|
* Publish one captured model call: append it to the run's list (which becomes the terminal
|
|
413
409
|
* result's `callMetrics`) AND hand the SAME object to the live stream, where the job registry
|
|
@@ -593,15 +589,25 @@ export function runPi(opts) {
|
|
|
593
589
|
// 'close'/'error' handlers below own the actual failure reporting.
|
|
594
590
|
child.stdin.on('error', () => { });
|
|
595
591
|
child.stdin.end(opts.userPrompt);
|
|
596
|
-
|
|
597
|
-
|
|
592
|
+
// The close-of-run answers (summary, stats, diagnostics, terminal error), FOLDED as the
|
|
593
|
+
// records stream instead of re-parsing the whole of stdout two more times at close: those
|
|
594
|
+
// passes were O(entire output) on the event loop the watchdog timers and the poll endpoints
|
|
595
|
+
// share, at exactly the moment the job is settling (stuck-run audit F6). Folding rather than
|
|
596
|
+
// retaining the parsed records is the other half of that bound — it is what makes this not a
|
|
597
|
+
// second copy of the run, which an unbounded array of parsed objects would have been.
|
|
598
|
+
const reduction = new PiRunReducer();
|
|
599
|
+
// This stream's tool-silence window: Pi reports every completed tool call, so each one below
|
|
600
|
+
// beats it. Closed on BOTH terminal paths (`error` and `close`) — a window outliving the
|
|
601
|
+
// process it watches would expire against a run that is already over.
|
|
602
|
+
const toolWindow = opts.beginToolWindow?.() ?? NO_TOOL_WINDOW;
|
|
603
|
+
// Raw output kept ONLY to quote on a failure, so a bounded tail is the whole requirement —
|
|
604
|
+
// the longest slice anyone takes below is 2 KB.
|
|
605
|
+
const stdout = new BoundedTail(OUTPUT_TAIL_CHARS);
|
|
606
|
+
const stderr = new BoundedTail(OUTPUT_TAIL_CHARS);
|
|
598
607
|
let aborted = false;
|
|
599
608
|
// Set when the no-progress guard kills Pi; carries the diagnostic the run
|
|
600
609
|
// fails with (distinct from an external watchdog abort).
|
|
601
610
|
let guardReason;
|
|
602
|
-
// Pi's json mode is strict LF-framed JSONL; buffer partial lines across
|
|
603
|
-
// chunks so we only ever parse complete records for progress + the guard.
|
|
604
|
-
let lineBuffer = '';
|
|
605
611
|
// Counters for silent losses, warned ONCE at close (not per-line, to avoid log
|
|
606
612
|
// spam): `{`-leading lines that failed to JSON.parse, and observer-callback throws.
|
|
607
613
|
let malformedLines = 0;
|
|
@@ -621,14 +627,14 @@ export function runPi(opts) {
|
|
|
621
627
|
// SIGTERM first, then SIGKILL if Pi ignores it. Shared by the watchdog abort
|
|
622
628
|
// and the no-progress guard; the `close` handler turns it into a rejection.
|
|
623
629
|
const killChild = () => killChildProcess(child);
|
|
624
|
-
// Parse each complete JSONL record once,
|
|
625
|
-
// emitter and the no-progress guard. A tripped guard kills Pi
|
|
626
|
-
// diagnostic the run then fails on.
|
|
627
|
-
// `
|
|
628
|
-
//
|
|
629
|
-
//
|
|
630
|
-
//
|
|
631
|
-
const processLine = (line,
|
|
630
|
+
// Parse each complete JSONL record once, retaining it for the close-of-run reductions and
|
|
631
|
+
// feeding the todo-progress emitter and the no-progress guard. A tripped guard kills Pi
|
|
632
|
+
// with a diagnostic the run then fails on.
|
|
633
|
+
// `final` marks the at-close flush of a final unterminated line: the process has already
|
|
634
|
+
// exited, so feeding that record to the no-progress guard could trip it and turn a clean
|
|
635
|
+
// (code 0) exit into a spurious "no progress" rejection. The flush still recovers the
|
|
636
|
+
// record's progress/span signal; only the kill decision is skipped.
|
|
637
|
+
const processLine = (line, final) => {
|
|
632
638
|
if (!line.startsWith('{'))
|
|
633
639
|
return;
|
|
634
640
|
let event;
|
|
@@ -641,6 +647,7 @@ export function runPi(opts) {
|
|
|
641
647
|
malformedLines++;
|
|
642
648
|
return;
|
|
643
649
|
}
|
|
650
|
+
reduction.observe(event);
|
|
644
651
|
if (opts.onEvent) {
|
|
645
652
|
try {
|
|
646
653
|
opts.onEvent(event);
|
|
@@ -655,12 +662,17 @@ export function runPi(opts) {
|
|
|
655
662
|
if (progress)
|
|
656
663
|
opts.onProgress(progress);
|
|
657
664
|
}
|
|
665
|
+
// A completed tool call is the progress the tool-silence watchdog measures. Detected
|
|
666
|
+
// OUTSIDE the span branch below: the trajectory is an observability opt-in, and a watchdog
|
|
667
|
+
// that only ran when someone wanted spans would be armed against a stream it could not see.
|
|
668
|
+
const signal = toolCallSignal(event);
|
|
669
|
+
if (signal?.name)
|
|
670
|
+
toolWindow.toolCompleted();
|
|
658
671
|
if (opts.onSpan) {
|
|
659
672
|
const start = toolCallStart(event);
|
|
660
673
|
if (start)
|
|
661
674
|
tools.started(start.id, start.name, start.args);
|
|
662
|
-
|
|
663
|
-
if (signal && signal.name) {
|
|
675
|
+
if (signal?.name) {
|
|
664
676
|
const call = tools.finished(readToolCallId(event), signal.name, toolCallResult(event), signal.isError);
|
|
665
677
|
try {
|
|
666
678
|
opts.onSpan({ ...call, bodies: 'stored' });
|
|
@@ -671,7 +683,7 @@ export function runPi(opts) {
|
|
|
671
683
|
}
|
|
672
684
|
}
|
|
673
685
|
}
|
|
674
|
-
if (
|
|
686
|
+
if (!final && !guardReason && !aborted) {
|
|
675
687
|
const reason = guard.observe(event);
|
|
676
688
|
if (reason) {
|
|
677
689
|
guardReason = reason;
|
|
@@ -679,16 +691,9 @@ export function runPi(opts) {
|
|
|
679
691
|
}
|
|
680
692
|
}
|
|
681
693
|
};
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
while (nl !== -1) {
|
|
686
|
-
const line = lineBuffer.slice(0, nl).trim();
|
|
687
|
-
lineBuffer = lineBuffer.slice(nl + 1);
|
|
688
|
-
nl = lineBuffer.indexOf('\n');
|
|
689
|
-
processLine(line);
|
|
690
|
-
}
|
|
691
|
-
};
|
|
694
|
+
// Pi's json mode is strict LF-framed JSONL; the reader buffers partial records across
|
|
695
|
+
// chunks (bounded — see `JsonlLineReader`) so we only ever parse complete ones.
|
|
696
|
+
const reader = new JsonlLineReader(processLine);
|
|
692
697
|
// When the watchdog aborts, terminate Pi: the `close` handler then rejects
|
|
693
698
|
// with the abort reason.
|
|
694
699
|
const onAbort = () => {
|
|
@@ -699,11 +704,11 @@ export function runPi(opts) {
|
|
|
699
704
|
const onChunk = (chunk, sink) => {
|
|
700
705
|
const text = chunk.toString();
|
|
701
706
|
if (sink === 'out') {
|
|
702
|
-
stdout
|
|
703
|
-
|
|
707
|
+
stdout.push(text);
|
|
708
|
+
reader.push(text);
|
|
704
709
|
}
|
|
705
710
|
else
|
|
706
|
-
stderr
|
|
711
|
+
stderr.push(text);
|
|
707
712
|
// Any output means progress: reset the inactivity watchdog.
|
|
708
713
|
opts.onActivity?.();
|
|
709
714
|
};
|
|
@@ -711,62 +716,98 @@ export function runPi(opts) {
|
|
|
711
716
|
child.stderr.on('data', (chunk) => onChunk(chunk, 'err'));
|
|
712
717
|
child.on('error', (error) => {
|
|
713
718
|
opts.signal?.removeEventListener('abort', onAbort);
|
|
719
|
+
toolWindow.close();
|
|
714
720
|
reject(error);
|
|
715
721
|
});
|
|
716
722
|
child.on('close', (code) => {
|
|
717
723
|
opts.signal?.removeEventListener('abort', onAbort);
|
|
724
|
+
toolWindow.close();
|
|
718
725
|
// Flush a final record that arrived without a trailing newline: Pi usually LF-frames
|
|
719
726
|
// every line, but a clean exit can leave the last event (often `agent_end`) unterminated
|
|
720
727
|
// in the buffer, so without this its progress/span/guard signal would be silently lost.
|
|
721
|
-
|
|
722
|
-
processLine(lineBuffer.trim(), false);
|
|
723
|
-
lineBuffer = '';
|
|
724
|
-
}
|
|
728
|
+
reader.flush();
|
|
725
729
|
// Surface any silent stream losses ONCE (counts, not per-line), so a corrupted JSONL
|
|
726
|
-
// stream or a throwing observer is
|
|
727
|
-
|
|
728
|
-
|
|
730
|
+
// stream, an oversized record the reader refused to buffer, or a throwing observer is
|
|
731
|
+
// diagnosable rather than invisible.
|
|
732
|
+
if (malformedLines > 0 || observerErrors > 0 || reader.droppedLines > 0) {
|
|
733
|
+
log.warn('pi: skipped malformed/oversized JSONL lines or observer errors', {
|
|
729
734
|
malformedLines,
|
|
735
|
+
oversizedLines: reader.droppedLines,
|
|
730
736
|
observerErrors,
|
|
731
737
|
});
|
|
732
738
|
}
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
if (runError) {
|
|
750
|
-
const scrubbed = redactSecrets(runError).slice(0, 1000);
|
|
751
|
-
const detail = tail ? `${scrubbed} Agent stderr: ${tail}` : scrubbed;
|
|
752
|
-
reject(piRunFailure(detail, runError));
|
|
753
|
-
}
|
|
754
|
-
else {
|
|
755
|
-
resolve({ ...summarizePiRun(stdout), ...(tail ? { stderrTail: tail } : {}) });
|
|
756
|
-
}
|
|
757
|
-
}
|
|
758
|
-
else {
|
|
759
|
-
// A non-zero exit is the OTHER way a proxy refusal can surface (Pi crashing rather
|
|
760
|
-
// than exiting 0 after exhausting retries), so classify it here too — otherwise a
|
|
761
|
-
// 401/402/429 that happens to crash Pi would read as a generic agent failure. Redact
|
|
762
|
-
// the transcript slice before it becomes the detail: unlike the exit-0 path above, the
|
|
763
|
-
// raw `stderr`/`stdout` here was previously interpolated unscrubbed.
|
|
764
|
-
const raw = (stderr || stdout).slice(-500);
|
|
765
|
-
reject(piRunFailure(`pi exited with code ${code}: ${redactSecrets(raw)}`, raw));
|
|
766
|
-
}
|
|
739
|
+
const settled = settlePiRun({
|
|
740
|
+
code,
|
|
741
|
+
reduction,
|
|
742
|
+
droppedLines: reader.droppedLines,
|
|
743
|
+
aborted,
|
|
744
|
+
stdoutTail: stdout.toString(),
|
|
745
|
+
stderrTail: stderr.toString(),
|
|
746
|
+
...(guardReason ? { guardReason } : {}),
|
|
747
|
+
...(opts.signal?.reason instanceof Error
|
|
748
|
+
? { abortReason: opts.signal.reason.message }
|
|
749
|
+
: {}),
|
|
750
|
+
});
|
|
751
|
+
if (settled.ok)
|
|
752
|
+
resolve(settled.outcome);
|
|
753
|
+
else
|
|
754
|
+
reject(settled.error);
|
|
767
755
|
});
|
|
768
756
|
});
|
|
769
757
|
}
|
|
758
|
+
/**
|
|
759
|
+
* Turn an EXITED Pi process into the run's outcome or its failure. Split out of {@link runPi} for
|
|
760
|
+
* the per-function budget, and pure so the dispositions can be reasoned about (and tested) without
|
|
761
|
+
* spawning anything: everything it needs is already reduced by the time the process closes.
|
|
762
|
+
*
|
|
763
|
+
* The four dispositions, in the order they win: the no-progress guard's own kill, the external
|
|
764
|
+
* watchdog's abort, a crash, and a clean exit — which is where the run is CERTIFIED, below.
|
|
765
|
+
*/
|
|
766
|
+
function settlePiRun(args) {
|
|
767
|
+
const { code, reduction, droppedLines, aborted, guardReason, stdoutTail, stderrTail } = args;
|
|
768
|
+
const fail = (error) => ({ ok: false, error });
|
|
769
|
+
if (guardReason) {
|
|
770
|
+
const guardTail = redactSecrets(stderrTail.trim()).slice(-700);
|
|
771
|
+
return fail(new Error(guardTail ? `${guardReason} Agent stderr: ${guardTail}` : guardReason));
|
|
772
|
+
}
|
|
773
|
+
if (aborted)
|
|
774
|
+
return fail(new Error(args.abortReason ?? 'pi aborted'));
|
|
775
|
+
if (code !== 0) {
|
|
776
|
+
// A non-zero exit is the OTHER way a proxy refusal can surface (Pi crashing rather than
|
|
777
|
+
// exiting 0 after exhausting retries), so classify it here too — otherwise a 401/402/429 that
|
|
778
|
+
// happens to crash Pi would read as a generic agent failure. Redact the transcript slice
|
|
779
|
+
// before it becomes the detail: unlike the exit-0 path below, this was previously
|
|
780
|
+
// interpolated unscrubbed.
|
|
781
|
+
const raw = (stderrTail || stdoutTail).slice(-500);
|
|
782
|
+
return fail(piRunFailure(`pi exited with code ${code}: ${redactSecrets(raw)}`, raw));
|
|
783
|
+
}
|
|
784
|
+
const tail = redactSecrets(stderrTail.trim()).slice(-1500);
|
|
785
|
+
// Pi can exit 0 even when the agent run ended in a hard error (e.g. every model call failed and
|
|
786
|
+
// its retries were exhausted): the process completed, but the agent did not. Exit code alone
|
|
787
|
+
// then reads as success, and a run that RESUMED a branch with prior commits would even open a
|
|
788
|
+
// PR off work this pass never produced.
|
|
789
|
+
const runError = reduction.terminalError();
|
|
790
|
+
if (runError) {
|
|
791
|
+
const scrubbed = redactSecrets(runError).slice(0, 1000);
|
|
792
|
+
return fail(piRunFailure(tail ? `${scrubbed} Agent stderr: ${tail}` : scrubbed, runError));
|
|
793
|
+
}
|
|
794
|
+
if (!reduction.sawTerminalRecord && droppedLines > 0) {
|
|
795
|
+
// The check above answered "no terminal failure" from having seen no terminal record AT ALL,
|
|
796
|
+
// and the reader dropped at least one oversized one — so the record that decides this
|
|
797
|
+
// question is exactly the record most likely to have been dropped (`agent_end` carries the
|
|
798
|
+
// run's whole transcript). Resolving here would report a hard-failed run as a success, which
|
|
799
|
+
// is the case that check exists to prevent, so refuse to certify it instead.
|
|
800
|
+
// `no-usable-output` because that is literally what happened: the run finished and its
|
|
801
|
+
// terminal report never reached us.
|
|
802
|
+
const detail = `pi exited 0 but its terminal record was dropped for exceeding the JSONL line cap ` +
|
|
803
|
+
`(${droppedLines} oversized record(s)), so the run's outcome is unknown`;
|
|
804
|
+
return fail(new HarnessFailure('no-usable-output', tail ? `${detail}. ${tail}` : detail));
|
|
805
|
+
}
|
|
806
|
+
return {
|
|
807
|
+
ok: true,
|
|
808
|
+
outcome: { ...reduction.reduce(stdoutTail), ...(tail ? { stderrTail: tail } : {}) },
|
|
809
|
+
};
|
|
810
|
+
}
|
|
770
811
|
/**
|
|
771
812
|
* Build the rejection for a failed Pi run: if its terminal text points at the LLM proxy
|
|
772
813
|
* refusing every model call (auth/quota/rate-limit), stamp the structured `llm-upstream`
|
|
@@ -779,51 +820,6 @@ function piRunFailure(detail, sourceText) {
|
|
|
779
820
|
const remedy = classifyLlmUpstreamError(sourceText);
|
|
780
821
|
return remedy ? new HarnessFailure('llm-upstream', `${detail}\n${remedy}`) : new Error(detail);
|
|
781
822
|
}
|
|
782
|
-
/** Parse Pi's LF-framed JSONL stdout into its event records, skipping noise. */
|
|
783
|
-
function parsePiEvents(stdout) {
|
|
784
|
-
const events = [];
|
|
785
|
-
for (const raw of stdout.split('\n')) {
|
|
786
|
-
const line = raw.trim();
|
|
787
|
-
if (!line.startsWith('{'))
|
|
788
|
-
continue;
|
|
789
|
-
try {
|
|
790
|
-
events.push(JSON.parse(line));
|
|
791
|
-
}
|
|
792
|
-
catch {
|
|
793
|
-
// Not a JSON event line; skip.
|
|
794
|
-
}
|
|
795
|
-
}
|
|
796
|
-
return events;
|
|
797
|
-
}
|
|
798
|
-
/**
|
|
799
|
-
* The terminal-failure message when Pi's run ended in a hard error (the model was
|
|
800
|
-
* unreachable / refused, and Pi exhausted its auto-retries), else undefined. Only
|
|
801
|
-
* the FINAL outcome counts: a mid-run hiccup the agent recovered from leaves a clean
|
|
802
|
-
* terminal `agent_end`, so it returns undefined. Scans from the end and decides on
|
|
803
|
-
* the first terminal signal it meets — the trailing `auto_retry_end` (its `success`
|
|
804
|
-
* flag) or the last `agent_end` (its `stopReason`). Pure so it is unit-testable over
|
|
805
|
-
* a fixed event sequence.
|
|
806
|
-
*/
|
|
807
|
-
export function terminalRunError(stdout) {
|
|
808
|
-
const events = parsePiEvents(stdout);
|
|
809
|
-
for (let i = events.length - 1; i >= 0; i--) {
|
|
810
|
-
const e = events[i];
|
|
811
|
-
if (e.type === 'auto_retry_end') {
|
|
812
|
-
if (e.success === false) {
|
|
813
|
-
return typeof e.finalError === 'string'
|
|
814
|
-
? e.finalError
|
|
815
|
-
: 'the agent failed after exhausting its retries';
|
|
816
|
-
}
|
|
817
|
-
return undefined;
|
|
818
|
-
}
|
|
819
|
-
if (e.type === 'agent_end') {
|
|
820
|
-
return e.stopReason === 'error' && typeof e.errorMessage === 'string'
|
|
821
|
-
? e.errorMessage
|
|
822
|
-
: undefined;
|
|
823
|
-
}
|
|
824
|
-
}
|
|
825
|
-
return undefined;
|
|
826
|
-
}
|
|
827
823
|
/**
|
|
828
824
|
* Classify a terminal run error whose text points at the LLM PROXY rejecting every model call
|
|
829
825
|
* (auth / quota / rate-limit) into an actionable remedy, else undefined. All model traffic goes
|
|
@@ -855,185 +851,3 @@ export function classifyLlmUpstreamError(finalError) {
|
|
|
855
851
|
}
|
|
856
852
|
return undefined;
|
|
857
853
|
}
|
|
858
|
-
/**
|
|
859
|
-
* Pi's assistant summary plus {@link PiRunStats}, derived from one pass over its
|
|
860
|
-
* output — the canonical close-of-run signal the harness uses both to report the
|
|
861
|
-
* answer and to detect a no-op run (the agent never acted).
|
|
862
|
-
*/
|
|
863
|
-
export function summarizePiRun(stdout) {
|
|
864
|
-
const events = parsePiEvents(stdout);
|
|
865
|
-
return {
|
|
866
|
-
summary: summaryFromEvents(events, stdout),
|
|
867
|
-
stats: statsFromEvents(events),
|
|
868
|
-
diagnostics: diagnosticsFromEvents(events),
|
|
869
|
-
};
|
|
870
|
-
}
|
|
871
|
-
/**
|
|
872
|
-
* Output-quality signals over the canonical `agent_end` transcript: whether any
|
|
873
|
-
* completion hit the output ceiling (its content was cut off), whether the FINAL
|
|
874
|
-
* completion did, and whether that final turn carried no text at all. Pure so it is
|
|
875
|
-
* unit-testable over a fixed event sequence. Defaults to all-false when there is no
|
|
876
|
-
* terminal transcript (a no-op run is already caught by {@link agentNeverActed}).
|
|
877
|
-
*
|
|
878
|
-
* `cap` is the per-completion ceiling Pi requested ({@link PI_MAX_OUTPUT_TOKENS});
|
|
879
|
-
* truncation is detected by an assistant message whose `usage.output` reached it,
|
|
880
|
-
* which is reliable even when the model reports a non-`length` stop reason (Workers
|
|
881
|
-
* AI labelled a cut-off tool call `tool_calls`, not `length`).
|
|
882
|
-
*/
|
|
883
|
-
export function diagnosticsFromEvents(events, cap = PI_MAX_OUTPUT_TOKENS) {
|
|
884
|
-
let messages;
|
|
885
|
-
for (let i = events.length - 1; i >= 0; i--) {
|
|
886
|
-
const e = events[i];
|
|
887
|
-
if (e.type === 'agent_end' && Array.isArray(e.messages)) {
|
|
888
|
-
messages = e.messages;
|
|
889
|
-
break;
|
|
890
|
-
}
|
|
891
|
-
}
|
|
892
|
-
if (!messages)
|
|
893
|
-
return { truncated: false, finalTruncated: false, finalAnswerEmpty: false };
|
|
894
|
-
const assistants = messages.filter((m) => isObject(m) && m.role === 'assistant');
|
|
895
|
-
const truncated = assistants.some((m) => assistantOutputTokens(m) >= cap);
|
|
896
|
-
const last = assistants.at(-1);
|
|
897
|
-
return {
|
|
898
|
-
truncated,
|
|
899
|
-
finalTruncated: last ? assistantOutputTokens(last) >= cap : false,
|
|
900
|
-
finalAnswerEmpty: last ? messageText(last) === '' : false,
|
|
901
|
-
};
|
|
902
|
-
}
|
|
903
|
-
/** `usage.output` (completion tokens) reported on a Pi assistant message, or 0. */
|
|
904
|
-
function assistantOutputTokens(message) {
|
|
905
|
-
const usage = message.usage;
|
|
906
|
-
if (!isObject(usage))
|
|
907
|
-
return 0;
|
|
908
|
-
const output = usage.output;
|
|
909
|
-
return typeof output === 'number' ? output : 0;
|
|
910
|
-
}
|
|
911
|
-
/** {@link RunDiagnostics} over Pi's raw `--mode json` stdout (see {@link diagnosticsFromEvents}). */
|
|
912
|
-
export function runDiagnostics(stdout, cap = PI_MAX_OUTPUT_TOKENS) {
|
|
913
|
-
return diagnosticsFromEvents(parsePiEvents(stdout), cap);
|
|
914
|
-
}
|
|
915
|
-
/**
|
|
916
|
-
* Count what the agent actually did. Prefers the canonical `agent_end`
|
|
917
|
-
* transcript (assistant `toolCall` parts + text); falls back to the streamed
|
|
918
|
-
* `tool_execution_end` / `message_end` events when no terminal transcript was
|
|
919
|
-
* emitted, so a no-op is never mistaken for a real run because of a schema tweak.
|
|
920
|
-
*/
|
|
921
|
-
function statsFromEvents(events) {
|
|
922
|
-
for (let i = events.length - 1; i >= 0; i--) {
|
|
923
|
-
const e = events[i];
|
|
924
|
-
if (e.type === 'agent_end' && Array.isArray(e.messages)) {
|
|
925
|
-
return statsFromMessages(e.messages);
|
|
926
|
-
}
|
|
927
|
-
}
|
|
928
|
-
let toolCalls = 0;
|
|
929
|
-
let toolResults = 0;
|
|
930
|
-
let assistantChars = 0;
|
|
931
|
-
for (const e of events) {
|
|
932
|
-
if (e.type === 'tool_execution_end') {
|
|
933
|
-
toolCalls++;
|
|
934
|
-
}
|
|
935
|
-
else if (e.type === 'message_end' && isObject(e.message)) {
|
|
936
|
-
const m = e.message;
|
|
937
|
-
if (m.role === 'assistant')
|
|
938
|
-
assistantChars += messageText(m).length;
|
|
939
|
-
else if (m.role === 'toolResult')
|
|
940
|
-
toolResults++;
|
|
941
|
-
}
|
|
942
|
-
}
|
|
943
|
-
// The same call can surface as both a `tool_execution_end` and a toolResult
|
|
944
|
-
// `message_end`; prefer the former and only fall back to toolResult counts.
|
|
945
|
-
return { toolCalls: toolCalls || toolResults, assistantChars };
|
|
946
|
-
}
|
|
947
|
-
/** {@link PiRunStats} from a transcript: assistant `toolCall` parts + text length. */
|
|
948
|
-
function statsFromMessages(messages) {
|
|
949
|
-
let toolCalls = 0;
|
|
950
|
-
let assistantChars = 0;
|
|
951
|
-
for (const m of messages) {
|
|
952
|
-
if (!isObject(m) || m.role !== 'assistant')
|
|
953
|
-
continue;
|
|
954
|
-
const content = m.content;
|
|
955
|
-
if (typeof content === 'string') {
|
|
956
|
-
assistantChars += content.trim().length;
|
|
957
|
-
}
|
|
958
|
-
else if (Array.isArray(content)) {
|
|
959
|
-
for (const part of content) {
|
|
960
|
-
if (!isObject(part))
|
|
961
|
-
continue;
|
|
962
|
-
if (part.type === 'toolCall')
|
|
963
|
-
toolCalls++;
|
|
964
|
-
else if (typeof part.text === 'string')
|
|
965
|
-
assistantChars += part.text.length;
|
|
966
|
-
}
|
|
967
|
-
}
|
|
968
|
-
}
|
|
969
|
-
return { toolCalls, assistantChars };
|
|
970
|
-
}
|
|
971
|
-
/**
|
|
972
|
-
* Extract the assistant's final summary from Pi's JSON-lines output. Pi emits a
|
|
973
|
-
* terminal `agent_end` event whose `messages` is the full transcript, so the
|
|
974
|
-
* last assistant message there is the canonical answer. Falls back to scanning
|
|
975
|
-
* `message_end` events, then to a raw tail, so a schema tweak never loses output.
|
|
976
|
-
*/
|
|
977
|
-
export function parsePiOutput(stdout) {
|
|
978
|
-
return summaryFromEvents(parsePiEvents(stdout), stdout);
|
|
979
|
-
}
|
|
980
|
-
/** Shared summary extraction over already-parsed events (see {@link parsePiOutput}). */
|
|
981
|
-
function summaryFromEvents(events, stdout) {
|
|
982
|
-
// Preferred: the final transcript from the last agent_end event.
|
|
983
|
-
for (let i = events.length - 1; i >= 0; i--) {
|
|
984
|
-
const e = events[i];
|
|
985
|
-
if (e.type === 'agent_end' && Array.isArray(e.messages)) {
|
|
986
|
-
const text = lastAssistantText(e.messages);
|
|
987
|
-
if (text)
|
|
988
|
-
return text;
|
|
989
|
-
}
|
|
990
|
-
}
|
|
991
|
-
// Fallback: assistant text accumulated from message_end events.
|
|
992
|
-
const parts = [];
|
|
993
|
-
for (const e of events) {
|
|
994
|
-
if (e.type === 'message_end' &&
|
|
995
|
-
typeof e.message === 'object' &&
|
|
996
|
-
e.message !== null &&
|
|
997
|
-
e.message.role === 'assistant') {
|
|
998
|
-
const text = messageText(e.message);
|
|
999
|
-
if (text)
|
|
1000
|
-
parts.push(text);
|
|
1001
|
-
}
|
|
1002
|
-
}
|
|
1003
|
-
const joined = parts.join('\n').trim();
|
|
1004
|
-
if (joined)
|
|
1005
|
-
return joined;
|
|
1006
|
-
// Nothing structured matched — return a trimmed tail of the raw output.
|
|
1007
|
-
return stdout.trim().slice(-2000);
|
|
1008
|
-
}
|
|
1009
|
-
/** The text of the last assistant message in a transcript, or '' if none. */
|
|
1010
|
-
function lastAssistantText(messages) {
|
|
1011
|
-
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1012
|
-
const m = messages[i];
|
|
1013
|
-
if (typeof m === 'object' && m !== null && m.role === 'assistant') {
|
|
1014
|
-
const text = messageText(m);
|
|
1015
|
-
if (text)
|
|
1016
|
-
return text;
|
|
1017
|
-
}
|
|
1018
|
-
}
|
|
1019
|
-
return '';
|
|
1020
|
-
}
|
|
1021
|
-
/** Join the text parts of a Pi message whose content is a string or parts array. */
|
|
1022
|
-
function messageText(message) {
|
|
1023
|
-
if (typeof message !== 'object' || message === null)
|
|
1024
|
-
return '';
|
|
1025
|
-
const content = message.content;
|
|
1026
|
-
if (typeof content === 'string')
|
|
1027
|
-
return content.trim();
|
|
1028
|
-
if (Array.isArray(content)) {
|
|
1029
|
-
return content
|
|
1030
|
-
.map((part) => typeof part === 'object' &&
|
|
1031
|
-
part !== null &&
|
|
1032
|
-
typeof part.text === 'string'
|
|
1033
|
-
? part.text
|
|
1034
|
-
: '')
|
|
1035
|
-
.join('')
|
|
1036
|
-
.trim();
|
|
1037
|
-
}
|
|
1038
|
-
return '';
|
|
1039
|
-
}
|