@cat-factory/executor-harness 1.96.0 → 1.98.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/pi.d.ts CHANGED
@@ -1,17 +1,7 @@
1
1
  import type { EffortReport } from './effort.js';
2
2
  import { type ProgressGuardLimits } from './progress-guard.js';
3
- /**
4
- * Per-completion output-token ceiling Pi requests (its model-entry `maxTokens`).
5
- * Generous on purpose: a reasoning model (e.g. GLM-5.2) spends tokens on its
6
- * `<think>` trace before the answer + tool calls, so a tight cap truncates it
7
- * mid-reasoning and the agent never commits edits. It is a ceiling, not a target
8
- * — unused output tokens are not billed and Workers AI clamps the request to the
9
- * model's real max — so erring high is safe. Raised to 32k after a spec-writer run
10
- * truncated an intermediate tool call at the old 16k cap; the document itself
11
- * stopped well under it, so this is headroom for larger specs/diffs, with
12
- * {@link runDiagnostics} flagging the rare case where even 32k is not enough.
13
- */
14
- export declare const PI_MAX_OUTPUT_TOKENS = 32768;
3
+ import { type PiRunStats, type RunDiagnostics } from './pi-reduction.js';
4
+ import { type ToolProgressWindow } from './tool-silence.js';
15
5
  /**
16
6
  * Normalise a phase label to what the backend will actually store: trimmed, lowercased,
17
7
  * `[a-z0-9-]` only, bounded. `''` when the label is not a phase at all.
@@ -221,37 +211,6 @@ export interface ToolSpan {
221
211
  argsDropped: number;
222
212
  resultDropped: number;
223
213
  }
224
- /**
225
- * What the agent actually did this run, independent of any file changes. Used to
226
- * tell a genuine no-op (the agent never reached the model / never acted) apart
227
- * from a real run, so a bootstrap that produced nothing is failed rather than
228
- * pushed as an empty repo. `toolCalls === 0 && assistantChars === 0` is the
229
- * signature of a run where Pi never made a successful model call.
230
- */
231
- export interface PiRunStats {
232
- /** Tool calls the assistant emitted across the transcript (0 ⇒ it never acted). */
233
- toolCalls: number;
234
- /** Total characters of assistant text (0 ⇒ the model produced nothing). */
235
- assistantChars: number;
236
- }
237
- /**
238
- * Output-quality signals lifted from the agent's transcript, so the harness can fail
239
- * LOUDLY on a malformed run instead of silently handing a half-baked artifact to the
240
- * structured-output repair (which would manufacture a doc from garbage — the trap
241
- * behind the spec-writer ⇄ companion rework loop). Two distinct invalid states, both
242
- * seen in production from `kimi-k2.7-code`:
243
- * - a completion that hit the output ceiling (its answer/tool call was cut off), and
244
- * - a FINAL turn that carried no text at all (an empty `content: []` despite spending
245
- * output tokens), so there is no answer to parse.
246
- */
247
- export interface RunDiagnostics {
248
- /** Some completion ended at the output-token ceiling — its content was cut off. */
249
- truncated: boolean;
250
- /** The agent's FINAL completion hit the ceiling: its ANSWER (not a mid-run step) was cut off. */
251
- finalTruncated: boolean;
252
- /** The agent's final turn carried no text content (e.g. an empty `content: []`). */
253
- finalAnswerEmpty: boolean;
254
- }
255
214
  /**
256
215
  * One model call captured from a subscription harness's CLI event stream, shaped so
257
216
  * the backend can record it into the same `llm_call_metrics` telemetry the LLM proxy
@@ -426,6 +385,12 @@ export declare function runPi(opts: {
426
385
  * the container payload doesn't pass it, so production behaviour is unchanged.
427
386
  */
428
387
  onSpan?: (span: ToolSpan) => void;
388
+ /**
389
+ * Opens this stream's tool-silence window (see `RunOptions.beginToolWindow`), closed when Pi
390
+ * exits. Pi reports every completed tool call, so the window it opens is one this run can
391
+ * always beat; a caller that passes nothing leaves the watchdog silent for the run.
392
+ */
393
+ beginToolWindow?: () => ToolProgressWindow;
429
394
  /**
430
395
  * Called with every parsed Pi `--mode json` event, in stream order — the raw
431
396
  * observability seam over the run. Used by offline tooling (the smoketest
@@ -446,16 +411,6 @@ export declare function runPi(opts: {
446
411
  */
447
412
  extraEnv?: Record<string, string>;
448
413
  }): Promise<PiRunOutcome>;
449
- /**
450
- * The terminal-failure message when Pi's run ended in a hard error (the model was
451
- * unreachable / refused, and Pi exhausted its auto-retries), else undefined. Only
452
- * the FINAL outcome counts: a mid-run hiccup the agent recovered from leaves a clean
453
- * terminal `agent_end`, so it returns undefined. Scans from the end and decides on
454
- * the first terminal signal it meets — the trailing `auto_retry_end` (its `success`
455
- * flag) or the last `agent_end` (its `stopReason`). Pure so it is unit-testable over
456
- * a fixed event sequence.
457
- */
458
- export declare function terminalRunError(stdout: string): string | undefined;
459
414
  /**
460
415
  * Classify a terminal run error whose text points at the LLM PROXY rejecting every model call
461
416
  * (auth / quota / rate-limit) into an actionable remedy, else undefined. All model traffic goes
@@ -466,31 +421,3 @@ export declare function terminalRunError(stdout: string): string | undefined;
466
421
  * `llm-upstream` cause + this remedy. Pure, so it is unit-tested over fixed error strings.
467
422
  */
468
423
  export declare function classifyLlmUpstreamError(finalError: string): string | undefined;
469
- /**
470
- * Pi's assistant summary plus {@link PiRunStats}, derived from one pass over its
471
- * output — the canonical close-of-run signal the harness uses both to report the
472
- * answer and to detect a no-op run (the agent never acted).
473
- */
474
- export declare function summarizePiRun(stdout: string): PiRunOutcome;
475
- /**
476
- * Output-quality signals over the canonical `agent_end` transcript: whether any
477
- * completion hit the output ceiling (its content was cut off), whether the FINAL
478
- * completion did, and whether that final turn carried no text at all. Pure so it is
479
- * unit-testable over a fixed event sequence. Defaults to all-false when there is no
480
- * terminal transcript (a no-op run is already caught by {@link agentNeverActed}).
481
- *
482
- * `cap` is the per-completion ceiling Pi requested ({@link PI_MAX_OUTPUT_TOKENS});
483
- * truncation is detected by an assistant message whose `usage.output` reached it,
484
- * which is reliable even when the model reports a non-`length` stop reason (Workers
485
- * AI labelled a cut-off tool call `tool_calls`, not `length`).
486
- */
487
- export declare function diagnosticsFromEvents(events: Record<string, unknown>[], cap?: number): RunDiagnostics;
488
- /** {@link RunDiagnostics} over Pi's raw `--mode json` stdout (see {@link diagnosticsFromEvents}). */
489
- export declare function runDiagnostics(stdout: string, cap?: number): RunDiagnostics;
490
- /**
491
- * Extract the assistant's final summary from Pi's JSON-lines output. Pi emits a
492
- * terminal `agent_end` event whose `messages` is the full transcript, so the
493
- * last assistant message there is the canonical answer. Falls back to scanning
494
- * `message_end` events, then to a raw tail, so a schema tweak never loses output.
495
- */
496
- export declare function parsePiOutput(stdout: string): string;
package/dist/pi.js CHANGED
@@ -9,22 +9,21 @@ import { HarnessFailure } from './failure.js';
9
9
  import { log } from './logger.js';
10
10
  import { ProgressGuard, progressGuardLimitsFromEnv, toolCallSignal, } from './progress-guard.js';
11
11
  import { ToolCallTracker, readToolCallId, toolCallResult, toolCallStart, } from './tool-trajectory.js';
12
+ import { BoundedTail, JsonlLineReader } from './jsonl-stream.js';
13
+ import { PI_MAX_OUTPUT_TOKENS, PiRunReducer, isObject, } from './pi-reduction.js';
14
+ import { NO_TOOL_WINDOW } from './tool-silence.js';
12
15
  // Drives the Pi coding-agent CLI. Pi is pointed at the Worker's OpenAI-compatible
13
16
  // proxy via a custom provider in ~/.pi/agent/models.json, authenticated with the
14
17
  // per-job session token (interpolated from $PI_PROXY_TOKEN) — so no provider key
15
18
  // ever lives in the image or in Pi's config on disk.
16
19
  /**
17
- * Per-completion output-token ceiling Pi requests (its model-entry `maxTokens`).
18
- * Generous on purpose: a reasoning model (e.g. GLM-5.2) spends tokens on its
19
- * `<think>` trace before the answer + tool calls, so a tight cap truncates it
20
- * mid-reasoning and the agent never commits edits. It is a ceiling, not a target
21
- * unused output tokens are not billed and Workers AI clamps the request to the
22
- * model's real max — so erring high is safe. Raised to 32k after a spec-writer run
23
- * truncated an intermediate tool call at the old 16k cap; the document itself
24
- * stopped well under it, so this is headroom for larger specs/diffs, with
25
- * {@link runDiagnostics} flagging the rare case where even 32k is not enough.
20
+ * How much of Pi's raw stdout/stderr the run holds for diagnostics. Every consumer takes a tail
21
+ * of it (2 KB for the last-resort summary, 1.5 KB for a stderr quote, 500 B for a crash detail),
22
+ * so this is generous headroom over the largest of them rather than a number anything depends
23
+ * on. What it replaces is retaining the WHOLE of a chatty run's output to slice 2 KB off the end
24
+ * (stuck-run audit F6).
26
25
  */
27
- export const PI_MAX_OUTPUT_TOKENS = 32_768;
26
+ const OUTPUT_TAIL_CHARS = 64 * 1024;
28
27
  /**
29
28
  * Longest phase label the backend keeps. Mirrors kernel's `MAX_PHASE_CHARS`; see
30
29
  * {@link normalizeProxyPhase} for why this is a copy rather than an import.
@@ -405,9 +404,6 @@ export async function writeWebToolsConfig(config) {
405
404
  await writeFile(path, JSON.stringify({ provider: config.provider }, null, 2), { mode: 0o600 });
406
405
  return path;
407
406
  }
408
- function isObject(value) {
409
- return typeof value === 'object' && value !== null;
410
- }
411
407
  /**
412
408
  * Publish one captured model call: append it to the run's list (which becomes the terminal
413
409
  * result's `callMetrics`) AND hand the SAME object to the live stream, where the job registry
@@ -593,15 +589,25 @@ export function runPi(opts) {
593
589
  // 'close'/'error' handlers below own the actual failure reporting.
594
590
  child.stdin.on('error', () => { });
595
591
  child.stdin.end(opts.userPrompt);
596
- let stdout = '';
597
- let stderr = '';
592
+ // The close-of-run answers (summary, stats, diagnostics, terminal error), FOLDED as the
593
+ // records stream instead of re-parsing the whole of stdout two more times at close: those
594
+ // passes were O(entire output) on the event loop the watchdog timers and the poll endpoints
595
+ // share, at exactly the moment the job is settling (stuck-run audit F6). Folding rather than
596
+ // retaining the parsed records is the other half of that bound — it is what makes this not a
597
+ // second copy of the run, which an unbounded array of parsed objects would have been.
598
+ const reduction = new PiRunReducer();
599
+ // This stream's tool-silence window: Pi reports every completed tool call, so each one below
600
+ // beats it. Closed on BOTH terminal paths (`error` and `close`) — a window outliving the
601
+ // process it watches would expire against a run that is already over.
602
+ const toolWindow = opts.beginToolWindow?.() ?? NO_TOOL_WINDOW;
603
+ // Raw output kept ONLY to quote on a failure, so a bounded tail is the whole requirement —
604
+ // the longest slice anyone takes below is 2 KB.
605
+ const stdout = new BoundedTail(OUTPUT_TAIL_CHARS);
606
+ const stderr = new BoundedTail(OUTPUT_TAIL_CHARS);
598
607
  let aborted = false;
599
608
  // Set when the no-progress guard kills Pi; carries the diagnostic the run
600
609
  // fails with (distinct from an external watchdog abort).
601
610
  let guardReason;
602
- // Pi's json mode is strict LF-framed JSONL; buffer partial lines across
603
- // chunks so we only ever parse complete records for progress + the guard.
604
- let lineBuffer = '';
605
611
  // Counters for silent losses, warned ONCE at close (not per-line, to avoid log
606
612
  // spam): `{`-leading lines that failed to JSON.parse, and observer-callback throws.
607
613
  let malformedLines = 0;
@@ -621,14 +627,14 @@ export function runPi(opts) {
621
627
  // SIGTERM first, then SIGKILL if Pi ignores it. Shared by the watchdog abort
622
628
  // and the no-progress guard; the `close` handler turns it into a rejection.
623
629
  const killChild = () => killChildProcess(child);
624
- // Parse each complete JSONL record once, feeding both the todo-progress
625
- // emitter and the no-progress guard. A tripped guard kills Pi with a
626
- // diagnostic the run then fails on.
627
- // `runGuard` is false only for the at-close flush of a final unterminated line: the
628
- // process has already exited, so feeding that record to the no-progress guard could trip
629
- // it and turn a clean (code 0) exit into a spurious "no progress" rejection. The flush
630
- // still recovers the record's progress/span signal; only the kill decision is skipped.
631
- const processLine = (line, runGuard = true) => {
630
+ // Parse each complete JSONL record once, retaining it for the close-of-run reductions and
631
+ // feeding the todo-progress emitter and the no-progress guard. A tripped guard kills Pi
632
+ // with a diagnostic the run then fails on.
633
+ // `final` marks the at-close flush of a final unterminated line: the process has already
634
+ // exited, so feeding that record to the no-progress guard could trip it and turn a clean
635
+ // (code 0) exit into a spurious "no progress" rejection. The flush still recovers the
636
+ // record's progress/span signal; only the kill decision is skipped.
637
+ const processLine = (line, final) => {
632
638
  if (!line.startsWith('{'))
633
639
  return;
634
640
  let event;
@@ -641,6 +647,7 @@ export function runPi(opts) {
641
647
  malformedLines++;
642
648
  return;
643
649
  }
650
+ reduction.observe(event);
644
651
  if (opts.onEvent) {
645
652
  try {
646
653
  opts.onEvent(event);
@@ -655,12 +662,17 @@ export function runPi(opts) {
655
662
  if (progress)
656
663
  opts.onProgress(progress);
657
664
  }
665
+ // A completed tool call is the progress the tool-silence watchdog measures. Detected
666
+ // OUTSIDE the span branch below: the trajectory is an observability opt-in, and a watchdog
667
+ // that only ran when someone wanted spans would be armed against a stream it could not see.
668
+ const signal = toolCallSignal(event);
669
+ if (signal?.name)
670
+ toolWindow.toolCompleted();
658
671
  if (opts.onSpan) {
659
672
  const start = toolCallStart(event);
660
673
  if (start)
661
674
  tools.started(start.id, start.name, start.args);
662
- const signal = toolCallSignal(event);
663
- if (signal && signal.name) {
675
+ if (signal?.name) {
664
676
  const call = tools.finished(readToolCallId(event), signal.name, toolCallResult(event), signal.isError);
665
677
  try {
666
678
  opts.onSpan({ ...call, bodies: 'stored' });
@@ -671,7 +683,7 @@ export function runPi(opts) {
671
683
  }
672
684
  }
673
685
  }
674
- if (runGuard && !guardReason && !aborted) {
686
+ if (!final && !guardReason && !aborted) {
675
687
  const reason = guard.observe(event);
676
688
  if (reason) {
677
689
  guardReason = reason;
@@ -679,16 +691,9 @@ export function runPi(opts) {
679
691
  }
680
692
  }
681
693
  };
682
- const consumeStdout = (text) => {
683
- lineBuffer += text;
684
- let nl = lineBuffer.indexOf('\n');
685
- while (nl !== -1) {
686
- const line = lineBuffer.slice(0, nl).trim();
687
- lineBuffer = lineBuffer.slice(nl + 1);
688
- nl = lineBuffer.indexOf('\n');
689
- processLine(line);
690
- }
691
- };
694
+ // Pi's json mode is strict LF-framed JSONL; the reader buffers partial records across
695
+ // chunks (bounded — see `JsonlLineReader`) so we only ever parse complete ones.
696
+ const reader = new JsonlLineReader(processLine);
692
697
  // When the watchdog aborts, terminate Pi: the `close` handler then rejects
693
698
  // with the abort reason.
694
699
  const onAbort = () => {
@@ -699,11 +704,11 @@ export function runPi(opts) {
699
704
  const onChunk = (chunk, sink) => {
700
705
  const text = chunk.toString();
701
706
  if (sink === 'out') {
702
- stdout += text;
703
- consumeStdout(text);
707
+ stdout.push(text);
708
+ reader.push(text);
704
709
  }
705
710
  else
706
- stderr += text;
711
+ stderr.push(text);
707
712
  // Any output means progress: reset the inactivity watchdog.
708
713
  opts.onActivity?.();
709
714
  };
@@ -711,62 +716,98 @@ export function runPi(opts) {
711
716
  child.stderr.on('data', (chunk) => onChunk(chunk, 'err'));
712
717
  child.on('error', (error) => {
713
718
  opts.signal?.removeEventListener('abort', onAbort);
719
+ toolWindow.close();
714
720
  reject(error);
715
721
  });
716
722
  child.on('close', (code) => {
717
723
  opts.signal?.removeEventListener('abort', onAbort);
724
+ toolWindow.close();
718
725
  // Flush a final record that arrived without a trailing newline: Pi usually LF-frames
719
726
  // every line, but a clean exit can leave the last event (often `agent_end`) unterminated
720
727
  // in the buffer, so without this its progress/span/guard signal would be silently lost.
721
- if (lineBuffer.trim()) {
722
- processLine(lineBuffer.trim(), false);
723
- lineBuffer = '';
724
- }
728
+ reader.flush();
725
729
  // Surface any silent stream losses ONCE (counts, not per-line), so a corrupted JSONL
726
- // stream or a throwing observer is diagnosable rather than invisible.
727
- if (malformedLines > 0 || observerErrors > 0) {
728
- log.warn('pi: skipped malformed JSONL lines / observer errors', {
730
+ // stream, an oversized record the reader refused to buffer, or a throwing observer is
731
+ // diagnosable rather than invisible.
732
+ if (malformedLines > 0 || observerErrors > 0 || reader.droppedLines > 0) {
733
+ log.warn('pi: skipped malformed/oversized JSONL lines or observer errors', {
729
734
  malformedLines,
735
+ oversizedLines: reader.droppedLines,
730
736
  observerErrors,
731
737
  });
732
738
  }
733
- if (guardReason) {
734
- const tail = redactSecrets(stderr.trim()).slice(-700);
735
- reject(new Error(tail ? `${guardReason} Agent stderr: ${tail}` : guardReason));
736
- }
737
- else if (aborted) {
738
- reject(new Error(opts.signal?.reason instanceof Error ? opts.signal.reason.message : 'pi aborted'));
739
- }
740
- else if (code === 0) {
741
- const tail = redactSecrets(stderr.trim()).slice(-1500);
742
- // Pi can exit 0 even when the agent run ended in a hard error (e.g. every
743
- // model call failed and its retries were exhausted): the process completed,
744
- // but the agent did not. Exit code alone then reads as success, and a run
745
- // that RESUMED a branch with prior commits would even open a PR off work this
746
- // pass never produced. Inspect the terminal transcript and fail loudly so the
747
- // step is marked failed instead of masking a total failure as green.
748
- const runError = terminalRunError(stdout);
749
- if (runError) {
750
- const scrubbed = redactSecrets(runError).slice(0, 1000);
751
- const detail = tail ? `${scrubbed} Agent stderr: ${tail}` : scrubbed;
752
- reject(piRunFailure(detail, runError));
753
- }
754
- else {
755
- resolve({ ...summarizePiRun(stdout), ...(tail ? { stderrTail: tail } : {}) });
756
- }
757
- }
758
- else {
759
- // A non-zero exit is the OTHER way a proxy refusal can surface (Pi crashing rather
760
- // than exiting 0 after exhausting retries), so classify it here too — otherwise a
761
- // 401/402/429 that happens to crash Pi would read as a generic agent failure. Redact
762
- // the transcript slice before it becomes the detail: unlike the exit-0 path above, the
763
- // raw `stderr`/`stdout` here was previously interpolated unscrubbed.
764
- const raw = (stderr || stdout).slice(-500);
765
- reject(piRunFailure(`pi exited with code ${code}: ${redactSecrets(raw)}`, raw));
766
- }
739
+ const settled = settlePiRun({
740
+ code,
741
+ reduction,
742
+ droppedLines: reader.droppedLines,
743
+ aborted,
744
+ stdoutTail: stdout.toString(),
745
+ stderrTail: stderr.toString(),
746
+ ...(guardReason ? { guardReason } : {}),
747
+ ...(opts.signal?.reason instanceof Error
748
+ ? { abortReason: opts.signal.reason.message }
749
+ : {}),
750
+ });
751
+ if (settled.ok)
752
+ resolve(settled.outcome);
753
+ else
754
+ reject(settled.error);
767
755
  });
768
756
  });
769
757
  }
758
+ /**
759
+ * Turn an EXITED Pi process into the run's outcome or its failure. Split out of {@link runPi} for
760
+ * the per-function budget, and pure so the dispositions can be reasoned about (and tested) without
761
+ * spawning anything: everything it needs is already reduced by the time the process closes.
762
+ *
763
+ * The four dispositions, in the order they win: the no-progress guard's own kill, the external
764
+ * watchdog's abort, a crash, and a clean exit — which is where the run is CERTIFIED, below.
765
+ */
766
+ function settlePiRun(args) {
767
+ const { code, reduction, droppedLines, aborted, guardReason, stdoutTail, stderrTail } = args;
768
+ const fail = (error) => ({ ok: false, error });
769
+ if (guardReason) {
770
+ const guardTail = redactSecrets(stderrTail.trim()).slice(-700);
771
+ return fail(new Error(guardTail ? `${guardReason} Agent stderr: ${guardTail}` : guardReason));
772
+ }
773
+ if (aborted)
774
+ return fail(new Error(args.abortReason ?? 'pi aborted'));
775
+ if (code !== 0) {
776
+ // A non-zero exit is the OTHER way a proxy refusal can surface (Pi crashing rather than
777
+ // exiting 0 after exhausting retries), so classify it here too — otherwise a 401/402/429 that
778
+ // happens to crash Pi would read as a generic agent failure. Redact the transcript slice
779
+ // before it becomes the detail: unlike the exit-0 path below, this was previously
780
+ // interpolated unscrubbed.
781
+ const raw = (stderrTail || stdoutTail).slice(-500);
782
+ return fail(piRunFailure(`pi exited with code ${code}: ${redactSecrets(raw)}`, raw));
783
+ }
784
+ const tail = redactSecrets(stderrTail.trim()).slice(-1500);
785
+ // Pi can exit 0 even when the agent run ended in a hard error (e.g. every model call failed and
786
+ // its retries were exhausted): the process completed, but the agent did not. Exit code alone
787
+ // then reads as success, and a run that RESUMED a branch with prior commits would even open a
788
+ // PR off work this pass never produced.
789
+ const runError = reduction.terminalError();
790
+ if (runError) {
791
+ const scrubbed = redactSecrets(runError).slice(0, 1000);
792
+ return fail(piRunFailure(tail ? `${scrubbed} Agent stderr: ${tail}` : scrubbed, runError));
793
+ }
794
+ if (!reduction.sawTerminalRecord && droppedLines > 0) {
795
+ // The check above answered "no terminal failure" from having seen no terminal record AT ALL,
796
+ // and the reader dropped at least one oversized one — so the record that decides this
797
+ // question is exactly the record most likely to have been dropped (`agent_end` carries the
798
+ // run's whole transcript). Resolving here would report a hard-failed run as a success, which
799
+ // is the case that check exists to prevent, so refuse to certify it instead.
800
+ // `no-usable-output` because that is literally what happened: the run finished and its
801
+ // terminal report never reached us.
802
+ const detail = `pi exited 0 but its terminal record was dropped for exceeding the JSONL line cap ` +
803
+ `(${droppedLines} oversized record(s)), so the run's outcome is unknown`;
804
+ return fail(new HarnessFailure('no-usable-output', tail ? `${detail}. ${tail}` : detail));
805
+ }
806
+ return {
807
+ ok: true,
808
+ outcome: { ...reduction.reduce(stdoutTail), ...(tail ? { stderrTail: tail } : {}) },
809
+ };
810
+ }
770
811
  /**
771
812
  * Build the rejection for a failed Pi run: if its terminal text points at the LLM proxy
772
813
  * refusing every model call (auth/quota/rate-limit), stamp the structured `llm-upstream`
@@ -779,51 +820,6 @@ function piRunFailure(detail, sourceText) {
779
820
  const remedy = classifyLlmUpstreamError(sourceText);
780
821
  return remedy ? new HarnessFailure('llm-upstream', `${detail}\n${remedy}`) : new Error(detail);
781
822
  }
782
- /** Parse Pi's LF-framed JSONL stdout into its event records, skipping noise. */
783
- function parsePiEvents(stdout) {
784
- const events = [];
785
- for (const raw of stdout.split('\n')) {
786
- const line = raw.trim();
787
- if (!line.startsWith('{'))
788
- continue;
789
- try {
790
- events.push(JSON.parse(line));
791
- }
792
- catch {
793
- // Not a JSON event line; skip.
794
- }
795
- }
796
- return events;
797
- }
798
- /**
799
- * The terminal-failure message when Pi's run ended in a hard error (the model was
800
- * unreachable / refused, and Pi exhausted its auto-retries), else undefined. Only
801
- * the FINAL outcome counts: a mid-run hiccup the agent recovered from leaves a clean
802
- * terminal `agent_end`, so it returns undefined. Scans from the end and decides on
803
- * the first terminal signal it meets — the trailing `auto_retry_end` (its `success`
804
- * flag) or the last `agent_end` (its `stopReason`). Pure so it is unit-testable over
805
- * a fixed event sequence.
806
- */
807
- export function terminalRunError(stdout) {
808
- const events = parsePiEvents(stdout);
809
- for (let i = events.length - 1; i >= 0; i--) {
810
- const e = events[i];
811
- if (e.type === 'auto_retry_end') {
812
- if (e.success === false) {
813
- return typeof e.finalError === 'string'
814
- ? e.finalError
815
- : 'the agent failed after exhausting its retries';
816
- }
817
- return undefined;
818
- }
819
- if (e.type === 'agent_end') {
820
- return e.stopReason === 'error' && typeof e.errorMessage === 'string'
821
- ? e.errorMessage
822
- : undefined;
823
- }
824
- }
825
- return undefined;
826
- }
827
823
  /**
828
824
  * Classify a terminal run error whose text points at the LLM PROXY rejecting every model call
829
825
  * (auth / quota / rate-limit) into an actionable remedy, else undefined. All model traffic goes
@@ -855,185 +851,3 @@ export function classifyLlmUpstreamError(finalError) {
855
851
  }
856
852
  return undefined;
857
853
  }
858
- /**
859
- * Pi's assistant summary plus {@link PiRunStats}, derived from one pass over its
860
- * output — the canonical close-of-run signal the harness uses both to report the
861
- * answer and to detect a no-op run (the agent never acted).
862
- */
863
- export function summarizePiRun(stdout) {
864
- const events = parsePiEvents(stdout);
865
- return {
866
- summary: summaryFromEvents(events, stdout),
867
- stats: statsFromEvents(events),
868
- diagnostics: diagnosticsFromEvents(events),
869
- };
870
- }
871
- /**
872
- * Output-quality signals over the canonical `agent_end` transcript: whether any
873
- * completion hit the output ceiling (its content was cut off), whether the FINAL
874
- * completion did, and whether that final turn carried no text at all. Pure so it is
875
- * unit-testable over a fixed event sequence. Defaults to all-false when there is no
876
- * terminal transcript (a no-op run is already caught by {@link agentNeverActed}).
877
- *
878
- * `cap` is the per-completion ceiling Pi requested ({@link PI_MAX_OUTPUT_TOKENS});
879
- * truncation is detected by an assistant message whose `usage.output` reached it,
880
- * which is reliable even when the model reports a non-`length` stop reason (Workers
881
- * AI labelled a cut-off tool call `tool_calls`, not `length`).
882
- */
883
- export function diagnosticsFromEvents(events, cap = PI_MAX_OUTPUT_TOKENS) {
884
- let messages;
885
- for (let i = events.length - 1; i >= 0; i--) {
886
- const e = events[i];
887
- if (e.type === 'agent_end' && Array.isArray(e.messages)) {
888
- messages = e.messages;
889
- break;
890
- }
891
- }
892
- if (!messages)
893
- return { truncated: false, finalTruncated: false, finalAnswerEmpty: false };
894
- const assistants = messages.filter((m) => isObject(m) && m.role === 'assistant');
895
- const truncated = assistants.some((m) => assistantOutputTokens(m) >= cap);
896
- const last = assistants.at(-1);
897
- return {
898
- truncated,
899
- finalTruncated: last ? assistantOutputTokens(last) >= cap : false,
900
- finalAnswerEmpty: last ? messageText(last) === '' : false,
901
- };
902
- }
903
- /** `usage.output` (completion tokens) reported on a Pi assistant message, or 0. */
904
- function assistantOutputTokens(message) {
905
- const usage = message.usage;
906
- if (!isObject(usage))
907
- return 0;
908
- const output = usage.output;
909
- return typeof output === 'number' ? output : 0;
910
- }
911
- /** {@link RunDiagnostics} over Pi's raw `--mode json` stdout (see {@link diagnosticsFromEvents}). */
912
- export function runDiagnostics(stdout, cap = PI_MAX_OUTPUT_TOKENS) {
913
- return diagnosticsFromEvents(parsePiEvents(stdout), cap);
914
- }
915
- /**
916
- * Count what the agent actually did. Prefers the canonical `agent_end`
917
- * transcript (assistant `toolCall` parts + text); falls back to the streamed
918
- * `tool_execution_end` / `message_end` events when no terminal transcript was
919
- * emitted, so a no-op is never mistaken for a real run because of a schema tweak.
920
- */
921
- function statsFromEvents(events) {
922
- for (let i = events.length - 1; i >= 0; i--) {
923
- const e = events[i];
924
- if (e.type === 'agent_end' && Array.isArray(e.messages)) {
925
- return statsFromMessages(e.messages);
926
- }
927
- }
928
- let toolCalls = 0;
929
- let toolResults = 0;
930
- let assistantChars = 0;
931
- for (const e of events) {
932
- if (e.type === 'tool_execution_end') {
933
- toolCalls++;
934
- }
935
- else if (e.type === 'message_end' && isObject(e.message)) {
936
- const m = e.message;
937
- if (m.role === 'assistant')
938
- assistantChars += messageText(m).length;
939
- else if (m.role === 'toolResult')
940
- toolResults++;
941
- }
942
- }
943
- // The same call can surface as both a `tool_execution_end` and a toolResult
944
- // `message_end`; prefer the former and only fall back to toolResult counts.
945
- return { toolCalls: toolCalls || toolResults, assistantChars };
946
- }
947
- /** {@link PiRunStats} from a transcript: assistant `toolCall` parts + text length. */
948
- function statsFromMessages(messages) {
949
- let toolCalls = 0;
950
- let assistantChars = 0;
951
- for (const m of messages) {
952
- if (!isObject(m) || m.role !== 'assistant')
953
- continue;
954
- const content = m.content;
955
- if (typeof content === 'string') {
956
- assistantChars += content.trim().length;
957
- }
958
- else if (Array.isArray(content)) {
959
- for (const part of content) {
960
- if (!isObject(part))
961
- continue;
962
- if (part.type === 'toolCall')
963
- toolCalls++;
964
- else if (typeof part.text === 'string')
965
- assistantChars += part.text.length;
966
- }
967
- }
968
- }
969
- return { toolCalls, assistantChars };
970
- }
971
- /**
972
- * Extract the assistant's final summary from Pi's JSON-lines output. Pi emits a
973
- * terminal `agent_end` event whose `messages` is the full transcript, so the
974
- * last assistant message there is the canonical answer. Falls back to scanning
975
- * `message_end` events, then to a raw tail, so a schema tweak never loses output.
976
- */
977
- export function parsePiOutput(stdout) {
978
- return summaryFromEvents(parsePiEvents(stdout), stdout);
979
- }
980
- /** Shared summary extraction over already-parsed events (see {@link parsePiOutput}). */
981
- function summaryFromEvents(events, stdout) {
982
- // Preferred: the final transcript from the last agent_end event.
983
- for (let i = events.length - 1; i >= 0; i--) {
984
- const e = events[i];
985
- if (e.type === 'agent_end' && Array.isArray(e.messages)) {
986
- const text = lastAssistantText(e.messages);
987
- if (text)
988
- return text;
989
- }
990
- }
991
- // Fallback: assistant text accumulated from message_end events.
992
- const parts = [];
993
- for (const e of events) {
994
- if (e.type === 'message_end' &&
995
- typeof e.message === 'object' &&
996
- e.message !== null &&
997
- e.message.role === 'assistant') {
998
- const text = messageText(e.message);
999
- if (text)
1000
- parts.push(text);
1001
- }
1002
- }
1003
- const joined = parts.join('\n').trim();
1004
- if (joined)
1005
- return joined;
1006
- // Nothing structured matched — return a trimmed tail of the raw output.
1007
- return stdout.trim().slice(-2000);
1008
- }
1009
- /** The text of the last assistant message in a transcript, or '' if none. */
1010
- function lastAssistantText(messages) {
1011
- for (let i = messages.length - 1; i >= 0; i--) {
1012
- const m = messages[i];
1013
- if (typeof m === 'object' && m !== null && m.role === 'assistant') {
1014
- const text = messageText(m);
1015
- if (text)
1016
- return text;
1017
- }
1018
- }
1019
- return '';
1020
- }
1021
- /** Join the text parts of a Pi message whose content is a string or parts array. */
1022
- function messageText(message) {
1023
- if (typeof message !== 'object' || message === null)
1024
- return '';
1025
- const content = message.content;
1026
- if (typeof content === 'string')
1027
- return content.trim();
1028
- if (Array.isArray(content)) {
1029
- return content
1030
- .map((part) => typeof part === 'object' &&
1031
- part !== null &&
1032
- typeof part.text === 'string'
1033
- ? part.text
1034
- : '')
1035
- .join('')
1036
- .trim();
1037
- }
1038
- return '';
1039
- }