@forwardimpact/libeval 0.1.8 → 0.1.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.8",
+  "version": "0.1.9",
   "description": "Process Claude Code stream-json output into structured traces",
   "license": "Apache-2.0",
   "author": "D. Olsson <hi@senzilla.io>",

package/src/agent-runner.js

@@ -17,7 +17,8 @@ export class AgentRunner {
    * @param {string[]} [deps.allowedTools] - Tools the agent may use
    * @param {string} [deps.permissionMode] - SDK permission mode
    * @param {function} [deps.onLine] - Callback invoked with each NDJSON line as it's produced
-   * @param {function} [deps.onBatch] - Async callback invoked with a batch of NDJSON lines at flush boundaries (assistant text blocks and result messages). Receives `(lines, { abort })` where calling `abort()` stops the in-flight SDK session via the AbortController. Optional; assignable at runtime so the Supervisor can swap it per turn.
+   * @param {function} [deps.onBatch] - Async callback invoked with a batch of NDJSON lines at flush boundaries: every `batchSize` assistant text blocks, the terminal `result` message, and — on iterator crash/abort — once more in a final flush carrying any lines that never reached a boundary. Receives `(lines, { abort })` where calling `abort()` stops the in-flight SDK session via the AbortController. Optional; assignable at runtime so the Supervisor can swap it per turn.
+   * @param {number} [deps.batchSize] - Assistant text-block messages to accumulate before firing onBatch. Tool-only assistant messages ride along without counting. Default 3: the supervisor reviews the agent every three text turns instead of every turn. The terminal `result` always flushes regardless of count.
    * @param {string[]} [deps.settingSources] - SDK setting sources (e.g. ['project'] to load CLAUDE.md)
    * @param {string} [deps.agentProfile] - Agent profile name to pass as --agent to the Claude CLI
    * @param {string|object} [deps.systemPrompt] - SDK system prompt (string replaces default; {type:'preset', preset:'claude_code', append} appends)
@@ -33,6 +34,7 @@ export class AgentRunner {
     permissionMode,
     onLine,
     onBatch,
+    batchSize,
     settingSources,
     agentProfile,
     systemPrompt,
@@ -57,6 +59,7 @@ export class AgentRunner {
     this.permissionMode = permissionMode ?? "bypassPermissions";
     this.onLine = onLine ?? null;
     this.onBatch = onBatch ?? null;
+    this.batchSize = batchSize ?? 3;
     this.settingSources = settingSources ?? [];
     this.agentProfile = agentProfile ?? null;
     this.systemPrompt = systemPrompt ?? null;
@@ -128,8 +131,18 @@ export class AgentRunner {
    * Shared consumer for both `run()` and `resume()`. Iterates the SDK query
    * iterator, mirroring every line to the output stream / buffer / onLine
    * callback, and — when `onBatch` is set — flushes accumulated lines to it
-   * at natural boundaries (assistant messages with text blocks, and the
-   * terminal `result` message).
+   * at coarse boundaries: every `batchSize` assistant text-block messages,
+   * and the terminal `result` message. Tool-only assistant messages still
+   * accumulate in the pending batch and ride along in the next flush, so
+   * the supervisor always sees the tool calls that led up to each text
+   * block. Raising `batchSize` above 1 is the knob that makes the mid-turn
+   * supervisor review less chatty — with the default of 3, the supervisor
+   * sees the agent in chunks of three text turns instead of every turn.
+   *
+   * Corollary: a turn that is *entirely* tool_use with no text blocks and
+   * then hits `result` produces exactly one flush at `result` regardless
+   * of how many tools ran. That is deliberate — the supervisor only needs
+   * to weigh in when the agent surfaces something text-like to react to.
    *
    * INVARIANT: the `await this.onBatch(...)` call below is the ONLY
    * suspension point in this loop. While it is pending, no further lines
@@ -143,6 +156,13 @@ export class AgentRunner {
    * `currentAbortController.signal.aborted` (avoiding fragility around
    * AbortError vs DOMException shapes), and report `aborted: true` so the
    * caller can distinguish "supervisor asked us to stop" from a real error.
+   *
+   * If the iterator throws before a flush boundary, any lines still in the
+   * pending batch would otherwise vanish without the supervisor seeing
+   * them. The `finally` block emits a terminal batch so the supervisor can
+   * observe the partial state (e.g. note a crash or react to an external
+   * abort). A throw from that final flush becomes the returned `error`
+   * only if no earlier error was captured — the original failure wins.
    * @param {AsyncIterable<object>} iterator
    * @returns {Promise<{success: boolean, text: string, sessionId: string|null, error: Error|null, aborted: boolean}>}
    */
@@ -151,34 +171,16 @@ export class AgentRunner {
     let stopReason = null;
     let error = null;
     let aborted = false;
-    const pendingBatch = [];
+    const state = { pendingBatch: [], assistantTextCount: 0 };
 
     try {
       for await (const message of iterator) {
-        const line = JSON.stringify(message);
-        this.output.write(line + "\n");
-        this.buffer.push(line);
-        if (this.onLine) this.onLine(line);
-        if (this.onBatch) pendingBatch.push(line);
-
-        if (message.type === "system" && message.subtype === "init") {
-          this.sessionId = message.session_id;
-        }
+        this.#recordLine(message, state);
         if (message.type === "result") {
           text = message.result ?? "";
           stopReason = message.subtype;
         }
-
-        const shouldFlush =
-          this.onBatch &&
-          (message.type === "result" ||
-            (message.type === "assistant" && hasTextBlock(message)));
-        if (shouldFlush) {
-          const batchLines = pendingBatch.splice(0, pendingBatch.length);
-          await this.onBatch(batchLines, {
-            abort: () => this.currentAbortController?.abort(),
-          });
-        }
+        await this.#maybeFlushBatch(message, state);
       }
     } catch (err) {
       if (this.currentAbortController?.signal.aborted) {
@@ -188,10 +190,84 @@ export class AgentRunner {
       }
     }
 
+    const flushErr = await this.#terminalFlush(state, { error, aborted });
+    if (flushErr && !error) error = flushErr;
+
     const success = stopReason === "success";
     return { success, text, sessionId: this.sessionId, error, aborted };
   }
 
+  /**
+   * Mirror a single SDK message to the output stream, buffer, onLine
+   * callback, and (when set) the pending-batch state. Also handles
+   * session id capture and text-block counting so `#consumeQuery` can
+   * stay within the complexity budget.
+   * @param {object} message
+   * @param {{pendingBatch: string[], assistantTextCount: number}} state
+   */
+  #recordLine(message, state) {
+    const line = JSON.stringify(message);
+    this.output.write(line + "\n");
+    this.buffer.push(line);
+    if (this.onLine) this.onLine(line);
+    if (this.onBatch) state.pendingBatch.push(line);
+
+    if (message.type === "system" && message.subtype === "init") {
+      this.sessionId = message.session_id;
+    }
+    if (message.type === "assistant" && hasTextBlock(message)) {
+      state.assistantTextCount++;
+    }
+  }
+
+  /**
+   * Terminal flush — only fires on the abnormal-end paths (iterator
+   * threw or was aborted mid-stream). Delivers any pending lines so the
+   * supervisor sees the partial state instead of losing the tail of
+   * the run. A natural-end iterator that simply ran out of messages
+   * without a `result` marker is treated as an incomplete stub (the
+   * real SDK always terminates with `result`) and its pending batch is
+   * not re-flushed. Returns an error thrown by the flush callback, or
+   * `null` if the flush succeeded or did not fire.
+   * @param {{pendingBatch: string[], assistantTextCount: number}} state
+   * @param {{error: Error|null, aborted: boolean}} outcome
+   * @returns {Promise<Error|null>}
+   */
+  async #terminalFlush(state, { error, aborted }) {
+    const loopEndedAbnormally = Boolean(error || aborted);
+    if (!loopEndedAbnormally) return null;
+    if (!this.onBatch || state.pendingBatch.length === 0) return null;
+    try {
+      const batchLines = state.pendingBatch.splice(0);
+      await this.onBatch(batchLines, {
+        abort: () => this.currentAbortController?.abort(),
+      });
+      return null;
+    } catch (flushErr) {
+      return flushErr;
+    }
+  }
+
+  /**
+   * Flush the pending batch to `onBatch` if either the batchSize threshold
+   * has been reached or the current message is the terminal `result`.
+   * Extracted so that `#consumeQuery` stays within the project's complexity
+   * budget — the flush is one cohesive unit of logic in its own right.
+   * @param {object} message
+   * @param {{pendingBatch: string[], assistantTextCount: number}} state
+   */
+  async #maybeFlushBatch(message, state) {
+    if (!this.onBatch) return;
+    const shouldFlush =
+      message.type === "result" || state.assistantTextCount >= this.batchSize;
+    if (!shouldFlush) return;
+    state.assistantTextCount = 0;
+    const batchLines = state.pendingBatch.splice(0);
+    await this.onBatch(batchLines, {
+      abort: () => this.currentAbortController?.abort(),
+    });
+  }
+
   /**
    * Drain buffered output lines. Used by Supervisor to tag and re-emit lines.
    * @returns {string[]}
@@ -205,13 +281,14 @@ export class AgentRunner {
 
 /**
  * Whether an SDK assistant message contains at least one text block.
- * Tool-only assistant messages return false so they accumulate into the
- * pending batch and flush with the next text block (or with the terminal
- * `result` message), keeping supervisor LLM cost bounded.
+ * Only text-block messages count toward the `batchSize` threshold — tool-only
+ * assistant messages accumulate silently into the pending batch and ride along
+ * in the next flush, keeping supervisor LLM cost bounded. Exported so the mock
+ * runner can mirror the real flush predicate without duplicating the logic.
  * @param {object} message
  * @returns {boolean}
  */
-function hasTextBlock(message) {
+export function hasTextBlock(message) {
   const content = message.message?.content ?? message.content;
   if (!Array.isArray(content)) return false;
   for (const block of content) {

package/test/agent-runner-batching.test.js

@@ -0,0 +1,271 @@
+import { describe, test } from "node:test";
+import assert from "node:assert";
+import { PassThrough } from "node:stream";
+
+import { AgentRunner } from "@forwardimpact/libeval";
+
+/**
+ * Create a mock query function that yields canned messages.
+ * @param {object[]} messages - Messages to yield
+ * @returns {function}
+ */
+function mockQuery(messages) {
+  return async function* () {
+    for (const msg of messages) {
+      yield msg;
+    }
+  };
+}
+
+const textBlock = (t) => ({
+  type: "assistant",
+  message: { content: [{ type: "text", text: t }] },
+});
+
+const toolOnly = (name) => ({
+  type: "assistant",
+  message: {
+    content: [{ type: "tool_use", id: "tu_" + name, name, input: {} }],
+  },
+});
+
+describe("AgentRunner - onBatch batching", () => {
+  test("batchSize defaults to 3", () => {
+    const runner = new AgentRunner({
+      cwd: "/tmp",
+      query: async function* () {},
+      output: new PassThrough(),
+    });
+    assert.strictEqual(runner.batchSize, 3);
+  });
+
+  test("onBatch fires every 3 assistant text-block messages by default", async () => {
+    // 5 text-block messages + terminal result. With the default batchSize
+    // of 3, onBatch should fire on the 3rd text message and again on the
+    // terminal result (flushing the remaining 2).
+    const messages = [
+      { type: "system", subtype: "init", session_id: "sess-batch" },
+      textBlock("one"),
+      textBlock("two"),
+      textBlock("three"),
+      textBlock("four"),
+      textBlock("five"),
+      { type: "result", subtype: "success", result: "Done." },
+    ];
+
+    const batches = [];
+    const runner = new AgentRunner({
+      cwd: "/tmp",
+      query: mockQuery(messages),
+      output: new PassThrough(),
+    });
+    runner.onBatch = async (lines) => {
+      batches.push(lines.map((l) => JSON.parse(l)));
+    };
+
+    await runner.run("Task");
+
+    // First flush carries init + first 3 text messages; second carries
+    // remaining 2 text messages + the result.
+    assert.strictEqual(batches.length, 2);
+    assert.strictEqual(batches[0].length, 4);
+    assert.strictEqual(batches[1].length, 3);
+  });
+
+  test("onBatch honours custom batchSize", async () => {
+    // batchSize = 2: 4 text messages produce 2 flushes; result adds a 3rd.
+    const messages = [
+      textBlock("a"),
+      textBlock("b"),
+      textBlock("c"),
+      textBlock("d"),
+      { type: "result", subtype: "success", result: "Done." },
+    ];
+
+    const batches = [];
+    const runner = new AgentRunner({
+      cwd: "/tmp",
+      query: mockQuery(messages),
+      output: new PassThrough(),
+      batchSize: 2,
+    });
+    runner.onBatch = async (lines) => {
+      batches.push(lines.length);
+    };
+
+    await runner.run("Task");
+
+    assert.deepStrictEqual(batches, [2, 2, 1]);
+  });
+
+  test("tool-only assistant messages ride along in the next flush", async () => {
+    // Tool-only assistant messages accumulate without incrementing the
+    // counter. The supervisor sees the preceding tool calls when the
+    // flush eventually fires.
+    const messages = [
+      toolOnly("Read"),
+      toolOnly("Grep"),
+      textBlock("found it"),
+      { type: "result", subtype: "success", result: "Done." },
+    ];
+
+    const batches = [];
+    const runner = new AgentRunner({
+      cwd: "/tmp",
+      query: mockQuery(messages),
+      output: new PassThrough(),
+      batchSize: 1,
+    });
+    runner.onBatch = async (lines) => {
+      batches.push(lines.map((l) => JSON.parse(l)));
+    };
+
+    await runner.run("Task");
+
+    // First flush triggered by the single text-block message; it carries
+    // the two preceding tool-only messages with it.
+    assert.strictEqual(batches.length, 2);
+    assert.strictEqual(batches[0].length, 3);
+    assert.strictEqual(batches[0][0].message.content[0].type, "tool_use");
+    assert.strictEqual(batches[0][1].message.content[0].type, "tool_use");
+    assert.strictEqual(batches[0][2].message.content[0].type, "text");
+    assert.strictEqual(batches[1].length, 1);
+    assert.strictEqual(batches[1][0].type, "result");
+  });
+
+  test("terminal result always flushes even if batchSize not yet reached", async () => {
+    // 1 text-block + result, batchSize = 5. The counter only reaches 1
+    // but the terminal result must still flush.
+    const messages = [
+      textBlock("only one"),
+      { type: "result", subtype: "success", result: "Done." },
+    ];
+
+    const batches = [];
+    const runner = new AgentRunner({
+      cwd: "/tmp",
+      query: mockQuery(messages),
+      output: new PassThrough(),
+      batchSize: 5,
+    });
+    runner.onBatch = async (lines) => {
+      batches.push(lines.length);
+    };
+
+    await runner.run("Task");
+
+    assert.deepStrictEqual(batches, [2]);
+  });
+});
+
+describe("AgentRunner - terminal flush on abnormal end", () => {
+  test("iterator crash before a flush boundary still delivers the pending batch", async () => {
+    // batchSize = 3: the first two text messages accumulate without
+    // flushing. The iterator then throws before the threshold — the
+    // pending batch must ship in a terminal flush.
+    async function* crashingQuery() {
+      yield { type: "system", subtype: "init", session_id: "sess-crash" };
+      yield textBlock("step 1");
+      yield textBlock("step 2");
+      throw new Error("Claude Code process exited with code 1");
+    }
+
+    const batches = [];
+    const runner = new AgentRunner({
+      cwd: "/tmp",
+      query: () => crashingQuery(),
+      output: new PassThrough(),
+    });
+    runner.onBatch = async (lines) => {
+      batches.push(lines.map((l) => JSON.parse(l)));
+    };
+
+    const result = await runner.run("Task");
+
+    assert.ok(result.error);
+    assert.match(result.error.message, /exited with code 1/);
+    assert.strictEqual(batches.length, 1);
+    assert.strictEqual(batches[0].length, 3);
+    assert.strictEqual(batches[0][0].type, "system");
+    assert.strictEqual(batches[0][1].type, "assistant");
+    assert.strictEqual(batches[0][2].type, "assistant");
+  });
+
+  test("iterator crash after a completed batch does not re-flush", async () => {
+    // batchSize = 2: two text messages trigger a normal flush, emptying
+    // the pending batch. The iterator then throws with nothing pending —
+    // the terminal flush must be a no-op, not an empty call.
+    async function* crashingQuery() {
+      yield textBlock("a");
+      yield textBlock("b");
+      throw new Error("boom");
+    }
+
+    const batches = [];
+    const runner = new AgentRunner({
+      cwd: "/tmp",
+      query: () => crashingQuery(),
+      output: new PassThrough(),
+      batchSize: 2,
+    });
+    runner.onBatch = async (lines) => {
+      batches.push(lines.length);
+    };
+
+    const result = await runner.run("Task");
+    assert.ok(result.error);
+    assert.match(result.error.message, /boom/);
+    assert.deepStrictEqual(batches, [2]);
+  });
+
+  test("natural-end iterator without a result does not trigger terminal flush", async () => {
+    // The real SDK always terminates with `result`. A mock that ends
+    // naturally with pending lines is treated as an incomplete stub —
+    // no phantom flush, since nothing about a natural end warrants a
+    // new mid-turn review.
+    async function* noResultQuery() {
+      yield textBlock("one");
+      yield textBlock("two");
+      // No result, no error — just ends.
+    }
+
+    const batches = [];
+    const runner = new AgentRunner({
+      cwd: "/tmp",
+      query: () => noResultQuery(),
+      output: new PassThrough(),
+      batchSize: 3,
+    });
+    runner.onBatch = async (lines) => {
+      batches.push(lines.length);
+    };
+
+    const result = await runner.run("Task");
+    assert.strictEqual(result.error, null);
+    assert.strictEqual(batches.length, 0);
+  });
+
+  test("onBatch throw during terminal flush does not mask an earlier error", async () => {
+    // The iterator threw first; the terminal flush also throws. The
+    // original iterator error must win — it is the more actionable
+    // condition to surface to the caller.
+    async function* crashingQuery() {
+      yield textBlock("partial");
+      throw new Error("original failure");
+    }
+
+    const runner = new AgentRunner({
+      cwd: "/tmp",
+      query: () => crashingQuery(),
+      output: new PassThrough(),
+      batchSize: 3,
+    });
+    runner.onBatch = async () => {
+      throw new Error("flush failure");
+    };
+
+    const result = await runner.run("Task");
+    assert.ok(result.error);
+    assert.match(result.error.message, /original failure/);
+  });
+});

package/test/mock-runner.js

@@ -1,10 +1,13 @@
 /**
  * Test-only mock factory for AgentRunner. Yields pre-scripted responses,
  * and (when an `onBatch` callback is set) fires it at the same boundaries
- * the real AgentRunner would: assistant messages with at least one text
- * block, and the terminal `result` message. If the callback calls
- * `abort()`, the mock stops iterating that response's messages and
- * reports `aborted: true`.
+ * the real AgentRunner would: every `runner.batchSize` assistant messages
+ * with a text block, and the terminal `result` message. Tool-only
+ * assistant messages accumulate into the pending batch without counting
+ * toward the threshold. If the callback calls `abort()`, the mock stops
+ * iterating that response's messages and reports `aborted: true` — any
+ * lines that never made it through a flush boundary then ship in a
+ * terminal batch, mirroring the real runner's finally-flush.
  *
  * Intentionally a regular module (not a test file) so describe/test blocks
  * here would not run. Lives under test/ to make its scope explicit.
@@ -12,24 +15,7 @@
 
 import { PassThrough } from "node:stream";
 import { AgentRunner } from "@forwardimpact/libeval";
-
-/**
- * Whether a scripted message should trigger an onBatch flush. Mirrors the
- * real AgentRunner: assistant-with-text-block or terminal `result` message.
- * Tool-only or string-content messages accumulate without flushing.
- * @param {object} message
- * @returns {boolean}
- */
-export function shouldFlush(message) {
-  if (message.type === "result") return true;
-  if (message.type !== "assistant") return false;
-  const content = message.message?.content ?? message.content;
-  if (!Array.isArray(content)) return false;
-  for (const block of content) {
-    if (block.type === "text" && block.text) return true;
-  }
-  return false;
-}
+import { hasTextBlock } from "../src/agent-runner.js";
 
 /**
  * Create a mock AgentRunner that yields pre-scripted responses. Each call
@@ -50,12 +36,25 @@ export function createMockRunner(responses, messages) {
 
   const consume = async (msgs) => {
     let aborted = false;
+    const pendingBatch = [];
+    let assistantTextCount = 0;
     for (const m of msgs) {
       const line = JSON.stringify(m);
       runner.buffer.push(line);
       if (runner.onLine) runner.onLine(line);
-      if (runner.onBatch && shouldFlush(m)) {
-        await runner.onBatch([line], {
+      if (runner.onBatch) pendingBatch.push(line);
+
+      if (hasTextBlock(m)) {
+        assistantTextCount++;
+      }
+
+      const shouldFlush =
+        runner.onBatch &&
+        (m.type === "result" || assistantTextCount >= runner.batchSize);
+      if (shouldFlush) {
+        assistantTextCount = 0;
+        const batchLines = pendingBatch.splice(0);
+        await runner.onBatch(batchLines, {
           abort: () => {
             aborted = true;
           },
@@ -63,6 +62,19 @@ export function createMockRunner(responses, messages) {
         if (aborted) break;
       }
     }
+    // Terminal flush: mirror the real AgentRunner's abnormal-end path —
+    // an aborted scripted run delivers any pending tail so the supervisor
+    // sees the partial state. Natural-end without a `result` marker is
+    // treated as a simplified stub (no phantom flush), matching the real
+    // runner's rule that terminal flush only fires on error/abort.
+    if (aborted && runner.onBatch && pendingBatch.length > 0) {
+      const batchLines = pendingBatch.splice(0);
+      await runner.onBatch(batchLines, {
+        abort: () => {
+          aborted = true;
+        },
+      });
+    }
     return aborted;
   };
 

package/test/supervisor-batching.test.js

@@ -0,0 +1,175 @@
+import { describe, test } from "node:test";
+import assert from "node:assert";
+import { PassThrough } from "node:stream";
+
+import { Supervisor } from "@forwardimpact/libeval";
+import { createMockRunner } from "./mock-runner.js";
+
+const textBlock = (t) => ({
+  type: "assistant",
+  message: { content: [{ type: "text", text: t }] },
+});
+
+describe("Supervisor - batching at the default batchSize", () => {
+  test("mid-turn review fires once per 3 agent text messages", async () => {
+    // Agent emits 7 text-block assistant messages in one turn. With the
+    // default batchSize of 3 the supervisor's mid-turn review should fire
+    // twice (after messages 3 and 6) plus once more from the terminal
+    // result flush carrying the remaining message — not seven times, as
+    // the old per-message flushing would have done.
+    const agentMessages = [
+      [
+        textBlock("step 1"),
+        textBlock("step 2"),
+        textBlock("step 3"),
+        textBlock("step 4"),
+        textBlock("step 5"),
+        textBlock("step 6"),
+        textBlock("step 7"),
+        { type: "result", subtype: "success", result: "Done." },
+      ],
+    ];
+
+    const agentRunner = createMockRunner(
+      [{ text: "Finished." }],
+      agentMessages,
+    );
+    // Leave batchSize at the default (3) — this is the behaviour we're
+    // verifying end-to-end through the supervisor loop.
+    assert.strictEqual(agentRunner.batchSize, 3);
+
+    const supervisorRunner = createMockRunner([
+      { text: "Welcome. Begin." },
+      { text: "Keep going." }, // mid-turn batch 1 (messages 1-3)
+      { text: "Keep going." }, // mid-turn batch 2 (messages 4-6)
+      { text: "Keep going." }, // terminal result flush (message 7 + result)
+      { text: "Good work.\n\nEVALUATION_COMPLETE" }, // end-of-turn review
+    ]);
+
+    const output = new PassThrough();
+    const supervisor = new Supervisor({
+      agentRunner,
+      supervisorRunner,
+      output,
+      maxTurns: 10,
+    });
+    agentRunner.onLine = (line) => supervisor.emitLine(line);
+    supervisorRunner.onLine = (line) => supervisor.emitLine(line);
+
+    const result = await supervisor.run("Do the task");
+    assert.strictEqual(result.success, true);
+
+    const midTurnReviews = (output.read()?.toString() ?? "")
+      .trim()
+      .split("\n")
+      .filter((l) => l.length > 0)
+      .map((l) => JSON.parse(l))
+      .filter(
+        (l) =>
+          l.source === "orchestrator" && l.event?.type === "mid_turn_review",
+      );
+
+    // 3 flushes total: two at the batchSize threshold (messages 3 and 6),
+    // one at the terminal result (trailing message + result marker).
+    assert.strictEqual(
+      midTurnReviews.length,
+      3,
+      "Supervisor should review 3 times per turn, not 7",
+    );
+  });
+
+  test("EVALUATION_INTERVENTION at the default batchSize still aborts and relays", async () => {
+    // Companion to the observation test above: the 3-message batching and
+    // the intervention path exercised together.
+    //
+    // Agent call 1 emits 3 text-block messages (triggering a flush at the
+    // 3rd). The supervisor intervenes; the agent SDK session aborts and
+    // the supervisor's intervention text is relayed into resume(). Agent
+    // call 2 has 1 text block — below the batchSize threshold — so no
+    // extra mid-turn flush fires, and the supervisor jumps straight to
+    // the end-of-turn review.
+    const agentMessages = [
+      [
+        textBlock("reading docs"),
+        textBlock("running Bash"),
+        textBlock("found the wrong path"),
+      ],
+      [textBlock("corrected, using the documented path")],
+    ];
+
+    const agentRunner = createMockRunner(
+      [{ text: "wrong path" }, { text: "corrected" }],
+      agentMessages,
+    );
+    assert.strictEqual(agentRunner.batchSize, 3);
+
+    const supervisorMessages = [
+      undefined,
+      [
+        {
+          type: "assistant",
+          message: {
+            content: [
+              {
+                type: "text",
+                text: "EVALUATION_INTERVENTION Use the documented path.",
+              },
+            ],
+          },
+        },
+      ],
+      undefined,
+    ];
+
+    const supervisorRunner = createMockRunner(
+      [
+        { text: "Welcome. Begin." },
+        { text: "EVALUATION_INTERVENTION Use the documented path." },
+        { text: "Good.\n\nEVALUATION_COMPLETE" },
+      ],
+      supervisorMessages,
+    );
+
+    const output = new PassThrough();
+    const supervisor = new Supervisor({
+      agentRunner,
+      supervisorRunner,
+      output,
+      maxTurns: 10,
+    });
+    agentRunner.onLine = (line) => supervisor.emitLine(line);
+    supervisorRunner.onLine = (line) => supervisor.emitLine(line);
+
+    let resumePrompt = null;
+    const origResume = agentRunner.resume;
+    agentRunner.resume = async (prompt) => {
+      resumePrompt = prompt;
+      return origResume.call(agentRunner, prompt);
+    };
+
+    const result = await supervisor.run("Install");
+    assert.strictEqual(result.success, true);
+    assert.strictEqual(result.turns, 1);
+    assert.ok(
+      resumePrompt && resumePrompt.includes("documented path"),
+      "Resume prompt should carry the supervisor's intervention text",
+    );
+
+    const orchestratorEvents = (output.read()?.toString() ?? "")
+      .trim()
+      .split("\n")
+      .filter((l) => l.length > 0)
+      .map((l) => JSON.parse(l))
+      .filter((e) => e.source === "orchestrator");
+    assert.ok(
+      orchestratorEvents.some(
+        (e) => e.event?.type === "intervention_requested",
+      ),
+      "Trace should contain intervention_requested",
+    );
+    assert.ok(
+      orchestratorEvents.some((e) => e.event?.type === "intervention_relayed"),
+      "Trace should contain intervention_relayed",
+    );
+  });
+});

package/test/supervisor-intervention.test.js

@@ -65,6 +65,9 @@ describe("Supervisor - mid-turn intervention", () => {
     // Supervisor responds with "Keep going." — neither signal flag is set,
     // so the agent's SDK session completes naturally and the end-of-turn
     // review then emits EVALUATION_COMPLETE.
+    //
+    // batchSize = 1 keeps this test focused on intervention semantics, not
+    // on the coarser default batching (3) exercised by agent-runner.test.js.
     const agentMessages = [
       [
         {
@@ -80,6 +83,7 @@ describe("Supervisor - mid-turn intervention", () => {
       [{ text: "I'm working on it." }],
       agentMessages,
     );
+    agentRunner.batchSize = 1;
 
     const supervisorRunner = createMockRunner([
       { text: "Welcome! Please install." },
@@ -167,6 +171,7 @@ describe("Supervisor - mid-turn intervention", () => {
       ],
       agentMessages,
     );
+    agentRunner.batchSize = 1;
 
     // Supervisor responses (in order):
     //   0: turn 0 introduction
@@ -277,6 +282,7 @@ describe("Supervisor - mid-turn intervention", () => {
       [{ text: "Trying X." }, { text: "Trying Y." }],
       agentMessages,
     );
+    agentRunner.batchSize = 1;
 
     const supervisorMessages = [
       undefined,

package/test/supervisor-output.test.js

@@ -146,6 +146,7 @@ describe("Supervisor - output and events", () => {
       [{ text: "Trying the wrong thing." }, { text: "Switching." }],
       agentMessages,
     );
+    agentRunner.batchSize = 1;
     const supervisorRunner = createMockRunner(
       [
         { text: "Welcome." },