@forwardimpact/libeval 0.1.5 → 0.1.8

package/src/supervisor.js

@@ -13,25 +13,49 @@ import { TraceCollector } from "./trace-collector.js";
 
 /**
  * Check if the supervisor's response signals evaluation success.
- * Matches EVALUATION_SUCCESSFUL anywhere in the text, tolerating markdown
- * formatting (e.g. **EVALUATION_SUCCESSFUL**). Uses word boundaries to
+ * Matches EVALUATION_COMPLETE anywhere in the text, tolerating markdown
+ * formatting (e.g. **EVALUATION_COMPLETE**). Uses word boundaries to
  * avoid matching inside longer identifiers.
  * @param {string} text
  * @returns {boolean}
  */
-export function isSuccessful(text) {
-  return /(?:^|[\s*_~`])EVALUATION_SUCCESSFUL(?:[\s*_~`.,!?]|$)/m.test(text);
+export function isComplete(text) {
+  return /(?:^|[\s*_~`])EVALUATION_COMPLETE(?:[\s*_~`.,!?]|$)/m.test(text);
+}
+
+/**
+ * Check if the supervisor's response signals a mid-turn intervention.
+ * Same tolerance rules as isComplete (markdown formatting, word boundaries),
+ * but matches the EVALUATION_INTERVENTION keyword instead.
+ * @param {string} text
+ * @returns {boolean}
+ */
+export function isIntervention(text) {
+  return /(?:^|[\s*_~`])EVALUATION_INTERVENTION(?:[\s*_~`.,!?]|$)/m.test(text);
 }
 
 /** System prompt appended for the supervisor runner in supervise mode. */
 export const SUPERVISOR_SYSTEM_PROMPT =
-  "You supervise another AI agent through a relay — your output becomes the agent's next input. " +
-  "Guide the agent, answer its questions, and write EVALUATION_SUCCESSFUL when their task is complete.";
+  "You relay messages to one persistent agent session — your only output " +
+  "channel. Spawning sub-agents or restarting the agent is blocked. Do not " +
+  "do the work yourself. Reply briefly to let the agent continue, write " +
+  "EVALUATION_INTERVENTION + instructions to interrupt mid-turn, or " +
+  "EVALUATION_COMPLETE when done. Only your final message each turn is " +
+  "relayed.";
 
 /** System prompt appended for the agent runner in supervise mode. */
 export const AGENT_SYSTEM_PROMPT =
-  "You are being supervised by another AI agent. " +
-  "When requirements are ambiguous or you are uncertain, stop and ask a clarifying question before proceeding.";
+  "A supervisor watches your work and may interrupt with new instructions " +
+  "mid-task. Treat any new prompt as authoritative and adjust course. " +
+  "When uncertain, stop and ask a clarifying question.";
+
+/**
+ * Maximum number of mid-turn interventions allowed within a single agent turn.
+ * Bounded so a looping supervisor exhausts its quota fast (observability) but
+ * leaves headroom for legitimate "intervene, observe, intervene again" patterns.
+ * The outer maxTurns budget still bounds overall runtime.
+ */
+const MAX_INTERVENTIONS_PER_TURN = 5;
 
 export class Supervisor {
   /**
@@ -53,6 +77,31 @@ export class Supervisor {
     this.currentSource = "agent";
     /** @type {number} */
     this.currentTurn = 0;
+    /**
+     * Set to true when any supervisor message contains the success signal.
+     * The SDK result text only reflects the last assistant message, so when
+     * the supervisor writes EVALUATION_COMPLETE in an early message and
+     * then continues with follow-up work, the result text won't contain it.
+     * This flag captures the signal from the full message stream.
+     * @type {boolean}
+     */
+    this.completeSignalSeen = false;
+    /**
+     * Set to true when any supervisor message contains EVALUATION_INTERVENTION.
+     * Mirrors completeSignalSeen — populated by emitLine when a supervisor
+     * assistant text block matches isIntervention(...). The mid-turn loop
+     * reads this flag after each supervisor invocation to decide whether to
+     * abort the agent's in-flight SDK session.
+     * @type {boolean}
+     */
+    this.interventionSignalSeen = false;
+    /**
+     * The most recent supervisor SDK result captured inside the mid-turn
+     * onBatch callback. The outer loop reads this after the agent aborts to
+     * build the next relay prompt without re-running the supervisor.
+     * @type {{success: boolean, text: string}|null}
+     */
+    this.lastSupervisorResult = null;
   }
 
   /**
@@ -66,6 +115,9 @@ export class Supervisor {
     // Turn 0: Supervisor receives the task and introduces it to the agent
     this.currentSource = "supervisor";
     this.currentTurn = 0;
+    this.completeSignalSeen = false;
+    this.interventionSignalSeen = false;
+    this.lastSupervisorResult = null;
     let supervisorResult = await this.supervisorRunner.run(task);
 
     if (supervisorResult.error) {
@@ -73,55 +125,192 @@ export class Supervisor {
       return { success: false, turns: 0 };
     }
 
-    // The supervisor's turn is fully complete (all tool calls executed) by the
-    // time we check the signal — no work is interrupted.
-    if (isSuccessful(supervisorResult.text)) {
+    // Check for the success signal in either the SDK result text or the
+    // streamed message content. The SDK result text only reflects the last
+    // assistant message, so when the supervisor writes EVALUATION_COMPLETE
+    // early and then continues (e.g. filing issues), we must also check the
+    // flag set by emitLine during streaming.
+    if (this.completeSignalSeen || isComplete(supervisorResult.text)) {
       this.emitSummary({ success: true, turns: 0 });
       return { success: true, turns: 0 };
     }
 
-    for (let turn = 1; turn <= this.maxTurns; turn++) {
-      // Supervisor's output becomes the agent's input
-      this.currentSource = "agent";
-      this.currentTurn = turn;
-      let agentResult;
-      if (turn === 1) {
-        agentResult = await this.agentRunner.run(supervisorResult.text);
-      } else {
-        agentResult = await this.agentRunner.resume(supervisorResult.text);
-      }
+    const turnLimit = this.maxTurns === 0 ? Infinity : this.maxTurns;
+    for (let turn = 1; turn <= turnLimit; turn++) {
+      // Only the supervisor's final message is relayed to the agent.
+      // Extract the last assistant text block from the buffer to avoid
+      // leaking intermediate reasoning (research, tool calls, notes).
+      const relay = this.extractLastText(
+        this.supervisorRunner,
+        supervisorResult.text,
+      );
 
-      if (agentResult.error) {
-        this.emitSummary({ success: false, turns: turn });
-        return { success: false, turns: turn };
-      }
+      // Drive the agent through interventions until its SDK session ends
+      // naturally, the supervisor signals completion mid-turn, or the
+      // per-turn intervention budget is exhausted.
+      const turnOutcome = await this.#runAgentTurn(turn, relay);
+      if (turnOutcome.exit) return turnOutcome.exit;
 
-      // Build the full agent transcript from buffered NDJSON events so the
-      // supervisor sees tool calls and reasoning, not just the SDK result summary.
-      const agentTranscript = this.extractTranscript(this.agentRunner);
+      // End-of-turn review (existing behaviour). Returns either an exit
+      // outcome (error or completion) or the supervisor result for the
+      // next turn's relay.
+      const reviewOutcome = await this.#endOfTurnReview(turn);
+      if (reviewOutcome.exit) return reviewOutcome.exit;
+      supervisorResult = reviewOutcome.supervisorResult;
+    }
 
-      const supervisorPrompt =
-        `The agent reported:\n\n${agentTranscript}\n\n` +
-        `Review the agent's work and decide how to proceed.`;
+    this.emitSummary({ success: false, turns: this.maxTurns });
+    return { success: false, turns: this.maxTurns };
+  }
 
-      this.currentSource = "supervisor";
-      this.currentTurn = turn;
-      supervisorResult = await this.supervisorRunner.resume(supervisorPrompt);
+  /**
+   * Drive the agent through one turn, allowing the supervisor to interrupt
+   * mid-stream via EVALUATION_INTERVENTION. Returns either an `exit` outcome
+   * (the loop should return immediately) or `{exit: null}` (proceed to the
+   * end-of-turn review).
+   * @param {number} turn
+   * @param {string} initialRelay
+   * @returns {Promise<{exit: {success: boolean, turns: number}|null}>}
+   */
+  async #runAgentTurn(turn, initialRelay) {
+    let relay = initialRelay;
+    let interventions = 0;
 
-      if (supervisorResult.error) {
-        this.emitSummary({ success: false, turns: turn });
-        return { success: false, turns: turn };
-      }
+    // Wire the mid-turn observation hook on the agent runner. The bound
+    // callback captures `turn` so the inner loop's multiple resume(...)
+    // calls all see the same turn id. The supervisorRunner does NOT get
+    // an onBatch callback — it only fires onLine, which is enough for
+    // emitLine to detect EVALUATION_COMPLETE / EVALUATION_INTERVENTION.
+    this.agentRunner.onBatch = (batchLines, ctx) =>
+      this.#midTurnReview(turn, batchLines, ctx);
 
-      // The supervisor's turn is fully complete — check for success signal.
-      if (isSuccessful(supervisorResult.text)) {
-        this.emitSummary({ success: true, turns: turn });
-        return { success: true, turns: turn };
+    try {
+      while (true) {
+        this.currentSource = "agent";
+        this.currentTurn = turn;
+        const isFirstAgentCall = turn === 1 && interventions === 0;
+        const agentResult = isFirstAgentCall
+          ? await this.agentRunner.run(relay)
+          : await this.agentRunner.resume(relay);
+
+        if (agentResult.error && !agentResult.aborted) {
+          this.emitSummary({ success: false, turns: turn });
+          return { exit: { success: false, turns: turn } };
+        }
+
+        // Mid-turn EVALUATION_COMPLETE: end the session immediately.
+        if (this.completeSignalSeen) {
+          this.emitSummary({ success: true, turns: turn });
+          return { exit: { success: true, turns: turn } };
+        }
+
+        if (agentResult.aborted && this.interventionSignalSeen) {
+          interventions++;
+          if (interventions >= MAX_INTERVENTIONS_PER_TURN) {
+            this.emitOrchestratorEvent({ type: "intervention_limit", turn });
+            return { exit: null };
+          }
+          relay = this.extractLastText(
+            this.supervisorRunner,
+            this.lastSupervisorResult?.text ?? "",
+          );
+          this.emitOrchestratorEvent({ type: "intervention_relayed", turn });
+          continue;
+        }
+
+        // Agent's SDK session finished naturally — proceed to end-of-turn.
+        return { exit: null };
       }
+    } finally {
+      // Detach onBatch before the end-of-turn review so the supervisor's
+      // own SDK session does not trigger nested onBatch fires.
+      this.agentRunner.onBatch = null;
     }
+  }
 
-    this.emitSummary({ success: false, turns: this.maxTurns });
-    return { success: false, turns: this.maxTurns };
+  /**
+   * Mid-turn supervisor review fired from inside the agent's onBatch hook.
+   * Emits a `mid_turn_review` orchestrator marker, runs the supervisor's
+   * LLM against the batch, and aborts the agent if the supervisor signals
+   * EVALUATION_INTERVENTION or EVALUATION_COMPLETE.
+   * @param {number} turn
+   * @param {string[]} batchLines
+   * @param {{abort: () => void}} ctx
+   */
+  async #midTurnReview(turn, batchLines, { abort }) {
+    const batchTranscript = this.renderBatch(batchLines);
+
+    // Order matters: emit the orchestrator marker BEFORE the supervisor
+    // LLM call so the trace reads
+    //   agent line → orchestrator:mid_turn_review
+    //   → supervisor lines (tagged turn:N)
+    //   → orchestrator:intervention_requested|complete_requested
+    this.emitOrchestratorEvent({ type: "mid_turn_review", turn });
+
+    // currentTurn stays = turn so mid-turn supervisor lines share the
+    // agent's turn id. They are distinguishable from end-of-turn reviews
+    // by the surrounding orchestrator events emitted around this call.
+    this.currentSource = "supervisor";
+    this.completeSignalSeen = false;
+    this.interventionSignalSeen = false;
+
+    this.lastSupervisorResult = await this.supervisorRunner.resume(
+      `The agent is mid-turn. Latest batch:\n\n${batchTranscript}\n\n` +
+        `Respond with a brief acknowledgement to let it continue, or write ` +
+        `EVALUATION_INTERVENTION followed by a corrective message to stop ` +
+        `and relay a new instruction. Write EVALUATION_COMPLETE only when ` +
+        `the task is fully done.`,
+    );
+    this.currentSource = "agent";
+
+    if (this.interventionSignalSeen) {
+      this.emitOrchestratorEvent({ type: "intervention_requested", turn });
+      abort();
+      return;
+    }
+    if (this.completeSignalSeen) {
+      this.emitOrchestratorEvent({ type: "complete_requested", turn });
+      abort();
+    }
+    // Non-intervention: do nothing; the agent loop pulls the next line.
+  }
+
+  /**
+   * End-of-turn supervisor review (existing behaviour). Returns either an
+   * exit outcome (error or completion) or the supervisor result so the
+   * outer loop can build the next turn's relay.
+   * @param {number} turn
+   * @returns {Promise<{exit: {success: boolean, turns: number}|null, supervisorResult?: object}>}
+   */
+  async #endOfTurnReview(turn) {
+    // Build the full agent transcript from buffered NDJSON events so the
+    // supervisor sees tool calls and reasoning, not just the SDK result.
+    const agentTranscript = this.extractTranscript(this.agentRunner);
+
+    const supervisorPrompt =
+      `The agent reported:\n\n${agentTranscript}\n\n` +
+      `Review the agent's work and decide how to proceed.`;
+
+    this.currentSource = "supervisor";
+    this.currentTurn = turn;
+    this.completeSignalSeen = false;
+    this.interventionSignalSeen = false;
+    const supervisorResult =
+      await this.supervisorRunner.resume(supervisorPrompt);
+
+    if (supervisorResult.error) {
+      this.emitSummary({ success: false, turns: turn });
+      return { exit: { success: false, turns: turn } };
+    }
+
+    // The supervisor's turn is fully complete — check for success signal
+    // in either the SDK result text or streamed messages.
+    if (this.completeSignalSeen || isComplete(supervisorResult.text)) {
+      this.emitSummary({ success: true, turns: turn });
+      return { exit: { success: true, turns: turn } };
+    }
+
+    return { exit: null, supervisorResult };
   }
 
   /**
@@ -139,9 +328,38 @@ export class Supervisor {
     return collector.toText() || "[The agent produced no output.]";
   }
 
+  /**
+   * Extract only the last assistant text block from an AgentRunner's buffer.
+   * Scans buffered NDJSON events in reverse to find the final assistant message
+   * with a text content block. This prevents intermediate reasoning (tool calls,
+   * research notes) from leaking to the agent.
+   * @param {import("./agent-runner.js").AgentRunner} runner
+   * @param {string} fallback - Fallback text if no assistant text block is found
+   * @returns {string}
+   */
+  extractLastText(runner, fallback) {
+    const lines = runner.buffer;
+    for (let i = lines.length - 1; i >= 0; i--) {
+      const event = JSON.parse(lines[i]);
+      if (event.type !== "assistant") continue;
+      const content = event.message?.content ?? event.content;
+      if (!Array.isArray(content)) continue;
+      for (let j = content.length - 1; j >= 0; j--) {
+        if (content[j].type === "text" && content[j].text) {
+          return content[j].text;
+        }
+      }
+    }
+    return fallback;
+  }
+
   /**
    * Emit a single NDJSON line tagged with the current source and turn.
    * Called in real-time via the AgentRunner onLine callback.
+   *
+   * When the current source is the supervisor, also scans assistant text
+   * content for the EVALUATION_COMPLETE and EVALUATION_INTERVENTION signals,
+   * setting completeSignalSeen / interventionSignalSeen respectively.
    * @param {string} line - Raw NDJSON line from the runner
    */
   emitLine(line) {
@@ -152,6 +370,56 @@ export class Supervisor {
       event,
     };
     this.output.write(JSON.stringify(tagged) + "\n");
+
+    // Scan supervisor assistant messages for the signals in real time.
+    // The SDK result text only reflects the final assistant message, but the
+    // supervisor may write EVALUATION_COMPLETE / EVALUATION_INTERVENTION in
+    // an earlier message and then continue with follow-up tool calls.
+    if (this.currentSource === "supervisor" && event.type === "assistant") {
+      const content = event.message?.content ?? event.content ?? [];
+      if (Array.isArray(content)) {
+        for (const block of content) {
+          if (block.type !== "text" || !block.text) continue;
+          if (isComplete(block.text)) this.completeSignalSeen = true;
+          if (isIntervention(block.text)) this.interventionSignalSeen = true;
+        }
+      }
+    }
+  }
+
+  /**
+   * Render a batch of buffered NDJSON lines as human-readable text for the
+   * mid-turn supervisor prompt. Reuses the TraceCollector pipeline so the
+   * supervisor sees tool calls and reasoning, not just raw events.
+   * @param {string[]} batchLines
+   * @returns {string}
+   */
+  renderBatch(batchLines) {
+    if (batchLines.length === 0) return "[empty]";
+    const collector = new TraceCollector();
+    for (const line of batchLines) {
+      collector.addLine(line);
+    }
+    return collector.toText() || "[empty]";
+  }
+
+  /**
+   * Emit an orchestrator-source NDJSON line. Used by the mid-turn loop to
+   * mark mid_turn_review / intervention_requested / intervention_relayed /
+   * intervention_limit / complete_requested boundaries in the trace, so the
+   * improvement coach can distinguish mid-turn supervisor activity from
+   * end-of-turn reviews. Additive to existing trace shape — the parser
+   * already reads `source` and ignores unknown event types.
+   * @param {{type: string, turn?: number}} event
+   */
+  emitOrchestratorEvent(event) {
+    this.output.write(
+      JSON.stringify({
+        source: "orchestrator",
+        turn: this.currentTurn,
+        event,
+      }) + "\n",
+    );
   }
 
   /**
@@ -220,10 +488,14 @@ export function createSupervisor({
     },
   });
 
-  // Block Task/TaskOutput so the supervisor cannot spawn its own sub-agents.
-  // The relay loop handles agent communication — letting the supervisor use
-  // Task would bypass the relay and produce an empty agent trace.
-  const defaultDisallowed = ["Task", "TaskOutput"];
+  // Block every sub-agent spawning tool so the supervisor cannot bypass the
+  // relay loop. The current Claude Agent SDK exposes the spawn tool to the
+  // model as `Agent`; older versions called it `Task`. Both are blocked
+  // (along with TaskOutput/TaskStop) so the supervisor sees no spawn tool
+  // regardless of which SDK version is installed. Letting the supervisor
+  // spawn its own sub-agent would bypass the relay and produce an empty
+  // agent trace, which is the failure mode that motivated this default.
+  const defaultDisallowed = ["Agent", "Task", "TaskOutput", "TaskStop"];
   const disallowedTools = supervisorDisallowedTools
     ? [...new Set([...defaultDisallowed, ...supervisorDisallowedTools])]
     : defaultDisallowed;

package/src/trace-collector.js

@@ -38,6 +38,13 @@ export class TraceCollector {
       return;
     }
 
+    // Unwrap combined supervised trace format {source, turn, event}.
+    // The Supervisor emits this wrapper; when replayed through addLine the
+    // inner event is the one we need.
+    if (event.event && !event.type && typeof event.source === "string") {
+      event = event.event;
+    }
+
     switch (event.type) {
       case "system":
         this.handleSystem(event);

package/test/mock-runner.js

@@ -0,0 +1,101 @@
+/**
+ * Test-only mock factory for AgentRunner. Yields pre-scripted responses,
+ * and (when an `onBatch` callback is set) fires it at the same boundaries
+ * the real AgentRunner would: assistant messages with at least one text
+ * block, and the terminal `result` message. If the callback calls
+ * `abort()`, the mock stops iterating that response's messages and
+ * reports `aborted: true`.
+ *
+ * Intentionally a regular module (not a test file) so describe/test blocks
+ * here would not run. Lives under test/ to make its scope explicit.
+ */
+
+import { PassThrough } from "node:stream";
+import { AgentRunner } from "@forwardimpact/libeval";
+
+/**
+ * Whether a scripted message should trigger an onBatch flush. Mirrors the
+ * real AgentRunner: assistant-with-text-block or terminal `result` message.
+ * Tool-only or string-content messages accumulate without flushing.
+ * @param {object} message
+ * @returns {boolean}
+ */
+export function shouldFlush(message) {
+  if (message.type === "result") return true;
+  if (message.type !== "assistant") return false;
+  const content = message.message?.content ?? message.content;
+  if (!Array.isArray(content)) return false;
+  for (const block of content) {
+    if (block.type === "text" && block.text) return true;
+  }
+  return false;
+}
+
+/**
+ * Create a mock AgentRunner that yields pre-scripted responses. Each call
+ * to `run()` or `resume()` pops the next response from the array.
+ * @param {object[]} responses - Array of {text, success} objects
+ * @param {object[]} [messages] - Messages to buffer per response
+ * @returns {AgentRunner}
+ */
+export function createMockRunner(responses, messages) {
+  const output = new PassThrough();
+  let callIndex = 0;
+
+  const runner = new AgentRunner({
+    cwd: "/tmp",
+    query: async function* () {},
+    output,
+  });
+
+  const consume = async (msgs) => {
+    let aborted = false;
+    for (const m of msgs) {
+      const line = JSON.stringify(m);
+      runner.buffer.push(line);
+      if (runner.onLine) runner.onLine(line);
+      if (runner.onBatch && shouldFlush(m)) {
+        await runner.onBatch([line], {
+          abort: () => {
+            aborted = true;
+          },
+        });
+        if (aborted) break;
+      }
+    }
+    return aborted;
+  };
+
+  runner.run = async (_task) => {
+    const resp = responses[callIndex++];
+    const msgs = messages?.[callIndex - 1] ?? [
+      { type: "assistant", content: resp.text },
+    ];
+    const aborted = await consume(msgs);
+    runner.sessionId = "mock-session";
+    return {
+      success: resp.success ?? true,
+      text: resp.text,
+      sessionId: "mock-session",
+      aborted,
+      error: null,
+    };
+  };
+
+  runner.resume = async (_prompt) => {
+    const resp = responses[callIndex++];
+    const msgs = messages?.[callIndex - 1] ?? [
+      { type: "assistant", content: resp.text },
+    ];
+    const aborted = await consume(msgs);
+    return {
+      success: resp.success ?? true,
+      text: resp.text,
+      sessionId: runner.sessionId,
+      aborted,
+      error: null,
+    };
+  };
+
+  return runner;
+}