@forwardimpact/libeval 0.1.6 → 0.1.9

package/test/{supervisor.test.js → supervisor-output.test.js}

@@ -3,317 +3,23 @@ import assert from "node:assert";
 import { PassThrough } from "node:stream";
 
 import {
-  AgentRunner,
   Supervisor,
   createSupervisor,
   SUPERVISOR_SYSTEM_PROMPT,
   AGENT_SYSTEM_PROMPT,
 } from "@forwardimpact/libeval";
-import { isSuccessful } from "../src/supervisor.js";
-
-/**
- * Create a mock AgentRunner that yields pre-scripted responses.
- * Each call to run() or resume() pops the next response from the array.
- * @param {object[]} responses - Array of {text, success} objects
- * @param {object[]} [messages] - Messages to buffer per turn
- * @returns {AgentRunner}
- */
-function createMockRunner(responses, messages) {
-  const output = new PassThrough();
-  let callIndex = 0;
-
-  const runner = new AgentRunner({
-    cwd: "/tmp",
-    query: async function* () {},
-    output,
-  });
-
-  // Override run and resume to return scripted responses
-  runner.run = async (_task) => {
-    const resp = responses[callIndex++];
-    const msgs = messages?.[callIndex - 1] ?? [
-      { type: "assistant", content: resp.text },
-    ];
-    for (const m of msgs) {
-      const line = JSON.stringify(m);
-      runner.buffer.push(line);
-      if (runner.onLine) runner.onLine(line);
-    }
-    runner.sessionId = "mock-session";
-    return {
-      success: resp.success ?? true,
-      text: resp.text,
-      sessionId: "mock-session",
-    };
-  };
-
-  runner.resume = async (_prompt) => {
-    const resp = responses[callIndex++];
-    const msgs = messages?.[callIndex - 1] ?? [
-      { type: "assistant", content: resp.text },
-    ];
-    for (const m of msgs) {
-      const line = JSON.stringify(m);
-      runner.buffer.push(line);
-      if (runner.onLine) runner.onLine(line);
-    }
-    return { success: resp.success ?? true, text: resp.text };
-  };
-
-  return runner;
-}
-
-describe("isSuccessful", () => {
-  test("detects EVALUATION_SUCCESSFUL on its own line", () => {
-    assert.strictEqual(isSuccessful("EVALUATION_SUCCESSFUL"), true);
-    assert.strictEqual(
-      isSuccessful("Some text\nEVALUATION_SUCCESSFUL\nMore text"),
-      true,
-    );
-    assert.strictEqual(isSuccessful("Done.\n\nEVALUATION_SUCCESSFUL"), true);
-  });
-
-  test("tolerates markdown formatting around the signal", () => {
-    assert.strictEqual(isSuccessful("**EVALUATION_SUCCESSFUL**"), true);
-    assert.strictEqual(isSuccessful("*EVALUATION_SUCCESSFUL*"), true);
-    assert.strictEqual(isSuccessful("__EVALUATION_SUCCESSFUL__"), true);
-    assert.strictEqual(isSuccessful("_EVALUATION_SUCCESSFUL_"), true);
-    assert.strictEqual(isSuccessful("`EVALUATION_SUCCESSFUL`"), true);
-    assert.strictEqual(
-      isSuccessful(
-        "Good work.\n\n**EVALUATION_SUCCESSFUL**\n\nNow filing issues.",
-      ),
-      true,
-    );
-  });
-
-  test("matches EVALUATION_SUCCESSFUL anywhere in text", () => {
-    assert.strictEqual(isSuccessful("not EVALUATION_SUCCESSFUL yet"), true);
-    assert.strictEqual(
-      isSuccessful("The agent is EVALUATION_SUCCESSFUL done"),
-      true,
-    );
-    assert.strictEqual(
-      isSuccessful("Great work! EVALUATION_SUCCESSFUL. Now filing issues."),
-      true,
-    );
-  });
-
-  test("does not match empty or unrelated text", () => {
-    assert.strictEqual(isSuccessful(""), false);
-    assert.strictEqual(isSuccessful("All done!"), false);
-    assert.strictEqual(isSuccessful("DONE"), false);
-  });
-
-  test("does not match old EVALUATION_COMPLETE signal", () => {
-    assert.strictEqual(isSuccessful("EVALUATION_COMPLETE"), false);
-  });
-});
-
-describe("Supervisor", () => {
-  test("constructor throws on missing agentRunner", () => {
-    assert.throws(
-      () =>
-        new Supervisor({
-          supervisorRunner: createMockRunner([]),
-          output: new PassThrough(),
-        }),
-      /agentRunner is required/,
-    );
-  });
-
-  test("constructor throws on missing supervisorRunner", () => {
-    assert.throws(
-      () =>
-        new Supervisor({
-          agentRunner: createMockRunner([]),
-          output: new PassThrough(),
-        }),
-      /supervisorRunner is required/,
-    );
-  });
-
-  test("constructor throws on missing output", () => {
-    assert.throws(
-      () =>
-        new Supervisor({
-          agentRunner: createMockRunner([]),
-          supervisorRunner: createMockRunner([]),
-        }),
-      /output is required/,
-    );
-  });
-
-  test("completes on EVALUATION_SUCCESSFUL from supervisor at turn 0", async () => {
-    const agentRunner = createMockRunner([]);
-
-    const supervisorRunner = createMockRunner([
-      { text: "EVALUATION_SUCCESSFUL" },
-    ]);
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 10,
-    });
-
-    const result = await supervisor.run("Install stuff");
-
-    assert.strictEqual(result.success, true);
-    assert.strictEqual(result.turns, 0);
-  });
-
-  test("completes after one agent turn", async () => {
-    const agentRunner = createMockRunner([
-      { text: "I installed the packages." },
-    ]);
-
-    const supervisorRunner = createMockRunner([
-      { text: "Welcome! Please install the packages." },
-      { text: "Good work.\n\nEVALUATION_SUCCESSFUL" },
-    ]);
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 10,
-    });
-
-    const result = await supervisor.run("Install stuff");
-
-    assert.strictEqual(result.success, true);
-    assert.strictEqual(result.turns, 1);
-  });
-
-  test("detects EVALUATION_SUCCESSFUL in streamed messages when result text differs", async () => {
-    // Simulates the real failure: supervisor writes EVALUATION_SUCCESSFUL in
-    // an early message, then continues with follow-up work (e.g. filing issues).
-    // The SDK result text reflects only the final message, which does NOT
-    // contain the signal.
-    const agentRunner = createMockRunner([
-      { text: "I installed the packages." },
-    ]);
-
-    // The supervisor's result text is the Summary (no signal), but messages
-    // include one with EVALUATION_SUCCESSFUL.
-    const supervisorMessages = [
-      undefined, // turn 0: use default
-      [
-        {
-          type: "assistant",
-          message: {
-            content: [
-              {
-                type: "text",
-                text: "Good work.\n\nEVALUATION_SUCCESSFUL\n\nNow filing issues.",
-              },
-            ],
-          },
-        },
-        {
-          type: "assistant",
-          message: {
-            content: [
-              { type: "text", text: "## Summary\n\nAll issues filed." },
-            ],
-          },
-        },
-      ],
-    ];
-
-    const supervisorRunner = createMockRunner(
-      [
-        { text: "Welcome! Please install the packages." },
-        // Result text is the final message — does NOT contain the signal
-        { text: "## Summary\n\nAll issues filed." },
-      ],
-      supervisorMessages,
-    );
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 10,
-    });
-    agentRunner.onLine = (line) => supervisor.emitLine(line);
-    supervisorRunner.onLine = (line) => supervisor.emitLine(line);
-
-    const result = await supervisor.run("Install stuff");
-
-    assert.strictEqual(result.success, true);
-    assert.strictEqual(result.turns, 1);
-  });
-
-  test("runs multiple turns before completion", async () => {
-    const agentRunner = createMockRunner([
-      { text: "Started working." },
-      { text: "Made progress." },
-      { text: "Finished everything." },
-    ]);
-
-    const supervisorRunner = createMockRunner([
-      { text: "Here is your task. Do the work." },
-      { text: "Keep going, you need to do more." },
-      { text: "Almost there, continue." },
-      { text: "EVALUATION_SUCCESSFUL" },
-    ]);
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 10,
-    });
-
-    const result = await supervisor.run("Do the work");
-
-    assert.strictEqual(result.success, true);
-    assert.strictEqual(result.turns, 3);
-  });
-
-  test("enforces maxTurns limit", async () => {
-    // Supervisor starts, agent responds each turn, supervisor never says done
-    const agentRunner = createMockRunner([
-      { text: "Turn 1" },
-      { text: "Turn 2" },
-    ]);
-
-    const supervisorRunner = createMockRunner([
-      { text: "Start working." },
-      { text: "Continue." },
-      { text: "Continue." },
-    ]);
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 2,
-    });
-
-    const result = await supervisor.run("Endless task");
-
-    assert.strictEqual(result.success, false);
-    assert.strictEqual(result.turns, 2);
-  });
+import { createMockRunner } from "./mock-runner.js";
 
+describe("Supervisor - output and events", () => {
   test("output contains tagged lines with correct source and turn", async () => {
     const supervisorMessages = [
       [{ type: "assistant", content: "Go ahead" }],
-      [{ type: "assistant", content: "EVALUATION_SUCCESSFUL" }],
+      [{ type: "assistant", content: "EVALUATION_COMPLETE" }],
     ];
     const agentMessages = [[{ type: "assistant", content: "Working" }]];
 
     const supervisorRunner = createMockRunner(
-      [{ text: "Go ahead" }, { text: "EVALUATION_SUCCESSFUL" }],
+      [{ text: "Go ahead" }, { text: "EVALUATION_COMPLETE" }],
       supervisorMessages,
     );
     const agentRunner = createMockRunner([{ text: "Working" }], agentMessages);
@@ -362,7 +68,7 @@ describe("Supervisor", () => {
       content: "test",
     };
     const supervisorRunner = createMockRunner(
-      [{ text: "Go" }, { text: "EVALUATION_SUCCESSFUL" }],
+      [{ text: "Go" }, { text: "EVALUATION_COMPLETE" }],
       [
         [{ type: "assistant", content: "Go" }],
         [{ type: "assistant", content: "ok" }],
@@ -390,11 +96,118 @@ describe("Supervisor", () => {
 
     // First line is supervisor turn 0, second is agent turn 1
     const tagged = JSON.parse(lines[1]);
-    // The original event's `source` field is preserved inside `event`
     assert.strictEqual(tagged.source, "agent");
     assert.strictEqual(tagged.event.source, "sdk-internal");
   });
 
+  test("mid-turn intervention emits orchestrator events and shares the agent's turn id", async () => {
+    // Agent emits one structured assistant text block on its first call —
+    // supervisor intervenes mid-turn. Resume then completes naturally and
+    // the end-of-turn review signals EVALUATION_COMPLETE.
+    const agentMessages = [
+      [
+        {
+          type: "assistant",
+          message: {
+            content: [{ type: "text", text: "Trying the wrong thing." }],
+          },
+        },
+      ],
+      [
+        {
+          type: "assistant",
+          message: {
+            content: [{ type: "text", text: "Switching to the right thing." }],
+          },
+        },
+      ],
+    ];
+
+    const supervisorMessages = [
+      undefined,
+      [
+        {
+          type: "assistant",
+          message: {
+            content: [
+              {
+                type: "text",
+                text: "EVALUATION_INTERVENTION Switch to the right path.",
+              },
+            ],
+          },
+        },
+      ],
+      undefined,
+      undefined,
+    ];
+
+    const agentRunner = createMockRunner(
+      [{ text: "Trying the wrong thing." }, { text: "Switching." }],
+      agentMessages,
+    );
+    agentRunner.batchSize = 1;
+    const supervisorRunner = createMockRunner(
+      [
+        { text: "Welcome." },
+        { text: "EVALUATION_INTERVENTION Switch to the right path." },
+        { text: "Keep going." },
+        { text: "Done. EVALUATION_COMPLETE" },
+      ],
+      supervisorMessages,
+    );
+
+    const output = new PassThrough();
+    const supervisor = new Supervisor({
+      agentRunner,
+      supervisorRunner,
+      output,
+      maxTurns: 10,
+    });
+    agentRunner.onLine = (line) => supervisor.emitLine(line);
+    supervisorRunner.onLine = (line) => supervisor.emitLine(line);
+
+    const result = await supervisor.run("Task");
+    assert.strictEqual(result.success, true);
+
+    const lines = (output.read()?.toString() ?? "")
+      .trim()
+      .split("\n")
+      .filter((l) => l.length > 0)
+      .map((l) => JSON.parse(l));
+
+    // (1) Orchestrator event with intervention_requested.
+    const interventionRequested = lines.find(
+      (l) =>
+        l.source === "orchestrator" &&
+        l.event?.type === "intervention_requested",
+    );
+    assert.ok(
+      interventionRequested,
+      "Trace must contain intervention_requested orchestrator event",
+    );
+
+    // (2) At least one agent line and one supervisor line share a turn id —
+    //     mid-turn supervisor activity is tagged with the agent's turn.
+    const agentTurns = new Set(
+      lines.filter((l) => l.source === "agent").map((l) => l.turn),
+    );
+    const supervisorTurns = new Set(
+      lines.filter((l) => l.source === "supervisor").map((l) => l.turn),
+    );
+    const sharedTurns = [...agentTurns].filter((t) => supervisorTurns.has(t));
+    assert.ok(
+      sharedTurns.length > 0,
+      "At least one turn id must appear on both agent and supervisor lines",
+    );
+
+    // (3) Final summary line still emitted.
+    const summary = lines[lines.length - 1];
+    assert.strictEqual(summary.source, "orchestrator");
+    assert.strictEqual(summary.type, "summary");
+    assert.strictEqual(summary.success, true);
+  });
+
   test("emits supervisor output and summary when supervisor errors on turn 0", async () => {
     const supervisorMessages = [
       [{ type: "assistant", content: "Starting..." }],
@@ -404,7 +217,6 @@ describe("Supervisor", () => {
       supervisorMessages,
     );
 
-    // Override run to simulate an error return
     const origRun = supervisorRunner.run;
     supervisorRunner.run = async (task) => {
       const result = await origRun.call(supervisorRunner, task);
@@ -428,7 +240,6 @@ describe("Supervisor", () => {
     assert.strictEqual(result.success, false);
     assert.strictEqual(result.turns, 0);
 
-    // Output should still contain the supervisor's buffered lines + summary
     const data = output.read()?.toString() ?? "";
     const lines = data
       .trim()
@@ -446,7 +257,9 @@ describe("Supervisor", () => {
     assert.strictEqual(summaryLine.success, false);
     assert.strictEqual(summaryLine.turns, 0);
   });
+});
 
+describe("Supervisor - createSupervisor factory", () => {
   test("createSupervisor factory returns a Supervisor instance", () => {
     const supervisor = createSupervisor({
       supervisorCwd: "/tmp/sup",
@@ -509,7 +322,7 @@ describe("Supervisor", () => {
     });
   });
 
-  test("createSupervisor blocks Task and TaskOutput on supervisor by default", () => {
+  test("createSupervisor blocks sub-agent spawn tools on supervisor by default", () => {
     const supervisor = createSupervisor({
       supervisorCwd: "/tmp/sup",
       agentCwd: "/tmp/agent",
@@ -517,10 +330,11 @@ describe("Supervisor", () => {
       output: new PassThrough(),
     });
     assert.deepStrictEqual(supervisor.supervisorRunner.disallowedTools, [
+      "Agent",
       "Task",
       "TaskOutput",
+      "TaskStop",
     ]);
-    // Agent should not have disallowed tools
     assert.deepStrictEqual(supervisor.agentRunner.disallowedTools, []);
   });
 
@@ -533,10 +347,11 @@ describe("Supervisor", () => {
       supervisorDisallowedTools: ["WebSearch", "Task"],
     });
     const disallowed = supervisor.supervisorRunner.disallowedTools;
+    assert.ok(disallowed.includes("Agent"));
     assert.ok(disallowed.includes("Task"));
     assert.ok(disallowed.includes("TaskOutput"));
+    assert.ok(disallowed.includes("TaskStop"));
     assert.ok(disallowed.includes("WebSearch"));
-    // No duplicates
     assert.strictEqual(disallowed.length, new Set(disallowed).size);
   });
 
@@ -549,6 +364,6 @@ describe("Supervisor", () => {
 
   test("SUPERVISOR_SYSTEM_PROMPT explains relay mechanism", () => {
     assert.ok(SUPERVISOR_SYSTEM_PROMPT.includes("relay"));
-    assert.ok(SUPERVISOR_SYSTEM_PROMPT.includes("EVALUATION_SUCCESSFUL"));
+    assert.ok(SUPERVISOR_SYSTEM_PROMPT.includes("EVALUATION_COMPLETE"));
   });
 });