@forwardimpact/libeval 0.1.9 → 0.1.12

package/test/supervisor-output.test.js

@@ -1,369 +0,0 @@
-import { describe, test } from "node:test";
-import assert from "node:assert";
-import { PassThrough } from "node:stream";
-
-import {
-  Supervisor,
-  createSupervisor,
-  SUPERVISOR_SYSTEM_PROMPT,
-  AGENT_SYSTEM_PROMPT,
-} from "@forwardimpact/libeval";
-import { createMockRunner } from "./mock-runner.js";
-
-describe("Supervisor - output and events", () => {
-  test("output contains tagged lines with correct source and turn", async () => {
-    const supervisorMessages = [
-      [{ type: "assistant", content: "Go ahead" }],
-      [{ type: "assistant", content: "EVALUATION_COMPLETE" }],
-    ];
-    const agentMessages = [[{ type: "assistant", content: "Working" }]];
-
-    const supervisorRunner = createMockRunner(
-      [{ text: "Go ahead" }, { text: "EVALUATION_COMPLETE" }],
-      supervisorMessages,
-    );
-    const agentRunner = createMockRunner([{ text: "Working" }], agentMessages);
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 10,
-    });
-    agentRunner.onLine = (line) => supervisor.emitLine(line);
-    supervisorRunner.onLine = (line) => supervisor.emitLine(line);
-
-    await supervisor.run("Task");
-
-    const data = output.read()?.toString() ?? "";
-    const lines = data
-      .trim()
-      .split("\n")
-      .filter((l) => l.length > 0);
-
-    // Should have: supervisor turn 0, agent turn 1, supervisor turn 1, orchestrator summary
-    assert.ok(lines.length >= 4);
-
-    const supervisorLine = JSON.parse(lines[0]);
-    assert.strictEqual(supervisorLine.source, "supervisor");
-    assert.strictEqual(supervisorLine.turn, 0);
-    assert.ok("event" in supervisorLine);
-
-    const agentLine = JSON.parse(lines[1]);
-    assert.strictEqual(agentLine.source, "agent");
-    assert.strictEqual(agentLine.turn, 1);
-    assert.ok("event" in agentLine);
-
-    const summaryLine = JSON.parse(lines[lines.length - 1]);
-    assert.strictEqual(summaryLine.source, "orchestrator");
-    assert.strictEqual(summaryLine.type, "summary");
-    assert.strictEqual(summaryLine.success, true);
-  });
-
-  test("events are nested under event key (no field collisions)", async () => {
-    const sourceEvent = {
-      type: "assistant",
-      source: "sdk-internal",
-      content: "test",
-    };
-    const supervisorRunner = createMockRunner(
-      [{ text: "Go" }, { text: "EVALUATION_COMPLETE" }],
-      [
-        [{ type: "assistant", content: "Go" }],
-        [{ type: "assistant", content: "ok" }],
-      ],
-    );
-    const agentRunner = createMockRunner([{ text: "Done" }], [[sourceEvent]]);
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 10,
-    });
-    agentRunner.onLine = (line) => supervisor.emitLine(line);
-    supervisorRunner.onLine = (line) => supervisor.emitLine(line);
-
-    await supervisor.run("Task");
-
-    const data = output.read()?.toString() ?? "";
-    const lines = data
-      .trim()
-      .split("\n")
-      .filter((l) => l.length > 0);
-
-    // First line is supervisor turn 0, second is agent turn 1
-    const tagged = JSON.parse(lines[1]);
-    assert.strictEqual(tagged.source, "agent");
-    assert.strictEqual(tagged.event.source, "sdk-internal");
-  });
-
-  test("mid-turn intervention emits orchestrator events and shares the agent's turn id", async () => {
-    // Agent emits one structured assistant text block on its first call —
-    // supervisor intervenes mid-turn. Resume then completes naturally and
-    // the end-of-turn review signals EVALUATION_COMPLETE.
-    const agentMessages = [
-      [
-        {
-          type: "assistant",
-          message: {
-            content: [{ type: "text", text: "Trying the wrong thing." }],
-          },
-        },
-      ],
-      [
-        {
-          type: "assistant",
-          message: {
-            content: [{ type: "text", text: "Switching to the right thing." }],
-          },
-        },
-      ],
-    ];
-
-    const supervisorMessages = [
-      undefined,
-      [
-        {
-          type: "assistant",
-          message: {
-            content: [
-              {
-                type: "text",
-                text: "EVALUATION_INTERVENTION Switch to the right path.",
-              },
-            ],
-          },
-        },
-      ],
-      undefined,
-      undefined,
-    ];
-
-    const agentRunner = createMockRunner(
-      [{ text: "Trying the wrong thing." }, { text: "Switching." }],
-      agentMessages,
-    );
-    agentRunner.batchSize = 1;
-    const supervisorRunner = createMockRunner(
-      [
-        { text: "Welcome." },
-        { text: "EVALUATION_INTERVENTION Switch to the right path." },
-        { text: "Keep going." },
-        { text: "Done. EVALUATION_COMPLETE" },
-      ],
-      supervisorMessages,
-    );
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 10,
-    });
-    agentRunner.onLine = (line) => supervisor.emitLine(line);
-    supervisorRunner.onLine = (line) => supervisor.emitLine(line);
-
-    const result = await supervisor.run("Task");
-    assert.strictEqual(result.success, true);
-
-    const lines = (output.read()?.toString() ?? "")
-      .trim()
-      .split("\n")
-      .filter((l) => l.length > 0)
-      .map((l) => JSON.parse(l));
-
-    // (1) Orchestrator event with intervention_requested.
-    const interventionRequested = lines.find(
-      (l) =>
-        l.source === "orchestrator" &&
-        l.event?.type === "intervention_requested",
-    );
-    assert.ok(
-      interventionRequested,
-      "Trace must contain intervention_requested orchestrator event",
-    );
-
-    // (2) At least one agent line and one supervisor line share a turn id —
-    //     mid-turn supervisor activity is tagged with the agent's turn.
-    const agentTurns = new Set(
-      lines.filter((l) => l.source === "agent").map((l) => l.turn),
-    );
-    const supervisorTurns = new Set(
-      lines.filter((l) => l.source === "supervisor").map((l) => l.turn),
-    );
-    const sharedTurns = [...agentTurns].filter((t) => supervisorTurns.has(t));
-    assert.ok(
-      sharedTurns.length > 0,
-      "At least one turn id must appear on both agent and supervisor lines",
-    );
-
-    // (3) Final summary line still emitted.
-    const summary = lines[lines.length - 1];
-    assert.strictEqual(summary.source, "orchestrator");
-    assert.strictEqual(summary.type, "summary");
-    assert.strictEqual(summary.success, true);
-  });
-
-  test("emits supervisor output and summary when supervisor errors on turn 0", async () => {
-    const supervisorMessages = [
-      [{ type: "assistant", content: "Starting..." }],
-    ];
-    const supervisorRunner = createMockRunner(
-      [{ text: "Starting...", success: false }],
-      supervisorMessages,
-    );
-
-    const origRun = supervisorRunner.run;
-    supervisorRunner.run = async (task) => {
-      const result = await origRun.call(supervisorRunner, task);
-      return { ...result, error: new Error("Process exited with code 1") };
-    };
-
-    const agentRunner = createMockRunner([]);
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 10,
-    });
-    agentRunner.onLine = (line) => supervisor.emitLine(line);
-    supervisorRunner.onLine = (line) => supervisor.emitLine(line);
-
-    const result = await supervisor.run("Task");
-
-    assert.strictEqual(result.success, false);
-    assert.strictEqual(result.turns, 0);
-
-    const data = output.read()?.toString() ?? "";
-    const lines = data
-      .trim()
-      .split("\n")
-      .filter((l) => l.length > 0);
-
-    assert.ok(lines.length >= 2, "Expected at least supervisor line + summary");
-
-    const supervisorLine = JSON.parse(lines[0]);
-    assert.strictEqual(supervisorLine.source, "supervisor");
-    assert.strictEqual(supervisorLine.turn, 0);
-
-    const summaryLine = JSON.parse(lines[lines.length - 1]);
-    assert.strictEqual(summaryLine.source, "orchestrator");
-    assert.strictEqual(summaryLine.success, false);
-    assert.strictEqual(summaryLine.turns, 0);
-  });
-});
-
-describe("Supervisor - createSupervisor factory", () => {
-  test("createSupervisor factory returns a Supervisor instance", () => {
-    const supervisor = createSupervisor({
-      supervisorCwd: "/tmp/sup",
-      agentCwd: "/tmp/agent",
-      query: async function* () {},
-      output: new PassThrough(),
-    });
-    assert.ok(supervisor instanceof Supervisor);
-  });
-
-  test("createSupervisor uses default supervisor tools when none specified", () => {
-    const supervisor = createSupervisor({
-      supervisorCwd: "/tmp/sup",
-      agentCwd: "/tmp/agent",
-      query: async function* () {},
-      output: new PassThrough(),
-    });
-    assert.deepStrictEqual(supervisor.supervisorRunner.allowedTools, [
-      "Bash",
-      "Read",
-      "Glob",
-      "Grep",
-      "Write",
-      "Edit",
-    ]);
-  });
-
-  test("createSupervisor passes custom supervisor tools", () => {
-    const supervisor = createSupervisor({
-      supervisorCwd: "/tmp/sup",
-      agentCwd: "/tmp/agent",
-      query: async function* () {},
-      output: new PassThrough(),
-      supervisorAllowedTools: ["Read", "Glob", "Grep"],
-    });
-    assert.deepStrictEqual(supervisor.supervisorRunner.allowedTools, [
-      "Read",
-      "Glob",
-      "Grep",
-    ]);
-  });
-
-  test("createSupervisor wires system prompts to both runners", () => {
-    const supervisor = createSupervisor({
-      supervisorCwd: "/tmp/sup",
-      agentCwd: "/tmp/agent",
-      query: async function* () {},
-      output: new PassThrough(),
-    });
-
-    assert.deepStrictEqual(supervisor.agentRunner.systemPrompt, {
-      type: "preset",
-      preset: "claude_code",
-      append: AGENT_SYSTEM_PROMPT,
-    });
-    assert.deepStrictEqual(supervisor.supervisorRunner.systemPrompt, {
-      type: "preset",
-      preset: "claude_code",
-      append: SUPERVISOR_SYSTEM_PROMPT,
-    });
-  });
-
-  test("createSupervisor blocks sub-agent spawn tools on supervisor by default", () => {
-    const supervisor = createSupervisor({
-      supervisorCwd: "/tmp/sup",
-      agentCwd: "/tmp/agent",
-      query: async function* () {},
-      output: new PassThrough(),
-    });
-    assert.deepStrictEqual(supervisor.supervisorRunner.disallowedTools, [
-      "Agent",
-      "Task",
-      "TaskOutput",
-      "TaskStop",
-    ]);
-    assert.deepStrictEqual(supervisor.agentRunner.disallowedTools, []);
-  });
-
-  test("createSupervisor merges custom supervisorDisallowedTools with defaults", () => {
-    const supervisor = createSupervisor({
-      supervisorCwd: "/tmp/sup",
-      agentCwd: "/tmp/agent",
-      query: async function* () {},
-      output: new PassThrough(),
-      supervisorDisallowedTools: ["WebSearch", "Task"],
-    });
-    const disallowed = supervisor.supervisorRunner.disallowedTools;
-    assert.ok(disallowed.includes("Agent"));
-    assert.ok(disallowed.includes("Task"));
-    assert.ok(disallowed.includes("TaskOutput"));
-    assert.ok(disallowed.includes("TaskStop"));
-    assert.ok(disallowed.includes("WebSearch"));
-    assert.strictEqual(disallowed.length, new Set(disallowed).size);
-  });
-
-  test("system prompt constants are non-empty strings", () => {
-    assert.ok(typeof SUPERVISOR_SYSTEM_PROMPT === "string");
-    assert.ok(typeof AGENT_SYSTEM_PROMPT === "string");
-    assert.ok(SUPERVISOR_SYSTEM_PROMPT.length > 0);
-    assert.ok(AGENT_SYSTEM_PROMPT.length > 0);
-  });
-
-  test("SUPERVISOR_SYSTEM_PROMPT explains relay mechanism", () => {
-    assert.ok(SUPERVISOR_SYSTEM_PROMPT.includes("relay"));
-    assert.ok(SUPERVISOR_SYSTEM_PROMPT.includes("EVALUATION_COMPLETE"));
-  });
-});

package/test/supervisor-run.test.js

@@ -1,310 +0,0 @@
-import { describe, test } from "node:test";
-import assert from "node:assert";
-import { PassThrough } from "node:stream";
-
-import { Supervisor } from "@forwardimpact/libeval";
-import { isComplete } from "../src/supervisor.js";
-import { createMockRunner } from "./mock-runner.js";
-
-describe("isComplete", () => {
-  test("detects EVALUATION_COMPLETE on its own line", () => {
-    assert.strictEqual(isComplete("EVALUATION_COMPLETE"), true);
-    assert.strictEqual(
-      isComplete("Some text\nEVALUATION_COMPLETE\nMore text"),
-      true,
-    );
-    assert.strictEqual(isComplete("Done.\n\nEVALUATION_COMPLETE"), true);
-  });
-
-  test("tolerates markdown formatting around the signal", () => {
-    assert.strictEqual(isComplete("**EVALUATION_COMPLETE**"), true);
-    assert.strictEqual(isComplete("*EVALUATION_COMPLETE*"), true);
-    assert.strictEqual(isComplete("__EVALUATION_COMPLETE__"), true);
-    assert.strictEqual(isComplete("_EVALUATION_COMPLETE_"), true);
-    assert.strictEqual(isComplete("`EVALUATION_COMPLETE`"), true);
-    assert.strictEqual(
-      isComplete("Good work.\n\n**EVALUATION_COMPLETE**\n\nNow filing issues."),
-      true,
-    );
-  });
-
-  test("matches EVALUATION_COMPLETE anywhere in text", () => {
-    assert.strictEqual(isComplete("not EVALUATION_COMPLETE yet"), true);
-    assert.strictEqual(
-      isComplete("The agent is EVALUATION_COMPLETE done"),
-      true,
-    );
-    assert.strictEqual(
-      isComplete("Great work! EVALUATION_COMPLETE. Now filing issues."),
-      true,
-    );
-  });
-
-  test("does not match empty or unrelated text", () => {
-    assert.strictEqual(isComplete(""), false);
-    assert.strictEqual(isComplete("All done!"), false);
-    assert.strictEqual(isComplete("DONE"), false);
-  });
-
-  test("does not match old EVALUATION_SUCCESSFUL signal", () => {
-    assert.strictEqual(isComplete("EVALUATION_SUCCESSFUL"), false);
-  });
-});
-
-describe("Supervisor - run and turns", () => {
-  test("constructor throws on missing agentRunner", () => {
-    assert.throws(
-      () =>
-        new Supervisor({
-          supervisorRunner: createMockRunner([]),
-          output: new PassThrough(),
-        }),
-      /agentRunner is required/,
-    );
-  });
-
-  test("constructor throws on missing supervisorRunner", () => {
-    assert.throws(
-      () =>
-        new Supervisor({
-          agentRunner: createMockRunner([]),
-          output: new PassThrough(),
-        }),
-      /supervisorRunner is required/,
-    );
-  });
-
-  test("constructor throws on missing output", () => {
-    assert.throws(
-      () =>
-        new Supervisor({
-          agentRunner: createMockRunner([]),
-          supervisorRunner: createMockRunner([]),
-        }),
-      /output is required/,
-    );
-  });
-
-  test("completes on EVALUATION_COMPLETE from supervisor at turn 0", async () => {
-    const agentRunner = createMockRunner([]);
-
-    const supervisorRunner = createMockRunner([
-      { text: "EVALUATION_COMPLETE" },
-    ]);
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 10,
-    });
-
-    const result = await supervisor.run("Install stuff");
-
-    assert.strictEqual(result.success, true);
-    assert.strictEqual(result.turns, 0);
-  });
-
-  test("completes after one agent turn", async () => {
-    const agentRunner = createMockRunner([
-      { text: "I installed the packages." },
-    ]);
-
-    const supervisorRunner = createMockRunner([
-      { text: "Welcome! Please install the packages." },
-      { text: "Good work.\n\nEVALUATION_COMPLETE" },
-    ]);
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 10,
-    });
-
-    const result = await supervisor.run("Install stuff");
-
-    assert.strictEqual(result.success, true);
-    assert.strictEqual(result.turns, 1);
-  });
-
-  test("detects EVALUATION_COMPLETE in streamed messages when result text differs", async () => {
-    const agentRunner = createMockRunner([
-      { text: "I installed the packages." },
-    ]);
-
-    const supervisorMessages = [
-      undefined,
-      [
-        {
-          type: "assistant",
-          message: {
-            content: [
-              {
-                type: "text",
-                text: "Good work.\n\nEVALUATION_COMPLETE\n\nNow filing issues.",
-              },
-            ],
-          },
-        },
-        {
-          type: "assistant",
-          message: {
-            content: [
-              { type: "text", text: "## Summary\n\nAll issues filed." },
-            ],
-          },
-        },
-      ],
-    ];
-
-    const supervisorRunner = createMockRunner(
-      [
-        { text: "Welcome! Please install the packages." },
-        { text: "## Summary\n\nAll issues filed." },
-      ],
-      supervisorMessages,
-    );
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 10,
-    });
-    agentRunner.onLine = (line) => supervisor.emitLine(line);
-    supervisorRunner.onLine = (line) => supervisor.emitLine(line);
-
-    const result = await supervisor.run("Install stuff");
-
-    assert.strictEqual(result.success, true);
-    assert.strictEqual(result.turns, 1);
-  });
-
-  test("relays only the last assistant text block to the agent", async () => {
-    // Supervisor emits reasoning text ("Let me research...") then a tool call,
-    // then a final task message. Only the final message should reach the agent.
-    const supervisorMessages = [
-      // Turn 0: multiple assistant messages with reasoning + task
-      [
-        {
-          type: "assistant",
-          message: {
-            content: [
-              { type: "text", text: "Let me research the product first." },
-            ],
-          },
-        },
-        {
-          type: "assistant",
-          message: {
-            content: [
-              {
-                type: "text",
-                text: "Hello! Here is your task: install the packages.",
-              },
-            ],
-          },
-        },
-      ],
-      // Turn 1: evaluation
-      undefined,
-    ];
-
-    let capturedAgentPrompt = null;
-    const agentRunner = createMockRunner([
-      { text: "I installed the packages." },
-    ]);
-    const origRun = agentRunner.run;
-    agentRunner.run = async (task) => {
-      capturedAgentPrompt = task;
-      return origRun.call(agentRunner, task);
-    };
-
-    const supervisorRunner = createMockRunner(
-      [
-        // SDK result text = last message text (but relay should use buffer)
-        { text: "Hello! Here is your task: install the packages." },
-        { text: "EVALUATION_COMPLETE" },
-      ],
-      supervisorMessages,
-    );
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 10,
-    });
-
-    await supervisor.run("Evaluate the product");
-
-    // Agent should receive only the final text, not the reasoning
-    assert.strictEqual(
-      capturedAgentPrompt,
-      "Hello! Here is your task: install the packages.",
-    );
-    assert.ok(
-      !capturedAgentPrompt.includes("research"),
-      "Reasoning text should not leak to agent",
-    );
-  });
-
-  test("runs multiple turns before completion", async () => {
-    const agentRunner = createMockRunner([
-      { text: "Started working." },
-      { text: "Made progress." },
-      { text: "Finished everything." },
-    ]);
-
-    const supervisorRunner = createMockRunner([
-      { text: "Here is your task. Do the work." },
-      { text: "Keep going, you need to do more." },
-      { text: "Almost there, continue." },
-      { text: "EVALUATION_COMPLETE" },
-    ]);
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 10,
-    });
-
-    const result = await supervisor.run("Do the work");
-
-    assert.strictEqual(result.success, true);
-    assert.strictEqual(result.turns, 3);
-  });
-
-  test("enforces maxTurns limit", async () => {
-    const agentRunner = createMockRunner([
-      { text: "Turn 1" },
-      { text: "Turn 2" },
-    ]);
-
-    const supervisorRunner = createMockRunner([
-      { text: "Start working." },
-      { text: "Continue." },
-      { text: "Continue." },
-    ]);
-
-    const output = new PassThrough();
-    const supervisor = new Supervisor({
-      agentRunner,
-      supervisorRunner,
-      output,
-      maxTurns: 2,
-    });
-
-    const result = await supervisor.run("Endless task");
-
-    assert.strictEqual(result.success, false);
-    assert.strictEqual(result.turns, 2);
-  });
-});