@forwardimpact/libeval 0.1.5 → 0.1.8

package/test/supervisor-run.test.js

@@ -0,0 +1,310 @@
+import { describe, test } from "node:test";
+import assert from "node:assert";
+import { PassThrough } from "node:stream";
+
+import { Supervisor } from "@forwardimpact/libeval";
+import { isComplete } from "../src/supervisor.js";
+import { createMockRunner } from "./mock-runner.js";
+
+describe("isComplete", () => {
+  test("detects EVALUATION_COMPLETE on its own line", () => {
+    assert.strictEqual(isComplete("EVALUATION_COMPLETE"), true);
+    assert.strictEqual(
+      isComplete("Some text\nEVALUATION_COMPLETE\nMore text"),
+      true,
+    );
+    assert.strictEqual(isComplete("Done.\n\nEVALUATION_COMPLETE"), true);
+  });
+
+  test("tolerates markdown formatting around the signal", () => {
+    assert.strictEqual(isComplete("**EVALUATION_COMPLETE**"), true);
+    assert.strictEqual(isComplete("*EVALUATION_COMPLETE*"), true);
+    assert.strictEqual(isComplete("__EVALUATION_COMPLETE__"), true);
+    assert.strictEqual(isComplete("_EVALUATION_COMPLETE_"), true);
+    assert.strictEqual(isComplete("`EVALUATION_COMPLETE`"), true);
+    assert.strictEqual(
+      isComplete("Good work.\n\n**EVALUATION_COMPLETE**\n\nNow filing issues."),
+      true,
+    );
+  });
+
+  test("matches EVALUATION_COMPLETE anywhere in text", () => {
+    assert.strictEqual(isComplete("not EVALUATION_COMPLETE yet"), true);
+    assert.strictEqual(
+      isComplete("The agent is EVALUATION_COMPLETE done"),
+      true,
+    );
+    assert.strictEqual(
+      isComplete("Great work! EVALUATION_COMPLETE. Now filing issues."),
+      true,
+    );
+  });
+
+  test("does not match empty or unrelated text", () => {
+    assert.strictEqual(isComplete(""), false);
+    assert.strictEqual(isComplete("All done!"), false);
+    assert.strictEqual(isComplete("DONE"), false);
+  });
+
+  test("does not match old EVALUATION_SUCCESSFUL signal", () => {
+    assert.strictEqual(isComplete("EVALUATION_SUCCESSFUL"), false);
+  });
+});
+
+describe("Supervisor - run and turns", () => {
+  test("constructor throws on missing agentRunner", () => {
+    assert.throws(
+      () =>
+        new Supervisor({
+          supervisorRunner: createMockRunner([]),
+          output: new PassThrough(),
+        }),
+      /agentRunner is required/,
+    );
+  });
+
+  test("constructor throws on missing supervisorRunner", () => {
+    assert.throws(
+      () =>
+        new Supervisor({
+          agentRunner: createMockRunner([]),
+          output: new PassThrough(),
+        }),
+      /supervisorRunner is required/,
+    );
+  });
+
+  test("constructor throws on missing output", () => {
+    assert.throws(
+      () =>
+        new Supervisor({
+          agentRunner: createMockRunner([]),
+          supervisorRunner: createMockRunner([]),
+        }),
+      /output is required/,
+    );
+  });
+
+  test("completes on EVALUATION_COMPLETE from supervisor at turn 0", async () => {
+    const agentRunner = createMockRunner([]);
+
+    const supervisorRunner = createMockRunner([
+      { text: "EVALUATION_COMPLETE" },
+    ]);
+
+    const output = new PassThrough();
+    const supervisor = new Supervisor({
+      agentRunner,
+      supervisorRunner,
+      output,
+      maxTurns: 10,
+    });
+
+    const result = await supervisor.run("Install stuff");
+
+    assert.strictEqual(result.success, true);
+    assert.strictEqual(result.turns, 0);
+  });
+
+  test("completes after one agent turn", async () => {
+    const agentRunner = createMockRunner([
+      { text: "I installed the packages." },
+    ]);
+
+    const supervisorRunner = createMockRunner([
+      { text: "Welcome! Please install the packages." },
+      { text: "Good work.\n\nEVALUATION_COMPLETE" },
+    ]);
+
+    const output = new PassThrough();
+    const supervisor = new Supervisor({
+      agentRunner,
+      supervisorRunner,
+      output,
+      maxTurns: 10,
+    });
+
+    const result = await supervisor.run("Install stuff");
+
+    assert.strictEqual(result.success, true);
+    assert.strictEqual(result.turns, 1);
+  });
+
+  test("detects EVALUATION_COMPLETE in streamed messages when result text differs", async () => {
+    const agentRunner = createMockRunner([
+      { text: "I installed the packages." },
+    ]);
+
+    const supervisorMessages = [
+      undefined,
+      [
+        {
+          type: "assistant",
+          message: {
+            content: [
+              {
+                type: "text",
+                text: "Good work.\n\nEVALUATION_COMPLETE\n\nNow filing issues.",
+              },
+            ],
+          },
+        },
+        {
+          type: "assistant",
+          message: {
+            content: [
+              { type: "text", text: "## Summary\n\nAll issues filed." },
+            ],
+          },
+        },
+      ],
+    ];
+
+    const supervisorRunner = createMockRunner(
+      [
+        { text: "Welcome! Please install the packages." },
+        { text: "## Summary\n\nAll issues filed." },
+      ],
+      supervisorMessages,
+    );
+
+    const output = new PassThrough();
+    const supervisor = new Supervisor({
+      agentRunner,
+      supervisorRunner,
+      output,
+      maxTurns: 10,
+    });
+    agentRunner.onLine = (line) => supervisor.emitLine(line);
+    supervisorRunner.onLine = (line) => supervisor.emitLine(line);
+
+    const result = await supervisor.run("Install stuff");
+
+    assert.strictEqual(result.success, true);
+    assert.strictEqual(result.turns, 1);
+  });
+
+  test("relays only the last assistant text block to the agent", async () => {
+    // Supervisor emits reasoning text ("Let me research...") then a tool call,
+    // then a final task message. Only the final message should reach the agent.
+    const supervisorMessages = [
+      // Turn 0: multiple assistant messages with reasoning + task
+      [
+        {
+          type: "assistant",
+          message: {
+            content: [
+              { type: "text", text: "Let me research the product first." },
+            ],
+          },
+        },
+        {
+          type: "assistant",
+          message: {
+            content: [
+              {
+                type: "text",
+                text: "Hello! Here is your task: install the packages.",
+              },
+            ],
+          },
+        },
+      ],
+      // Turn 1: evaluation
+      undefined,
+    ];
+
+    let capturedAgentPrompt = null;
+    const agentRunner = createMockRunner([
+      { text: "I installed the packages." },
+    ]);
+    const origRun = agentRunner.run;
+    agentRunner.run = async (task) => {
+      capturedAgentPrompt = task;
+      return origRun.call(agentRunner, task);
+    };
+
+    const supervisorRunner = createMockRunner(
+      [
+        // SDK result text = last message text (but relay should use buffer)
+        { text: "Hello! Here is your task: install the packages." },
+        { text: "EVALUATION_COMPLETE" },
+      ],
+      supervisorMessages,
+    );
+
+    const output = new PassThrough();
+    const supervisor = new Supervisor({
+      agentRunner,
+      supervisorRunner,
+      output,
+      maxTurns: 10,
+    });
+
+    await supervisor.run("Evaluate the product");
+
+    // Agent should receive only the final text, not the reasoning
+    assert.strictEqual(
+      capturedAgentPrompt,
+      "Hello! Here is your task: install the packages.",
+    );
+    assert.ok(
+      !capturedAgentPrompt.includes("research"),
+      "Reasoning text should not leak to agent",
+    );
+  });
+
+  test("runs multiple turns before completion", async () => {
+    const agentRunner = createMockRunner([
+      { text: "Started working." },
+      { text: "Made progress." },
+      { text: "Finished everything." },
+    ]);
+
+    const supervisorRunner = createMockRunner([
+      { text: "Here is your task. Do the work." },
+      { text: "Keep going, you need to do more." },
+      { text: "Almost there, continue." },
+      { text: "EVALUATION_COMPLETE" },
+    ]);
+
+    const output = new PassThrough();
+    const supervisor = new Supervisor({
+      agentRunner,
+      supervisorRunner,
+      output,
+      maxTurns: 10,
+    });
+
+    const result = await supervisor.run("Do the work");
+
+    assert.strictEqual(result.success, true);
+    assert.strictEqual(result.turns, 3);
+  });
+
+  test("enforces maxTurns limit", async () => {
+    const agentRunner = createMockRunner([
+      { text: "Turn 1" },
+      { text: "Turn 2" },
+    ]);
+
+    const supervisorRunner = createMockRunner([
+      { text: "Start working." },
+      { text: "Continue." },
+      { text: "Continue." },
+    ]);
+
+    const output = new PassThrough();
+    const supervisor = new Supervisor({
+      agentRunner,
+      supervisorRunner,
+      output,
+      maxTurns: 2,
+    });
+
+    const result = await supervisor.run("Endless task");
+
+    assert.strictEqual(result.success, false);
+    assert.strictEqual(result.turns, 2);
+  });
+});

package/test/trace-collector.test.js

@@ -149,6 +149,102 @@ describe("TraceCollector", () => {
       assert.strictEqual(trace.summary.tokenUsage.inputTokens, 5000);
     });
 
+    test("unwraps combined supervised trace format {source, turn, event}", () => {
+      const collector = new TraceCollector();
+
+      // System init wrapped in supervisor envelope
+      collector.addLine(
+        JSON.stringify({
+          source: "agent",
+          turn: 0,
+          event: {
+            type: "system",
+            subtype: "init",
+            session_id: "sess-supervised",
+            model: "claude-opus-4-6",
+            tools: ["Bash"],
+          },
+        }),
+      );
+
+      // Assistant message wrapped in supervisor envelope
+      collector.addLine(
+        JSON.stringify({
+          source: "agent",
+          turn: 1,
+          event: {
+            type: "assistant",
+            message: {
+              content: [{ type: "text", text: "I ran the tests." }],
+              usage: { input_tokens: 100, output_tokens: 50 },
+            },
+          },
+        }),
+      );
+
+      // Tool result wrapped in supervisor envelope
+      collector.addLine(
+        JSON.stringify({
+          source: "agent",
+          turn: 1,
+          event: {
+            type: "user",
+            message: {
+              role: "user",
+              content: [
+                {
+                  type: "tool_result",
+                  tool_use_id: "toolu_sup",
+                  content: "All tests passed",
+                },
+              ],
+            },
+          },
+        }),
+      );
+
+      // Result event wrapped in supervisor envelope
+      collector.addLine(
+        JSON.stringify({
+          source: "supervisor",
+          turn: 1,
+          event: {
+            type: "result",
+            subtype: "success",
+            total_cost_usd: 0.44,
+            duration_ms: 30000,
+            num_turns: 2,
+          },
+        }),
+      );
+
+      const trace = collector.toJSON();
+      assert.strictEqual(trace.metadata.sessionId, "sess-supervised");
+      assert.strictEqual(trace.turns.length, 2);
+      assert.strictEqual(trace.turns[0].role, "assistant");
+      assert.strictEqual(trace.turns[0].content[0].text, "I ran the tests.");
+      assert.strictEqual(trace.turns[1].role, "tool_result");
+      assert.strictEqual(trace.turns[1].content, "All tests passed");
+      assert.strictEqual(trace.summary.result, "success");
+      assert.strictEqual(trace.summary.totalCostUsd, 0.44);
+    });
+
+    test("skips orchestrator summary lines from supervised traces", () => {
+      const collector = new TraceCollector();
+      collector.addLine(
+        JSON.stringify({
+          source: "orchestrator",
+          type: "summary",
+          success: true,
+          turns: 3,
+        }),
+      );
+
+      // Orchestrator summaries have no inner event and no recognized type
+      // after unwrap — they should be silently skipped.
+      assert.strictEqual(collector.toJSON().turns.length, 0);
+    });
+
     test("skips rate_limit_event and unknown types", () => {
       const collector = new TraceCollector();
       collector.addLine(