@forwardimpact/libeval 0.1.9 → 0.1.12

package/test/agent-runner-batching.test.js

@@ -1,271 +0,0 @@
-import { describe, test } from "node:test";
-import assert from "node:assert";
-import { PassThrough } from "node:stream";
-
-import { AgentRunner } from "@forwardimpact/libeval";
-
-/**
- * Create a mock query function that yields canned messages.
- * @param {object[]} messages - Messages to yield
- * @returns {function}
- */
-function mockQuery(messages) {
-  return async function* () {
-    for (const msg of messages) {
-      yield msg;
-    }
-  };
-}
-
-const textBlock = (t) => ({
-  type: "assistant",
-  message: { content: [{ type: "text", text: t }] },
-});
-
-const toolOnly = (name) => ({
-  type: "assistant",
-  message: {
-    content: [{ type: "tool_use", id: "tu_" + name, name, input: {} }],
-  },
-});
-
-describe("AgentRunner - onBatch batching", () => {
-  test("batchSize defaults to 3", () => {
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: async function* () {},
-      output: new PassThrough(),
-    });
-    assert.strictEqual(runner.batchSize, 3);
-  });
-
-  test("onBatch fires every 3 assistant text-block messages by default", async () => {
-    // 5 text-block messages + terminal result. With the default batchSize
-    // of 3, onBatch should fire on the 3rd text message and again on the
-    // terminal result (flushing the remaining 2).
-    const messages = [
-      { type: "system", subtype: "init", session_id: "sess-batch" },
-      textBlock("one"),
-      textBlock("two"),
-      textBlock("three"),
-      textBlock("four"),
-      textBlock("five"),
-      { type: "result", subtype: "success", result: "Done." },
-    ];
-
-    const batches = [];
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: mockQuery(messages),
-      output: new PassThrough(),
-    });
-    runner.onBatch = async (lines) => {
-      batches.push(lines.map((l) => JSON.parse(l)));
-    };
-
-    await runner.run("Task");
-
-    // First flush carries init + first 3 text messages; second carries
-    // remaining 2 text messages + the result.
-    assert.strictEqual(batches.length, 2);
-    assert.strictEqual(batches[0].length, 4);
-    assert.strictEqual(batches[1].length, 3);
-  });
-
-  test("onBatch honours custom batchSize", async () => {
-    // batchSize = 2: 4 text messages produce 2 flushes; result adds a 3rd.
-    const messages = [
-      textBlock("a"),
-      textBlock("b"),
-      textBlock("c"),
-      textBlock("d"),
-      { type: "result", subtype: "success", result: "Done." },
-    ];
-
-    const batches = [];
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: mockQuery(messages),
-      output: new PassThrough(),
-      batchSize: 2,
-    });
-    runner.onBatch = async (lines) => {
-      batches.push(lines.length);
-    };
-
-    await runner.run("Task");
-
-    assert.deepStrictEqual(batches, [2, 2, 1]);
-  });
-
-  test("tool-only assistant messages ride along in the next flush", async () => {
-    // Tool-only assistant messages accumulate without incrementing the
-    // counter. The supervisor sees the preceding tool calls when the
-    // flush eventually fires.
-    const messages = [
-      toolOnly("Read"),
-      toolOnly("Grep"),
-      textBlock("found it"),
-      { type: "result", subtype: "success", result: "Done." },
-    ];
-
-    const batches = [];
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: mockQuery(messages),
-      output: new PassThrough(),
-      batchSize: 1,
-    });
-    runner.onBatch = async (lines) => {
-      batches.push(lines.map((l) => JSON.parse(l)));
-    };
-
-    await runner.run("Task");
-
-    // First flush triggered by the single text-block message; it carries
-    // the two preceding tool-only messages with it.
-    assert.strictEqual(batches.length, 2);
-    assert.strictEqual(batches[0].length, 3);
-    assert.strictEqual(batches[0][0].message.content[0].type, "tool_use");
-    assert.strictEqual(batches[0][1].message.content[0].type, "tool_use");
-    assert.strictEqual(batches[0][2].message.content[0].type, "text");
-    assert.strictEqual(batches[1].length, 1);
-    assert.strictEqual(batches[1][0].type, "result");
-  });
-
-  test("terminal result always flushes even if batchSize not yet reached", async () => {
-    // 1 text-block + result, batchSize = 5. The counter only reaches 1
-    // but the terminal result must still flush.
-    const messages = [
-      textBlock("only one"),
-      { type: "result", subtype: "success", result: "Done." },
-    ];
-
-    const batches = [];
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: mockQuery(messages),
-      output: new PassThrough(),
-      batchSize: 5,
-    });
-    runner.onBatch = async (lines) => {
-      batches.push(lines.length);
-    };
-
-    await runner.run("Task");
-
-    assert.deepStrictEqual(batches, [2]);
-  });
-});
-
-describe("AgentRunner - terminal flush on abnormal end", () => {
-  test("iterator crash before a flush boundary still delivers the pending batch", async () => {
-    // batchSize = 3: the first two text messages accumulate without
-    // flushing. The iterator then throws before the threshold — the
-    // pending batch must ship in a terminal flush.
-    async function* crashingQuery() {
-      yield { type: "system", subtype: "init", session_id: "sess-crash" };
-      yield textBlock("step 1");
-      yield textBlock("step 2");
-      throw new Error("Claude Code process exited with code 1");
-    }
-
-    const batches = [];
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: () => crashingQuery(),
-      output: new PassThrough(),
-    });
-    runner.onBatch = async (lines) => {
-      batches.push(lines.map((l) => JSON.parse(l)));
-    };
-
-    const result = await runner.run("Task");
-
-    assert.ok(result.error);
-    assert.match(result.error.message, /exited with code 1/);
-    assert.strictEqual(batches.length, 1);
-    assert.strictEqual(batches[0].length, 3);
-    assert.strictEqual(batches[0][0].type, "system");
-    assert.strictEqual(batches[0][1].type, "assistant");
-    assert.strictEqual(batches[0][2].type, "assistant");
-  });
-
-  test("iterator crash after a completed batch does not re-flush", async () => {
-    // batchSize = 2: two text messages trigger a normal flush, emptying
-    // the pending batch. The iterator then throws with nothing pending —
-    // the terminal flush must be a no-op, not an empty call.
-    async function* crashingQuery() {
-      yield textBlock("a");
-      yield textBlock("b");
-      throw new Error("boom");
-    }
-
-    const batches = [];
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: () => crashingQuery(),
-      output: new PassThrough(),
-      batchSize: 2,
-    });
-    runner.onBatch = async (lines) => {
-      batches.push(lines.length);
-    };
-
-    const result = await runner.run("Task");
-    assert.ok(result.error);
-    assert.match(result.error.message, /boom/);
-    assert.deepStrictEqual(batches, [2]);
-  });
-
-  test("natural-end iterator without a result does not trigger terminal flush", async () => {
-    // The real SDK always terminates with `result`. A mock that ends
-    // naturally with pending lines is treated as an incomplete stub —
-    // no phantom flush, since nothing about a natural end warrants a
-    // new mid-turn review.
-    async function* noResultQuery() {
-      yield textBlock("one");
-      yield textBlock("two");
-      // No result, no error — just ends.
-    }
-
-    const batches = [];
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: () => noResultQuery(),
-      output: new PassThrough(),
-      batchSize: 3,
-    });
-    runner.onBatch = async (lines) => {
-      batches.push(lines.length);
-    };
-
-    const result = await runner.run("Task");
-    assert.strictEqual(result.error, null);
-    assert.strictEqual(batches.length, 0);
-  });
-
-  test("onBatch throw during terminal flush does not mask an earlier error", async () => {
-    // The iterator threw first; the terminal flush also throws. The
-    // original iterator error must win — it is the more actionable
-    // condition to surface to the caller.
-    async function* crashingQuery() {
-      yield textBlock("partial");
-      throw new Error("original failure");
-    }
-
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: () => crashingQuery(),
-      output: new PassThrough(),
-      batchSize: 3,
-    });
-    runner.onBatch = async () => {
-      throw new Error("flush failure");
-    };
-
-    const result = await runner.run("Task");
-    assert.ok(result.error);
-    assert.match(result.error.message, /original failure/);
-  });
-});

package/test/agent-runner.test.js

@@ -1,317 +0,0 @@
-import { describe, test } from "node:test";
-import assert from "node:assert";
-import { PassThrough } from "node:stream";
-
-import { AgentRunner, createAgentRunner } from "@forwardimpact/libeval";
-
-/**
- * Create a mock query function that yields canned messages.
- * @param {object[]} messages - Messages to yield
- * @param {function} [captureOptions] - Callback to capture query options
- * @returns {function}
- */
-function mockQuery(messages, captureOptions) {
-  return async function* (params) {
-    if (captureOptions) captureOptions(params);
-    for (const msg of messages) {
-      yield msg;
-    }
-  };
-}
-
-/**
- * Collect all NDJSON lines written to a PassThrough stream.
- * @param {PassThrough} stream
- * @returns {string[]}
- */
-function collectLines(stream) {
-  const data = stream.read();
-  if (!data) return [];
-  return data
-    .toString()
-    .trim()
-    .split("\n")
-    .filter((l) => l.length > 0);
-}
-
-describe("AgentRunner", () => {
-  test("constructor throws on missing cwd", () => {
-    assert.throws(
-      () =>
-        new AgentRunner({
-          query: async function* () {},
-          output: new PassThrough(),
-        }),
-      /cwd is required/,
-    );
-  });
-
-  test("constructor throws on missing query", () => {
-    assert.throws(
-      () => new AgentRunner({ cwd: "/tmp", output: new PassThrough() }),
-      /query is required/,
-    );
-  });
-
-  test("constructor throws on missing output", () => {
-    assert.throws(
-      () =>
-        new AgentRunner({
-          cwd: "/tmp",
-          query: async function* () {},
-        }),
-      /output is required/,
-    );
-  });
-
-  test("constructor uses defaults for optional params", () => {
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: async function* () {},
-      output: new PassThrough(),
-    });
-    assert.strictEqual(runner.model, "opus");
-    assert.strictEqual(runner.maxTurns, 50);
-    assert.deepStrictEqual(runner.allowedTools, [
-      "Bash",
-      "Read",
-      "Glob",
-      "Grep",
-      "Write",
-      "Edit",
-    ]);
-    assert.strictEqual(runner.permissionMode, "bypassPermissions");
-    assert.deepStrictEqual(runner.settingSources, []);
-    assert.strictEqual(runner.sessionId, null);
-  });
-
-  test("run() writes NDJSON lines to output stream", async () => {
-    const messages = [
-      { type: "system", subtype: "init", session_id: "sess-1" },
-      { type: "assistant", content: "Working on it..." },
-      { type: "result", subtype: "success", result: "Done." },
-    ];
-
-    const output = new PassThrough();
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: mockQuery(messages),
-      output,
-    });
-
-    const result = await runner.run("Test task");
-    const lines = collectLines(output);
-
-    assert.strictEqual(lines.length, 3);
-    assert.deepStrictEqual(JSON.parse(lines[0]), messages[0]);
-    assert.deepStrictEqual(JSON.parse(lines[1]), messages[1]);
-    assert.deepStrictEqual(JSON.parse(lines[2]), messages[2]);
-    assert.strictEqual(result.success, true);
-    assert.strictEqual(result.text, "Done.");
-    assert.strictEqual(result.sessionId, "sess-1");
-  });
-
-  test("run() captures sessionId from init event", async () => {
-    const messages = [
-      { type: "system", subtype: "init", session_id: "my-session" },
-      { type: "result", subtype: "success", result: "OK" },
-    ];
-
-    const output = new PassThrough();
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: mockQuery(messages),
-      output,
-    });
-
-    await runner.run("Task");
-    assert.strictEqual(runner.sessionId, "my-session");
-  });
-
-  test("run() passes options to query", async () => {
-    let captured = null;
-    const query = mockQuery(
-      [{ type: "result", subtype: "success", result: "OK" }],
-      (params) => {
-        captured = params;
-      },
-    );
-
-    const output = new PassThrough();
-    const runner = new AgentRunner({
-      cwd: "/work",
-      query,
-      output,
-      model: "sonnet",
-      maxTurns: 10,
-      allowedTools: ["Read", "Grep"],
-      permissionMode: "plan",
-      settingSources: ["project"],
-    });
-
-    await runner.run("My task");
-
-    assert.strictEqual(captured.prompt, "My task");
-    assert.strictEqual(captured.options.cwd, "/work");
-    assert.strictEqual(captured.options.model, "sonnet");
-    assert.strictEqual(captured.options.maxTurns, 10);
-    assert.deepStrictEqual(captured.options.allowedTools, ["Read", "Grep"]);
-    assert.strictEqual(captured.options.permissionMode, "plan");
-    assert.strictEqual(captured.options.allowDangerouslySkipPermissions, true);
-    assert.deepStrictEqual(captured.options.settingSources, ["project"]);
-  });
-
-  test("run() returns success=false on non-success subtype", async () => {
-    const messages = [{ type: "result", subtype: "error", result: "Stopped" }];
-
-    const output = new PassThrough();
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: mockQuery(messages),
-      output,
-    });
-
-    const result = await runner.run("Task");
-    assert.strictEqual(result.success, false);
-    assert.strictEqual(result.text, "Stopped");
-  });
-
-  test("resume() passes sessionId via options.resume", async () => {
-    let resumeCapture = null;
-
-    const initMessages = [
-      { type: "system", subtype: "init", session_id: "sess-42" },
-      { type: "result", subtype: "success", result: "First done" },
-    ];
-
-    let callCount = 0;
-    const query = async function* (params) {
-      callCount++;
-      if (callCount === 1) {
-        for (const m of initMessages) yield m;
-      } else {
-        resumeCapture = params;
-        yield { type: "result", subtype: "success", result: "Resumed" };
-      }
-    };
-
-    const output = new PassThrough();
-    const runner = new AgentRunner({ cwd: "/tmp", query, output });
-
-    await runner.run("Initial task");
-    const result = await runner.resume("Follow up");
-
-    assert.strictEqual(resumeCapture.options.resume, "sess-42");
-    assert.strictEqual(resumeCapture.prompt, "Follow up");
-    assert.strictEqual(result.success, true);
-    assert.strictEqual(result.text, "Resumed");
-  });
-
-  test("drainOutput() returns buffered lines and clears buffer", async () => {
-    const messages = [
-      { type: "assistant", content: "Line 1" },
-      { type: "result", subtype: "success", result: "Line 2" },
-    ];
-
-    const output = new PassThrough();
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: mockQuery(messages),
-      output,
-    });
-
-    await runner.run("Task");
-
-    const drained = runner.drainOutput();
-    assert.strictEqual(drained.length, 2);
-    assert.deepStrictEqual(JSON.parse(drained[0]), messages[0]);
-    assert.deepStrictEqual(JSON.parse(drained[1]), messages[1]);
-
-    // Buffer should be empty after drain
-    const secondDrain = runner.drainOutput();
-    assert.strictEqual(secondDrain.length, 0);
-  });
-
-  test("run() captures error when query throws and returns buffered output", async () => {
-    async function* failingQuery() {
-      yield { type: "system", subtype: "init", session_id: "sess-err" };
-      yield { type: "assistant", content: "Partial work" };
-      throw new Error("Claude Code process exited with code 1");
-    }
-
-    const output = new PassThrough();
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: () => failingQuery(),
-      output,
-    });
-
-    const result = await runner.run("Task");
-    assert.strictEqual(result.success, false);
-    assert.ok(result.error);
-    assert.match(result.error.message, /exited with code 1/);
-    assert.strictEqual(result.sessionId, "sess-err");
-
-    // Buffered output should contain the messages yielded before the error
-    const drained = runner.drainOutput();
-    assert.strictEqual(drained.length, 2);
-  });
-
-  test("resume() captures error when query throws", async () => {
-    const initMessages = [
-      { type: "system", subtype: "init", session_id: "sess-r" },
-      { type: "result", subtype: "success", result: "OK" },
-    ];
-
-    let callCount = 0;
-    const query = async function* () {
-      callCount++;
-      if (callCount === 1) {
-        for (const m of initMessages) yield m;
-      } else {
-        yield { type: "assistant", content: "Resuming..." };
-        throw new Error("Process crashed");
-      }
-    };
-
-    const output = new PassThrough();
-    const runner = new AgentRunner({ cwd: "/tmp", query, output });
-
-    await runner.run("Task");
-    const result = await runner.resume("Continue");
-    assert.strictEqual(result.success, false);
-    assert.ok(result.error);
-    assert.match(result.error.message, /Process crashed/);
-  });
-
-  test("run() succeeds when SDK throws after emitting successful result", async () => {
-    async function* creditExhaustedQuery() {
-      yield { type: "system", subtype: "init", session_id: "sess-credit" };
-      yield { type: "assistant", content: "Analysis complete." };
-      yield { type: "result", subtype: "success", result: "Done." };
-      throw new Error("Credit balance is too low");
-    }
-
-    const output = new PassThrough();
-    const runner = new AgentRunner({
-      cwd: "/tmp",
-      query: () => creditExhaustedQuery(),
-      output,
-    });
-
-    const result = await runner.run("Task");
-    assert.strictEqual(result.success, true);
-    assert.strictEqual(result.text, "Done.");
-    assert.ok(result.error);
-    assert.match(result.error.message, /Credit balance/);
-  });
-
-  test("createAgentRunner factory returns an AgentRunner instance", () => {
-    const runner = createAgentRunner({
-      cwd: "/tmp",
-      query: async function* () {},
-      output: new PassThrough(),
-    });
-    assert.ok(runner instanceof AgentRunner);
-  });
-});

package/test/fixtures/stream.ndjson

@@ -1,7 +0,0 @@
-{"type":"system","subtype":"init","cwd":"/home/user/monorepo","session_id":"abc-123","tools":["Bash","Read","Glob","Grep","Write","Edit"],"mcp_servers":[],"model":"claude-opus-4-6","permissionMode":"default","claude_code_version":"2.1.87","uuid":"evt-1"}
-{"type":"assistant","message":{"model":"claude-opus-4-6","id":"msg_01","type":"message","role":"assistant","content":[{"type":"text","text":"I'll start by checking the repository structure."}],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":100,"cache_creation_input_tokens":500,"cache_read_input_tokens":200,"output_tokens":15,"service_tier":"standard"}},"session_id":"abc-123","uuid":"evt-2"}
-{"type":"assistant","message":{"model":"claude-opus-4-6","id":"msg_01","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01","name":"Bash","input":{"command":"ls -la"}}],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":100,"cache_creation_input_tokens":500,"cache_read_input_tokens":200,"output_tokens":15,"service_tier":"standard"}},"session_id":"abc-123","uuid":"evt-3"}
-{"type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_01","type":"tool_result","content":"total 42\ndrwxr-xr-x  5 user user 4096 Mar 29 12:00 .\ndrwxr-xr-x  3 user user 4096 Mar 29 12:00 .."}]},"uuid":"evt-4","session_id":"abc-123"}
-{"type":"assistant","message":{"model":"claude-opus-4-6","id":"msg_02","type":"message","role":"assistant","content":[{"type":"text","text":"The repository looks good. No security issues found."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":150,"cache_creation_input_tokens":0,"cache_read_input_tokens":700,"output_tokens":12,"service_tier":"standard"}},"session_id":"abc-123","uuid":"evt-5"}
-{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"},"uuid":"evt-6","session_id":"abc-123"}
-{"type":"result","subtype":"success","is_error":false,"duration_ms":5200,"duration_api_ms":4800,"num_turns":3,"result":"The repository looks good. No security issues found.","stop_reason":"end_turn","session_id":"abc-123","total_cost_usd":0.0523,"usage":{"input_tokens":350,"cache_creation_input_tokens":500,"cache_read_input_tokens":900,"output_tokens":42,"service_tier":"standard"},"modelUsage":{"claude-opus-4-6":{"inputTokens":350,"outputTokens":42,"cacheReadInputTokens":900,"cacheCreationInputTokens":500,"costUSD":0.0523}},"uuid":"evt-7"}