@crewhaus/eval-runner 0.1.0 → 0.1.2

package/src/run-sample.eventlog.test.ts

@@ -0,0 +1,124 @@
+/**
+ * Isolated tests for the `readTranscript` catch arm in `runSample`.
+ *
+ * `readTranscript` opens the per-sample event-log to reconstruct the
+ * transcript when the invoker didn't supply one. A missing log is tolerated
+ * only when it surfaces as an "invalid sessionId" error (test stubs that skip
+ * persistence); any other failure is rethrown as a `RunnerError`. Both arms
+ * require `openEventLog` to throw, so we stub `@crewhaus/event-log`.
+ *
+ * `mock.module` is process-global and Bun evaluates every test file's body in
+ * one up-front load phase, so this stub can leak into sibling suites whose
+ * real `runSample` legitimately opens an event-log. The stub must therefore be
+ * BOTH deterministic for this file and transparent when leaked:
+ *
+ *   - Its throwing modes ("invalid"/"generic") are armed only for the single
+ *     test that needs them and disarmed again in `afterEach`. In its default
+ *     "off" state it is a pure pass-through to the real implementation, so any
+ *     leak into a sibling suite behaves exactly like the real module.
+ *   - The pass-through calls a *snapshot* of the real `openEventLog`
+ *     (`realOpenEventLog`), captured into a local before `mock.module` runs —
+ *     NOT `realEventLog.openEventLog`. Reading the property at call time would
+ *     self-recurse, because `mock.module` redirects the captured namespace's
+ *     own `openEventLog` binding back into this stub (the `RangeError:
+ *     Maximum call stack size exceeded` previously seen in sibling suites). A
+ *     plain function value captured pre-mock is immune to that redirection.
+ *
+ * `throwMode` is disarmed in `afterEach` and the real module is reinstalled in
+ * `afterAll` so the override is fully inert once this file's tests finish.
+ */
+import { afterAll, afterEach, describe, expect, mock, test } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { Sample } from "@crewhaus/eval-dataset";
+import { parseGradersConfig } from "@crewhaus/eval-grader";
+
+// Capture the real module (for `afterAll` restoration) AND snapshot the real
+// `openEventLog` function value *before* installing the override, so the
+// pass-through can call it without re-entering this stub. The module capture
+// is a plain-object SNAPSHOT (`{ ...ns }`): an ESM namespace is a live view
+// that resolves to the stub once mock.module patches the module, so restoring
+// from the namespace itself would silently reinstall the stub.
+const realEventLog = { ...(await import("@crewhaus/event-log")) };
+const realOpenEventLog = realEventLog.openEventLog;
+
+// "off" → transparent pass-through to the real log (safe for any leaked call).
+let throwMode: "off" | "invalid" | "generic" = "off";
+
+mock.module("@crewhaus/event-log", () => ({
+  ...realEventLog,
+  openEventLog: async (sessionId: string, opts?: { rootDir?: string }) => {
+    if (throwMode === "invalid") {
+      // Tolerated: matches the /invalid sessionId/ guard in readTranscript.
+      throw new Error('event-log: invalid sessionId "nope" — expected sess_<16 hex>');
+    }
+    if (throwMode === "generic") {
+      // Rethrown as RunnerError.
+      throw new Error("disk on fire");
+    }
+    // Default: real behaviour against the (possibly empty) on-disk log.
+    return realOpenEventLog(sessionId, opts ?? {});
+  },
+}));
+
+const { runSample } = await import("./run-sample");
+
+const SAMPLE: Sample = { id: "s1", input: "hi", expected_output: "ok" };
+const EXACT = (() => {
+  const { compiled } = parseGradersConfig("graders:\n  - name: m\n    type: exact_match\n");
+  return compiled.map((g) => ({ name: g.name, grader: g.grader }));
+})();
+
+const TMP_ROOTS: string[] = [];
+function newTempRoot(): string {
+  const dir = mkdtempSync(join(tmpdir(), "crewhaus-rs-eventlog-"));
+  TMP_ROOTS.push(dir);
+  return dir;
+}
+afterEach(() => {
+  // Disarm immediately so the stub is inert (returns an empty log) outside the
+  // test that armed a throwing mode. The stub itself stays installed across
+  // this file's tests — both of them rely on it — but in its "off" state it is
+  // behaviourally identical to the real module against a not-yet-written log,
+  // so any leak into a sibling suite is harmless.
+  throwMode = "off";
+});
+afterAll(() => {
+  for (const dir of TMP_ROOTS) rmSync(dir, { recursive: true, force: true });
+  mock.module("@crewhaus/event-log", () => realEventLog);
+});
+
+describe("runSample — readTranscript failure handling", () => {
+  test("tolerates an 'invalid sessionId' error (missing-log stub case)", async () => {
+    throwMode = "invalid";
+    const outDir = newTempRoot();
+    // Invoker supplies no transcript/events and no error → readTranscript runs.
+    const invoker = async () => ({ agentOutput: "ok" });
+    const result = await runSample({
+      sample: SAMPLE,
+      invoker,
+      graders: EXACT,
+      outDir,
+      model: "claude-test",
+    });
+    // Tolerated: transcript stays empty, sample still grades.
+    expect(result.turns).toBe(0);
+    expect(result.grades.overall.passed).toBe(true);
+  });
+
+  test("rethrows any other open failure as a RunnerError", async () => {
+    throwMode = "generic";
+    const outDir = newTempRoot();
+    const invoker = async () => ({ agentOutput: "ok" });
+    await expect(
+      runSample({
+        sample: SAMPLE,
+        invoker,
+        graders: EXACT,
+        outDir,
+        model: "claude-test",
+      }),
+    ).rejects.toThrow(/failed to read transcript for/);
+  });
+});

package/src/run-sample.test.ts

@@ -0,0 +1,273 @@
+import { afterAll, describe, expect, test } from "bun:test";
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { Sample } from "@crewhaus/eval-dataset";
+import { parseGradersConfig } from "@crewhaus/eval-grader";
+import type {
+  ModelResponseEvent,
+  ToolCallEndEvent,
+  ToolCallStartEvent,
+  TraceEvent,
+} from "@crewhaus/trace-event-bus";
+import { combineGraderEntries, runSample } from "./run-sample";
+import type { AgentInvoker, GraderEntry } from "./types";
+
+const TMP_ROOTS: string[] = [];
+function newTempRoot(): string {
+  const dir = mkdtempSync(join(tmpdir(), "crewhaus-run-sample-"));
+  TMP_ROOTS.push(dir);
+  return dir;
+}
+afterAll(() => {
+  for (const dir of TMP_ROOTS) rmSync(dir, { recursive: true, force: true });
+});
+
+const SAMPLE: Sample = { id: "s1", input: "hi", expected_output: "ok" };
+
+/** Build a fully-formed envelope so published events validate as TraceEvents. */
+function envelope(timestamp: string) {
+  return {
+    runId: "run_test",
+    sessionId: "sess_0000000000000000",
+    turnNumber: 1,
+    traceId: "0".repeat(32),
+    spanId: "0".repeat(16),
+    timestamp,
+  };
+}
+
+function toolStart(toolUseId: string, toolName: string, ts: string): ToolCallStartEvent {
+  return { ...envelope(ts), kind: "tool_call_start", toolUseId, toolName, inputBytes: 1 };
+}
+function toolEnd(
+  toolUseId: string,
+  toolName: string,
+  ts: string,
+  isError = false,
+): ToolCallEndEvent {
+  return {
+    ...envelope(ts),
+    kind: "tool_call_end",
+    toolUseId,
+    toolName,
+    isError,
+    outputBytes: 1,
+    durationMs: 1,
+  };
+}
+function modelResponse(input: number, output: number, ts: string): ModelResponseEvent {
+  return {
+    ...envelope(ts),
+    kind: "model_response",
+    model: "claude-test",
+    stopReason: "end_turn",
+    usage: { input, output },
+    durationMs: 1,
+  };
+}
+
+const EXACT: GraderEntry[] = (() => {
+  const { compiled } = parseGradersConfig("graders:\n  - name: m\n    type: exact_match\n");
+  return compiled.map((g) => ({ name: g.name, grader: g.grader }));
+})();
+
+describe("runSample — bus capture, token/tool extraction", () => {
+  test("captures bus events, sums tokens, extracts ordered tool calls", async () => {
+    const outDir = newTempRoot();
+    // Invoker that publishes trace events through the per-sample bus and does
+    // NOT supply `events` (so runSample falls back to the captured array,
+    // exercising the subscribe callback + extractToolCalls + sumTokens).
+    const invoker: AgentInvoker = async ({ runContext }) => {
+      const bus = runContext.eventBus;
+      // Emit tool calls out of timestamp order to prove the sort in
+      // extractToolCalls; the "later" start has an earlier timestamp.
+      bus.publish(toolStart("u2", "bash", "2026-01-01T00:00:02.000Z"));
+      bus.publish(toolStart("u1", "read", "2026-01-01T00:00:01.000Z"));
+      bus.publish(toolEnd("u2", "bash", "2026-01-01T00:00:03.000Z", true));
+      bus.publish(toolEnd("u1", "read", "2026-01-01T00:00:04.000Z"));
+      // A tool_call_end with no matching start → falls back to its own ts.
+      bus.publish(toolEnd("orphan", "glob", "2026-01-01T00:00:00.500Z"));
+      bus.publish(modelResponse(10, 5, "2026-01-01T00:00:05.000Z"));
+      bus.publish(modelResponse(3, 2, "2026-01-01T00:00:06.000Z"));
+      return { agentOutput: "ok" };
+    };
+
+    const result = await runSample({
+      sample: SAMPLE,
+      invoker,
+      graders: EXACT,
+      outDir,
+      model: "claude-test",
+    });
+
+    expect(result.tokens).toEqual({ input: 13, output: 7 });
+    // events.jsonl persisted with the captured events.
+    const eventsRaw = readFileSync(join(outDir, "s1", "events.jsonl"), "utf-8");
+    const lines = eventsRaw
+      .trim()
+      .split("\n")
+      .map((l) => JSON.parse(l) as TraceEvent);
+    expect(lines).toHaveLength(7);
+    // grades.json + meta.json written.
+    expect(existsSync(join(outDir, "s1", "grades.json"))).toBe(true);
+    const meta = JSON.parse(readFileSync(join(outDir, "s1", "meta.json"), "utf-8"));
+    expect(meta.tokens).toEqual({ input: 13, output: 7 });
+    expect(result.grades.overall.passed).toBe(true);
+  });
+
+  test("stub-supplied transcript is written verbatim and turns counted", async () => {
+    const outDir = newTempRoot();
+    const transcript = [
+      { ts: 1, version: 1 as const, kind: "user_message" as const, payload: { text: "hi" } },
+      {
+        ts: 2,
+        version: 1 as const,
+        kind: "assistant_message" as const,
+        payload: { text: "ok" },
+      },
+      {
+        ts: 3,
+        version: 1 as const,
+        kind: "assistant_message" as const,
+        payload: { text: "more" },
+      },
+    ];
+    const invoker: AgentInvoker = async () => ({ agentOutput: "ok", transcript, events: [] });
+    const result = await runSample({
+      sample: SAMPLE,
+      invoker,
+      graders: EXACT,
+      outDir,
+      model: "claude-test",
+    });
+    // transcript.jsonl written by runSample (stub branch).
+    const tRaw = readFileSync(join(outDir, "s1", "transcript.jsonl"), "utf-8");
+    expect(tRaw.trim().split("\n")).toHaveLength(3);
+    expect(result.turns).toBe(2); // two assistant_message events
+  });
+
+  test("renames an invoker-written auto-named event-log file to transcript.jsonl", async () => {
+    const outDir = newTempRoot();
+    // Invoker writes <sessionId>.jsonl into sessionRootDir (mimics runChatLoop).
+    const invoker: AgentInvoker = async ({ runContext, sessionRootDir }) => {
+      const auto = join(sessionRootDir, `${runContext.sessionId}.jsonl`);
+      writeFileSync(
+        auto,
+        `${JSON.stringify({ ts: 1, version: 1, kind: "assistant_message", payload: {} })}\n`,
+      );
+      return { agentOutput: "ok" };
+    };
+    const result = await runSample({
+      sample: SAMPLE,
+      invoker,
+      graders: EXACT,
+      outDir,
+      model: "claude-test",
+    });
+    expect(existsSync(join(outDir, "s1", "transcript.jsonl"))).toBe(true);
+    expect(existsSync(join(outDir, "s1", `${result.sessionId}.jsonl`))).toBe(false);
+    expect(result.turns).toBe(1);
+  });
+
+  test("an empty captured run writes an empty transcript.jsonl", async () => {
+    const outDir = newTempRoot();
+    // No transcript, no events, no on-disk log → final else branch writes "".
+    const invoker: AgentInvoker = async () => ({ agentOutput: "ok", events: [] });
+    await runSample({
+      sample: SAMPLE,
+      invoker,
+      graders: EXACT,
+      outDir,
+      model: "claude-test",
+    });
+    expect(readFileSync(join(outDir, "s1", "transcript.jsonl"), "utf-8")).toBe("");
+    expect(readFileSync(join(outDir, "s1", "events.jsonl"), "utf-8")).toBe("");
+  });
+
+  test("a grader that throws is captured as a failed perGrader entry", async () => {
+    const outDir = newTempRoot();
+    const graders: GraderEntry[] = [
+      {
+        name: "boom",
+        grader: async () => {
+          throw new Error("grader kaboom");
+        },
+      },
+      {
+        name: "nonError",
+        grader: async () => {
+          // Throw a non-Error to exercise the String(err) branch.
+          throw "stringly-typed";
+        },
+      },
+    ];
+    const invoker: AgentInvoker = async () => ({ agentOutput: "x", events: [] });
+    const result = await runSample({
+      sample: SAMPLE,
+      invoker,
+      graders,
+      outDir,
+      model: "claude-test",
+    });
+    const boom = result.grades.perGrader.find((g) => g.name === "boom");
+    expect(boom?.passed).toBe(false);
+    expect(boom?.rationale).toContain("grader threw: grader kaboom");
+    const nonError = result.grades.perGrader.find((g) => g.name === "nonError");
+    expect(nonError?.rationale).toContain("stringly-typed");
+    expect(result.grades.overall.passed).toBe(false);
+  });
+
+  test("sanitizes path-separator-laden sample ids into a flat dir", async () => {
+    const outDir = newTempRoot();
+    const invoker: AgentInvoker = async () => ({ agentOutput: "ok", events: [] });
+    const result = await runSample({
+      sample: { id: "a/b:c d", input: "x", expected_output: "y" },
+      invoker,
+      graders: EXACT,
+      outDir,
+      model: "claude-test",
+    });
+    // sampleId is preserved in the result, but the dir name is sanitized.
+    expect(result.sampleId).toBe("a/b:c d");
+    expect(existsSync(join(outDir, "a_b_c_d", "meta.json"))).toBe(true);
+  });
+
+  test("seed is threaded into the request and persisted in meta.json", async () => {
+    const outDir = newTempRoot();
+    let seenSeed: number | undefined;
+    const invoker: AgentInvoker = async ({ seed }) => {
+      seenSeed = seed;
+      return { agentOutput: "ok", events: [] };
+    };
+    await runSample({
+      sample: SAMPLE,
+      invoker,
+      graders: EXACT,
+      outDir,
+      model: "claude-test",
+      seed: 1234,
+    });
+    expect(seenSeed).toBe(1234);
+    const meta = JSON.parse(readFileSync(join(outDir, "s1", "meta.json"), "utf-8"));
+    expect(meta.seed).toBe(1234);
+  });
+});
+
+describe("combineGraderEntries", () => {
+  test("combines compiled graders into a single Grader", async () => {
+    const { compiled } = parseGradersConfig(
+      "graders:\n  - name: a\n    type: contains\n    substring: ok\n",
+    );
+    const combined = combineGraderEntries(compiled);
+    const res = await combined(SAMPLE, {
+      agentOutput: "this is ok",
+      events: [],
+      transcript: [],
+      toolCalls: [],
+      turns: 1,
+      latencyMs: 0,
+    });
+    expect(res.passed).toBe(true);
+  });
+});

package/src/semaphore.test.ts

@@ -0,0 +1,58 @@
+import { describe, expect, test } from "bun:test";
+import { Semaphore } from "./semaphore";
+
+describe("Semaphore — queueing and introspection", () => {
+  test("queues over-capacity acquirers and wakes them FIFO on release", async () => {
+    const sem = new Semaphore(1);
+    const order: number[] = [];
+
+    const r0 = await sem.acquire(); // takes the only slot
+    expect(sem.active).toBe(1);
+    expect(sem.pending).toBe(0);
+
+    // Two more acquirers must queue.
+    const p1 = sem.acquire().then((release) => {
+      order.push(1);
+      return release;
+    });
+    const p2 = sem.acquire().then((release) => {
+      order.push(2);
+      return release;
+    });
+
+    // Let the microtask queue settle; both are still waiting.
+    await Promise.resolve();
+    expect(sem.pending).toBe(2);
+    expect(sem.active).toBe(1);
+
+    r0(); // releasing wakes the first waiter (FIFO)
+    const r1 = await p1;
+    expect(order).toEqual([1]);
+    expect(sem.active).toBe(1);
+    expect(sem.pending).toBe(1);
+
+    r1(); // wakes the second waiter
+    const r2 = await p2;
+    expect(order).toEqual([1, 2]);
+    expect(sem.pending).toBe(0);
+
+    r2();
+    expect(sem.active).toBe(0);
+    expect(sem.pending).toBe(0);
+  });
+
+  test("release with an empty queue simply frees a slot", async () => {
+    const sem = new Semaphore(2);
+    const a = await sem.acquire();
+    const b = await sem.acquire();
+    expect(sem.active).toBe(2);
+    a();
+    expect(sem.active).toBe(1);
+    b();
+    expect(sem.active).toBe(0);
+    // A subsequent acquire takes the fast path (inflight < capacity).
+    const c = await sem.acquire();
+    expect(sem.active).toBe(1);
+    c();
+  });
+});

package/src/tenant-roots.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, test } from "bun:test";
+import { buildTenant, withTenant } from "@crewhaus/tenancy";
+import { resolveEvalOutDir } from "./index";
+
+// Regression — issue #150 (CWE-1230). Eval artifacts must land under the active
+// tenant's evalRoot so one tenant's eval data never shares a directory with
+// another's.
+describe("resolveEvalOutDir — tenant isolation (#150)", () => {
+  test("each tenant resolves under its own evalRoot, disjoint", () => {
+    const a = buildTenant("tenant-a", { tenantsRoot: "/tmp/ch-eval-test" });
+    const b = buildTenant("tenant-b", { tenantsRoot: "/tmp/ch-eval-test" });
+    const outA = withTenant(a, () => resolveEvalOutDir("run1")) as string;
+    const outB = withTenant(b, () => resolveEvalOutDir("run1")) as string;
+    expect(outA.startsWith(`${a.evalRoot}/`)).toBe(true);
+    expect(outB.startsWith(`${b.evalRoot}/`)).toBe(true);
+    expect(outA).not.toBe(outB);
+  });
+
+  test("outside a tenant scope, uses the global default", () => {
+    const out = resolveEvalOutDir("runX");
+    expect(out).toContain(".crewhaus");
+    expect(out).toContain("runX");
+  });
+
+  test("an explicit outDir always wins (trusted caller override)", () => {
+    const a = buildTenant("tenant-a", { tenantsRoot: "/tmp/ch-eval-test" });
+    const out = withTenant(a, () => resolveEvalOutDir("runX", "/tmp/explicit-out")) as string;
+    expect(out).toBe("/tmp/explicit-out");
+  });
+});