@workos/quickstudy 0.0.1

package/src/diagnose/run.ts

@@ -0,0 +1,290 @@
+/**
+ * Diagnosis orchestration: load one run's persisted evidence, select the
+ * diagnosable pairs, bundle harness-selected excerpts, ask the analyst per
+ * pair, map index-based findings back to attempt ids, derive confidence from
+ * rules, and write `results/<runId>/diagnosis.json` beside report.json.
+ *
+ * Everything reads the STORE (rows + artifacts) — never consumer experiment
+ * modules, never `src/report/` output. The model client is injected; the
+ * Anthropic-backed client is constructed only in the CLI command. Per-pair
+ * model failures land in `errors[]` and the artifact is still written —
+ * evidence of a failure is never lost.
+ */
+
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { requestValidated, type StructuredModelClient } from "../llm.ts";
+import type { AttemptRecord, ResultsStore, RunRecord } from "../store/db.ts";
+import {
+  excerptDiff,
+  excerptTranscript,
+  readArtifact,
+} from "./excerpt.ts";
+import {
+  DIAGNOSE_JSON_SCHEMA,
+  DIAGNOSE_MAX_TOKENS,
+  DIAGNOSE_SYSTEM,
+  diagnoseResponseSchema,
+  renderPairPrompt,
+  type BundleAttempt,
+  type PairBundle,
+} from "./prompt.ts";
+import { selectDiagnosablePairs, type DiagnosablePair } from "./select.ts";
+import {
+  DIAGNOSIS_DISCLAIMER,
+  DIAGNOSIS_PROMPT_VERSION,
+  DIAGNOSIS_SCHEMA_VERSION,
+  DiagnoseError,
+  type DiagnosisConfidence,
+  type DiagnosisEvidence,
+  type DiagnosisFinding,
+  type DiagnosisJson,
+  type PairDiagnosisError,
+} from "./types.ts";
+
+/** Analyst model default; overridable via QUICKSTUDY_ANALYST_MODEL (deprecation resilience). */
+export const DEFAULT_ANALYST_MODEL = "claude-sonnet-5";
+
+export function analystModelFromEnv(env: Record<string, string | undefined> = process.env): string {
+  const model = env["QUICKSTUDY_ANALYST_MODEL"];
+  return model !== undefined && model !== "" ? model : DEFAULT_ANALYST_MODEL;
+}
+
+/** Canonical location of a run's diagnosis.json (mirrors reportJsonPath). */
+export function diagnosisJsonPath(resultsDir: string, runId: string): string {
+  return join(resultsDir, runId, "diagnosis.json");
+}
+
+/** Write diagnosis.json into the run's results dir. Returns the path. */
+export function writeDiagnosisJson(diagnosis: DiagnosisJson, resultsDir: string): string {
+  const dir = join(resultsDir, diagnosis.run_id);
+  mkdirSync(dir, { recursive: true });
+  const path = diagnosisJsonPath(resultsDir, diagnosis.run_id);
+  writeFileSync(path, `${JSON.stringify(diagnosis, null, 2)}\n`, "utf8");
+  return path;
+}
+
+function mustResolveRun(store: ResultsStore, runId: string | undefined): RunRecord {
+  if (runId === undefined) {
+    const latest = store.latestRun();
+    if (latest === null) {
+      throw new DiagnoseError(
+        "this results database has no runs yet — run `quickstudy run --eval <id> --experiment <id>` first",
+      );
+    }
+    return latest;
+  }
+  const run = store.getRun(runId);
+  if (run) return run;
+  const known = store
+    .listRuns()
+    .slice(0, 5)
+    .map((candidate) => `  ${candidate.id} (started ${new Date(candidate.startedAt).toISOString()})`);
+  throw new DiagnoseError(
+    `no run "${runId}" in this results database${known.length > 0 ? `; most recent runs:\n${known.join("\n")}` : ""}`,
+  );
+}
+
+function attemptLabel(cls: "failed" | "error" | "incomplete" | "passed", contrast: boolean): string {
+  if (contrast) return "passing contrast";
+  if (cls === "error") return "errored";
+  return cls;
+}
+
+/**
+ * Read a bundle attempt's artifacts and excerpt them. Paths derive from the
+ * results dir layout (the report's egress-denials precedent) with the stored
+ * ref as fallback, and only refs the runner actually recorded are read.
+ */
+function bundleAttemptFor(
+  resultsDir: string,
+  runId: string,
+  selection: { attempt: AttemptRecord; class: "failed" | "error" | "incomplete" | "passed"; contrast: boolean },
+  index: number,
+): BundleAttempt {
+  const { attempt } = selection;
+  const bundleAttempt: BundleAttempt = {
+    index,
+    attemptId: attempt.id,
+    label: attemptLabel(selection.class, selection.contrast),
+  };
+  if (attempt.transcriptRef !== null) {
+    const raw =
+      readArtifact(join(resultsDir, runId, attempt.id, "transcript.jsonl")) ?? readArtifact(attempt.transcriptRef);
+    if (raw !== undefined && raw !== "") bundleAttempt.transcript = excerptTranscript(raw);
+  }
+  if (attempt.diffRef !== null) {
+    const raw = readArtifact(join(resultsDir, runId, attempt.id, "diff.patch")) ?? readArtifact(attempt.diffRef);
+    if (raw !== undefined && raw !== "") bundleAttempt.diff = excerptDiff(raw);
+  }
+  return bundleAttempt;
+}
+
+/** Assemble one pair's prompt bundle. Exported for tests. */
+export function buildPairBundle(options: {
+  run: RunRecord;
+  resultsDir: string;
+  pair: DiagnosablePair;
+}): PairBundle {
+  const { run, resultsDir, pair } = options;
+  const treatment = run.manifest.experiments[pair.experimentId]?.components.treatment ?? null;
+  const surfaces = run.config.surfaces?.[pair.experimentId];
+  return {
+    evalId: pair.evalId,
+    experimentId: pair.experimentId,
+    treatment,
+    tallies: pair.tallies,
+    failedChecks: pair.failedChecks,
+    ...(surfaces !== undefined ? { surfaces } : {}),
+    attempts: pair.bundle.map((selection, position) => bundleAttemptFor(resultsDir, run.id, selection, position + 1)),
+  };
+}
+
+/**
+ * Map the model's index-only references back to attempt ids + the excerpt
+ * windows the model was actually shown. Out-of-range indexes are dropped; a
+ * finding whose indexes all fail to map falls back to the bundle's first
+ * failing attempt's excerpt — degrade, never fabricate.
+ */
+export function mapEvidence(bundle: PairBundle, attemptIndexes: number[]): DiagnosisEvidence[] {
+  const mapped: DiagnosisEvidence[] = [];
+  const seen = new Set<string>();
+  for (const index of attemptIndexes) {
+    const attempt = bundle.attempts.find((candidate) => candidate.index === index);
+    if (!attempt) continue;
+    const excerpt = attempt.transcript ?? attempt.diff;
+    if (!excerpt) continue;
+    if (seen.has(attempt.attemptId)) continue;
+    seen.add(attempt.attemptId);
+    mapped.push({
+      attempt_id: attempt.attemptId,
+      artifact: attempt.transcript ? "transcript" : "diff",
+      lines: excerpt.lines,
+    });
+  }
+  if (mapped.length === 0) {
+    const first = bundle.attempts.find((attempt) => attempt.label !== "passing contrast" && (attempt.transcript ?? attempt.diff));
+    const excerpt = first?.transcript ?? first?.diff;
+    if (first && excerpt) {
+      mapped.push({
+        attempt_id: first.attemptId,
+        artifact: first.transcript ? "transcript" : "diff",
+        lines: excerpt.lines,
+      });
+    }
+  }
+  return mapped;
+}
+
+/**
+ * Rule-derived confidence — never asked of the model:
+ *   high   = pair has zero passes and >= 2 bundled attempts mapped as evidence;
+ *   medium = zero passes with 1 mapped attempt, or flaky with >= 2;
+ *   low    = otherwise.
+ */
+export function deriveConfidence(pair: DiagnosablePair, evidence: DiagnosisEvidence[]): DiagnosisConfidence {
+  const mappedAttempts = new Set(evidence.map((entry) => entry.attempt_id)).size;
+  const zeroPasses = pair.tallies.passed === 0;
+  if (zeroPasses && mappedAttempts >= 2) return "high";
+  if ((zeroPasses && mappedAttempts === 1) || (pair.flaky && mappedAttempts >= 2)) return "medium";
+  return "low";
+}
+
+export interface RunDiagnoseOptions {
+  store: ResultsStore;
+  /** Artifacts root (the harness `--results` dir) — transcripts and diffs live here. */
+  resultsDir: string;
+  /** Omit to diagnose the most recent run (`--latest`). */
+  runId?: string;
+  /** Injected model client — the CLI is the only place the Anthropic client is constructed. */
+  client: StructuredModelClient;
+  /** Overrides the analyst model (default: QUICKSTUDY_ANALYST_MODEL or the pinned default). */
+  model?: string;
+  /** Overrides the `generated_at` stamp — fixture seeding only. */
+  generatedAt?: string;
+  /** Progress lines (pair counts) — the CLI passes console.error. */
+  log?: (line: string) => void;
+}
+
+/**
+ * Diagnose one run and write diagnosis.json. Throws DiagnoseError only when
+ * the run cannot be resolved; per-pair failures are recorded in `errors[]`
+ * and the artifact is still written.
+ */
+export async function runDiagnose(options: RunDiagnoseOptions): Promise<DiagnosisJson> {
+  const { store, resultsDir, client } = options;
+  const log = options.log ?? ((): void => {});
+  const model = options.model ?? analystModelFromEnv();
+
+  const run = mustResolveRun(store, options.runId);
+  const attempts = store.listAttempts(run.id);
+  const selection = selectDiagnosablePairs(attempts);
+  log(
+    `diagnose: run ${run.id} — ${selection.pairs.length} diagnosable pair(s), ` +
+      `${selection.skipped} passing pair(s) skipped`,
+  );
+
+  const findings: DiagnosisFinding[] = [];
+  const errors: PairDiagnosisError[] = [];
+
+  // Sequential by design: a sweep is at most a handful of requests and the
+  // SDK owns transport retries — no concurrency machinery.
+  for (const pair of selection.pairs) {
+    const bundle = buildPairBundle({ run, resultsDir, pair });
+    if (bundle.attempts.every((attempt) => attempt.transcript === undefined && attempt.diff === undefined)) {
+      errors.push({
+        eval_id: pair.evalId,
+        experiment_id: pair.experimentId,
+        error: "no readable transcript or diff artifacts for any bundled attempt",
+      });
+      continue;
+    }
+    try {
+      const response = await requestValidated(
+        client,
+        {
+          model,
+          system: DIAGNOSE_SYSTEM,
+          user: renderPairPrompt(bundle),
+          maxTokens: DIAGNOSE_MAX_TOKENS,
+          schema: DIAGNOSE_JSON_SCHEMA,
+        },
+        diagnoseResponseSchema,
+      );
+      for (const draft of response.findings) {
+        const evidence = mapEvidence(bundle, draft.attempt_indexes);
+        if (evidence.length === 0) continue; // nothing mappable at all — unusable
+        findings.push({
+          eval_id: pair.evalId,
+          experiment_id: pair.experimentId,
+          target: draft.target,
+          claim: draft.claim,
+          suggested_change: draft.suggested_change,
+          confidence: deriveConfidence(pair, evidence),
+          evidence,
+        });
+      }
+    } catch (err) {
+      errors.push({
+        eval_id: pair.evalId,
+        experiment_id: pair.experimentId,
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+
+  const diagnosis: DiagnosisJson = {
+    schema_version: DIAGNOSIS_SCHEMA_VERSION,
+    run_id: run.id,
+    generated_at: options.generatedAt ?? new Date().toISOString(),
+    model: { requested: model, resolved: client.resolvedModel?.() ?? null },
+    prompt_version: DIAGNOSIS_PROMPT_VERSION,
+    disclaimer: DIAGNOSIS_DISCLAIMER,
+    pairs_diagnosed: selection.pairs.length - errors.length,
+    pairs_skipped: selection.skipped,
+    findings,
+    errors,
+  };
+  writeDiagnosisJson(diagnosis, resultsDir);
+  return diagnosis;
+}

package/src/diagnose/select.ts

@@ -0,0 +1,110 @@
+/**
+ * Pair selection for diagnosis: which (eval × experiment) pairs are worth
+ * model spend, and which attempts feed each pair's evidence bundle.
+ *
+ * Pair counts and flaky/check aggregation reuse the report's `collectPairs`
+ * helper so the two views cannot drift.
+ */
+
+import { collectPairs, type ReportPair } from "../report/report.ts";
+import type { AttemptRecord } from "../store/db.ts";
+
+export type AttemptClass = "passed" | "failed" | "error" | "incomplete";
+
+/** At most this many non-passing attempts feed one pair bundle. */
+export const MAX_BUNDLE_ATTEMPTS = 3;
+
+/** Report-semantics attempt classification (see module docblock). */
+export function classifyAttempt(attempt: AttemptRecord): AttemptClass {
+  if (attempt.status === "incomplete") return "incomplete";
+  if (attempt.status === "error") return "error";
+  return attempt.result?.passed ? "passed" : "failed";
+}
+
+export type PairTallies = Pick<ReportPair, "total" | "passed" | "failed" | "errors" | "incomplete">;
+
+/** One attempt selected into a pair's bundle. */
+export interface BundleSelection {
+  attempt: AttemptRecord;
+  class: AttemptClass;
+  /** True for the single passing attempt included as flaky-pair contrast. */
+  contrast: boolean;
+}
+
+/** A pair with at least one failed/errored/incomplete attempt. */
+export interface DiagnosablePair {
+  evalId: string;
+  experimentId: string;
+  tallies: PairTallies;
+  flaky: boolean;
+  /** Check names that failed at least once across the pair's attempts, sorted. */
+  failedChecks: string[];
+  /** ≤ MAX_BUNDLE_ATTEMPTS non-passing attempts (+ 1 passing contrast when flaky). */
+  bundle: BundleSelection[];
+}
+
+export interface PairSelection {
+  pairs: DiagnosablePair[];
+  /** Passing-only pairs — skipped, never sent to the model. */
+  skipped: number;
+}
+
+/** Bundle ordering: failed first, then errors, then incomplete. */
+const CLASS_ORDER: Record<Exclude<AttemptClass, "passed">, number> = {
+  failed: 0,
+  error: 1,
+  incomplete: 2,
+};
+
+/**
+ * Group attempts into pairs, classify, and select each diagnosable pair's
+ * bundle. Deterministic: pairs sort by eval id then experiment id (the
+ * report's order); within a class, attempts sort by ULID (id) order.
+ */
+export function selectDiagnosablePairs(attempts: AttemptRecord[]): PairSelection {
+  const byPair = new Map<string, AttemptRecord[]>();
+  for (const attempt of attempts) {
+    const key = `${attempt.evalId}|${attempt.experimentId}`;
+    const group = byPair.get(key);
+    if (group) group.push(attempt);
+    else byPair.set(key, [attempt]);
+  }
+
+  const pairs: DiagnosablePair[] = [];
+  let skipped = 0;
+
+  for (const group of byPair.values()) {
+    const report = collectPairs(group)[0]!;
+    if (report.failed + report.errors + report.incomplete === 0) {
+      skipped += 1;
+      continue;
+    }
+
+    const classified = group
+      .map((attempt) => ({ attempt, class: classifyAttempt(attempt) }))
+      .sort((a, b) => a.attempt.id.localeCompare(b.attempt.id));
+    const nonPassing = classified
+      .filter((entry): entry is { attempt: AttemptRecord; class: Exclude<AttemptClass, "passed"> } => entry.class !== "passed")
+      .sort((a, b) => CLASS_ORDER[a.class] - CLASS_ORDER[b.class] || a.attempt.id.localeCompare(b.attempt.id));
+    const bundle: BundleSelection[] = nonPassing
+      .slice(0, MAX_BUNDLE_ATTEMPTS)
+      .map((entry) => ({ attempt: entry.attempt, class: entry.class, contrast: false }));
+
+    if (report.flaky) {
+      const passing = classified.find((entry) => entry.class === "passed");
+      if (passing) bundle.push({ attempt: passing.attempt, class: "passed", contrast: true });
+    }
+
+    pairs.push({
+      evalId: report.eval_id,
+      experimentId: report.experiment_id,
+      tallies: report,
+      flaky: report.flaky,
+      failedChecks: report.checks.filter((check) => check.passed < check.total).map((check) => check.name),
+      bundle,
+    });
+  }
+
+  pairs.sort((a, b) => a.evalId.localeCompare(b.evalId) || a.experimentId.localeCompare(b.experimentId));
+  return { pairs, skipped };
+}

package/src/diagnose/types.ts

@@ -0,0 +1,88 @@
+/**
+ * The diagnosis.json contract: LLM-drafted, harness-evidenced hypotheses for
+ * one run's failing (eval × experiment) pairs.
+ *
+ * diagnosis.json is a SEPARATE artifact from report.json, deliberately exempt
+ * from the report's BANNED_LANGUAGE guard — it self-labels with
+ * {@link DIAGNOSIS_DISCLAIMER} instead, and nothing under `src/report/`
+ * reads or writes it. Evidence is harness-owned: every finding's evidence
+ * points at a deterministic excerpt window the harness selected; the model
+ * only ever referenced attempts by 1-based index.
+ */
+
+/** Loud, CLI-friendly diagnose failures (unknown run, empty evidence, ...). */
+export class DiagnoseError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "DiagnoseError";
+  }
+}
+
+export const DIAGNOSIS_SCHEMA_VERSION = 1;
+
+/** Bumped whenever the system/user prompt text changes shape. */
+export const DIAGNOSIS_PROMPT_VERSION = 1;
+
+/** Fixed self-labeling carried inside every diagnosis.json. */
+export const DIAGNOSIS_DISCLAIMER =
+  "These findings are model-drafted hypotheses, not verdicts. Scope is this single run: " +
+  "no cross-run claims, no attribution beyond the cited evidence. Each evidence link names a " +
+  "harness-selected excerpt (artifact + 1-indexed line range) — verify the cited lines before acting.";
+
+/**
+ * The generic surface taxonomy a finding can implicate. Concrete surface
+ * names (hosts, commands, server names) come from the run's recorded config —
+ * the taxonomy itself stays consumer-neutral.
+ */
+export type DiagnosisTarget = "docs" | "mcp" | "cli" | "sdk" | "environment" | "eval-defect";
+
+export const DIAGNOSIS_TARGETS: readonly DiagnosisTarget[] = [
+  "docs",
+  "mcp",
+  "cli",
+  "sdk",
+  "environment",
+  "eval-defect",
+];
+
+/** Rule-derived from pair shape + mapped evidence — never model-asserted. */
+export type DiagnosisConfidence = "high" | "medium" | "low";
+
+export interface DiagnosisEvidence {
+  attempt_id: string;
+  artifact: "transcript" | "diff";
+  /** 1-indexed inclusive line range inside the artifact — always a harness-selected window. */
+  lines: [number, number];
+}
+
+export interface DiagnosisFinding {
+  eval_id: string;
+  experiment_id: string;
+  target: DiagnosisTarget;
+  claim: string;
+  suggested_change: string;
+  confidence: DiagnosisConfidence;
+  evidence: DiagnosisEvidence[];
+}
+
+/** A pair whose analysis failed (model error after retries) — recorded, never silently dropped. */
+export interface PairDiagnosisError {
+  eval_id: string;
+  experiment_id: string;
+  error: string;
+}
+
+export interface DiagnosisJson {
+  schema_version: typeof DIAGNOSIS_SCHEMA_VERSION;
+  run_id: string;
+  generated_at: string;
+  /** Requested model (pin or env override) and the model id the API reported back. */
+  model: { requested: string; resolved: string | null };
+  prompt_version: typeof DIAGNOSIS_PROMPT_VERSION;
+  /** Fixed self-labeling: hypotheses with evidence links, not verdicts; single-sweep scope. */
+  disclaimer: string;
+  pairs_diagnosed: number;
+  pairs_skipped: number;
+  findings: DiagnosisFinding[];
+  errors: PairDiagnosisError[];
+}

package/src/evals/discovery.ts

@@ -0,0 +1,173 @@
+/**
+ * Eval discovery: scan each `<root>/<dir>/PROMPT.md`, pair it with its EVAL.ts, and
+ * detect the optional `local/` starting-state directory. Discovery is dumb on
+ * purpose — one visible expansion (a `frameworks:` frontmatter list yields
+ * one eval per framework, id `<dirname>-<framework>`), no placeholders, no
+ * binding — and loud: the first error aborts the scan.
+ */
+
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { basename, dirname, join, relative, resolve, sep } from "node:path";
+import { pathToFileURL } from "node:url";
+import { EvalLoadError, parsePrompt } from "./prompt.ts";
+import type { EvalScorer, LoadedEval } from "./types.ts";
+
+/** One collected eval; `framework` is set for variant-expanded entries. */
+interface CollectedEval {
+  loaded: LoadedEval;
+  framework?: string;
+}
+
+/**
+ * Discover every eval under `root`. A subdirectory containing a PROMPT.md is
+ * an eval; it must also carry an EVAL.ts, its frontmatter `id` must equal the
+ * directory name, and ids must be unique across the root.
+ *
+ * A `frameworks:` frontmatter list expands the directory into one eval per
+ * framework: id `<dirname>-<framework>`, `metadata.framework` set, prompt and
+ * scorer shared verbatim, starting state resolved per framework from
+ * `local/<framework>` — or `<local:>/<framework>` when the eval points at a
+ * shared fixture root. Everything downstream consumes the expanded
+ * {@link LoadedEval}s exactly like single-directory ones.
+ */
+export function discoverEvals(root: string): LoadedEval[] {
+  const rootDir = resolve(root);
+  if (!existsSync(rootDir) || !statSync(rootDir).isDirectory()) {
+    throw new Error(`evals root "${root}" is not a directory`);
+  }
+
+  const collected: CollectedEval[] = [];
+  for (const entry of readdirSync(rootDir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+    if (!entry.isDirectory()) continue;
+    const dir = join(rootDir, entry.name);
+    const promptPath = join(dir, "PROMPT.md");
+    if (!existsSync(promptPath)) continue;
+
+    const { metadata, promptBody, frameworks, local } = parsePrompt(readFileSync(promptPath, "utf8"), promptPath);
+
+    const scorerPath = join(dir, "EVAL.ts");
+    if (!existsSync(scorerPath)) {
+      throw new EvalLoadError(promptPath, 1, `eval "${metadata.id}" has no EVAL.ts next to its PROMPT.md (expected ${scorerPath})`);
+    }
+
+    if (frameworks === undefined) {
+      const localDir = join(dir, "local");
+      const hasLocal = existsSync(localDir) && statSync(localDir).isDirectory();
+      collected.push({
+        loaded: { metadata, dir, promptPath, promptBody, scorerPath, localDir: hasLocal ? localDir : null },
+      });
+      continue;
+    }
+
+    // Variant form: one eval per framework. The starting-state base is the
+    // conventional local/ directory or the explicit `local:` path (shared
+    // fixtures), resolved from the eval directory. hashTree receives the
+    // resolved per-framework root, so unchanged fixture trees keep
+    // byte-stable hashes wherever they live.
+    const localBase = local === undefined ? join(dir, "local") : resolve(dir, local);
+    const hasBase = existsSync(localBase) && statSync(localBase).isDirectory();
+    if (local !== undefined) {
+      const rootParent = dirname(rootDir);
+      const escape = relative(rootParent, localBase);
+      if (escape === ".." || escape.startsWith(`..${sep}`)) {
+        throw new EvalLoadError(
+          promptPath,
+          1,
+          `"local" path ${JSON.stringify(local)} resolves to ${localBase}, ` +
+            `outside the evals root's parent (${rootParent})`,
+        );
+      }
+      if (!hasBase) {
+        throw new EvalLoadError(
+          promptPath,
+          1,
+          `"local" path ${JSON.stringify(local)} resolves to ${localBase}, which is not a directory`,
+        );
+      }
+    }
+    // No starting state at all (no local/ and no `local:`) is valid — every
+    // variant starts from an empty workspace, like a tools-only eval.
+    for (const framework of frameworks) {
+      let localDir: string | null = null;
+      if (hasBase) {
+        const frameworkDir = join(localBase, framework);
+        if (!existsSync(frameworkDir) || !statSync(frameworkDir).isDirectory()) {
+          throw new EvalLoadError(
+            promptPath,
+            1,
+            `eval "${metadata.id}" lists framework "${framework}" but has no starting state for it ` +
+              `(expected ${frameworkDir})`,
+          );
+        }
+        localDir = frameworkDir;
+      }
+      collected.push({
+        loaded: {
+          metadata: { ...metadata, id: `${metadata.id}-${framework}`, framework },
+          dir,
+          promptPath,
+          promptBody,
+          scorerPath,
+          localDir,
+        },
+        framework,
+      });
+    }
+  }
+
+  // Duplicates are checked BEFORE the id/dirname match so two directories
+  // claiming one id — or a variant expansion colliding with a literal
+  // directory — fail with both paths named, not with a mismatch message
+  // that points at only one of them.
+  const byId = new Map<string, CollectedEval>();
+  for (const entry of collected) {
+    const existing = byId.get(entry.loaded.metadata.id);
+    if (existing) {
+      throw new EvalLoadError(
+        entry.loaded.promptPath,
+        1,
+        `duplicate eval id "${entry.loaded.metadata.id}" — also declared by ${existing.loaded.promptPath}`,
+      );
+    }
+    byId.set(entry.loaded.metadata.id, entry);
+  }
+  for (const entry of collected) {
+    const dirName = basename(entry.loaded.dir);
+    const expected = entry.framework === undefined ? dirName : `${dirName}-${entry.framework}`;
+    if (entry.loaded.metadata.id !== expected) {
+      throw new EvalLoadError(
+        entry.loaded.promptPath,
+        1,
+        entry.framework === undefined
+          ? `frontmatter id "${entry.loaded.metadata.id}" must equal the directory name "${dirName}" (prevents silent aliasing)`
+          : `expanded eval id "${entry.loaded.metadata.id}" must equal "<dirname>-<framework>" ("${expected}") — ` +
+            `a variant eval's frontmatter id must equal its directory name (prevents silent aliasing)`,
+      );
+    }
+  }
+
+  return collected.map((entry) => entry.loaded).sort((a, b) => a.metadata.id.localeCompare(b.metadata.id));
+}
+
+/**
+ * Import an eval's scorer module and validate its default export is a
+ * function. Called by `validate` (so a broken EVAL.ts fails before any
+ * attempt runs) and by the runner at score time.
+ */
+export async function importEvalScorer(scorerPath: string): Promise<EvalScorer> {
+  let module: Record<string, unknown>;
+  try {
+    module = (await import(pathToFileURL(resolve(scorerPath)).href)) as Record<string, unknown>;
+  } catch (err) {
+    throw new EvalLoadError(scorerPath, 1, `EVAL.ts failed to import: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  const scorer = module["default"];
+  if (typeof scorer !== "function") {
+    throw new EvalLoadError(
+      scorerPath,
+      1,
+      `EVAL.ts must default-export an async scorer function (ctx) => EvalResult; got ${typeof scorer}`,
+    );
+  }
+  return scorer as EvalScorer;
+}