@michaelfromyeg/loom-eval 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Michael DeMarco
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/index.d.ts

@@ -0,0 +1,164 @@
+import { Transcript, HarnessDriver, ToolCall } from '@michaelfromyeg/loom-adapter-kit';
+import { Assertion, Target, EvalFile } from '@michaelfromyeg/loom-schema';
+import { AdapterRegistry } from '@michaelfromyeg/loom-core';
+
+type AssertStatus = "pass" | "fail" | "degraded" | "skipped";
+interface AssertResult {
+    kind: Assertion["kind"];
+    status: AssertStatus;
+    detail: string;
+}
+interface JudgeInput {
+    candidate: string;
+    reference?: string;
+    rubric: string;
+    mode: "absolute" | "pairwise";
+    samples: number;
+}
+interface JudgeVerdict {
+    pass: boolean;
+    detail?: string;
+}
+/** A judge model. Injected so evals run offline/deterministically in tests. */
+type JudgeFn = (input: JudgeInput) => Promise<JudgeVerdict>;
+interface AssertContext {
+    judge?: JudgeFn;
+    /** Deterministic score of the case (fraction of trace/output passing) -- differential. */
+    caseScore?: number;
+    /** Baseline score for (component, harness) from evals/.baselines/ -- differential. */
+    baselineScore?: number;
+}
+/**
+ * Evaluate one assertion against a case's sample transcripts (spec §9.5). Trace
+ * and output are deterministic; judge is advisory unless `gate:true` and needs an
+ * injected judge model; differential compares the case score to a committed
+ * baseline (the "vibes" no-regression gate).
+ */
+declare function evaluateAssertion(a: Assertion, transcripts: Transcript[], ctx?: AssertContext): Promise<AssertResult>;
+
+interface Baseline {
+    version: string;
+    score: number;
+}
+/** Load the committed baseline score for (component, harness), or null. */
+declare function loadBaseline(pluginDir: string, component: string, harness: Target): Baseline | null;
+/**
+ * Snapshot a baseline (spec §9.5). Called on `loom publish` so the next release's
+ * differential evals compare against this score.
+ */
+declare function writeBaseline(pluginDir: string, component: string, harness: Target, baseline: Baseline): string;
+
+declare const claudeDriver: HarnessDriver;
+
+declare const codexDriver: HarnessDriver;
+
+/**
+ * GitHub Copilot's headless `-p` mode exposes NO structured tool-call trace today
+ * (only a Markdown transcript via --share). So `run` returns the final text with
+ * `traceUnavailable: true`, and the runner degrades `trace` assertions to `output`
+ * for this harness rather than faking a pass (spec §14, harness-research.md).
+ */
+declare const copilotDriver: HarnessDriver;
+
+declare const cursorDriver: HarnessDriver;
+
+declare const opencodeDriver: HarnessDriver;
+
+/** Split NDJSON/JSONL into parsed objects, skipping blank/garbage lines. */
+declare function parseLines(raw: string): Record<string, unknown>[];
+/**
+ * Parse Claude's `--output-format stream-json --verbose` NDJSON. Tool calls are
+ * `assistant` message content blocks `{type:"tool_use", name, input}`; the final
+ * text is the `result` event's `result` field (see harness-research.md).
+ */
+declare function parseClaudeStream(raw: string): {
+    finalText: string;
+    toolCalls: ToolCall[];
+};
+/**
+ * Parse Codex `exec --json` JSONL. Tool calls live in `item.completed` events by
+ * `item.type`; the final text is the `agent_message` item. Only completed items
+ * are counted to avoid double-counting the matching `item.started`.
+ */
+declare function parseCodexStream(raw: string): {
+    finalText: string;
+    toolCalls: ToolCall[];
+};
+/**
+ * Parse Cursor `--output-format stream-json` NDJSON. Tool calls are `tool_call`
+ * events (count the `completed` subtype); the final text is the `result` event.
+ */
+declare function parseCursorStream(raw: string): {
+    finalText: string;
+    toolCalls: ToolCall[];
+};
+/**
+ * Parse OpenCode `run --format json` JSONL. Tool calls are `tool_use` events
+ * carrying a ToolPart (`part.tool`, `part.state.{input,output}`); the final text
+ * is the concatenation of `text` events.
+ */
+declare function parseOpencodeStream(raw: string): {
+    finalText: string;
+    toolCalls: ToolCall[];
+};
+
+interface CliResult {
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+}
+
+/** Every built-in headless driver, keyed by Target. */
+declare const drivers: Record<Target, HarnessDriver>;
+
+interface CaseResult {
+    name: string;
+    assertions: AssertResult[];
+    verifyPassed?: boolean;
+    /** Deterministic score: fraction of trace/output assertions that passed. */
+    score: number;
+    pass: boolean;
+}
+interface HarnessReport {
+    harness: Target;
+    status: "tested" | "untested";
+    /** Why a harness was not tested (no driver / CLI absent / install failed). */
+    reason?: string;
+    cases: CaseResult[];
+    /** Mean deterministic case score (snapshotted as the next release's baseline). */
+    score: number;
+    pass: boolean;
+}
+interface EvalReport {
+    component: string;
+    harnesses: HarnessReport[];
+}
+interface DiscoveredEval {
+    componentLeaf: string;
+    evalsPath: string;
+    evalFile: EvalFile;
+}
+/** Find the components in a plugin that declare an `evals` file and load each. */
+declare function discoverEvals(pluginDir: string): DiscoveredEval[];
+interface RunEvalOptions {
+    evalFile: EvalFile;
+    pluginDir: string;
+    componentLeaf: string;
+    registry: AdapterRegistry;
+    drivers: Record<Target, HarnessDriver>;
+    scratchRoot?: string;
+    timeoutMs?: number;
+    /** Judge model for `judge` assertions (advisory unless gated). Omit to skip. */
+    judge?: JudgeFn;
+    /** Snapshot each harness's mean score into evals/.baselines/ (spec §9.5). */
+    snapshotBaselines?: boolean;
+}
+/**
+ * Drive the real harnesses headlessly and assert over what each did (spec §9.5).
+ * A harness with no available driver is reported UNTESTED -- never faked. For each
+ * tested harness the component is installed into a throwaway scratch project so the
+ * harness loads it, then every case runs and is evaluated.
+ */
+declare function runEval(opts: RunEvalOptions): Promise<EvalReport>;
+
+export { type AssertContext, type AssertResult, type AssertStatus, type Baseline, type CaseResult, type CliResult, type DiscoveredEval, type EvalReport, type HarnessReport, type JudgeFn, type JudgeInput, type JudgeVerdict, type RunEvalOptions, claudeDriver, codexDriver, copilotDriver, cursorDriver, discoverEvals, drivers, evaluateAssertion, loadBaseline, opencodeDriver, parseClaudeStream, parseCodexStream, parseCursorStream, parseLines, parseOpencodeStream, runEval, writeBaseline };

package/dist/index.js

@@ -0,0 +1,479 @@
+import { existsSync, readFileSync, mkdirSync, writeFileSync, mkdtempSync, rmSync } from 'fs';
+import { dirname, join } from 'path';
+import { execa } from 'execa';
+import { tmpdir } from 'os';
+import { loadPluginDir, install } from '@michaelfromyeg/loom-core';
+import { loadManifest, EvalFile, leafNameOf } from '@michaelfromyeg/loom-schema';
+
+// src/assert.ts
+function isSubsequence(seq, arr) {
+  let i = 0;
+  for (const item of arr) {
+    if (item === seq[i]) i++;
+    if (i === seq.length) return true;
+  }
+  return seq.length === 0;
+}
+function traceSamplePasses(a, t) {
+  const names = t.toolCalls.map((c) => c.name);
+  if (a.toolCalled && !names.includes(a.toolCalled)) return false;
+  if (a.toolNotCalled && names.includes(a.toolNotCalled)) return false;
+  if (a.maxCalls !== void 0 && t.toolCalls.length > a.maxCalls) return false;
+  if (a.sequence && !isSubsequence(a.sequence, names)) return false;
+  return true;
+}
+function evaluateTrace(a, transcripts) {
+  if (transcripts.length > 0 && transcripts.every((t) => t.traceUnavailable)) {
+    return {
+      kind: "trace",
+      status: "degraded",
+      detail: "harness exposes no tool-call trace; trace assertion not evaluated"
+    };
+  }
+  const passes = transcripts.filter((t) => traceSamplePasses(a, t)).length;
+  const rate = transcripts.length > 0 ? passes / transcripts.length : 0;
+  return {
+    kind: "trace",
+    status: rate >= a.minPassRate ? "pass" : "fail",
+    detail: `${passes}/${transcripts.length} samples passed (minPassRate ${a.minPassRate})`
+  };
+}
+function compileRegex(pattern) {
+  const m = /^\(\?([a-z]+)\)/.exec(pattern);
+  try {
+    return m ? new RegExp(pattern.slice(m[0].length), m[1]) : new RegExp(pattern);
+  } catch {
+    return null;
+  }
+}
+function outputSamplePasses(a, t) {
+  const text = t.finalText;
+  if (a.equals !== void 0 && text.trim() !== a.equals.trim()) return false;
+  if (a.matches !== void 0 && !compileRegex(a.matches)?.test(text)) return false;
+  if (a.jsonSchema !== void 0) {
+    try {
+      JSON.parse(text);
+    } catch {
+      return false;
+    }
+  }
+  return true;
+}
+function evaluateOutput(a, transcripts) {
+  if (transcripts.length === 0) return { kind: "output", status: "fail", detail: "no samples" };
+  const passes = transcripts.filter((t) => outputSamplePasses(a, t)).length;
+  return {
+    kind: "output",
+    status: passes === transcripts.length ? "pass" : "fail",
+    detail: `${passes}/${transcripts.length} samples matched`
+  };
+}
+async function evaluateJudge(a, transcripts, ctx) {
+  if (!ctx.judge) {
+    return { kind: "judge", status: "skipped", detail: "no judge model configured (advisory)" };
+  }
+  let passes = 0;
+  for (const t of transcripts) {
+    const v = await ctx.judge({
+      candidate: t.finalText,
+      reference: a.reference,
+      rubric: a.rubric,
+      mode: a.mode,
+      samples: a.samples
+    });
+    if (v.pass) passes++;
+  }
+  const ok = passes * 2 > transcripts.length;
+  const status = a.gate ? ok ? "pass" : "fail" : ok ? "pass" : "skipped";
+  return {
+    kind: "judge",
+    status,
+    detail: `${passes}/${transcripts.length} judge verdicts pass${a.gate ? "" : " (advisory)"}`
+  };
+}
+function evaluateDifferential(a, ctx) {
+  if (ctx.baselineScore === void 0) {
+    return {
+      kind: "differential",
+      status: "skipped",
+      detail: "no baseline (run loom publish to snapshot one)"
+    };
+  }
+  const score = ctx.caseScore ?? 0;
+  const ok = score - ctx.baselineScore >= -a.noWorseThan;
+  return {
+    kind: "differential",
+    status: ok ? "pass" : "fail",
+    detail: `score ${score.toFixed(2)} vs baseline ${ctx.baselineScore.toFixed(2)} (noWorseThan ${a.noWorseThan})`
+  };
+}
+async function evaluateAssertion(a, transcripts, ctx = {}) {
+  switch (a.kind) {
+    case "trace":
+      return evaluateTrace(a, transcripts);
+    case "output":
+      return evaluateOutput(a, transcripts);
+    case "judge":
+      return evaluateJudge(a, transcripts, ctx);
+    case "differential":
+      return evaluateDifferential(a, ctx);
+  }
+}
+function baselineFile(pluginDir, component, harness) {
+  const safe = component.replace(/[^\w.-]/g, "_");
+  return join(pluginDir, "evals", ".baselines", safe, `${harness}.json`);
+}
+function loadBaseline(pluginDir, component, harness) {
+  const f = baselineFile(pluginDir, component, harness);
+  if (!existsSync(f)) return null;
+  try {
+    return JSON.parse(readFileSync(f, "utf8"));
+  } catch {
+    return null;
+  }
+}
+function writeBaseline(pluginDir, component, harness, baseline) {
+  const f = baselineFile(pluginDir, component, harness);
+  mkdirSync(dirname(f), { recursive: true });
+  writeFileSync(f, `${JSON.stringify(baseline, null, 2)}
+`);
+  return f;
+}
+
+// src/drivers/parse.ts
+function parseLines(raw) {
+  const out = [];
+  for (const line of raw.split("\n")) {
+    const t = line.trim();
+    if (!t) continue;
+    try {
+      out.push(JSON.parse(t));
+    } catch {
+    }
+  }
+  return out;
+}
+function parseClaudeStream(raw) {
+  const toolCalls = [];
+  let finalText = "";
+  let ts = 0;
+  for (const evt of parseLines(raw)) {
+    if (evt.type === "assistant") {
+      const content = evt.message?.content ?? [];
+      for (const block of content) {
+        if (block?.type === "tool_use") {
+          toolCalls.push({ name: String(block.name), args: block.input, ts: ts++ });
+        }
+      }
+    } else if (evt.type === "result" && typeof evt.result === "string") {
+      finalText = evt.result;
+    }
+  }
+  return { finalText, toolCalls };
+}
+function parseCodexStream(raw) {
+  const toolCalls = [];
+  let finalText = "";
+  let ts = 0;
+  for (const evt of parseLines(raw)) {
+    if (evt.type !== "item.completed") continue;
+    const item = evt.item;
+    if (!item) continue;
+    const kind = String(item.type);
+    if (kind === "agent_message") {
+      finalText = String(item.text ?? item.message ?? finalText);
+    } else if (kind !== "reasoning") {
+      const name = kind === "mcp_tool_call" ? String(item.tool ?? item.name ?? kind) : kind;
+      toolCalls.push({ name, args: item, ts: ts++ });
+    }
+  }
+  return { finalText, toolCalls };
+}
+function cursorToolName(evt) {
+  const tc = evt.tool_call ?? evt.toolCall;
+  if (tc && typeof tc === "object") {
+    const key = Object.keys(tc).find((k) => /ToolCall$/.test(k));
+    if (key) return key.replace(/ToolCall$/, "");
+    if (typeof tc.name === "string") return tc.name;
+  }
+  return typeof evt.name === "string" ? evt.name : "tool";
+}
+function parseCursorStream(raw) {
+  const toolCalls = [];
+  let finalText = "";
+  let ts = 0;
+  for (const evt of parseLines(raw)) {
+    if (evt.type === "tool_call" && (evt.subtype === "completed" || evt.subtype === void 0)) {
+      toolCalls.push({ name: cursorToolName(evt), args: evt.tool_call ?? evt, ts: ts++ });
+    } else if (evt.type === "result" && typeof evt.result === "string") {
+      finalText = evt.result;
+    }
+  }
+  return { finalText, toolCalls };
+}
+function parseOpencodeStream(raw) {
+  const toolCalls = [];
+  let finalText = "";
+  let ts = 0;
+  for (const evt of parseLines(raw)) {
+    if (evt.type === "tool_use") {
+      const part = evt.part ?? evt;
+      const state = part.state;
+      toolCalls.push({
+        name: String(part.tool ?? "tool"),
+        args: state?.input,
+        result: state?.output,
+        ts: ts++
+      });
+    } else if (evt.type === "text") {
+      const part = evt.part;
+      finalText += String(evt.text ?? part?.text ?? "");
+    }
+  }
+  return { finalText, toolCalls };
+}
+async function cliAvailable(cmd, versionArgs) {
+  try {
+    const r = await execa(cmd, versionArgs, { reject: false, timeout: 1e4 });
+    return r.exitCode === 0;
+  } catch {
+    return false;
+  }
+}
+async function runCli(cmd, args, opts) {
+  try {
+    const r = await execa(cmd, args, {
+      cwd: opts.cwd,
+      reject: false,
+      timeout: opts.timeoutMs ?? 12e4,
+      env: opts.config ?? {}
+    });
+    return { stdout: r.stdout ?? "", stderr: r.stderr ?? "", exitCode: r.exitCode ?? 1 };
+  } catch (err) {
+    return { stdout: "", stderr: err.message, exitCode: 1 };
+  }
+}
+
+// src/drivers/claude.ts
+var ALLOWED_TOOLS = "Read,Edit,Write,Bash,Grep,Glob,WebFetch";
+var claudeDriver = {
+  target: "claude",
+  available: () => cliAvailable("claude", ["--version"]),
+  async run({ prompt, cwd, config, timeoutMs }) {
+    const res = await runCli(
+      "claude",
+      [
+        "-p",
+        prompt,
+        "--output-format",
+        "stream-json",
+        "--verbose",
+        "--permission-mode",
+        "acceptEdits",
+        "--allowedTools",
+        ALLOWED_TOOLS
+      ],
+      { cwd, config, timeoutMs }
+    );
+    const { finalText, toolCalls } = parseClaudeStream(res.stdout);
+    return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };
+  }
+};
+
+// src/drivers/codex.ts
+var codexDriver = {
+  target: "codex",
+  available: () => cliAvailable("codex", ["--version"]),
+  async run({ prompt, cwd, config, timeoutMs }) {
+    const res = await runCli(
+      "codex",
+      ["exec", "--json", "--dangerously-bypass-approvals-and-sandbox", prompt],
+      { cwd, config, timeoutMs }
+    );
+    const { finalText, toolCalls } = parseCodexStream(res.stdout);
+    return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };
+  }
+};
+
+// src/drivers/copilot.ts
+var copilotDriver = {
+  target: "copilot",
+  available: () => cliAvailable("copilot", ["--version"]),
+  async run({ prompt, cwd, config, timeoutMs }) {
+    const res = await runCli("copilot", ["-p", prompt, "-s", "--allow-all"], {
+      cwd,
+      config,
+      timeoutMs
+    });
+    return {
+      finalText: res.stdout.trim(),
+      toolCalls: [],
+      exitCode: res.exitCode,
+      raw: res.stdout,
+      traceUnavailable: true
+    };
+  }
+};
+
+// src/drivers/cursor.ts
+var cursorDriver = {
+  target: "cursor",
+  available: () => cliAvailable("cursor-agent", ["--version"]),
+  async run({ prompt, cwd, config, timeoutMs }) {
+    const res = await runCli(
+      "cursor-agent",
+      ["-p", prompt, "--force", "--output-format", "stream-json"],
+      { cwd, config, timeoutMs }
+    );
+    const { finalText, toolCalls } = parseCursorStream(res.stdout);
+    return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };
+  }
+};
+
+// src/drivers/opencode.ts
+var opencodeDriver = {
+  target: "opencode",
+  available: () => cliAvailable("opencode", ["--version"]),
+  async run({ prompt, cwd, config, timeoutMs }) {
+    const res = await runCli("opencode", ["run", prompt, "--format", "json"], {
+      cwd,
+      config,
+      timeoutMs
+    });
+    const { finalText, toolCalls } = parseOpencodeStream(res.stdout);
+    return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };
+  }
+};
+
+// src/drivers/index.ts
+var drivers = {
+  claude: claudeDriver,
+  codex: codexDriver,
+  cursor: cursorDriver,
+  copilot: copilotDriver,
+  opencode: opencodeDriver
+};
+function discoverEvals(pluginDir) {
+  const loaded = loadPluginDir(pluginDir);
+  if (!loaded.ok) {
+    throw new Error(loaded.issues.map((i) => `${i.path}: ${i.message}`).join("\n"));
+  }
+  const out = [];
+  for (const c of loaded.value.plugin.components) {
+    const evalsRel = "evals" in c ? c.evals : void 0;
+    if (!evalsRel) continue;
+    const text = loaded.value.read(evalsRel).toString("utf8");
+    const parsed = loadManifest(EvalFile, text, { filename: evalsRel });
+    if (!parsed.ok) {
+      throw new Error(
+        `invalid evals "${evalsRel}": ${parsed.issues.map((i) => i.message).join("; ")}`
+      );
+    }
+    out.push({ componentLeaf: leafNameOf(c), evalsPath: evalsRel, evalFile: parsed.value });
+  }
+  return out;
+}
+async function sh(cmd, cwd) {
+  const r = await execa("bash", ["-lc", cmd], { cwd, reject: false, timeout: 6e4 });
+  return r.exitCode ?? 1;
+}
+async function runCase(c, driver, cwd, ctx) {
+  if (c.setup) await sh(c.setup, cwd);
+  const transcripts = [];
+  for (let i = 0; i < c.samples; i++) {
+    transcripts.push(await driver.run({ prompt: c.prompt, cwd, timeoutMs: ctx.timeoutMs }));
+  }
+  const results = new Array(c.assert.length);
+  let detTotal = 0;
+  let detPass = 0;
+  for (let i = 0; i < c.assert.length; i++) {
+    const a = c.assert[i];
+    if (a.kind === "trace" || a.kind === "output") {
+      const r = await evaluateAssertion(a, transcripts);
+      results[i] = r;
+      detTotal++;
+      if (r.status === "pass") detPass++;
+    }
+  }
+  const score = detTotal > 0 ? detPass / detTotal : 1;
+  for (let i = 0; i < c.assert.length; i++) {
+    const a = c.assert[i];
+    if (a.kind === "judge" || a.kind === "differential") {
+      results[i] = await evaluateAssertion(a, transcripts, {
+        judge: ctx.judge,
+        caseScore: score,
+        baselineScore: ctx.baselineScore
+      });
+    }
+  }
+  const assertions = results.filter((r) => r !== void 0);
+  let verifyPassed;
+  if (c.verify) verifyPassed = await sh(c.verify, cwd) === 0;
+  if (c.cleanup) await sh(c.cleanup, cwd);
+  const pass = !assertions.some((a) => a.status === "fail") && (verifyPassed ?? true);
+  return { name: c.name, assertions, verifyPassed, score, pass };
+}
+async function runEval(opts) {
+  const { evalFile, drivers: drivers2 } = opts;
+  const harnesses = [];
+  for (const harness of evalFile.harnesses) {
+    const driver = drivers2[harness];
+    if (!driver || !await driver.available()) {
+      harnesses.push({
+        harness,
+        status: "untested",
+        reason: driver ? "CLI not installed or not headless-capable" : "no driver for this harness",
+        cases: [],
+        score: 0,
+        pass: false
+      });
+      continue;
+    }
+    const baselineScore = loadBaseline(opts.pluginDir, evalFile.component, harness)?.score;
+    const scratch = mkdtempSync(join(opts.scratchRoot ?? tmpdir(), `loom-eval-${harness}-`));
+    try {
+      await install({
+        pluginDir: opts.pluginDir,
+        scope: "project",
+        cwd: scratch,
+        registry: opts.registry,
+        targets: [harness],
+        only: [opts.componentLeaf],
+        // Keep the source plugin pristine -- write the eval lock into the scratch dir.
+        lockDir: scratch
+      });
+      const cases = [];
+      for (const c of evalFile.cases) {
+        cases.push(
+          await runCase(c, driver, scratch, {
+            judge: opts.judge,
+            baselineScore,
+            timeoutMs: opts.timeoutMs
+          })
+        );
+      }
+      const score = cases.length > 0 ? cases.reduce((s, c) => s + c.score, 0) / cases.length : 1;
+      if (opts.snapshotBaselines) {
+        const snapshot = { version: "current", score };
+        writeBaseline(opts.pluginDir, evalFile.component, harness, snapshot);
+      }
+      harnesses.push({ harness, status: "tested", cases, score, pass: cases.every((c) => c.pass) });
+    } catch (err) {
+      harnesses.push({
+        harness,
+        status: "untested",
+        reason: `install failed: ${err.message}`,
+        cases: [],
+        score: 0,
+        pass: false
+      });
+    } finally {
+      rmSync(scratch, { recursive: true, force: true });
+    }
+  }
+  return { component: evalFile.component, harnesses };
+}
+
+export { claudeDriver, codexDriver, copilotDriver, cursorDriver, discoverEvals, drivers, evaluateAssertion, loadBaseline, opencodeDriver, parseClaudeStream, parseCodexStream, parseCursorStream, parseLines, parseOpencodeStream, runEval, writeBaseline };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/assert.ts","../src/baselines.ts","../src/drivers/parse.ts","../src/drivers/util.ts","../src/drivers/claude.ts","../src/drivers/codex.ts","../src/drivers/copilot.ts","../src/drivers/cursor.ts","../src/drivers/opencode.ts","../src/drivers/index.ts","../src/runner.ts"],"names":["execa","drivers","join"],"mappings":";;;;;;;;AAwCA,SAAS,aAAA,CAAc,KAAe,GAAA,EAAwB;AAC5D,EAAA,IAAI,CAAA,GAAI,CAAA;AACR,EAAA,KAAA,MAAW,QAAQ,GAAA,EAAK;AACtB,IAAA,IAAI,IAAA,KAAS,GAAA,CAAI,CAAC,CAAA,EAAG,CAAA,EAAA;AACrB,IAAA,IAAI,CAAA,KAAM,GAAA,CAAI,MAAA,EAAQ,OAAO,IAAA;AAAA,EAC/B;AACA,EAAA,OAAO,IAAI,MAAA,KAAW,CAAA;AACxB;AAEA,SAAS,iBAAA,CAAkB,GAAgB,CAAA,EAAwB;AACjE,EAAA,MAAM,QAAQ,CAAA,CAAE,SAAA,CAAU,IAAI,CAAC,CAAA,KAAM,EAAE,IAAI,CAAA;AAC3C,EAAA,IAAI,CAAA,CAAE,cAAc,CAAC,KAAA,CAAM,SAAS,CAAA,CAAE,UAAU,GAAG,OAAO,KAAA;AAC1D,EAAA,IAAI,EAAE,aAAA,IAAiB,KAAA,CAAM,SAAS,CAAA,CAAE,aAAa,GAAG,OAAO,KAAA;AAC/D,EAAA,IAAI,CAAA,CAAE,aAAa,MAAA,IAAa,CAAA,CAAE,UAAU,MAAA,GAAS,CAAA,CAAE,UAAU,OAAO,KAAA;AACxE,EAAA,IAAI,CAAA,CAAE,YAAY,CAAC,aAAA,CAAc,EAAE,QAAA,EAAU,KAAK,GAAG,OAAO,KAAA;AAC5D,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,aAAA,CAAc,GAAgB,WAAA,EAAyC;AAC9E,EAAA,IAAI,WAAA,CAAY,SAAS,CAAA,IAAK,WAAA,CAAY,MAAM,CAAC,CAAA,KAAM,CAAA,CAAE,gBAAgB,CAAA,EAAG;AAC1E,IAAA,OAAO;AAAA,MACL,IAAA,EAAM,OAAA;AAAA,MACN,MAAA,EAAQ,UAAA;AAAA,MACR,MAAA,EAAQ;AAAA,KACV;AAAA,EACF;AACA,EAAA,MAAM,MAAA,GAAS,YAAY,MAAA,CAAO,CAAC,MAAM,iBAAA,CAAkB,CAAA,EAAG,CAAC,CAAC,CAAA,CAAE,MAAA;AAClE,EAAA,MAAM,OAAO,WAAA,CAAY,MAAA,GAAS,CAAA,GAAI,MAAA,GAAS,YAAY,MAAA,GAAS,CAAA;AACpE,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,OAAA;AAAA,IACN,MAAA,EAAQ,IAAA,IAAQ,CAAA,CAAE,WAAA,GAAc,MAAA,GAAS,MAAA;AAAA,IACzC,MAAA,EAAQ,GAAG,MAAM,CAAA,CAAA,EAAI,YAAY,MAAM,CAAA,6BAAA,EAAgC,EAAE,WAAW,CAAA,CAAA;AAAA,GACtF;AACF;AAEA,SAAS,aAAa,OAAA,EAAgC;AACpD,EAAA,MAAM,CAAA,GAAI,iBAAA,CAAkB,IAAA,CAAK,OAAO,CAAA;AACxC,EAAA,IAAI;AACF,IAAA,OAAO,IAAI,IAAI,MAAA,CAAO,OAAA,CAAQ,KAAA,CAAM,EAAE,CAAC,CAAA,CAAE,MAAM,CAAA,EAAG,EAAE,CAAC,CAAC,CAAA,GAAI,IAAI,OAAO,OAAO,CAAA;AAAA,EAC9E,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,IAAA;AAAA,EACT;AACF;AAEA,SAAS,kBAAA,CAAmB,GAAiB,CAAA,EAAwB;AACnE,EAAA,MAAM,OAAO,CAAA,CAAE,SAAA;AACf,EAAA,IAAI,CAAA,CAAE,MAAA,KAAW,MAAA,IAAa,IAAA,CAAK,IAAA,OAAW,CAAA,CAAE,MAAA,CAAO,IAAA,EAAK,EAAG,OAAO,KAAA;AACtE,EAAA,IAAI,CAAA,CAAE,OAAA,KAAY,MAAA,IAAa,CAAC,YAAA,CAAa,CAAA,CAAE,OAAO,CAAA,EAAG,IAAA,CAAK,IAAI,CAAA,EAAG,OAAO,KAAA;AAC5E,EAAA,IAAI,CAAA,CAAE,eAAe,MAAA,EAAW;AAC9B,IAAA,IAAI;AACF,MAAA,IAAA,CAAK,MAAM,IAAI,CAAA;AAAA,IACjB,CAAA,CAAA,MAAQ;AACN,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AACA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,cAAA,CAAe,GAAiB,WAAA,EAAyC;AAChF,EAAA,IAAI,WAAA,CAAY,MAAA,KAAW,CAAA,EAAG,OAAO,EAAE,MAAM,QAAA,EAAU,MAAA,EAAQ,MAAA,EAAQ,MAAA,EAAQ,YAAA,EAAa;AAC5F,EAAA,MAAM,MAAA,GAAS,YAAY,MAAA,CAAO,CAAC,MAAM,kBAAA,CAAmB,CAAA,EAAG,CAAC,CAAC,CAAA,CAAE,MAAA;AACnE,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,QAAA;AAAA,IACN,MAAA,EAAQ,MAAA,KAAW,WAAA,CAAY,MAAA,GAAS,MAAA,GAAS,MAAA;AAAA,IACjD,MAAA,EAAQ,CAAA,EAAG,MAAM,CAAA,CAAA,EAAI,YAAY,MAAM,CAAA,gBAAA;AAAA,GACzC;AACF;AAEA,eAAe,aAAA,CACb,CAAA,EACA,WAAA,EACA,GAAA,EACuB;AACvB,EAAA,IAAI,CAAC,IAAI,KAAA,EAAO;AACd,IAAA,OAAO,EAAE,IAAA,EAAM,OAAA,EAAS,MAAA,EAAQ,SAAA,EAAW,QAAQ,sCAAA,EAAuC;AAAA,EAC5F;AACA,EAAA,IAAI,MAAA,GAAS,CAAA;AACb,EAAA,KAAA,MAAW,KAAK,WAAA,EAAa;AAC3B,IAAA,MAAM,CAAA,GAAI,MAAM,GAAA,CAAI,KAAA,CAAM;AAAA,MACxB,WAAW,CAAA,CAAE,SAAA;AAAA,MACb,WAAW,CAAA,CAAE,SAAA;AAAA,MACb,QAAQ,CAAA,CAAE,MAAA;AAAA,MACV,MAAM,CAAA,CAAE,IAAA;AAAA,MACR,SAAS,CAAA,CAAE;AAAA,KACZ,CAAA;AACD,IAAA,IAAI,EAAE,IAAA,EAAM,MAAA,EAAA;AAAA,EACd;AACA,EAAA,MAAM,EAAA,GAAK,MAAA,GAAS,CAAA,GAAI,WAAA,CAAY,MAAA;AAEpC,EAAA,MAAM,SAAuB,CAAA,CAAE,IAAA,GAAQ,KAAK,MAAA,GAAS,MAAA,GAAU,KAAK,MAAA,GAAS,SAAA;AAC7E,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,OAAA;AAAA,IACN,MAAA;AAAA,IACA,MAAA,EAAQ,CAAA,EAAG,MAAM,CAAA,CAAA,EAAI,WAAA,CAAY,MAAM,CAAA,oBAAA,EAAuB,CAAA,CAAE,IAAA,GAAO,EAAA,GAAK,aAAa,CAAA;AAAA,GAC3F;AACF;AAEA,SAAS,oBAAA,CAAqB,GAAuB,GAAA,EAAkC;AACrF,EAAA,IAAI,GAAA,CAAI,kBAAkB,MAAA,EAAW;AACnC,IAAA,OAAO;AAAA,MACL,IAAA,EAAM,cAAA;AAAA,MACN,MAAA,EAAQ,SAAA;AAAA,MACR,MAAA,EAAQ;AAAA,KACV;AAAA,EACF;AACA,EAAA,MAAM,KAAA,GAAQ,IAAI,SAAA,IAAa,CAAA;AAC/B,EAAA,MAAM,EAAA,GAAK,KAAA,GAAQ,GAAA,CAAI,aAAA,IAAiB,CAAC,CAAA,CAAE,WAAA;AAC3C,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,cAAA;AAAA,IACN,MAAA,EAAQ,KAAK,MAAA,GAAS,MAAA;AAAA,IACtB,MAAA,EAAQ,CAAA,MAAA,EAAS,KAAA,CAAM,OAAA,CAAQ,CAAC,CAAC,CAAA,aAAA,EAAgB,GAAA,CAAI,aAAA,CAAc,OAAA,CAAQ,CAAC,CAAC,CAAA,cAAA,EAAiB,EAAE,WAAW,CAAA,CAAA;AAAA,GAC7G;AACF;AAQA,eAAsB,iBAAA,CACpB,CAAA,EACA,WAAA,EACA,GAAA,GAAqB,EAAC,EACC;AACvB,EAAA,QAAQ,EAAE,IAAA;AAAM,IACd,KAAK,OAAA;AACH,MAAA,OAAO,aAAA,CAAc,GAAG,WAAW,CAAA;AAAA,IACrC,KAAK,QAAA;AACH,MAAA,OAAO,cAAA,CAAe,GAAG,WAAW,CAAA;AAAA,IACtC,KAAK,OAAA;AACH,MAAA,OAAO,aAAA,CAAc,CAAA,EAAG,WAAA,EAAa,GAAG,CAAA;AAAA,IAC1C,KAAK,cAAA;AACH,MAAA,OAAO,oBAAA,CAAqB,GAAG,GAAG,CAAA;AAAA;AAExC;ACtKA,SAAS,YAAA,CAAa,SAAA,EAAmB,SAAA,EAAmB,OAAA,EAAyB;AACnF,EAAA,MAAM,IAAA,GAAO,SAAA,CAAU,OAAA,CAAQ,UAAA,EAAY,GAAG,CAAA;AAC9C,EAAA,OAAO,KAAK,SAAA,EAAW,OAAA,EAAS,cAAc,IAAA,EAAM,CAAA,EAAG,OAAO,CAAA,KAAA,CAAO,CAAA;AACvE;AAGO,SAAS,YAAA,CACd,SAAA,EACA,SAAA,EACA,OAAA,EACiB;AACjB,EAAA,MAAM,CAAA,GAAI,YAAA,CAAa,SAAA,EAAW,SAAA,EAAW,OAAO,CAAA;AACpD,EAAA,IAAI,CAAC,UAAA,CAAW,CAAC,CAAA,EAAG,OAAO,IAAA;AAC3B,EAAA,IAAI;AACF,IAAA,OAAO,IAAA,CAAK,KAAA,CAAM,YAAA,CAAa,CAAA,EAAG,MAAM,CAAC,CAAA;AAAA,EAC3C,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,IAAA;AAAA,EACT;AACF;AAMO,SAAS,aAAA,CACd,SAAA,EACA,SAAA,EACA,OAAA,EACA,QAAA,EACQ;AACR,EAAA,MAAM,CAAA,GAAI,YAAA,CAAa,SAAA,EAAW,SAAA,EAAW,OAAO,CAAA;AACpD,EAAA,SAAA,CAAU,QAAQ,CAAC,CAAA,EAAG,EAAE,SAAA,EAAW,MAAM,CAAA;AACzC,EAAA,aAAA,CAAc,GAAG,CAAA,EAAG,IAAA,CAAK,UAAU,QAAA,EAAU,IAAA,EAAM,CAAC,CAAC;AAAA,CAAI,CAAA;AACzD,EAAA,OAAO,CAAA;AACT;;;ACxCO,SAAS,WAAW,GAAA,EAAwC;AACjE,EAAA,MAAM,MAAiC,EAAC;AACxC,EAAA,KAAA,MAAW,IAAA,IAAQ,GAAA,CAAI,KAAA,CAAM,IAAI,CAAA,EAAG;AAClC,IAAA,MAAM,CAAA,GAAI,KAAK,IAAA,EAAK;AACpB,IAAA,IAAI,CAAC,CAAA,EAAG;AACR,IAAA,IAAI;AACF,MAAA,GAAA,CAAI,IAAA,CAAK,IAAA,CAAK,KAAA,CAAM,CAAC,CAAC,CAAA;AAAA,IACxB,CAAA,CAAA,MAAQ;AAAA,IAER;AAAA,EACF;AACA,EAAA,OAAO,GAAA;AACT;AAOO,SAAS,kBAAkB,GAAA,EAA2D;AAC3F,EAAA,MAAM,YAAwB,EAAC;AAC/B,EAAA,IAAI,SAAA,GAAY,EAAA;AAChB,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,MAAW,GAAA,IAAO,UAAA,CAAW,GAAG,CAAA,EAAG;AACjC,IAAA,IAAI,GAAA,CAAI,SAAS,WAAA,EAAa;AAC5B,MAAA,MAAM,OAAA,GAAW,GAAA,CAAI,OAAA,EAAiD,OAAA,IAAW,EAAC;AAClF,MAAA,KAAA,MAAW,SAAS,OAAA,EAA2C;AAC7D,QAAA,IAAI,KAAA,EAAO,SAAS,UAAA,EAAY;AAC9B,UAAA,SAAA,CAAU,IAAA,CAAK,EAAE,IAAA,EAAM,MAAA,CAAO,KAAA,CAAM,IAAI,CAAA,EAAG,IAAA,EAAM,KAAA,CAAM,KAAA,EAAO,EAAA,EAAI,EAAA,EAAA,EAAM,CAAA;AAAA,QAC1E;AAAA,MACF;AAAA,IACF,WAAW,GAAA,CAAI,IAAA,KAAS,YAAY,OAAO,GAAA,CAAI,WAAW,QAAA,EAAU;AAClE,MAAA,SAAA,GAAY,GAAA,CAAI,MAAA;AAAA,IAClB;AAAA,EACF;AACA,EAAA,OAAO,EAAE,WAAW,SAAA,EAAU;AAChC;AAOO,SAAS,iBAAiB,GAAA,EAA2D;AAC1F,EAAA,MAAM,YAAwB,EAAC;AAC/B,EAAA,IAAI,SAAA,GAAY,EAAA;AAChB,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,MAAW,GAAA,IAAO,UAAA,CAAW,GAAG,CAAA,EAAG;AACjC,IAAA,IAAI,GAAA,CAAI,SAAS,gBAAA,EAAkB;AACnC,IAAA,MAAM,OAAO,GAAA,CAAI,IAAA;AACjB,IAAA,IAAI,CAAC,IAAA,EAAM;AACX,IAAA,MAAM,IAAA,GAAO,MAAA,CAAO,IAAA,CAAK,IAAI,CAAA;AAC7B,IAAA,IAAI,SAAS,eAAA,EAAiB;AAC5B,MAAA,SAAA,GAAY,MAAA,CAAO,IAAA,CAAK,IAAA,IAAQ,IAAA,CAAK,WAAW,SAAS,CAAA;AAAA,IAC3D,CAAA,MAAA,IAAW,SAAS,WAAA,EAAa;AAC/B,MAAA,MAAM,IAAA,GAAO,SAAS,eAAA,GAAkB,MAAA,CAAO,KAAK,IAAA,IAAQ,IAAA,CAAK,IAAA,IAAQ,IAAI,CAAA,GAAI,IAAA;AACjF,MAAA,SAAA,CAAU,KAAK,EAAE,IAAA,EAAM,MAAM,IAAA,EAAM,EAAA,EAAI,MAAM,CAAA;AAAA,IAC/C;AAAA,EACF;AACA,EAAA,OAAO,EAAE,WAAW,SAAA,EAAU;AAChC;AAGA,SAAS,eAAe,GAAA,EAAsC;AAC5D,EAAA,MAAM,EAAA,GAAM,GAAA,CAAI,SAAA,IAAa,GAAA,CAAI,QAAA;AACjC,EAAA,IAAI,EAAA,IAAM,OAAO,EAAA,KAAO,QAAA,EAAU;AAEhC,IAAA,MAAM,GAAA,GAAM,MAAA,CAAO,IAAA,CAAK,EAAE,CAAA,CAAE,IAAA,CAAK,CAAC,CAAA,KAAM,WAAA,CAAY,IAAA,CAAK,CAAC,CAAC,CAAA;AAC3D,IAAA,IAAI,GAAA,EAAK,OAAO,GAAA,CAAI,OAAA,CAAQ,aAAa,EAAE,CAAA;AAC3C,IAAA,IAAI,OAAO,EAAA,CAAG,IAAA,KAAS,QAAA,SAAiB,EAAA,CAAG,IAAA;AAAA,EAC7C;AACA,EAAA,OAAO,OAAO,GAAA,CAAI,IAAA,KAAS,QAAA,GAAW,IAAI,IAAA,GAAO,MAAA;AACnD;AAMO,SAAS,kBAAkB,GAAA,EAA2D;AAC3F,EAAA,MAAM,YAAwB,EAAC;AAC/B,EAAA,IAAI,SAAA,GAAY,EAAA;AAChB,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,MAAW,GAAA,IAAO,UAAA,CAAW,GAAG,CAAA,EAAG;AACjC,IAAA,IAAI,GAAA,CAAI,SAAS,WAAA,KAAgB,GAAA,CAAI,YAAY,WAAA,IAAe,GAAA,CAAI,YAAY,MAAA,CAAA,EAAY;AAC1F,MAAA,SAAA,CAAU,IAAA,CAAK,EAAE,IAAA,EAAM,cAAA,CAAe,GAAG,CAAA,EAAG,IAAA,EAAM,GAAA,CAAI,SAAA,IAAa,GAAA,EAAK,EAAA,EAAI,EAAA,EAAA,EAAM,CAAA;AAAA,IACpF,WAAW,GAAA,CAAI,IAAA,KAAS,YAAY,OAAO,GAAA,CAAI,WAAW,QAAA,EAAU;AAClE,MAAA,SAAA,GAAY,GAAA,CAAI,MAAA;AAAA,IAClB;AAAA,EACF;AACA,EAAA,OAAO,EAAE,WAAW,SAAA,EAAU;AAChC;AAOO,SAAS,oBAAoB,GAAA,EAA2D;AAC7F,EAAA,MAAM,YAAwB,EAAC;AAC/B,EAAA,IAAI,SAAA,GAAY,EAAA;AAChB,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,MAAW,GAAA,IAAO,UAAA,CAAW,GAAG,CAAA,EAAG;AACjC,IAAA,IAAI,GAAA,CAAI,SAAS,UAAA,EAAY;AAC3B,MAAA,MAAM,IAAA,GAAQ,IAAI,IAAA,IAAQ,GAAA;AAC1B,MAAA,MAAM,QAAQ,IAAA,CAAK,KAAA;AACnB,MAAA,SAAA,CAAU,IAAA,CAAK;AAAA,QACb,IAAA,EAAM,MAAA,CAAO,IAAA,CAAK,IAAA,IAAQ,MAAM,CAAA;AAAA,QAChC,MAAM,KAAA,EAAO,KAAA;AAAA,QACb,QAAQ,KAAA,EAAO,MAAA;AAAA,QACf,EAAA,EAAI,EAAA;AAAA,OACL,CAAA;AAAA,IACH,CAAA,MAAA,IAAW,GAAA,CAAI,IAAA,KAAS,MAAA,EAAQ;AAC9B,MAAA,MAAM,OAAO,GAAA,CAAI,IAAA;AACjB,MAAA,SAAA,IAAa,MAAA,CAAO,GAAA,CAAI,IAAA,IAAQ,IAAA,EAAM,QAAQ,EAAE,CAAA;AAAA,IAClD;AAAA,EACF;AACA,EAAA,OAAO,EAAE,WAAW,SAAA,EAAU;AAChC;ACrHA,eAAsB,YAAA,CAAa,KAAa,WAAA,EAAyC;AACvF,EAAA,IAAI;AACF,IAAA,MAAM,CAAA,GAAI,MAAM,KAAA,CAAM,GAAA,EAAK,WAAA,EAAa,EAAE,MAAA,EAAQ,KAAA,EAAO,OAAA,EAAS,GAAA,EAAQ,CAAA;AAC1E,IAAA,OAAO,EAAE,QAAA,KAAa,CAAA;AAAA,EACxB,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,KAAA;AAAA,EACT;AACF;AASA,eAAsB,MAAA,CACpB,GAAA,EACA,IAAA,EACA,IAAA,EACoB;AACpB,EAAA,IAAI;AACF,IAAA,MAAM,CAAA,GAAI,MAAM,KAAA,CAAM,GAAA,EAAK,IAAA,EAAM;AAAA,MAC/B,KAAK,IAAA,CAAK,GAAA;AAAA,MACV,MAAA,EAAQ,KAAA;AAAA,MACR,OAAA,EAAS,KAAK,SAAA,IAAa,IAAA;AAAA,MAC3B,GAAA,EAAK,IAAA,CAAK,MAAA,IAAU;AAAC,KACtB,CAAA;AACD,IAAA,OAAO,EAAE,MAAA,EAAQ,CAAA,CAAE,MAAA,IAAU,EAAA,EAAI,MAAA,EAAQ,CAAA,CAAE,MAAA,IAAU,EAAA,EAAI,QAAA,EAAU,CAAA,CAAE,QAAA,IAAY,CAAA,EAAE;AAAA,EACrF,SAAS,GAAA,EAAK;AACZ,IAAA,OAAO,EAAE,MAAA,EAAQ,EAAA,EAAI,QAAS,GAAA,CAAc,OAAA,EAAS,UAAU,CAAA,EAAE;AAAA,EACnE;AACF;;;AC/BA,IAAM,aAAA,GAAgB,yCAAA;AAEf,IAAM,YAAA,GAA8B;AAAA,EACzC,MAAA,EAAQ,QAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,QAAA,EAAU,CAAC,WAAW,CAAC,CAAA;AAAA,EACrD,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,MAAM,MAAM,MAAA;AAAA,MAChB,QAAA;AAAA,MACA;AAAA,QACE,IAAA;AAAA,QACA,MAAA;AAAA,QACA,iBAAA;AAAA,QACA,aAAA;AAAA,QACA,WAAA;AAAA,QACA,mBAAA;AAAA,QACA,aAAA;AAAA,QACA,gBAAA;AAAA,QACA;AAAA,OACF;AAAA,MACA,EAAE,GAAA,EAAK,MAAA,EAAQ,SAAA;AAAU,KAC3B;AACA,IAAA,MAAM,EAAE,SAAA,EAAW,SAAA,EAAU,GAAI,iBAAA,CAAkB,IAAI,MAAM,CAAA;AAC7D,IAAA,OAAO,EAAE,WAAW,SAAA,EAAW,QAAA,EAAU,IAAI,QAAA,EAAU,GAAA,EAAK,IAAI,MAAA,EAAO;AAAA,EACzE;AACF;;;ACxBO,IAAM,WAAA,GAA6B;AAAA,EACxC,MAAA,EAAQ,OAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,OAAA,EAAS,CAAC,WAAW,CAAC,CAAA;AAAA,EACpD,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,MAAM,MAAM,MAAA;AAAA,MAChB,OAAA;AAAA,MACA,CAAC,MAAA,EAAQ,QAAA,EAAU,4CAAA,EAA8C,MAAM,CAAA;AAAA,MACvE,EAAE,GAAA,EAAK,MAAA,EAAQ,SAAA;AAAU,KAC3B;AACA,IAAA,MAAM,EAAE,SAAA,EAAW,SAAA,EAAU,GAAI,gBAAA,CAAiB,IAAI,MAAM,CAAA;AAC5D,IAAA,OAAO,EAAE,WAAW,SAAA,EAAW,QAAA,EAAU,IAAI,QAAA,EAAU,GAAA,EAAK,IAAI,MAAA,EAAO;AAAA,EACzE;AACF;;;ACPO,IAAM,aAAA,GAA+B;AAAA,EAC1C,MAAA,EAAQ,SAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,SAAA,EAAW,CAAC,WAAW,CAAC,CAAA;AAAA,EACtD,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,GAAA,GAAM,MAAM,MAAA,CAAO,SAAA,EAAW,CAAC,IAAA,EAAM,MAAA,EAAQ,IAAA,EAAM,aAAa,CAAA,EAAG;AAAA,MACvE,GAAA;AAAA,MACA,MAAA;AAAA,MACA;AAAA,KACD,CAAA;AACD,IAAA,OAAO;AAAA,MACL,SAAA,EAAW,GAAA,CAAI,MAAA,CAAO,IAAA,EAAK;AAAA,MAC3B,WAAW,EAAC;AAAA,MACZ,UAAU,GAAA,CAAI,QAAA;AAAA,MACd,KAAK,GAAA,CAAI,MAAA;AAAA,MACT,gBAAA,EAAkB;AAAA,KACpB;AAAA,EACF;AACF;;;ACtBO,IAAM,YAAA,GAA8B;AAAA,EACzC,MAAA,EAAQ,QAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,cAAA,EAAgB,CAAC,WAAW,CAAC,CAAA;AAAA,EAC3D,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,MAAM,MAAM,MAAA;AAAA,MAChB,cAAA;AAAA,MACA,CAAC,IAAA,EAAM,MAAA,EAAQ,SAAA,EAAW,mBAAmB,aAAa,CAAA;AAAA,MAC1D,EAAE,GAAA,EAAK,MAAA,EAAQ,SAAA;AAAU,KAC3B;AACA,IAAA,MAAM,EAAE,SAAA,EAAW,SAAA,EAAU,GAAI,iBAAA,CAAkB,IAAI,MAAM,CAAA;AAC7D,IAAA,OAAO,EAAE,WAAW,SAAA,EAAW,QAAA,EAAU,IAAI,QAAA,EAAU,GAAA,EAAK,IAAI,MAAA,EAAO;AAAA,EACzE;AACF;;;ACZO,IAAM,cAAA,GAAgC;AAAA,EAC3C,MAAA,EAAQ,UAAA;AAAA,EACR,WAAW,MAAM,YAAA,CAAa,UAAA,EAAY,CAAC,WAAW,CAAC,CAAA;AAAA,EACvD,MAAM,GAAA,CAAI,EAAE,QAAQ,GAAA,EAAK,MAAA,EAAQ,WAAU,EAAwB;AACjE,IAAA,MAAM,GAAA,GAAM,MAAM,MAAA,CAAO,UAAA,EAAY,CAAC,KAAA,EAAO,MAAA,EAAQ,UAAA,EAAY,MAAM,CAAA,EAAG;AAAA,MACxE,GAAA;AAAA,MACA,MAAA;AAAA,MACA;AAAA,KACD,CAAA;AACD,IAAA,MAAM,EAAE,SAAA,EAAW,SAAA,EAAU,GAAI,mBAAA,CAAoB,IAAI,MAAM,CAAA;AAC/D,IAAA,OAAO,EAAE,WAAW,SAAA,EAAW,QAAA,EAAU,IAAI,QAAA,EAAU,GAAA,EAAK,IAAI,MAAA,EAAO;AAAA,EACzE;AACF;;;ACCO,IAAM,OAAA,GAAyC;AAAA,EACpD,MAAA,EAAQ,YAAA;AAAA,EACR,KAAA,EAAO,WAAA;AAAA,EACP,MAAA,EAAQ,YAAA;AAAA,EACR,OAAA,EAAS,aAAA;AAAA,EACT,QAAA,EAAU;AACZ;ACyBO,SAAS,cAAc,SAAA,EAAqC;AACjE,EAAA,MAAM,MAAA,GAAS,cAAc,SAAS,CAAA;AACtC,EAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AACd,IAAA,MAAM,IAAI,KAAA,CAAM,MAAA,CAAO,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,EAAG,CAAA,CAAE,IAAI,KAAK,CAAA,CAAE,OAAO,EAAE,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,EAChF;AACA,EAAA,MAAM,MAAwB,EAAC;AAC/B,EAAA,KAAA,MAAW,CAAA,IAAK,MAAA,CAAO,KAAA,CAAM,MAAA,CAAO,UAAA,EAAY;AAC9C,IAAA,MAAM,QAAA,GAAW,OAAA,IAAW,CAAA,GAAI,CAAA,CAAE,KAAA,GAAQ,MAAA;AAC1C,IAAA,IAAI,CAAC,QAAA,EAAU;AACf,IAAA,MAAM,OAAO,MAAA,CAAO,KAAA,CAAM,KAAK,QAAQ,CAAA,CAAE,SAAS,MAAM,CAAA;AACxD,IAAA,MAAM,SAAS,YAAA,CAAa,QAAA,EAAU,MAAM,EAAE,QAAA,EAAU,UAAU,CAAA;AAClE,IAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AACd,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,eAAA,EAAkB,QAAQ,CAAA,GAAA,EAAM,MAAA,CAAO,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,OAChF;AAAA,IACF;AACA,IAAA,GAAA,CAAI,IAAA,CAAK,EAAE,aAAA,EAAe,UAAA,CAAW,CAAC,CAAA,EAAG,SAAA,EAAW,QAAA,EAAU,QAAA,EAAU,MAAA,CAAO,KAAA,EAAO,CAAA;AAAA,EACxF;AACA,EAAA,OAAO,GAAA;AACT;AAEA,eAAe,EAAA,CAAG,KAAa,GAAA,EAA8B;AAC3D,EAAA,MAAM,CAAA,GAAI,MAAMA,KAAAA,CAAM,MAAA,EAAQ,CAAC,KAAA,EAAO,GAAG,CAAA,EAAG,EAAE,GAAA,EAAK,MAAA,EAAQ,KAAA,EAAO,OAAA,EAAS,KAAQ,CAAA;AACnF,EAAA,OAAO,EAAE,QAAA,IAAY,CAAA;AACvB;AAQA,eAAe,OAAA,CACb,CAAA,EACA,MAAA,EACA,GAAA,EACA,GAAA,EACqB;AACrB,EAAA,IAAI,EAAE,KAAA,EAAO,MAAM,EAAA,CAAG,CAAA,CAAE,OAAO,GAAG,CAAA;AAElC,EAAA,MAAM,cAA4B,EAAC;AACnC,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,SAAS,CAAA,EAAA,EAAK;AAClC,IAAA,WAAA,CAAY,IAAA,CAAK,MAAM,MAAA,CAAO,GAAA,CAAI,EAAE,MAAA,EAAQ,CAAA,CAAE,MAAA,EAAQ,GAAA,EAAK,SAAA,EAAW,GAAA,CAAI,SAAA,EAAW,CAAC,CAAA;AAAA,EACxF;AAGA,EAAA,MAAM,OAAA,GAAwC,IAAI,KAAA,CAAM,CAAA,CAAE,OAAO,MAAM,CAAA;AACvE,EAAA,IAAI,QAAA,GAAW,CAAA;AACf,EAAA,IAAI,OAAA,GAAU,CAAA;AACd,EAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,CAAO,QAAQ,CAAA,EAAA,EAAK;AACxC,IAAA,MAAM,CAAA,GAAI,CAAA,CAAE,MAAA,CAAO,CAAC,CAAA;AACpB,IAAA,IAAI,CAAA,CAAE,IAAA,KAAS,OAAA,IAAW,CAAA,CAAE,SAAS,QAAA,EAAU;AAC7C,MAAA,MAAM,CAAA,GAAI,MAAM,iBAAA,CAAkB,CAAA,EAAG,WAAW,CAAA;AAChD,MAAA,OAAA,CAAQ,CAAC,CAAA,GAAI,CAAA;AACb,MAAA,QAAA,EAAA;AACA,MAAA,IAAI,CAAA,CAAE,WAAW,MAAA,EAAQ,OAAA,EAAA;AAAA,IAC3B;AAAA,EACF;AACA,EAAA,MAAM,KAAA,GAAQ,QAAA,GAAW,CAAA,GAAI,OAAA,GAAU,QAAA,GAAW,CAAA;AAGlD,EAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,CAAO,QAAQ,CAAA,EAAA,EAAK;AACxC,IAAA,MAAM,CAAA,GAAI,CAAA,CAAE,MAAA,CAAO,CAAC,CAAA;AACpB,IAAA,IAAI,CAAA,CAAE,IAAA,KAAS,OAAA,IAAW,CAAA,CAAE,SAAS,cAAA,EAAgB;AACnD,MAAA,OAAA,CAAQ,CAAC,CAAA,GAAI,MAAM,iBAAA,CAAkB,GAAG,WAAA,EAAa;AAAA,QACnD,OAAO,GAAA,CAAI,KAAA;AAAA,QACX,SAAA,EAAW,KAAA;AAAA,QACX,eAAe,GAAA,CAAI;AAAA,OACpB,CAAA;AAAA,IACH;AAAA,EACF;AACA,EAAA,MAAM,aAAa,OAAA,CAAQ,MAAA,CAAO,CAAC,CAAA,KAAyB,MAAM,MAAS,CAAA;AAE3E,EAAA,IAAI,YAAA;AACJ,EAAA,IAAI,CAAA,CAAE,QAAQ,YAAA,GAAgB,MAAM,GAAG,CAAA,CAAE,MAAA,EAAQ,GAAG,CAAA,KAAO,CAAA;AAC3D,EAAA,IAAI,EAAE,OAAA,EAAS,MAAM,EAAA,CAAG,CAAA,CAAE,SAAS,GAAG,CAAA;AAItC,EAAA,MAAM,IAAA,GAAO,CAAC,UAAA,CAAW,IAAA,CAAK,CAAC,MAAM,CAAA,CAAE,MAAA,KAAW,MAAM,CAAA,KAAM,YAAA,IAAgB,IAAA,CAAA;AAC9E,EAAA,OAAO,EAAE,IAAA,EAAM,CAAA,CAAE,MAAM,UAAA,EAAY,YAAA,EAAc,OAAO,IAAA,EAAK;AAC/D;AAsBA,eAAsB,QAAQ,IAAA,EAA2C;AACvE,EAAA,MAAM,EAAE,QAAA,EAAU,OAAA,EAAAC,QAAAA,EAAQ,GAAI,IAAA;AAC9B,EAAA,MAAM,YAA6B,EAAC;AAEpC,EAAA,KAAA,MAAW,OAAA,IAAW,SAAS,SAAA,EAAW;AACxC,IAAA,MAAM,MAAA,GAASA,SAAQ,OAAO,CAAA;AAC9B,IAAA,IAAI,CAAC,MAAA,IAAU,CAAE,MAAM,MAAA,CAAO,WAAU,EAAI;AAC1C,MAAA,SAAA,CAAU,IAAA,CAAK;AAAA,QACb,OAAA;AAAA,QACA,MAAA,EAAQ,UAAA;AAAA,QACR,MAAA,EAAQ,SAAS,2CAAA,GAA8C,4BAAA;AAAA,QAC/D,OAAO,EAAC;AAAA,QACR,KAAA,EAAO,CAAA;AAAA,QACP,IAAA,EAAM;AAAA,OACP,CAAA;AACD,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,gBAAgB,YAAA,CAAa,IAAA,CAAK,WAAW,QAAA,CAAS,SAAA,EAAW,OAAO,CAAA,EAAG,KAAA;AACjF,IAAA,MAAM,OAAA,GAAU,WAAA,CAAYC,IAAAA,CAAK,IAAA,CAAK,WAAA,IAAe,QAAO,EAAG,CAAA,UAAA,EAAa,OAAO,CAAA,CAAA,CAAG,CAAC,CAAA;AACvF,IAAA,IAAI;AACF,MAAA,MAAM,OAAA,CAAQ;AAAA,QACZ,WAAW,IAAA,CAAK,SAAA;AAAA,QAChB,KAAA,EAAO,SAAA;AAAA,QACP,GAAA,EAAK,OAAA;AAAA,QACL,UAAU,IAAA,CAAK,QAAA;AAAA,QACf,OAAA,EAAS,CAAC,OAAO,CAAA;AAAA,QACjB,IAAA,EAAM,CAAC,IAAA,CAAK,aAAa,CAAA;AAAA;AAAA,QAEzB,OAAA,EAAS;AAAA,OACV,CAAA;AACD,MAAA,MAAM,QAAsB,EAAC;AAC7B,MAAA,KAAA,MAAW,CAAA,IAAK,SAAS,KAAA,EAAO;AAC9B,QAAA,KAAA,CAAM,IAAA;AAAA,UACJ,MAAM,OAAA,CAAQ,CAAA,EAAG,MAAA,EAAQ,OAAA,EAAS;AAAA,YAChC,OAAO,IAAA,CAAK,KAAA;AAAA,YACZ,aAAA;AAAA,YACA,WAAW,IAAA,CAAK;AAAA,WACjB;AAAA,SACH;AAAA,MACF;AACA,MAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,MAAA,GAAS,CAAA,GAAI,MAAM,MAAA,CAAO,CAAC,CAAA,EAAG,CAAA,KAAM,IAAI,CAAA,CAAE,KAAA,EAAO,CAAC,CAAA,GAAI,MAAM,MAAA,GAAS,CAAA;AACzF,MAAA,IAAI,KAAK,iBAAA,EAAmB;AAC1B,QAAA,MAAM,QAAA,GAAqB,EAAE,OAAA,EAAS,SAAA,EAAW,KAAA,EAAM;AACvD,QAAA,aAAA,CAAc,IAAA,CAAK,SAAA,EAAW,QAAA,CAAS,SAAA,EAAW,SAAS,QAAQ,CAAA;AAAA,MACrE;AACA,MAAA,SAAA,CAAU,IAAA,CAAK,EAAE,OAAA,EAAS,MAAA,EAAQ,UAAU,KAAA,EAAO,KAAA,EAAO,IAAA,EAAM,KAAA,CAAM,MAAM,CAAC,CAAA,KAAM,CAAA,CAAE,IAAI,GAAG,CAAA;AAAA,IAC9F,SAAS,GAAA,EAAK;AACZ,MAAA,SAAA,CAAU,IAAA,CAAK;AAAA,QACb,OAAA;AAAA,QACA,MAAA,EAAQ,UAAA;AAAA,QACR,MAAA,EAAQ,CAAA,gBAAA,EAAoB,GAAA,CAAc,OAAO,CAAA,CAAA;AAAA,QACjD,OAAO,EAAC;AAAA,QACR,KAAA,EAAO,CAAA;AAAA,QACP,IAAA,EAAM;AAAA,OACP,CAAA;AAAA,IACH,CAAA,SAAE;AACA,MAAA,MAAA,CAAO,SAAS,EAAE,SAAA,EAAW,IAAA,EAAM,KAAA,EAAO,MAAM,CAAA;AAAA,IAClD;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,SAAA,EAAW,QAAA,CAAS,SAAA,EAAW,SAAA,EAAU;AACpD","file":"index.js","sourcesContent":["import type { Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport type {\n  Assertion,\n  DifferentialAssert,\n  JudgeAssert,\n  OutputAssert,\n  TraceAssert,\n} from \"@michaelfromyeg/loom-schema\";\n\nexport type AssertStatus = \"pass\" | \"fail\" | \"degraded\" | \"skipped\";\n\nexport interface AssertResult {\n  kind: Assertion[\"kind\"];\n  status: AssertStatus;\n  detail: string;\n}\n\nexport interface JudgeInput {\n  candidate: string;\n  reference?: string;\n  rubric: string;\n  mode: \"absolute\" | \"pairwise\";\n  samples: number;\n}\nexport interface JudgeVerdict {\n  pass: boolean;\n  detail?: string;\n}\n/** A judge model. Injected so evals run offline/deterministically in tests. */\nexport type JudgeFn = (input: JudgeInput) => Promise<JudgeVerdict>;\n\nexport interface AssertContext {\n  judge?: JudgeFn;\n  /** Deterministic score of the case (fraction of trace/output passing) -- differential. */\n  caseScore?: number;\n  /** Baseline score for (component, harness) from evals/.baselines/ -- differential. */\n  baselineScore?: number;\n}\n\n/** True iff `seq` appears as an ordered subsequence of `arr`. */\nfunction isSubsequence(seq: string[], arr: string[]): boolean {\n  let i = 0;\n  for (const item of arr) {\n    if (item === seq[i]) i++;\n    if (i === seq.length) return true;\n  }\n  return seq.length === 0;\n}\n\nfunction traceSamplePasses(a: TraceAssert, t: Transcript): boolean {\n  const names = t.toolCalls.map((c) => c.name);\n  if (a.toolCalled && !names.includes(a.toolCalled)) return false;\n  if (a.toolNotCalled && names.includes(a.toolNotCalled)) return false;\n  if (a.maxCalls !== undefined && t.toolCalls.length > a.maxCalls) return false;\n  if (a.sequence && !isSubsequence(a.sequence, names)) return false;\n  return true;\n}\n\nfunction evaluateTrace(a: TraceAssert, transcripts: Transcript[]): AssertResult {\n  if (transcripts.length > 0 && transcripts.every((t) => t.traceUnavailable)) {\n    return {\n      kind: \"trace\",\n      status: \"degraded\",\n      detail: \"harness exposes no tool-call trace; trace assertion not evaluated\",\n    };\n  }\n  const passes = transcripts.filter((t) => traceSamplePasses(a, t)).length;\n  const rate = transcripts.length > 0 ? passes / transcripts.length : 0;\n  return {\n    kind: \"trace\",\n    status: rate >= a.minPassRate ? \"pass\" : \"fail\",\n    detail: `${passes}/${transcripts.length} samples passed (minPassRate ${a.minPassRate})`,\n  };\n}\n\nfunction compileRegex(pattern: string): RegExp | null {\n  const m = /^\\(\\?([a-z]+)\\)/.exec(pattern);\n  try {\n    return m ? new RegExp(pattern.slice(m[0].length), m[1]) : new RegExp(pattern);\n  } catch {\n    return null;\n  }\n}\n\nfunction outputSamplePasses(a: OutputAssert, t: Transcript): boolean {\n  const text = t.finalText;\n  if (a.equals !== undefined && text.trim() !== a.equals.trim()) return false;\n  if (a.matches !== undefined && !compileRegex(a.matches)?.test(text)) return false;\n  if (a.jsonSchema !== undefined) {\n    try {\n      JSON.parse(text);\n    } catch {\n      return false;\n    }\n  }\n  return true;\n}\n\nfunction evaluateOutput(a: OutputAssert, transcripts: Transcript[]): AssertResult {\n  if (transcripts.length === 0) return { kind: \"output\", status: \"fail\", detail: \"no samples\" };\n  const passes = transcripts.filter((t) => outputSamplePasses(a, t)).length;\n  return {\n    kind: \"output\",\n    status: passes === transcripts.length ? \"pass\" : \"fail\",\n    detail: `${passes}/${transcripts.length} samples matched`,\n  };\n}\n\nasync function evaluateJudge(\n  a: JudgeAssert,\n  transcripts: Transcript[],\n  ctx: AssertContext,\n): Promise<AssertResult> {\n  if (!ctx.judge) {\n    return { kind: \"judge\", status: \"skipped\", detail: \"no judge model configured (advisory)\" };\n  }\n  let passes = 0;\n  for (const t of transcripts) {\n    const v = await ctx.judge({\n      candidate: t.finalText,\n      reference: a.reference,\n      rubric: a.rubric,\n      mode: a.mode,\n      samples: a.samples,\n    });\n    if (v.pass) passes++;\n  }\n  const ok = passes * 2 > transcripts.length; // majority\n  // Advisory unless `gate:true` -- a non-gating judge never fails the case.\n  const status: AssertStatus = a.gate ? (ok ? \"pass\" : \"fail\") : ok ? \"pass\" : \"skipped\";\n  return {\n    kind: \"judge\",\n    status,\n    detail: `${passes}/${transcripts.length} judge verdicts pass${a.gate ? \"\" : \" (advisory)\"}`,\n  };\n}\n\nfunction evaluateDifferential(a: DifferentialAssert, ctx: AssertContext): AssertResult {\n  if (ctx.baselineScore === undefined) {\n    return {\n      kind: \"differential\",\n      status: \"skipped\",\n      detail: \"no baseline (run loom publish to snapshot one)\",\n    };\n  }\n  const score = ctx.caseScore ?? 0;\n  const ok = score - ctx.baselineScore >= -a.noWorseThan;\n  return {\n    kind: \"differential\",\n    status: ok ? \"pass\" : \"fail\",\n    detail: `score ${score.toFixed(2)} vs baseline ${ctx.baselineScore.toFixed(2)} (noWorseThan ${a.noWorseThan})`,\n  };\n}\n\n/**\n * Evaluate one assertion against a case's sample transcripts (spec §9.5). Trace\n * and output are deterministic; judge is advisory unless `gate:true` and needs an\n * injected judge model; differential compares the case score to a committed\n * baseline (the \"vibes\" no-regression gate).\n */\nexport async function evaluateAssertion(\n  a: Assertion,\n  transcripts: Transcript[],\n  ctx: AssertContext = {},\n): Promise<AssertResult> {\n  switch (a.kind) {\n    case \"trace\":\n      return evaluateTrace(a, transcripts);\n    case \"output\":\n      return evaluateOutput(a, transcripts);\n    case \"judge\":\n      return evaluateJudge(a, transcripts, ctx);\n    case \"differential\":\n      return evaluateDifferential(a, ctx);\n  }\n}\n","import { existsSync, mkdirSync, readFileSync, writeFileSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport type { Target } from \"@michaelfromyeg/loom-schema\";\n\nexport interface Baseline {\n  version: string;\n  score: number;\n}\n\nfunction baselineFile(pluginDir: string, component: string, harness: Target): string {\n  const safe = component.replace(/[^\\w.-]/g, \"_\");\n  return join(pluginDir, \"evals\", \".baselines\", safe, `${harness}.json`);\n}\n\n/** Load the committed baseline score for (component, harness), or null. */\nexport function loadBaseline(\n  pluginDir: string,\n  component: string,\n  harness: Target,\n): Baseline | null {\n  const f = baselineFile(pluginDir, component, harness);\n  if (!existsSync(f)) return null;\n  try {\n    return JSON.parse(readFileSync(f, \"utf8\")) as Baseline;\n  } catch {\n    return null;\n  }\n}\n\n/**\n * Snapshot a baseline (spec §9.5). Called on `loom publish` so the next release's\n * differential evals compare against this score.\n */\nexport function writeBaseline(\n  pluginDir: string,\n  component: string,\n  harness: Target,\n  baseline: Baseline,\n): string {\n  const f = baselineFile(pluginDir, component, harness);\n  mkdirSync(dirname(f), { recursive: true });\n  writeFileSync(f, `${JSON.stringify(baseline, null, 2)}\\n`);\n  return f;\n}\n","import type { ToolCall } from \"@michaelfromyeg/loom-adapter-kit\";\n\n/** Split NDJSON/JSONL into parsed objects, skipping blank/garbage lines. */\nexport function parseLines(raw: string): Record<string, unknown>[] {\n  const out: Record<string, unknown>[] = [];\n  for (const line of raw.split(\"\\n\")) {\n    const t = line.trim();\n    if (!t) continue;\n    try {\n      out.push(JSON.parse(t));\n    } catch {\n      // tolerate non-JSON noise lines\n    }\n  }\n  return out;\n}\n\n/**\n * Parse Claude's `--output-format stream-json --verbose` NDJSON. Tool calls are\n * `assistant` message content blocks `{type:\"tool_use\", name, input}`; the final\n * text is the `result` event's `result` field (see harness-research.md).\n */\nexport function parseClaudeStream(raw: string): { finalText: string; toolCalls: ToolCall[] } {\n  const toolCalls: ToolCall[] = [];\n  let finalText = \"\";\n  let ts = 0;\n  for (const evt of parseLines(raw)) {\n    if (evt.type === \"assistant\") {\n      const content = (evt.message as { content?: unknown[] } | undefined)?.content ?? [];\n      for (const block of content as Array<Record<string, unknown>>) {\n        if (block?.type === \"tool_use\") {\n          toolCalls.push({ name: String(block.name), args: block.input, ts: ts++ });\n        }\n      }\n    } else if (evt.type === \"result\" && typeof evt.result === \"string\") {\n      finalText = evt.result;\n    }\n  }\n  return { finalText, toolCalls };\n}\n\n/**\n * Parse Codex `exec --json` JSONL. Tool calls live in `item.completed` events by\n * `item.type`; the final text is the `agent_message` item. Only completed items\n * are counted to avoid double-counting the matching `item.started`.\n */\nexport function parseCodexStream(raw: string): { finalText: string; toolCalls: ToolCall[] } {\n  const toolCalls: ToolCall[] = [];\n  let finalText = \"\";\n  let ts = 0;\n  for (const evt of parseLines(raw)) {\n    if (evt.type !== \"item.completed\") continue;\n    const item = evt.item as Record<string, unknown> | undefined;\n    if (!item) continue;\n    const kind = String(item.type);\n    if (kind === \"agent_message\") {\n      finalText = String(item.text ?? item.message ?? finalText);\n    } else if (kind !== \"reasoning\") {\n      const name = kind === \"mcp_tool_call\" ? String(item.tool ?? item.name ?? kind) : kind;\n      toolCalls.push({ name, args: item, ts: ts++ });\n    }\n  }\n  return { finalText, toolCalls };\n}\n\n/** Best-effort tool name from a Cursor `tool_call` event (e.g. {readToolCall:{}} -> \"read\"). */\nfunction cursorToolName(evt: Record<string, unknown>): string {\n  const tc = (evt.tool_call ?? evt.toolCall) as Record<string, unknown> | undefined;\n  if (tc && typeof tc === \"object\") {\n    // TODO(verify): the exact tool-name field of cursor-agent stream-json events.\n    const key = Object.keys(tc).find((k) => /ToolCall$/.test(k));\n    if (key) return key.replace(/ToolCall$/, \"\");\n    if (typeof tc.name === \"string\") return tc.name;\n  }\n  return typeof evt.name === \"string\" ? evt.name : \"tool\";\n}\n\n/**\n * Parse Cursor `--output-format stream-json` NDJSON. Tool calls are `tool_call`\n * events (count the `completed` subtype); the final text is the `result` event.\n */\nexport function parseCursorStream(raw: string): { finalText: string; toolCalls: ToolCall[] } {\n  const toolCalls: ToolCall[] = [];\n  let finalText = \"\";\n  let ts = 0;\n  for (const evt of parseLines(raw)) {\n    if (evt.type === \"tool_call\" && (evt.subtype === \"completed\" || evt.subtype === undefined)) {\n      toolCalls.push({ name: cursorToolName(evt), args: evt.tool_call ?? evt, ts: ts++ });\n    } else if (evt.type === \"result\" && typeof evt.result === \"string\") {\n      finalText = evt.result;\n    }\n  }\n  return { finalText, toolCalls };\n}\n\n/**\n * Parse OpenCode `run --format json` JSONL. Tool calls are `tool_use` events\n * carrying a ToolPart (`part.tool`, `part.state.{input,output}`); the final text\n * is the concatenation of `text` events.\n */\nexport function parseOpencodeStream(raw: string): { finalText: string; toolCalls: ToolCall[] } {\n  const toolCalls: ToolCall[] = [];\n  let finalText = \"\";\n  let ts = 0;\n  for (const evt of parseLines(raw)) {\n    if (evt.type === \"tool_use\") {\n      const part = (evt.part ?? evt) as Record<string, unknown>;\n      const state = part.state as Record<string, unknown> | undefined;\n      toolCalls.push({\n        name: String(part.tool ?? \"tool\"),\n        args: state?.input,\n        result: state?.output,\n        ts: ts++,\n      });\n    } else if (evt.type === \"text\") {\n      const part = evt.part as { text?: string } | undefined;\n      finalText += String(evt.text ?? part?.text ?? \"\");\n    }\n  }\n  return { finalText, toolCalls };\n}\n","import { execa } from \"execa\";\n\n/** True iff `cmd <versionArgs>` exits 0 (CLI installed). Never throws. */\nexport async function cliAvailable(cmd: string, versionArgs: string[]): Promise<boolean> {\n  try {\n    const r = await execa(cmd, versionArgs, { reject: false, timeout: 10_000 });\n    return r.exitCode === 0;\n  } catch {\n    return false;\n  }\n}\n\nexport interface CliResult {\n  stdout: string;\n  stderr: string;\n  exitCode: number;\n}\n\n/** Run a harness CLI headlessly. Resolved config is passed as env; never throws. */\nexport async function runCli(\n  cmd: string,\n  args: string[],\n  opts: { cwd: string; config?: Record<string, string>; timeoutMs?: number },\n): Promise<CliResult> {\n  try {\n    const r = await execa(cmd, args, {\n      cwd: opts.cwd,\n      reject: false,\n      timeout: opts.timeoutMs ?? 120_000,\n      env: opts.config ?? {},\n    });\n    return { stdout: r.stdout ?? \"\", stderr: r.stderr ?? \"\", exitCode: r.exitCode ?? 1 };\n  } catch (err) {\n    return { stdout: \"\", stderr: (err as Error).message, exitCode: 1 };\n  }\n}\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { parseClaudeStream } from \"./parse\";\nimport { cliAvailable, runCli } from \"./util\";\n\nconst ALLOWED_TOOLS = \"Read,Edit,Write,Bash,Grep,Glob,WebFetch\";\n\nexport const claudeDriver: HarnessDriver = {\n  target: \"claude\",\n  available: () => cliAvailable(\"claude\", [\"--version\"]),\n  async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n    const res = await runCli(\n      \"claude\",\n      [\n        \"-p\",\n        prompt,\n        \"--output-format\",\n        \"stream-json\",\n        \"--verbose\",\n        \"--permission-mode\",\n        \"acceptEdits\",\n        \"--allowedTools\",\n        ALLOWED_TOOLS,\n      ],\n      { cwd, config, timeoutMs },\n    );\n    const { finalText, toolCalls } = parseClaudeStream(res.stdout);\n    return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };\n  },\n};\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { parseCodexStream } from \"./parse\";\nimport { cliAvailable, runCli } from \"./util\";\n\nexport const codexDriver: HarnessDriver = {\n  target: \"codex\",\n  available: () => cliAvailable(\"codex\", [\"--version\"]),\n  async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n    const res = await runCli(\n      \"codex\",\n      [\"exec\", \"--json\", \"--dangerously-bypass-approvals-and-sandbox\", prompt],\n      { cwd, config, timeoutMs },\n    );\n    const { finalText, toolCalls } = parseCodexStream(res.stdout);\n    return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };\n  },\n};\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { cliAvailable, runCli } from \"./util\";\n\n/**\n * GitHub Copilot's headless `-p` mode exposes NO structured tool-call trace today\n * (only a Markdown transcript via --share). So `run` returns the final text with\n * `traceUnavailable: true`, and the runner degrades `trace` assertions to `output`\n * for this harness rather than faking a pass (spec §14, harness-research.md).\n */\nexport const copilotDriver: HarnessDriver = {\n  target: \"copilot\",\n  available: () => cliAvailable(\"copilot\", [\"--version\"]),\n  async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n    const res = await runCli(\"copilot\", [\"-p\", prompt, \"-s\", \"--allow-all\"], {\n      cwd,\n      config,\n      timeoutMs,\n    });\n    return {\n      finalText: res.stdout.trim(),\n      toolCalls: [],\n      exitCode: res.exitCode,\n      raw: res.stdout,\n      traceUnavailable: true,\n    };\n  },\n};\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { parseCursorStream } from \"./parse\";\nimport { cliAvailable, runCli } from \"./util\";\n\nexport const cursorDriver: HarnessDriver = {\n  target: \"cursor\",\n  available: () => cliAvailable(\"cursor-agent\", [\"--version\"]),\n  async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n    const res = await runCli(\n      \"cursor-agent\",\n      [\"-p\", prompt, \"--force\", \"--output-format\", \"stream-json\"],\n      { cwd, config, timeoutMs },\n    );\n    const { finalText, toolCalls } = parseCursorStream(res.stdout);\n    return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };\n  },\n};\n","import type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { parseOpencodeStream } from \"./parse\";\nimport { cliAvailable, runCli } from \"./util\";\n\nexport const opencodeDriver: HarnessDriver = {\n  target: \"opencode\",\n  available: () => cliAvailable(\"opencode\", [\"--version\"]),\n  async run({ prompt, cwd, config, timeoutMs }): Promise<Transcript> {\n    const res = await runCli(\"opencode\", [\"run\", prompt, \"--format\", \"json\"], {\n      cwd,\n      config,\n      timeoutMs,\n    });\n    const { finalText, toolCalls } = parseOpencodeStream(res.stdout);\n    return { finalText, toolCalls, exitCode: res.exitCode, raw: res.stdout };\n  },\n};\n","import type { HarnessDriver } from \"@michaelfromyeg/loom-adapter-kit\";\nimport type { Target } from \"@michaelfromyeg/loom-schema\";\nimport { claudeDriver } from \"./claude\";\nimport { codexDriver } from \"./codex\";\nimport { copilotDriver } from \"./copilot\";\nimport { cursorDriver } from \"./cursor\";\nimport { opencodeDriver } from \"./opencode\";\n\nexport { claudeDriver } from \"./claude\";\nexport { codexDriver } from \"./codex\";\nexport { copilotDriver } from \"./copilot\";\nexport { cursorDriver } from \"./cursor\";\nexport { opencodeDriver } from \"./opencode\";\nexport * from \"./parse\";\nexport type { CliResult } from \"./util\";\n\n/** Every built-in headless driver, keyed by Target. */\nexport const drivers: Record<Target, HarnessDriver> = {\n  claude: claudeDriver,\n  codex: codexDriver,\n  cursor: cursorDriver,\n  copilot: copilotDriver,\n  opencode: opencodeDriver,\n};\n","import { mkdtempSync, rmSync } from \"node:fs\";\nimport { tmpdir } from \"node:os\";\nimport { join } from \"node:path\";\nimport type { HarnessDriver, Transcript } from \"@michaelfromyeg/loom-adapter-kit\";\nimport { type AdapterRegistry, install, loadPluginDir } from \"@michaelfromyeg/loom-core\";\nimport {\n  type Case,\n  EvalFile,\n  leafNameOf,\n  loadManifest,\n  type Target,\n} from \"@michaelfromyeg/loom-schema\";\nimport { execa } from \"execa\";\nimport { type AssertResult, evaluateAssertion, type JudgeFn } from \"./assert\";\nimport { type Baseline, loadBaseline, writeBaseline } from \"./baselines\";\n\nexport interface CaseResult {\n  name: string;\n  assertions: AssertResult[];\n  verifyPassed?: boolean;\n  /** Deterministic score: fraction of trace/output assertions that passed. */\n  score: number;\n  pass: boolean;\n}\n\nexport interface HarnessReport {\n  harness: Target;\n  status: \"tested\" | \"untested\";\n  /** Why a harness was not tested (no driver / CLI absent / install failed). */\n  reason?: string;\n  cases: CaseResult[];\n  /** Mean deterministic case score (snapshotted as the next release's baseline). */\n  score: number;\n  pass: boolean;\n}\n\nexport interface EvalReport {\n  component: string;\n  harnesses: HarnessReport[];\n}\n\nexport interface DiscoveredEval {\n  componentLeaf: string;\n  evalsPath: string;\n  evalFile: EvalFile;\n}\n\n/** Find the components in a plugin that declare an `evals` file and load each. */\nexport function discoverEvals(pluginDir: string): DiscoveredEval[] {\n  const loaded = loadPluginDir(pluginDir);\n  if (!loaded.ok) {\n    throw new Error(loaded.issues.map((i) => `${i.path}: ${i.message}`).join(\"\\n\"));\n  }\n  const out: DiscoveredEval[] = [];\n  for (const c of loaded.value.plugin.components) {\n    const evalsRel = \"evals\" in c ? c.evals : undefined;\n    if (!evalsRel) continue;\n    const text = loaded.value.read(evalsRel).toString(\"utf8\");\n    const parsed = loadManifest(EvalFile, text, { filename: evalsRel });\n    if (!parsed.ok) {\n      throw new Error(\n        `invalid evals \"${evalsRel}\": ${parsed.issues.map((i) => i.message).join(\"; \")}`,\n      );\n    }\n    out.push({ componentLeaf: leafNameOf(c), evalsPath: evalsRel, evalFile: parsed.value });\n  }\n  return out;\n}\n\nasync function sh(cmd: string, cwd: string): Promise<number> {\n  const r = await execa(\"bash\", [\"-lc\", cmd], { cwd, reject: false, timeout: 60_000 });\n  return r.exitCode ?? 1;\n}\n\ninterface CaseContext {\n  judge?: JudgeFn;\n  baselineScore?: number;\n  timeoutMs?: number;\n}\n\nasync function runCase(\n  c: Case,\n  driver: HarnessDriver,\n  cwd: string,\n  ctx: CaseContext,\n): Promise<CaseResult> {\n  if (c.setup) await sh(c.setup, cwd);\n\n  const transcripts: Transcript[] = [];\n  for (let i = 0; i < c.samples; i++) {\n    transcripts.push(await driver.run({ prompt: c.prompt, cwd, timeoutMs: ctx.timeoutMs }));\n  }\n\n  // Pass 1: evaluate the deterministic tier (trace/output) to get the case score.\n  const results: (AssertResult | undefined)[] = new Array(c.assert.length);\n  let detTotal = 0;\n  let detPass = 0;\n  for (let i = 0; i < c.assert.length; i++) {\n    const a = c.assert[i];\n    if (a.kind === \"trace\" || a.kind === \"output\") {\n      const r = await evaluateAssertion(a, transcripts);\n      results[i] = r;\n      detTotal++;\n      if (r.status === \"pass\") detPass++;\n    }\n  }\n  const score = detTotal > 0 ? detPass / detTotal : 1;\n\n  // Pass 2: judge (advisory unless gated) + differential (score vs baseline).\n  for (let i = 0; i < c.assert.length; i++) {\n    const a = c.assert[i];\n    if (a.kind === \"judge\" || a.kind === \"differential\") {\n      results[i] = await evaluateAssertion(a, transcripts, {\n        judge: ctx.judge,\n        caseScore: score,\n        baselineScore: ctx.baselineScore,\n      });\n    }\n  }\n  const assertions = results.filter((r): r is AssertResult => r !== undefined);\n\n  let verifyPassed: boolean | undefined;\n  if (c.verify) verifyPassed = (await sh(c.verify, cwd)) === 0;\n  if (c.cleanup) await sh(c.cleanup, cwd);\n\n  // A case passes iff no assertion FAILED and any post-state verify passed.\n  // degraded/skipped assertions do not fail the case (they are reported).\n  const pass = !assertions.some((a) => a.status === \"fail\") && (verifyPassed ?? true);\n  return { name: c.name, assertions, verifyPassed, score, pass };\n}\n\nexport interface RunEvalOptions {\n  evalFile: EvalFile;\n  pluginDir: string;\n  componentLeaf: string;\n  registry: AdapterRegistry;\n  drivers: Record<Target, HarnessDriver>;\n  scratchRoot?: string;\n  timeoutMs?: number;\n  /** Judge model for `judge` assertions (advisory unless gated). Omit to skip. */\n  judge?: JudgeFn;\n  /** Snapshot each harness's mean score into evals/.baselines/ (spec §9.5). */\n  snapshotBaselines?: boolean;\n}\n\n/**\n * Drive the real harnesses headlessly and assert over what each did (spec §9.5).\n * A harness with no available driver is reported UNTESTED -- never faked. For each\n * tested harness the component is installed into a throwaway scratch project so the\n * harness loads it, then every case runs and is evaluated.\n */\nexport async function runEval(opts: RunEvalOptions): Promise<EvalReport> {\n  const { evalFile, drivers } = opts;\n  const harnesses: HarnessReport[] = [];\n\n  for (const harness of evalFile.harnesses) {\n    const driver = drivers[harness];\n    if (!driver || !(await driver.available())) {\n      harnesses.push({\n        harness,\n        status: \"untested\",\n        reason: driver ? \"CLI not installed or not headless-capable\" : \"no driver for this harness\",\n        cases: [],\n        score: 0,\n        pass: false,\n      });\n      continue;\n    }\n\n    const baselineScore = loadBaseline(opts.pluginDir, evalFile.component, harness)?.score;\n    const scratch = mkdtempSync(join(opts.scratchRoot ?? tmpdir(), `loom-eval-${harness}-`));\n    try {\n      await install({\n        pluginDir: opts.pluginDir,\n        scope: \"project\",\n        cwd: scratch,\n        registry: opts.registry,\n        targets: [harness],\n        only: [opts.componentLeaf],\n        // Keep the source plugin pristine -- write the eval lock into the scratch dir.\n        lockDir: scratch,\n      });\n      const cases: CaseResult[] = [];\n      for (const c of evalFile.cases) {\n        cases.push(\n          await runCase(c, driver, scratch, {\n            judge: opts.judge,\n            baselineScore,\n            timeoutMs: opts.timeoutMs,\n          }),\n        );\n      }\n      const score = cases.length > 0 ? cases.reduce((s, c) => s + c.score, 0) / cases.length : 1;\n      if (opts.snapshotBaselines) {\n        const snapshot: Baseline = { version: \"current\", score };\n        writeBaseline(opts.pluginDir, evalFile.component, harness, snapshot);\n      }\n      harnesses.push({ harness, status: \"tested\", cases, score, pass: cases.every((c) => c.pass) });\n    } catch (err) {\n      harnesses.push({\n        harness,\n        status: \"untested\",\n        reason: `install failed: ${(err as Error).message}`,\n        cases: [],\n        score: 0,\n        pass: false,\n      });\n    } finally {\n      rmSync(scratch, { recursive: true, force: true });\n    }\n  }\n\n  return { component: evalFile.component, harnesses };\n}\n"]}

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@michaelfromyeg/loom-eval",
+  "version": "0.1.0",
+  "description": "Loom eval runner + headless HarnessDriver implementations (real harnesses, local-first).",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "execa": "^9.5.2",
+    "@michaelfromyeg/loom-core": "0.1.0",
+    "@michaelfromyeg/loom-adapter-kit": "0.1.0",
+    "@michaelfromyeg/loom-schema": "0.1.0"
+  },
+  "license": "MIT",
+  "author": "Michael DeMarco",
+  "homepage": "https://github.com/michaelfromyeg/loom#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/michaelfromyeg/loom.git",
+    "directory": "packages/eval"
+  },
+  "bugs": {
+    "url": "https://github.com/michaelfromyeg/loom/issues"
+  },
+  "keywords": [
+    "loom",
+    "coding-agent",
+    "ai-agent",
+    "claude-code",
+    "codex",
+    "cursor",
+    "copilot",
+    "opencode",
+    "mcp",
+    "plugin",
+    "skill",
+    "compiler"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsup"
+  }
+}