@sembl/testing 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sembl contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,88 @@
+# @sembl/testing
+
+Deterministic tests for code built on SEMBL, and a way to measure whether a
+prompt or schema change helped.
+
+```sh
+pnpm add -D @sembl/testing
+```
+
+## Record and replay
+
+```ts
+import { replayOrRecord } from "@sembl/testing";
+import { AnthropicProvider } from "@sembl/provider-anthropic";
+
+const live = process.env.ANTHROPIC_API_KEY
+  ? new AnthropicProvider({ model: "claude-sonnet-5", apiKey: process.env.ANTHROPIC_API_KEY })
+  : undefined;
+
+// Replays from ./recordings; records misses through `live` when it exists.
+const provider = replayOrRecord("./recordings", live);
+```
+
+`RecordingProvider(inner, dir)` writes every request/response pair to one
+JSON file, named by schema id and a hash of what reached the model — the
+system prompt, the user input and the JSON Schema. `ReplayProvider(dir)`
+answers from those files and never touches the network; a request with no
+recording throws `ReplayMissError`, which is what you want in CI: a miss
+means a fixture or a description changed and nobody re-recorded.
+
+Because the key covers everything the model sees, editing a field
+description invalidates only the recordings it affects.
+
+## Eval harness
+
+Nobody can tell whether a description change helped or hurt without
+measuring. Fixtures are `{ input, expected }` pairs in a directory:
+
+```json
+// evals/listing/sea-cabin.json
+{
+  "input": { "file": "sea-cabin.html", "label": "Airbnb listing" },
+  "expected": { "name": "Sea Cabin", "sleeps": 6, "amenities": ["sauna", "hot tub"] }
+}
+```
+
+A file may hold one fixture or an array; an input may be a string, a
+labelled source, a list of sources, or `{ "file": … }` to read a sibling
+file. Then either call the harness:
+
+```ts
+import { runEval, loadFixtures, formatReport } from "@sembl/testing";
+
+const report = await runEval({
+  fixtures: loadFixtures("./evals/listing"),
+  schema: Listing,
+  provider,
+  mode: "partialCoerce",
+  prices: { inputPerMTok: 3, outputPerMTok: 15, cacheReadPerMTok: 0.3 },
+});
+console.log(formatReport(report));
+```
+
+or run it from the CLI with a config module that exports the schema and the
+provider:
+
+```js
+// sembl.eval.mjs
+export { Listing as schema } from "./dist/schemas.js";
+export const provider = new AnthropicProvider({ model: "claude-sonnet-5", apiKey: process.env.ANTHROPIC_API_KEY });
+export const prices = { inputPerMTok: 3, outputPerMTok: 15 };
+export const coerceOptions = { onInvalidField: "clamp", maxInputChars: 40_000 };
+```
+
+```sh
+sembl eval --config sembl.eval.mjs --fixtures ./evals/listing --replay ./evals/listing/recordings
+```
+
+The report gives per-field precision and recall — a wrong value counts
+against both, a missing one against recall, an unexpected one against
+precision — plus token usage, cost when prices are given, and latency
+percentiles. Every run is written to `<fixtures>/.sembl-eval/last-run.json`
+(or `--out`) and the next run prints deltas against it, so a change shows up
+as `R -20pt` on the field it broke. `--replay` records the first run and
+replays it afterwards, which makes evals free and deterministic in CI;
+`--min-recall` and `--min-precision` turn a regression into a failing exit
+code. Arrays of primitives compare as sets, since the order amenities come
+back in is not a fact about the listing.

package/dist/index.cjs

@@ -0,0 +1,521 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  RecordingProvider: () => RecordingProvider,
+  ReplayMissError: () => ReplayMissError,
+  ReplayProvider: () => ReplayProvider,
+  compareLeaves: () => compareLeaves,
+  diffReports: () => diffReports,
+  estimateCost: () => estimateCost,
+  fieldStats: () => fieldStats,
+  flattenLeaves: () => flattenLeaves,
+  formatReport: () => formatReport,
+  leavesEqual: () => leavesEqual,
+  loadFixtures: () => loadFixtures,
+  loadReport: () => loadReport,
+  recordingKey: () => recordingKey,
+  recordingPath: () => recordingPath,
+  replayOrRecord: () => replayOrRecord,
+  runEval: () => runEval,
+  saveReport: () => saveReport
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/replay.ts
+var import_node_crypto = require("crypto");
+var import_node_fs = require("fs");
+var import_node_path = require("path");
+function stable(value) {
+  return JSON.stringify(
+    value,
+    (_key, v) => v && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v
+  );
+}
+function recordingKey(request) {
+  const material = stable({
+    systemPrompt: request.systemPrompt,
+    userInput: request.userInput,
+    jsonSchema: request.jsonSchema
+  });
+  return (0, import_node_crypto.createHash)("sha256").update(material).digest("hex").slice(0, 24);
+}
+function slug(id) {
+  return id.replace(/[^A-Za-z0-9_-]+/g, "_").slice(0, 40) || "schema";
+}
+function recordingPath(dir, request) {
+  return (0, import_node_path.join)(dir, `${slug(request.schema.id)}.${recordingKey(request)}.json`);
+}
+var ReplayMissError = class extends Error {
+  constructor(key, dir, schemaId) {
+    super(
+      `No recording for a "${schemaId}" request (key ${key}) in ${dir}. Run once with a RecordingProvider, or pass a fallback provider to record misses.`
+    );
+    this.key = key;
+    this.dir = dir;
+    this.schemaId = schemaId;
+    this.name = "ReplayMissError";
+  }
+};
+var RecordingProvider = class {
+  constructor(inner, dir) {
+    this.inner = inner;
+    this.dir = dir;
+    (0, import_node_fs.mkdirSync)(dir, { recursive: true });
+  }
+  async complete(request) {
+    const response = await this.inner.complete(request);
+    const recording = {
+      key: recordingKey(request),
+      schemaId: request.schema.id,
+      request: {
+        systemPrompt: request.systemPrompt,
+        userInput: request.userInput,
+        jsonSchema: request.jsonSchema
+      },
+      response,
+      recordedAt: (/* @__PURE__ */ new Date()).toISOString()
+    };
+    (0, import_node_fs.writeFileSync)(recordingPath(this.dir, request), JSON.stringify(recording, null, 2) + "\n");
+    return response;
+  }
+};
+var ReplayProvider = class {
+  constructor(dir, options = {}) {
+    this.dir = dir;
+    this.recorder = options.fallback ? new RecordingProvider(options.fallback, dir) : void 0;
+  }
+  recorder;
+  async complete(request) {
+    const path = recordingPath(this.dir, request);
+    if ((0, import_node_fs.existsSync)(path)) {
+      const recording = JSON.parse((0, import_node_fs.readFileSync)(path, "utf8"));
+      return recording.response;
+    }
+    if (this.recorder) {
+      return this.recorder.complete(request);
+    }
+    throw new ReplayMissError(recordingKey(request), this.dir, request.schema.id);
+  }
+  /** How many recordings the directory holds. */
+  size() {
+    if (!(0, import_node_fs.existsSync)(this.dir)) return 0;
+    return (0, import_node_fs.readdirSync)(this.dir).filter((f) => f.endsWith(".json")).length;
+  }
+};
+function replayOrRecord(dir, live) {
+  return new ReplayProvider(dir, live ? { fallback: live } : {});
+}
+
+// src/eval.ts
+var import_node_async_hooks = require("async_hooks");
+var import_node_fs2 = require("fs");
+var import_node_path2 = require("path");
+var import_core = require("@sembl/core");
+function isFileInput(value) {
+  return typeof value === "object" && value !== null && typeof value.file === "string";
+}
+function resolveInput(input, baseDir) {
+  const one = (value) => {
+    if (typeof value === "string") return value;
+    if (isFileInput(value)) {
+      const text = (0, import_node_fs2.readFileSync)((0, import_node_path2.resolve)(baseDir, value.file), "utf8");
+      return value.label ? { label: value.label, text } : { text };
+    }
+    return value;
+  };
+  if (Array.isArray(input)) {
+    return input.map((v) => {
+      const r = one(v);
+      return typeof r === "string" ? { text: r } : r;
+    });
+  }
+  return one(input);
+}
+function loadFixtures(dir) {
+  const root = (0, import_node_path2.resolve)(dir);
+  if (!(0, import_node_fs2.existsSync)(root)) {
+    throw new Error(`Fixture directory not found: ${root}`);
+  }
+  const fixtures = [];
+  const files = (0, import_node_fs2.readdirSync)(root).filter((f) => f.endsWith(".json") && !f.startsWith(".")).sort();
+  for (const file of files) {
+    const path = (0, import_node_path2.join)(root, file);
+    const parsed = JSON.parse((0, import_node_fs2.readFileSync)(path, "utf8"));
+    const list = Array.isArray(parsed) ? parsed : [parsed];
+    list.forEach((raw, index) => {
+      const fixture = raw;
+      if (!fixture || typeof fixture !== "object" || !("input" in fixture) || !("expected" in fixture)) {
+        throw new Error(`${path}${list.length > 1 ? `[${index}]` : ""}: a fixture needs "input" and "expected"`);
+      }
+      const stem = (0, import_node_path2.basename)(file, ".json");
+      fixtures.push({
+        name: fixture.name ?? (list.length > 1 ? `${stem}[${index}]` : stem),
+        input: resolveInput(fixture.input, (0, import_node_path2.dirname)(path)),
+        expected: fixture.expected
+      });
+    });
+  }
+  return fixtures;
+}
+function stable2(value) {
+  return JSON.stringify(
+    value,
+    (_k, v) => v && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v
+  );
+}
+function isPrimitive(value) {
+  return value === null || typeof value !== "object";
+}
+function flattenLeaves(value, prefix = "") {
+  const leaves = /* @__PURE__ */ new Map();
+  if (value === null || value === void 0) return leaves;
+  if (typeof value !== "object" || Array.isArray(value)) {
+    leaves.set(prefix, value);
+    return leaves;
+  }
+  for (const [key, child] of Object.entries(value)) {
+    const path = prefix ? `${prefix}.${key}` : key;
+    for (const [leafPath, leaf] of flattenLeaves(child, path)) {
+      leaves.set(leafPath, leaf);
+    }
+  }
+  return leaves;
+}
+function leavesEqual(a, b) {
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) return false;
+    if (a.every(isPrimitive) && b.every(isPrimitive)) {
+      const sortedA = [...a].map(String).sort();
+      const sortedB = [...b].map(String).sort();
+      return sortedA.every((v, i) => v === sortedB[i]);
+    }
+  }
+  return stable2(a) === stable2(b);
+}
+function compareLeaves(expected, actual, provenance = {}) {
+  const want = flattenLeaves(expected);
+  const got = flattenLeaves(actual ?? {});
+  const paths = [.../* @__PURE__ */ new Set([...want.keys(), ...got.keys()])].sort();
+  return paths.map((path) => {
+    const top = path.split(".")[0];
+    const confidence = provenance[top]?.confidence;
+    const result = { path, outcome: "match" };
+    if (confidence) result.confidence = confidence;
+    if (want.has(path) && got.has(path)) {
+      result.expected = want.get(path);
+      result.actual = got.get(path);
+      result.outcome = leavesEqual(want.get(path), got.get(path)) ? "match" : "wrong";
+    } else if (want.has(path)) {
+      result.expected = want.get(path);
+      result.outcome = "missing";
+    } else {
+      result.actual = got.get(path);
+      result.outcome = "extra";
+    }
+    return result;
+  });
+}
+function ratio(num, den) {
+  return den === 0 ? null : num / den;
+}
+function fieldStats(items) {
+  const byPath = /* @__PURE__ */ new Map();
+  for (const item of items) {
+    for (const leaf of item.leaves) {
+      const stats = byPath.get(leaf.path) ?? { tp: 0, fp: 0, fn: 0 };
+      switch (leaf.outcome) {
+        case "match":
+          stats.tp += 1;
+          break;
+        case "wrong":
+          stats.fp += 1;
+          stats.fn += 1;
+          break;
+        case "missing":
+          stats.fn += 1;
+          break;
+        case "extra":
+          stats.fp += 1;
+          break;
+      }
+      byPath.set(leaf.path, stats);
+    }
+  }
+  return [...byPath.entries()].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([path, { tp, fp, fn }]) => ({
+    path,
+    tp,
+    fp,
+    fn,
+    precision: ratio(tp, tp + fp),
+    recall: ratio(tp, tp + fn)
+  }));
+}
+function percentile(sorted, p) {
+  if (sorted.length === 0) return 0;
+  const index = Math.min(sorted.length - 1, Math.ceil(p / 100 * sorted.length) - 1);
+  return sorted[Math.max(0, index)];
+}
+function emptyUsage() {
+  return { promptTokens: 0, completionTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };
+}
+function addUsage(into, usage) {
+  if (!usage) return;
+  into.promptTokens += usage.promptTokens;
+  into.completionTokens += usage.completionTokens;
+  into.cacheReadTokens += usage.cacheReadTokens ?? 0;
+  into.cacheWriteTokens += usage.cacheWriteTokens ?? 0;
+}
+function estimateCost(usage, prices) {
+  const per = (tokens, price) => tokens / 1e6 * price;
+  return per(usage.promptTokens, prices.inputPerMTok) + per(usage.completionTokens, prices.outputPerMTok) + per(usage.cacheReadTokens, prices.cacheReadPerMTok ?? prices.inputPerMTok) + per(usage.cacheWriteTokens, prices.cacheWritePerMTok ?? prices.inputPerMTok);
+}
+var context = new import_node_async_hooks.AsyncLocalStorage();
+var MeteredProvider = class {
+  constructor(inner) {
+    this.inner = inner;
+  }
+  async complete(request) {
+    const response = await this.inner.complete(request);
+    const ctx = context.getStore();
+    if (ctx) {
+      ctx.calls += 1;
+      addUsage(ctx.usage, response.usage);
+    }
+    return response;
+  }
+};
+async function mapWithConcurrency(items, concurrency, fn) {
+  const results = new Array(items.length);
+  let next = 0;
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+    while (next < items.length) {
+      const index = next++;
+      results[index] = await fn(items[index], index);
+    }
+  });
+  await Promise.all(workers);
+  return results;
+}
+async function runEval(options) {
+  const {
+    fixtures,
+    mode = "coerce",
+    provenance = false,
+    concurrency = 1,
+    prices,
+    provider,
+    ...coerceOptions
+  } = options;
+  const metered = new MeteredProvider(provider);
+  const run = mode === "coerce" ? provenance ? import_core.coerceWithProvenance : import_core.coerce : provenance ? import_core.partialCoerceWithProvenance : import_core.partialCoerce;
+  const items = await mapWithConcurrency(fixtures, concurrency, async (fixture, index) => {
+    const ctx = { usage: emptyUsage(), calls: 0 };
+    const name = fixture.name ?? `fixture ${index + 1}`;
+    const started = performance.now();
+    return context.run(ctx, async () => {
+      try {
+        const result = await run(fixture.input, {
+          ...coerceOptions,
+          provider: metered
+        });
+        const data = provenance ? result.data : result;
+        const prov = provenance ? result.provenance : {};
+        const issues = provenance ? result.issues : [];
+        const leaves = compareLeaves(fixture.expected, data, prov);
+        return {
+          name,
+          ok: true,
+          exact: leaves.every((l) => l.outcome === "match"),
+          leaves,
+          issues,
+          latencyMs: Math.round(performance.now() - started),
+          calls: ctx.calls,
+          usage: ctx.usage
+        };
+      } catch (error) {
+        return {
+          name,
+          ok: false,
+          error: error instanceof Error ? error.message : String(error),
+          exact: false,
+          leaves: compareLeaves(fixture.expected, void 0),
+          issues: [],
+          latencyMs: Math.round(performance.now() - started),
+          calls: ctx.calls,
+          usage: ctx.usage
+        };
+      }
+    });
+  });
+  const fields = fieldStats(items);
+  const tp = fields.reduce((s, f) => s + f.tp, 0);
+  const fp = fields.reduce((s, f) => s + f.fp, 0);
+  const fn = fields.reduce((s, f) => s + f.fn, 0);
+  const usage = emptyUsage();
+  for (const item of items) {
+    usage.promptTokens += item.usage.promptTokens;
+    usage.completionTokens += item.usage.completionTokens;
+    usage.cacheReadTokens += item.usage.cacheReadTokens;
+    usage.cacheWriteTokens += item.usage.cacheWriteTokens;
+  }
+  const latencies = items.map((i) => i.latencyMs).sort((a, b) => a - b);
+  return {
+    schemaId: options.schema.id,
+    mode,
+    ranAt: (/* @__PURE__ */ new Date()).toISOString(),
+    items,
+    fields,
+    totals: {
+      fixtures: items.length,
+      ok: items.filter((i) => i.ok).length,
+      exact: items.filter((i) => i.exact).length,
+      precision: ratio(tp, tp + fp),
+      recall: ratio(tp, tp + fn),
+      usage,
+      calls: items.reduce((s, i) => s + i.calls, 0),
+      ...prices ? { cost: estimateCost(usage, prices) } : {},
+      latencyMs: {
+        p50: percentile(latencies, 50),
+        p95: percentile(latencies, 95),
+        max: latencies[latencies.length - 1] ?? 0,
+        total: latencies.reduce((s, l) => s + l, 0)
+      }
+    }
+  };
+}
+function saveReport(report, file) {
+  (0, import_node_fs2.mkdirSync)((0, import_node_path2.dirname)((0, import_node_path2.resolve)(file)), { recursive: true });
+  (0, import_node_fs2.writeFileSync)((0, import_node_path2.resolve)(file), JSON.stringify(report, null, 2) + "\n");
+}
+function loadReport(file) {
+  const path = (0, import_node_path2.resolve)(file);
+  if (!(0, import_node_fs2.existsSync)(path)) return void 0;
+  return JSON.parse((0, import_node_fs2.readFileSync)(path, "utf8"));
+}
+function delta(a, b) {
+  return a === null || a === void 0 || b === null || b === void 0 ? null : b - a;
+}
+function diffReports(previous, next) {
+  const prevFields = new Map(previous.fields.map((f) => [f.path, f]));
+  const fields = [];
+  for (const field of next.fields) {
+    const before = prevFields.get(field.path);
+    const precision = delta(before?.precision, field.precision);
+    const recall = delta(before?.recall, field.recall);
+    if (precision !== null && precision !== 0 || recall !== null && recall !== 0) {
+      fields.push({ path: field.path, precision, recall });
+    }
+  }
+  fields.sort((a, b) => Math.min(a.precision ?? 0, a.recall ?? 0) - Math.min(b.precision ?? 0, b.recall ?? 0));
+  return {
+    exact: next.totals.exact - previous.totals.exact,
+    ok: next.totals.ok - previous.totals.ok,
+    precision: delta(previous.totals.precision, next.totals.precision),
+    recall: delta(previous.totals.recall, next.totals.recall),
+    promptTokens: next.totals.usage.promptTokens - previous.totals.usage.promptTokens,
+    completionTokens: next.totals.usage.completionTokens - previous.totals.usage.completionTokens,
+    ...next.totals.cost !== void 0 && previous.totals.cost !== void 0 ? { cost: next.totals.cost - previous.totals.cost } : {},
+    p50Ms: next.totals.latencyMs.p50 - previous.totals.latencyMs.p50,
+    fields
+  };
+}
+function pct(value) {
+  return value === null ? "  \u2013  " : `${(value * 100).toFixed(0).padStart(3)}%`;
+}
+function signed(value, digits = 0, suffix = "") {
+  if (value === null || value === 0) return "";
+  const text = digits > 0 ? value.toFixed(digits) : String(value);
+  return ` (${value > 0 ? "+" : ""}${text}${suffix})`;
+}
+function signedPct(value) {
+  return value === null || value === 0 ? "" : ` (${value > 0 ? "+" : ""}${(value * 100).toFixed(0)}pt)`;
+}
+function formatReport(report, diff) {
+  const t = report.totals;
+  const lines = [];
+  lines.push(
+    `Eval: ${report.schemaId} (${report.mode}) \u2014 ${t.fixtures} fixture(s), ${t.ok} ran${signed(diff?.ok ?? null)}, ${t.exact} exact${signed(diff?.exact ?? null)}`
+  );
+  lines.push("");
+  const width = Math.max(5, ...report.fields.map((f) => f.path.length));
+  const deltas = new Map((diff?.fields ?? []).map((f) => [f.path, f]));
+  lines.push(`${"Field".padEnd(width)}  Prec  Recall   tp  fp  fn  \u0394`);
+  for (const field of report.fields) {
+    const d = deltas.get(field.path);
+    const change = d ? [d.precision !== null && d.precision !== 0 ? `P${signedPct(d.precision).trim()}` : "", d.recall !== null && d.recall !== 0 ? `R${signedPct(d.recall).trim()}` : ""].filter(Boolean).join(" ") : "";
+    lines.push(
+      `${field.path.padEnd(width)}  ${pct(field.precision)} ${pct(field.recall)}   ${String(field.tp).padStart(2)}  ${String(field.fp).padStart(2)}  ${String(field.fn).padStart(2)}  ${change}`
+    );
+  }
+  lines.push("");
+  lines.push(
+    `Overall: precision ${pct(t.precision).trim()}${signedPct(diff?.precision ?? null)}, recall ${pct(t.recall).trim()}${signedPct(diff?.recall ?? null)}`
+  );
+  const u = t.usage;
+  const cache = u.cacheReadTokens || u.cacheWriteTokens ? `, cache read ${u.cacheReadTokens.toLocaleString("en-US")} / write ${u.cacheWriteTokens.toLocaleString("en-US")}` : "";
+  lines.push(
+    `Tokens: ${u.promptTokens.toLocaleString("en-US")} prompt${signed(diff?.promptTokens ?? null)} / ${u.completionTokens.toLocaleString("en-US")} completion${signed(diff?.completionTokens ?? null)}${cache}; ${t.calls} call(s)` + (t.cost !== void 0 ? `; cost $${t.cost.toFixed(4)}${signed(diff?.cost ?? null, 4)}` : "")
+  );
+  lines.push(
+    `Latency: p50 ${t.latencyMs.p50}ms${signed(diff?.p50Ms ?? null, 0, "ms")}, p95 ${t.latencyMs.p95}ms, max ${t.latencyMs.max}ms`
+  );
+  const failed = report.items.filter((i) => !i.ok);
+  if (failed.length > 0) {
+    lines.push("");
+    lines.push("Failed:");
+    for (const item of failed) lines.push(`  ${item.name}: ${item.error}`);
+  }
+  const imperfect = report.items.filter((i) => i.ok && !i.exact);
+  if (imperfect.length > 0) {
+    lines.push("");
+    lines.push("Mismatches:");
+    for (const item of imperfect) {
+      for (const leaf of item.leaves.filter((l) => l.outcome !== "match")) {
+        const detail = leaf.outcome === "wrong" ? `expected ${JSON.stringify(leaf.expected)}, got ${JSON.stringify(leaf.actual)}` : leaf.outcome === "missing" ? `expected ${JSON.stringify(leaf.expected)}, got nothing` : `unexpected ${JSON.stringify(leaf.actual)}`;
+        const conf = leaf.confidence ? ` [${leaf.confidence}]` : "";
+        lines.push(`  ${item.name} \u203A ${leaf.path}: ${detail}${conf}`);
+      }
+    }
+  }
+  return lines.join("\n");
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  RecordingProvider,
+  ReplayMissError,
+  ReplayProvider,
+  compareLeaves,
+  diffReports,
+  estimateCost,
+  fieldStats,
+  flattenLeaves,
+  formatReport,
+  leavesEqual,
+  loadFixtures,
+  loadReport,
+  recordingKey,
+  recordingPath,
+  replayOrRecord,
+  runEval,
+  saveReport
+});
+//# sourceMappingURL=index.cjs.map