@tangle-network/agent-eval 0.36.0 → 0.38.0

package/dist/matrix/index.d.ts

@@ -1,113 +1,6 @@
-import { DefaultVerdict } from '@tangle-network/agent-runtime/loops';
+import { M as MatrixResult, a as MatrixAxis, A as AxisSummary, b as MatrixCell, C as CellResult, R as RunAgentMatrixOptions } from '../types-DHqkLwEU.js';
 export { DefaultVerdict } from '@tangle-network/agent-runtime/loops';
 
-/**
- * @experimental
- *
- * N-axis cartesian matrix over substrate types — types module.
- *
- * The matrix is a runner + aggregator. It iterates the cartesian product of
- * caller-provided axes (any value type — `AgentProfile` from sandbox, `Driver`
- * / `Validator` from agent-runtime, rubric records, thinking levels, anything)
- * and aggregates per-axis pass/score/cost summaries. Substrate types are
- * imported at the boundary by JSDoc only; the matrix never wraps them.
- */
-
-/** One axis = one dimension to iterate. `V` is the value type — pass any
- *  substrate type (AgentProfile, Driver, Validator, rubric record). */
-interface MatrixAxis<V> {
-    /** Axis name. Becomes the key in `MatrixResult.byAxis`. */
-    name: string;
-    /** Stable id per value. Used as the bucket key in aggregation. */
-    values: Array<{
-        id: string;
-        value: V;
-    }>;
-    /** Optional bucket label override. Receives the same `(value, id)` the
-     *  runner stored on the cell; default label is `id`. */
-    label?: (value: V, id: string) => string;
-}
-/** A cell carries one picked value from each axis, keyed by axis name. */
-interface MatrixCell {
-    axes: Record<string, {
-        id: string;
-        value: unknown;
-    }>;
-    /** 0-based replicate index within the same axis combination. */
-    rep: number;
-    /** Stable sort key — preserves cartesian order across concurrent execution. */
-    ordinal: number;
-}
-interface CellResult<Output> {
-    output: Output;
-    verdict: DefaultVerdict;
-    costUsd: number;
-    durationMs: number;
-    runId?: string;
-    /** Populated when `runCell` threw. The cell contributes 0 to passRate AND
-     *  meanScore regardless of `verdict`. */
-    error?: {
-        message: string;
-        kind: string;
-    };
-}
-interface AxisSummary {
-    axisName: string;
-    axisValue: string;
-    cells: number;
-    passRate: number;
-    meanScore: number;
-    p50Score: number;
-    p90Score: number;
-    totalCostUsd: number;
-    meanDurationMs: number;
-}
-interface MatrixResult<Output> {
-    cells: Array<{
-        cell: MatrixCell;
-        runs: CellResult<Output>[];
-    }>;
-    /** `byAxis[axisName][axisValueId] = summary`. Populated only for axes
-     *  named in `aggregateBy` (default = every axis in `axes`). */
-    byAxis: Record<string, Record<string, AxisSummary>>;
-    summary: {
-        totalCells: number;
-        runsExecuted: number;
-        /** Cells removed by `filter` plus cells unscheduled after the cost
-         *  ceiling or abort signal tripped. */
-        cellsSkipped: number;
-        overallPassRate: number;
-        overallMeanScore: number;
-        totalCostUsd: number;
-        durationMs: number;
-    };
-    /** Stable id-like string generated at the end of the run. */
-    matrixId: string;
-}
-interface RunAgentMatrixOptions<Output> {
-    axes: MatrixAxis<unknown>[];
-    /** User-supplied cell executor. May throw; the matrix captures throws as
-     *  `CellResult.error` and continues. */
-    runCell: (cell: MatrixCell) => Promise<CellResult<Output>>;
-    /** Replicates per cell. Default 1. */
-    reps?: number;
-    /** Prune cells from the cartesian BEFORE rep expansion. */
-    filter?: (cell: Omit<MatrixCell, 'rep' | 'ordinal'>) => boolean;
-    /** Axes to aggregate into `byAxis`. Default: every axis in `axes`. */
-    aggregateBy?: string[];
-    /** Max concurrent in-flight `runCell` invocations. Default 4. */
-    maxConcurrency?: number;
-    /** Cumulative-cost abort threshold (USD). When the running sum of
-     *  `result.costUsd` crosses this value, no new cells are scheduled.
-     *  In-flight cells finish. Default `Infinity`. */
-    costCeiling?: number;
-    /** Fires once per executed cell, after its promise settles. */
-    onCellComplete?: (cell: MatrixCell, result: CellResult<Output>) => void;
-    /** External cancellation. Aborts in-flight cells via a forwarded signal
-     *  and suppresses scheduling of new ones. */
-    signal?: AbortSignal;
-}
-
 /**
  * Per-axis aggregation of cell runs into `AxisSummary` rows.
  *
@@ -138,4 +31,4 @@ declare function buildByAxis<Output>(cells: MatrixResult<Output>['cells'], axes:
 
 declare function runAgentMatrix<Output>(opts: RunAgentMatrixOptions<Output>): Promise<MatrixResult<Output>>;
 
-export { type AxisSummary, type CellResult, type MatrixAxis, type MatrixCell, type MatrixResult, type RunAgentMatrixOptions, buildByAxis, runAgentMatrix, summariseRows };
+export { AxisSummary, CellResult, MatrixAxis, MatrixCell, MatrixResult, RunAgentMatrixOptions, buildByAxis, runAgentMatrix, summariseRows };

package/dist/matrix/index.js

@@ -1,274 +1,9 @@
+import {
+  buildByAxis,
+  runAgentMatrix,
+  summariseRows
+} from "../chunk-QWV226SL.js";
 import "../chunk-PZ5AY32C.js";
-
-// src/matrix/aggregation.ts
-function flattenRuns(cells) {
-  const rows = [];
-  for (const { cell, runs } of cells) {
-    for (const result of runs) rows.push({ cell, result });
-  }
-  return rows;
-}
-function quantile(sorted, q) {
-  if (sorted.length === 0) return 0;
-  if (sorted.length === 1) return sorted[0];
-  const pos = (sorted.length - 1) * q;
-  const lo = Math.floor(pos);
-  const hi = Math.ceil(pos);
-  if (lo === hi) return sorted[lo];
-  const frac = pos - lo;
-  return sorted[lo] * (1 - frac) + sorted[hi] * frac;
-}
-function summariseRows(rows, axisName, axisValue) {
-  if (rows.length === 0) {
-    return {
-      axisName,
-      axisValue,
-      cells: 0,
-      passRate: 0,
-      meanScore: 0,
-      p50Score: 0,
-      p90Score: 0,
-      totalCostUsd: 0,
-      meanDurationMs: 0
-    };
-  }
-  let pass = 0;
-  let scoreSum = 0;
-  let costSum = 0;
-  let durSum = 0;
-  const scores = [];
-  for (const { result } of rows) {
-    const errored = result.error !== void 0;
-    const score = errored ? 0 : result.verdict.score;
-    const valid = !errored && result.verdict.valid;
-    if (valid) pass++;
-    scoreSum += score;
-    scores.push(score);
-    costSum += result.costUsd;
-    durSum += result.durationMs;
-  }
-  scores.sort((a, b) => a - b);
-  return {
-    axisName,
-    axisValue,
-    cells: rows.length,
-    passRate: pass / rows.length,
-    meanScore: scoreSum / rows.length,
-    p50Score: quantile(scores, 0.5),
-    p90Score: quantile(scores, 0.9),
-    totalCostUsd: costSum,
-    meanDurationMs: durSum / rows.length
-  };
-}
-function bucketBy(rows, axisName, labelFor) {
-  const buckets = /* @__PURE__ */ new Map();
-  for (const row of rows) {
-    const slot = row.cell.axes[axisName];
-    if (!slot) continue;
-    const id = slot.id;
-    let arr = buckets.get(id);
-    if (!arr) {
-      arr = [];
-      buckets.set(id, arr);
-    }
-    arr.push(row);
-  }
-  const out = {};
-  for (const id of [...buckets.keys()].sort()) {
-    out[id] = summariseRows(buckets.get(id), axisName, labelFor(id));
-  }
-  return out;
-}
-function buildByAxis(cells, axes, aggregateBy) {
-  const rows = flattenRuns(cells);
-  const byName = new Map(axes.map((a) => [a.name, a]));
-  const byAxis = {};
-  for (const name of aggregateBy) {
-    const axis = byName.get(name);
-    const labelFor = (id) => {
-      if (!axis?.label) return id;
-      const found = axis.values.find((v) => v.id === id);
-      if (!found) return id;
-      return axis.label(found.value, id);
-    };
-    byAxis[name] = bucketBy(rows, name, labelFor);
-  }
-  return byAxis;
-}
-
-// src/matrix/runner.ts
-function cartesian(axes) {
-  if (axes.length === 0) return [{ axes: {} }];
-  for (const a of axes) if (a.values.length === 0) return [];
-  const out = [];
-  const idx = new Array(axes.length).fill(0);
-  while (true) {
-    const slot = {};
-    for (let i2 = 0; i2 < axes.length; i2++) {
-      const axis = axes[i2];
-      const v = axis.values[idx[i2]];
-      slot[axis.name] = { id: v.id, value: v.value };
-    }
-    out.push({ axes: slot });
-    let i = 0;
-    while (i < axes.length) {
-      const next = idx[i] + 1;
-      const axis = axes[i];
-      if (next < axis.values.length) {
-        idx[i] = next;
-        break;
-      }
-      idx[i] = 0;
-      i++;
-    }
-    if (i === axes.length) break;
-  }
-  return out;
-}
-function makeMatrixId() {
-  const t = Date.now().toString(36);
-  let r = "";
-  for (let i = 0; i < 8; i++) r += Math.floor(Math.random() * 16).toString(16);
-  return `mtx_${t}_${r}`;
-}
-function makeErrorResult(err) {
-  const e = err;
-  return {
-    output: void 0,
-    verdict: { valid: false, score: 0 },
-    costUsd: 0,
-    durationMs: 0,
-    error: {
-      message: typeof e?.message === "string" ? e.message : String(err),
-      kind: typeof e?.name === "string" ? e.name : "Error"
-    }
-  };
-}
-async function runAgentMatrix(opts) {
-  const startedAt = Date.now();
-  const reps = Math.max(1, opts.reps ?? 1);
-  const maxConcurrency = Math.max(1, opts.maxConcurrency ?? 4);
-  const costCeiling = opts.costCeiling ?? Number.POSITIVE_INFINITY;
-  const aggregateBy = opts.aggregateBy ?? opts.axes.map((a) => a.name);
-  const base = cartesian(opts.axes);
-  const filtered = opts.filter ? base.filter((c) => opts.filter(c)) : base;
-  const filteredOut = base.length - filtered.length;
-  const planned = [];
-  for (let i = 0; i < filtered.length; i++) {
-    for (let r = 0; r < reps; r++) {
-      planned.push({
-        axes: filtered[i].axes,
-        rep: r,
-        ordinal: i * reps + r
-      });
-    }
-  }
-  const cellRecords = [];
-  let cumulativeCost = 0;
-  let costCeilingReached = false;
-  let runsExecuted = 0;
-  let cellsUnscheduled = 0;
-  const aborted = () => opts.signal?.aborted === true;
-  let inFlight = 0;
-  let cursor = 0;
-  let resolveAll;
-  const done = new Promise((res) => {
-    resolveAll = res;
-  });
-  const pump = () => {
-    while (inFlight < maxConcurrency && cursor < planned.length) {
-      if (aborted() || costCeilingReached) {
-        const left = planned.length - cursor;
-        cellsUnscheduled += left;
-        cursor = planned.length;
-        break;
-      }
-      const cell = planned[cursor++];
-      inFlight++;
-      const record = { cell, runs: [] };
-      cellRecords.push(record);
-      const promise = (async () => {
-        try {
-          return await opts.runCell(cell);
-        } catch (err) {
-          return makeErrorResult(err);
-        }
-      })();
-      promise.then((result) => {
-        record.runs.push(result);
-        runsExecuted++;
-        cumulativeCost += result.costUsd;
-        if (cumulativeCost >= costCeiling && !costCeilingReached) {
-          costCeilingReached = true;
-          console.warn("[matrix] cost ceiling reached");
-        }
-        try {
-          opts.onCellComplete?.(cell, result);
-        } catch {
-        }
-        inFlight--;
-        if (cursor < planned.length) {
-          pump();
-        } else if (inFlight === 0) {
-          resolveAll?.();
-        }
-      });
-    }
-    if (cursor >= planned.length && inFlight === 0) resolveAll?.();
-  };
-  const onAbort = () => {
-    if (cursor < planned.length) {
-      cellsUnscheduled += planned.length - cursor;
-      cursor = planned.length;
-    }
-    if (inFlight === 0) resolveAll?.();
-  };
-  if (opts.signal) {
-    if (opts.signal.aborted) {
-      cellsUnscheduled = planned.length;
-      cursor = planned.length;
-      resolveAll?.();
-    } else {
-      opts.signal.addEventListener("abort", onAbort, { once: true });
-    }
-  }
-  if (planned.length === 0) {
-    resolveAll?.();
-  } else {
-    pump();
-  }
-  await done;
-  if (opts.signal) opts.signal.removeEventListener("abort", onAbort);
-  cellRecords.sort((a, b) => a.cell.ordinal - b.cell.ordinal);
-  let pass = 0;
-  let scoreSum = 0;
-  let totalCost = 0;
-  let runCount = 0;
-  for (const { runs } of cellRecords) {
-    for (const r of runs) {
-      runCount++;
-      const errored = r.error !== void 0;
-      if (!errored && r.verdict.valid) pass++;
-      scoreSum += errored ? 0 : r.verdict.score;
-      totalCost += r.costUsd;
-    }
-  }
-  const byAxis = buildByAxis(cellRecords, opts.axes, aggregateBy);
-  return {
-    cells: cellRecords,
-    byAxis,
-    summary: {
-      totalCells: planned.length,
-      runsExecuted,
-      cellsSkipped: cellsUnscheduled + filteredOut * reps,
-      overallPassRate: runCount === 0 ? 0 : pass / runCount,
-      overallMeanScore: runCount === 0 ? 0 : scoreSum / runCount,
-      totalCostUsd: totalCost,
-      durationMs: Date.now() - startedAt
-    },
-    matrixId: makeMatrixId()
-  };
-}
 export {
   buildByAxis,
   runAgentMatrix,

package/dist/matrix/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/matrix/aggregation.ts","../../src/matrix/runner.ts"],"sourcesContent":["/**\n * Per-axis aggregation of cell runs into `AxisSummary` rows.\n *\n * Pure: consumes the final `cells: [{cell, runs}]` array and returns the\n * `byAxis` table. Error runs contribute 0 to passRate and meanScore. Cost\n * and duration always count — the budget was spent regardless.\n */\n\nimport type { AxisSummary, CellResult, MatrixAxis, MatrixCell, MatrixResult } from './types'\n\ninterface Row<Output> {\n  cell: MatrixCell\n  result: CellResult<Output>\n}\n\nfunction flattenRuns<Output>(cells: MatrixResult<Output>['cells']): Row<Output>[] {\n  const rows: Row<Output>[] = []\n  for (const { cell, runs } of cells) {\n    for (const result of runs) rows.push({ cell, result })\n  }\n  return rows\n}\n\nfunction quantile(sorted: number[], q: number): number {\n  if (sorted.length === 0) return 0\n  if (sorted.length === 1) return sorted[0] as number\n  const pos = (sorted.length - 1) * q\n  const lo = Math.floor(pos)\n  const hi = Math.ceil(pos)\n  if (lo === hi) return sorted[lo] as number\n  const frac = pos - lo\n  return (sorted[lo] as number) * (1 - frac) + (sorted[hi] as number) * frac\n}\n\nexport function summariseRows<Output>(\n  rows: Row<Output>[],\n  axisName: string,\n  axisValue: string,\n): AxisSummary {\n  if (rows.length === 0) {\n    return {\n      axisName,\n      axisValue,\n      cells: 0,\n      passRate: 0,\n      meanScore: 0,\n      p50Score: 0,\n      p90Score: 0,\n      totalCostUsd: 0,\n      meanDurationMs: 0,\n    }\n  }\n  let pass = 0\n  let scoreSum = 0\n  let costSum = 0\n  let durSum = 0\n  const scores: number[] = []\n  for (const { result } of rows) {\n    const errored = result.error !== undefined\n    const score = errored ? 0 : result.verdict.score\n    const valid = !errored && result.verdict.valid\n    if (valid) pass++\n    scoreSum += score\n    scores.push(score)\n    costSum += result.costUsd\n    durSum += result.durationMs\n  }\n  scores.sort((a, b) => a - b)\n  return {\n    axisName,\n    axisValue,\n    cells: rows.length,\n    passRate: pass / rows.length,\n    meanScore: scoreSum / rows.length,\n    p50Score: quantile(scores, 0.5),\n    p90Score: quantile(scores, 0.9),\n    totalCostUsd: costSum,\n    meanDurationMs: durSum / rows.length,\n  }\n}\n\nfunction bucketBy<Output>(\n  rows: Row<Output>[],\n  axisName: string,\n  labelFor: (id: string) => string,\n): Record<string, AxisSummary> {\n  const buckets = new Map<string, Row<Output>[]>()\n  for (const row of rows) {\n    const slot = row.cell.axes[axisName]\n    if (!slot) continue\n    const id = slot.id\n    let arr = buckets.get(id)\n    if (!arr) {\n      arr = []\n      buckets.set(id, arr)\n    }\n    arr.push(row)\n  }\n  const out: Record<string, AxisSummary> = {}\n  // Sorted keys for deterministic JSON serialisation.\n  for (const id of [...buckets.keys()].sort()) {\n    out[id] = summariseRows(buckets.get(id) as Row<Output>[], axisName, labelFor(id))\n  }\n  return out\n}\n\nexport function buildByAxis<Output>(\n  cells: MatrixResult<Output>['cells'],\n  axes: MatrixAxis<unknown>[],\n  aggregateBy: string[],\n): Record<string, Record<string, AxisSummary>> {\n  const rows = flattenRuns(cells)\n  const byName = new Map(axes.map((a) => [a.name, a]))\n  const byAxis: Record<string, Record<string, AxisSummary>> = {}\n  for (const name of aggregateBy) {\n    const axis = byName.get(name)\n    const labelFor = (id: string): string => {\n      if (!axis?.label) return id\n      const found = axis.values.find((v) => v.id === id)\n      if (!found) return id\n      return axis.label(found.value, id)\n    }\n    byAxis[name] = bucketBy(rows, name, labelFor)\n  }\n  return byAxis\n}\n","/**\n * N-axis cartesian runner.\n *\n * Expansion order: cartesian over `axes` in declared order, then `reps` as the\n * inner-most dim → `ordinal = (cartIdx * reps) + rep`. The returned\n * `cells[]` is sorted by `ordinal` so concurrent execution does not reorder\n * the output.\n *\n * Scheduling is a sliding window of in-flight promises capped at\n * `maxConcurrency`. The window stops admitting new cells when the cost\n * ceiling trips or the abort signal fires; in-flight cells finish.\n */\n\nimport { buildByAxis } from './aggregation'\nimport type {\n  CellResult,\n  MatrixAxis,\n  MatrixCell,\n  MatrixResult,\n  RunAgentMatrixOptions,\n} from './types'\n\ninterface BaseCell {\n  axes: Record<string, { id: string; value: unknown }>\n}\n\nfunction cartesian(axes: MatrixAxis<unknown>[]): BaseCell[] {\n  // Empty axes (`values=[]`) collapse the whole product to zero cells. An\n  // empty `axes` array yields a single empty-axes cell — degenerate but\n  // valid (caller is iterating only reps).\n  if (axes.length === 0) return [{ axes: {} }]\n  for (const a of axes) if (a.values.length === 0) return []\n  const out: BaseCell[] = []\n  const idx = new Array(axes.length).fill(0)\n  while (true) {\n    const slot: Record<string, { id: string; value: unknown }> = {}\n    for (let i = 0; i < axes.length; i++) {\n      const axis = axes[i] as MatrixAxis<unknown>\n      const v = axis.values[idx[i] as number] as { id: string; value: unknown }\n      slot[axis.name] = { id: v.id, value: v.value }\n    }\n    out.push({ axes: slot })\n    // Increment like an odometer, left-most axis is fastest.\n    let i = 0\n    while (i < axes.length) {\n      const next = (idx[i] as number) + 1\n      const axis = axes[i] as MatrixAxis<unknown>\n      if (next < axis.values.length) {\n        idx[i] = next\n        break\n      }\n      idx[i] = 0\n      i++\n    }\n    if (i === axes.length) break\n  }\n  return out\n}\n\nfunction makeMatrixId(): string {\n  // Stable id-like string: time + 8 random hex chars. Avoids node:crypto\n  // import to keep the matrix dep-free.\n  const t = Date.now().toString(36)\n  let r = ''\n  for (let i = 0; i < 8; i++) r += Math.floor(Math.random() * 16).toString(16)\n  return `mtx_${t}_${r}`\n}\n\nfunction makeErrorResult<Output>(err: unknown): CellResult<Output> {\n  const e = err as { message?: string; name?: string }\n  return {\n    output: undefined as unknown as Output,\n    verdict: { valid: false, score: 0 },\n    costUsd: 0,\n    durationMs: 0,\n    error: {\n      message: typeof e?.message === 'string' ? e.message : String(err),\n      kind: typeof e?.name === 'string' ? e.name : 'Error',\n    },\n  }\n}\n\nexport async function runAgentMatrix<Output>(\n  opts: RunAgentMatrixOptions<Output>,\n): Promise<MatrixResult<Output>> {\n  const startedAt = Date.now()\n  const reps = Math.max(1, opts.reps ?? 1)\n  const maxConcurrency = Math.max(1, opts.maxConcurrency ?? 4)\n  const costCeiling = opts.costCeiling ?? Number.POSITIVE_INFINITY\n  const aggregateBy = opts.aggregateBy ?? opts.axes.map((a) => a.name)\n\n  const base = cartesian(opts.axes)\n  const filtered = opts.filter\n    ? base.filter((c) => (opts.filter as (b: BaseCell) => boolean)(c))\n    : base\n  const filteredOut = base.length - filtered.length\n\n  const planned: MatrixCell[] = []\n  for (let i = 0; i < filtered.length; i++) {\n    for (let r = 0; r < reps; r++) {\n      planned.push({\n        axes: (filtered[i] as BaseCell).axes,\n        rep: r,\n        ordinal: i * reps + r,\n      })\n    }\n  }\n\n  const cellRecords: Array<{ cell: MatrixCell; runs: CellResult<Output>[] }> = []\n  let cumulativeCost = 0\n  let costCeilingReached = false\n  let runsExecuted = 0\n  let cellsUnscheduled = 0\n\n  const aborted = (): boolean => opts.signal?.aborted === true\n\n  // Per-run abort controller forwards the external signal so cell executors\n  // see cancellation. We don't expose it on `MatrixCell` — the signature on\n  // `runCell` per the public API is `(cell) => Promise<...>`. Executors that\n  // need cancellation use the external signal directly via closure.\n\n  let inFlight = 0\n  let cursor = 0\n  let resolveAll: (() => void) | undefined\n  const done = new Promise<void>((res) => {\n    resolveAll = res\n  })\n\n  const pump = (): void => {\n    while (inFlight < maxConcurrency && cursor < planned.length) {\n      if (aborted() || costCeilingReached) {\n        // Drain remaining as unscheduled.\n        const left = planned.length - cursor\n        cellsUnscheduled += left\n        cursor = planned.length\n        break\n      }\n      const cell = planned[cursor++] as MatrixCell\n      inFlight++\n      // Lazily allocate the record so cells appear in `cells[]` in any\n      // arrival order; we sort by ordinal at the end.\n      const record = { cell, runs: [] as CellResult<Output>[] }\n      cellRecords.push(record)\n      const promise: Promise<CellResult<Output>> = (async () => {\n        try {\n          return await opts.runCell(cell)\n        } catch (err) {\n          return makeErrorResult<Output>(err)\n        }\n      })()\n      promise.then((result) => {\n        record.runs.push(result)\n        runsExecuted++\n        cumulativeCost += result.costUsd\n        if (cumulativeCost >= costCeiling && !costCeilingReached) {\n          costCeilingReached = true\n          // eslint-disable-next-line no-console\n          console.warn('[matrix] cost ceiling reached')\n        }\n        try {\n          opts.onCellComplete?.(cell, result)\n        } catch {\n          // onCellComplete is observational — swallow throws so a noisy\n          // callback can't tank the run.\n        }\n        inFlight--\n        if (cursor < planned.length) {\n          pump()\n        } else if (inFlight === 0) {\n          resolveAll?.()\n        }\n      })\n    }\n    if (cursor >= planned.length && inFlight === 0) resolveAll?.()\n  }\n\n  const onAbort = (): void => {\n    // External abort: stop scheduling. In-flight cells finish; their\n    // executors observe `opts.signal.aborted` directly via closure.\n    if (cursor < planned.length) {\n      cellsUnscheduled += planned.length - cursor\n      cursor = planned.length\n    }\n    if (inFlight === 0) resolveAll?.()\n  }\n  if (opts.signal) {\n    if (opts.signal.aborted) {\n      cellsUnscheduled = planned.length\n      cursor = planned.length\n      resolveAll?.()\n    } else {\n      opts.signal.addEventListener('abort', onAbort, { once: true })\n    }\n  }\n\n  if (planned.length === 0) {\n    resolveAll?.()\n  } else {\n    pump()\n  }\n\n  await done\n  if (opts.signal) opts.signal.removeEventListener('abort', onAbort)\n\n  cellRecords.sort((a, b) => a.cell.ordinal - b.cell.ordinal)\n\n  let pass = 0\n  let scoreSum = 0\n  let totalCost = 0\n  let runCount = 0\n  for (const { runs } of cellRecords) {\n    for (const r of runs) {\n      runCount++\n      const errored = r.error !== undefined\n      if (!errored && r.verdict.valid) pass++\n      scoreSum += errored ? 0 : r.verdict.score\n      totalCost += r.costUsd\n    }\n  }\n\n  const byAxis = buildByAxis(cellRecords, opts.axes, aggregateBy)\n\n  return {\n    cells: cellRecords,\n    byAxis,\n    summary: {\n      totalCells: planned.length,\n      runsExecuted,\n      cellsSkipped: cellsUnscheduled + filteredOut * reps,\n      overallPassRate: runCount === 0 ? 0 : pass / runCount,\n      overallMeanScore: runCount === 0 ? 0 : scoreSum / runCount,\n      totalCostUsd: totalCost,\n      durationMs: Date.now() - startedAt,\n    },\n    matrixId: makeMatrixId(),\n  }\n}\n"],"mappings":";;;AAeA,SAAS,YAAoB,OAAqD;AAChF,QAAM,OAAsB,CAAC;AAC7B,aAAW,EAAE,MAAM,KAAK,KAAK,OAAO;AAClC,eAAW,UAAU,KAAM,MAAK,KAAK,EAAE,MAAM,OAAO,CAAC;AAAA,EACvD;AACA,SAAO;AACT;AAEA,SAAS,SAAS,QAAkB,GAAmB;AACrD,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,MAAI,OAAO,WAAW,EAAG,QAAO,OAAO,CAAC;AACxC,QAAM,OAAO,OAAO,SAAS,KAAK;AAClC,QAAM,KAAK,KAAK,MAAM,GAAG;AACzB,QAAM,KAAK,KAAK,KAAK,GAAG;AACxB,MAAI,OAAO,GAAI,QAAO,OAAO,EAAE;AAC/B,QAAM,OAAO,MAAM;AACnB,SAAQ,OAAO,EAAE,KAAgB,IAAI,QAAS,OAAO,EAAE,IAAe;AACxE;AAEO,SAAS,cACd,MACA,UACA,WACa;AACb,MAAI,KAAK,WAAW,GAAG;AACrB,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO;AAAA,MACP,UAAU;AAAA,MACV,WAAW;AAAA,MACX,UAAU;AAAA,MACV,UAAU;AAAA,MACV,cAAc;AAAA,MACd,gBAAgB;AAAA,IAClB;AAAA,EACF;AACA,MAAI,OAAO;AACX,MAAI,WAAW;AACf,MAAI,UAAU;AACd,MAAI,SAAS;AACb,QAAM,SAAmB,CAAC;AAC1B,aAAW,EAAE,OAAO,KAAK,MAAM;AAC7B,UAAM,UAAU,OAAO,UAAU;AACjC,UAAM,QAAQ,UAAU,IAAI,OAAO,QAAQ;AAC3C,UAAM,QAAQ,CAAC,WAAW,OAAO,QAAQ;AACzC,QAAI,MAAO;AACX,gBAAY;AACZ,WAAO,KAAK,KAAK;AACjB,eAAW,OAAO;AAClB,cAAU,OAAO;AAAA,EACnB;AACA,SAAO,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3B,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,OAAO,KAAK;AAAA,IACZ,UAAU,OAAO,KAAK;AAAA,IACtB,WAAW,WAAW,KAAK;AAAA,IAC3B,UAAU,SAAS,QAAQ,GAAG;AAAA,IAC9B,UAAU,SAAS,QAAQ,GAAG;AAAA,IAC9B,cAAc;AAAA,IACd,gBAAgB,SAAS,KAAK;AAAA,EAChC;AACF;AAEA,SAAS,SACP,MACA,UACA,UAC6B;AAC7B,QAAM,UAAU,oBAAI,IAA2B;AAC/C,aAAW,OAAO,MAAM;AACtB,UAAM,OAAO,IAAI,KAAK,KAAK,QAAQ;AACnC,QAAI,CAAC,KAAM;AACX,UAAM,KAAK,KAAK;AAChB,QAAI,MAAM,QAAQ,IAAI,EAAE;AACxB,QAAI,CAAC,KAAK;AACR,YAAM,CAAC;AACP,cAAQ,IAAI,IAAI,GAAG;AAAA,IACrB;AACA,QAAI,KAAK,GAAG;AAAA,EACd;AACA,QAAM,MAAmC,CAAC;AAE1C,aAAW,MAAM,CAAC,GAAG,QAAQ,KAAK,CAAC,EAAE,KAAK,GAAG;AAC3C,QAAI,EAAE,IAAI,cAAc,QAAQ,IAAI,EAAE,GAAoB,UAAU,SAAS,EAAE,CAAC;AAAA,EAClF;AACA,SAAO;AACT;AAEO,SAAS,YACd,OACA,MACA,aAC6C;AAC7C,QAAM,OAAO,YAAY,KAAK;AAC9B,QAAM,SAAS,IAAI,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AACnD,QAAM,SAAsD,CAAC;AAC7D,aAAW,QAAQ,aAAa;AAC9B,UAAM,OAAO,OAAO,IAAI,IAAI;AAC5B,UAAM,WAAW,CAAC,OAAuB;AACvC,UAAI,CAAC,MAAM,MAAO,QAAO;AACzB,YAAM,QAAQ,KAAK,OAAO,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE;AACjD,UAAI,CAAC,MAAO,QAAO;AACnB,aAAO,KAAK,MAAM,MAAM,OAAO,EAAE;AAAA,IACnC;AACA,WAAO,IAAI,IAAI,SAAS,MAAM,MAAM,QAAQ;AAAA,EAC9C;AACA,SAAO;AACT;;;ACnGA,SAAS,UAAU,MAAyC;AAI1D,MAAI,KAAK,WAAW,EAAG,QAAO,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC;AAC3C,aAAW,KAAK,KAAM,KAAI,EAAE,OAAO,WAAW,EAAG,QAAO,CAAC;AACzD,QAAM,MAAkB,CAAC;AACzB,QAAM,MAAM,IAAI,MAAM,KAAK,MAAM,EAAE,KAAK,CAAC;AACzC,SAAO,MAAM;AACX,UAAM,OAAuD,CAAC;AAC9D,aAASA,KAAI,GAAGA,KAAI,KAAK,QAAQA,MAAK;AACpC,YAAM,OAAO,KAAKA,EAAC;AACnB,YAAM,IAAI,KAAK,OAAO,IAAIA,EAAC,CAAW;AACtC,WAAK,KAAK,IAAI,IAAI,EAAE,IAAI,EAAE,IAAI,OAAO,EAAE,MAAM;AAAA,IAC/C;AACA,QAAI,KAAK,EAAE,MAAM,KAAK,CAAC;AAEvB,QAAI,IAAI;AACR,WAAO,IAAI,KAAK,QAAQ;AACtB,YAAM,OAAQ,IAAI,CAAC,IAAe;AAClC,YAAM,OAAO,KAAK,CAAC;AACnB,UAAI,OAAO,KAAK,OAAO,QAAQ;AAC7B,YAAI,CAAC,IAAI;AACT;AAAA,MACF;AACA,UAAI,CAAC,IAAI;AACT;AAAA,IACF;AACA,QAAI,MAAM,KAAK,OAAQ;AAAA,EACzB;AACA,SAAO;AACT;AAEA,SAAS,eAAuB;AAG9B,QAAM,IAAI,KAAK,IAAI,EAAE,SAAS,EAAE;AAChC,MAAI,IAAI;AACR,WAAS,IAAI,GAAG,IAAI,GAAG,IAAK,MAAK,KAAK,MAAM,KAAK,OAAO,IAAI,EAAE,EAAE,SAAS,EAAE;AAC3E,SAAO,OAAO,CAAC,IAAI,CAAC;AACtB;AAEA,SAAS,gBAAwB,KAAkC;AACjE,QAAM,IAAI;AACV,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,SAAS,EAAE,OAAO,OAAO,OAAO,EAAE;AAAA,IAClC,SAAS;AAAA,IACT,YAAY;AAAA,IACZ,OAAO;AAAA,MACL,SAAS,OAAO,GAAG,YAAY,WAAW,EAAE,UAAU,OAAO,GAAG;AAAA,MAChE,MAAM,OAAO,GAAG,SAAS,WAAW,EAAE,OAAO;AAAA,IAC/C;AAAA,EACF;AACF;AAEA,eAAsB,eACpB,MAC+B;AAC/B,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,OAAO,KAAK,IAAI,GAAG,KAAK,QAAQ,CAAC;AACvC,QAAM,iBAAiB,KAAK,IAAI,GAAG,KAAK,kBAAkB,CAAC;AAC3D,QAAM,cAAc,KAAK,eAAe,OAAO;AAC/C,QAAM,cAAc,KAAK,eAAe,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,IAAI;AAEnE,QAAM,OAAO,UAAU,KAAK,IAAI;AAChC,QAAM,WAAW,KAAK,SAClB,KAAK,OAAO,CAAC,MAAO,KAAK,OAAoC,CAAC,CAAC,IAC/D;AACJ,QAAM,cAAc,KAAK,SAAS,SAAS;AAE3C,QAAM,UAAwB,CAAC;AAC/B,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,aAAS,IAAI,GAAG,IAAI,MAAM,KAAK;AAC7B,cAAQ,KAAK;AAAA,QACX,MAAO,SAAS,CAAC,EAAe;AAAA,QAChC,KAAK;AAAA,QACL,SAAS,IAAI,OAAO;AAAA,MACtB,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,cAAuE,CAAC;AAC9E,MAAI,iBAAiB;AACrB,MAAI,qBAAqB;AACzB,MAAI,eAAe;AACnB,MAAI,mBAAmB;AAEvB,QAAM,UAAU,MAAe,KAAK,QAAQ,YAAY;AAOxD,MAAI,WAAW;AACf,MAAI,SAAS;AACb,MAAI;AACJ,QAAM,OAAO,IAAI,QAAc,CAAC,QAAQ;AACtC,iBAAa;AAAA,EACf,CAAC;AAED,QAAM,OAAO,MAAY;AACvB,WAAO,WAAW,kBAAkB,SAAS,QAAQ,QAAQ;AAC3D,UAAI,QAAQ,KAAK,oBAAoB;AAEnC,cAAM,OAAO,QAAQ,SAAS;AAC9B,4BAAoB;AACpB,iBAAS,QAAQ;AACjB;AAAA,MACF;AACA,YAAM,OAAO,QAAQ,QAAQ;AAC7B;AAGA,YAAM,SAAS,EAAE,MAAM,MAAM,CAAC,EAA0B;AACxD,kBAAY,KAAK,MAAM;AACvB,YAAM,WAAwC,YAAY;AACxD,YAAI;AACF,iBAAO,MAAM,KAAK,QAAQ,IAAI;AAAA,QAChC,SAAS,KAAK;AACZ,iBAAO,gBAAwB,GAAG;AAAA,QACpC;AAAA,MACF,GAAG;AACH,cAAQ,KAAK,CAAC,WAAW;AACvB,eAAO,KAAK,KAAK,MAAM;AACvB;AACA,0BAAkB,OAAO;AACzB,YAAI,kBAAkB,eAAe,CAAC,oBAAoB;AACxD,+BAAqB;AAErB,kBAAQ,KAAK,+BAA+B;AAAA,QAC9C;AACA,YAAI;AACF,eAAK,iBAAiB,MAAM,MAAM;AAAA,QACpC,QAAQ;AAAA,QAGR;AACA;AACA,YAAI,SAAS,QAAQ,QAAQ;AAC3B,eAAK;AAAA,QACP,WAAW,aAAa,GAAG;AACzB,uBAAa;AAAA,QACf;AAAA,MACF,CAAC;AAAA,IACH;AACA,QAAI,UAAU,QAAQ,UAAU,aAAa,EAAG,cAAa;AAAA,EAC/D;AAEA,QAAM,UAAU,MAAY;AAG1B,QAAI,SAAS,QAAQ,QAAQ;AAC3B,0BAAoB,QAAQ,SAAS;AACrC,eAAS,QAAQ;AAAA,IACnB;AACA,QAAI,aAAa,EAAG,cAAa;AAAA,EACnC;AACA,MAAI,KAAK,QAAQ;AACf,QAAI,KAAK,OAAO,SAAS;AACvB,yBAAmB,QAAQ;AAC3B,eAAS,QAAQ;AACjB,mBAAa;AAAA,IACf,OAAO;AACL,WAAK,OAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAAA,IAC/D;AAAA,EACF;AAEA,MAAI,QAAQ,WAAW,GAAG;AACxB,iBAAa;AAAA,EACf,OAAO;AACL,SAAK;AAAA,EACP;AAEA,QAAM;AACN,MAAI,KAAK,OAAQ,MAAK,OAAO,oBAAoB,SAAS,OAAO;AAEjE,cAAY,KAAK,CAAC,GAAG,MAAM,EAAE,KAAK,UAAU,EAAE,KAAK,OAAO;AAE1D,MAAI,OAAO;AACX,MAAI,WAAW;AACf,MAAI,YAAY;AAChB,MAAI,WAAW;AACf,aAAW,EAAE,KAAK,KAAK,aAAa;AAClC,eAAW,KAAK,MAAM;AACpB;AACA,YAAM,UAAU,EAAE,UAAU;AAC5B,UAAI,CAAC,WAAW,EAAE,QAAQ,MAAO;AACjC,kBAAY,UAAU,IAAI,EAAE,QAAQ;AACpC,mBAAa,EAAE;AAAA,IACjB;AAAA,EACF;AAEA,QAAM,SAAS,YAAY,aAAa,KAAK,MAAM,WAAW;AAE9D,SAAO;AAAA,IACL,OAAO;AAAA,IACP;AAAA,IACA,SAAS;AAAA,MACP,YAAY,QAAQ;AAAA,MACpB;AAAA,MACA,cAAc,mBAAmB,cAAc;AAAA,MAC/C,iBAAiB,aAAa,IAAI,IAAI,OAAO;AAAA,MAC7C,kBAAkB,aAAa,IAAI,IAAI,WAAW;AAAA,MAClD,cAAc;AAAA,MACd,YAAY,KAAK,IAAI,IAAI;AAAA,IAC3B;AAAA,IACA,UAAU,aAAa;AAAA,EACzB;AACF;","names":["i"]}
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}