@tangle-network/agent-eval 0.35.0 → 0.36.0

package/README.md

@@ -222,6 +222,7 @@ const next = await analyzeOptimizationResult(campaign, { researcher })
 | `@tangle-network/agent-eval/rl` | adapters, verifiable rewards, preferences, OPE, PRM, contamination, tournaments, adversarial, compute curves, auto-research |
 | `@tangle-network/agent-eval/wire` | HTTP/RPC server + schemas (same protocol the Python client speaks) |
 | `@tangle-network/agent-eval/benchmarks` | benchmark adapter contracts and reference wrappers |
+| `@tangle-network/agent-eval/matrix` | N-axis cartesian runner over substrate types — see [`src/matrix/`](./src/matrix/) |
 
 The root export remains available for convenience; new code should prefer
 focused subpaths. Anything under `/rl`, `/pipelines`, `/meta-eval`, `/prm`,
@@ -294,6 +295,39 @@ and runtime. See [`examples/`](./examples/).
 - [`examples/production-loop`](./examples/production-loop/README.md):
   ingest prod traces + feedback, cluster failures, evolve, gate, open a PR.
 
+## Matrix
+
+`@tangle-network/agent-eval/matrix` is an N-axis cartesian runner over the
+substrate types you already use — `AgentProfile` from
+`@tangle-network/sandbox`, `Driver` / `Validator` from
+`@tangle-network/agent-runtime`, rubric records, anything. It does not wrap
+substrate types; the caller passes them in axis values, the runner iterates
+the cartesian, and the aggregator returns per-axis pass / score / cost /
+duration summaries.
+
+```ts
+import { runAgentMatrix } from '@tangle-network/agent-eval/matrix'
+
+const result = await runAgentMatrix({
+  axes: [
+    { name: 'scenario', values: scenarios.map((s) => ({ id: s.id, value: s })) },
+    { name: 'profile',  values: profiles.map((p)  => ({ id: p.name, value: p })) },
+    { name: 'thinking', values: [
+      { id: 'low', value: 'low' }, { id: 'high', value: 'high' },
+    ] },
+  ],
+  reps: 3,
+  maxConcurrency: 4,
+  costCeiling: 5.0,
+  filter: (cell) => !(cell.axes.scenario.value.hard === 5 && cell.axes.thinking.id === 'low'),
+  runCell: async (cell) => runScenario(cell.axes.scenario.value, cell.axes.profile.value),
+})
+
+console.log(result.byAxis.profile)  // per-profile passRate / meanScore / p90 / cost
+```
+
+See [`src/matrix/`](./src/matrix/) for the full surface.
+
 ## Docs
 
 Read in this order:

package/dist/matrix/index.d.ts

@@ -0,0 +1,141 @@
+import { DefaultVerdict } from '@tangle-network/agent-runtime/loops';
+export { DefaultVerdict } from '@tangle-network/agent-runtime/loops';
+
+/**
+ * @experimental
+ *
+ * N-axis cartesian matrix over substrate types — types module.
+ *
+ * The matrix is a runner + aggregator. It iterates the cartesian product of
+ * caller-provided axes (any value type — `AgentProfile` from sandbox, `Driver`
+ * / `Validator` from agent-runtime, rubric records, thinking levels, anything)
+ * and aggregates per-axis pass/score/cost summaries. Substrate types are
+ * imported at the boundary by JSDoc only; the matrix never wraps them.
+ */
+
+/** One axis = one dimension to iterate. `V` is the value type — pass any
+ *  substrate type (AgentProfile, Driver, Validator, rubric record). */
+interface MatrixAxis<V> {
+    /** Axis name. Becomes the key in `MatrixResult.byAxis`. */
+    name: string;
+    /** Stable id per value. Used as the bucket key in aggregation. */
+    values: Array<{
+        id: string;
+        value: V;
+    }>;
+    /** Optional bucket label override. Receives the same `(value, id)` the
+     *  runner stored on the cell; default label is `id`. */
+    label?: (value: V, id: string) => string;
+}
+/** A cell carries one picked value from each axis, keyed by axis name. */
+interface MatrixCell {
+    axes: Record<string, {
+        id: string;
+        value: unknown;
+    }>;
+    /** 0-based replicate index within the same axis combination. */
+    rep: number;
+    /** Stable sort key — preserves cartesian order across concurrent execution. */
+    ordinal: number;
+}
+interface CellResult<Output> {
+    output: Output;
+    verdict: DefaultVerdict;
+    costUsd: number;
+    durationMs: number;
+    runId?: string;
+    /** Populated when `runCell` threw. The cell contributes 0 to passRate AND
+     *  meanScore regardless of `verdict`. */
+    error?: {
+        message: string;
+        kind: string;
+    };
+}
+interface AxisSummary {
+    axisName: string;
+    axisValue: string;
+    cells: number;
+    passRate: number;
+    meanScore: number;
+    p50Score: number;
+    p90Score: number;
+    totalCostUsd: number;
+    meanDurationMs: number;
+}
+interface MatrixResult<Output> {
+    cells: Array<{
+        cell: MatrixCell;
+        runs: CellResult<Output>[];
+    }>;
+    /** `byAxis[axisName][axisValueId] = summary`. Populated only for axes
+     *  named in `aggregateBy` (default = every axis in `axes`). */
+    byAxis: Record<string, Record<string, AxisSummary>>;
+    summary: {
+        totalCells: number;
+        runsExecuted: number;
+        /** Cells removed by `filter` plus cells unscheduled after the cost
+         *  ceiling or abort signal tripped. */
+        cellsSkipped: number;
+        overallPassRate: number;
+        overallMeanScore: number;
+        totalCostUsd: number;
+        durationMs: number;
+    };
+    /** Stable id-like string generated at the end of the run. */
+    matrixId: string;
+}
+interface RunAgentMatrixOptions<Output> {
+    axes: MatrixAxis<unknown>[];
+    /** User-supplied cell executor. May throw; the matrix captures throws as
+     *  `CellResult.error` and continues. */
+    runCell: (cell: MatrixCell) => Promise<CellResult<Output>>;
+    /** Replicates per cell. Default 1. */
+    reps?: number;
+    /** Prune cells from the cartesian BEFORE rep expansion. */
+    filter?: (cell: Omit<MatrixCell, 'rep' | 'ordinal'>) => boolean;
+    /** Axes to aggregate into `byAxis`. Default: every axis in `axes`. */
+    aggregateBy?: string[];
+    /** Max concurrent in-flight `runCell` invocations. Default 4. */
+    maxConcurrency?: number;
+    /** Cumulative-cost abort threshold (USD). When the running sum of
+     *  `result.costUsd` crosses this value, no new cells are scheduled.
+     *  In-flight cells finish. Default `Infinity`. */
+    costCeiling?: number;
+    /** Fires once per executed cell, after its promise settles. */
+    onCellComplete?: (cell: MatrixCell, result: CellResult<Output>) => void;
+    /** External cancellation. Aborts in-flight cells via a forwarded signal
+     *  and suppresses scheduling of new ones. */
+    signal?: AbortSignal;
+}
+
+/**
+ * Per-axis aggregation of cell runs into `AxisSummary` rows.
+ *
+ * Pure: consumes the final `cells: [{cell, runs}]` array and returns the
+ * `byAxis` table. Error runs contribute 0 to passRate and meanScore. Cost
+ * and duration always count — the budget was spent regardless.
+ */
+
+interface Row<Output> {
+    cell: MatrixCell;
+    result: CellResult<Output>;
+}
+declare function summariseRows<Output>(rows: Row<Output>[], axisName: string, axisValue: string): AxisSummary;
+declare function buildByAxis<Output>(cells: MatrixResult<Output>['cells'], axes: MatrixAxis<unknown>[], aggregateBy: string[]): Record<string, Record<string, AxisSummary>>;
+
+/**
+ * N-axis cartesian runner.
+ *
+ * Expansion order: cartesian over `axes` in declared order, then `reps` as the
+ * inner-most dim → `ordinal = (cartIdx * reps) + rep`. The returned
+ * `cells[]` is sorted by `ordinal` so concurrent execution does not reorder
+ * the output.
+ *
+ * Scheduling is a sliding window of in-flight promises capped at
+ * `maxConcurrency`. The window stops admitting new cells when the cost
+ * ceiling trips or the abort signal fires; in-flight cells finish.
+ */
+
+declare function runAgentMatrix<Output>(opts: RunAgentMatrixOptions<Output>): Promise<MatrixResult<Output>>;
+
+export { type AxisSummary, type CellResult, type MatrixAxis, type MatrixCell, type MatrixResult, type RunAgentMatrixOptions, buildByAxis, runAgentMatrix, summariseRows };

package/dist/matrix/index.js

@@ -0,0 +1,277 @@
+import "../chunk-PZ5AY32C.js";
+
+// src/matrix/aggregation.ts
+function flattenRuns(cells) {
+  const rows = [];
+  for (const { cell, runs } of cells) {
+    for (const result of runs) rows.push({ cell, result });
+  }
+  return rows;
+}
+function quantile(sorted, q) {
+  if (sorted.length === 0) return 0;
+  if (sorted.length === 1) return sorted[0];
+  const pos = (sorted.length - 1) * q;
+  const lo = Math.floor(pos);
+  const hi = Math.ceil(pos);
+  if (lo === hi) return sorted[lo];
+  const frac = pos - lo;
+  return sorted[lo] * (1 - frac) + sorted[hi] * frac;
+}
+function summariseRows(rows, axisName, axisValue) {
+  if (rows.length === 0) {
+    return {
+      axisName,
+      axisValue,
+      cells: 0,
+      passRate: 0,
+      meanScore: 0,
+      p50Score: 0,
+      p90Score: 0,
+      totalCostUsd: 0,
+      meanDurationMs: 0
+    };
+  }
+  let pass = 0;
+  let scoreSum = 0;
+  let costSum = 0;
+  let durSum = 0;
+  const scores = [];
+  for (const { result } of rows) {
+    const errored = result.error !== void 0;
+    const score = errored ? 0 : result.verdict.score;
+    const valid = !errored && result.verdict.valid;
+    if (valid) pass++;
+    scoreSum += score;
+    scores.push(score);
+    costSum += result.costUsd;
+    durSum += result.durationMs;
+  }
+  scores.sort((a, b) => a - b);
+  return {
+    axisName,
+    axisValue,
+    cells: rows.length,
+    passRate: pass / rows.length,
+    meanScore: scoreSum / rows.length,
+    p50Score: quantile(scores, 0.5),
+    p90Score: quantile(scores, 0.9),
+    totalCostUsd: costSum,
+    meanDurationMs: durSum / rows.length
+  };
+}
+function bucketBy(rows, axisName, labelFor) {
+  const buckets = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    const slot = row.cell.axes[axisName];
+    if (!slot) continue;
+    const id = slot.id;
+    let arr = buckets.get(id);
+    if (!arr) {
+      arr = [];
+      buckets.set(id, arr);
+    }
+    arr.push(row);
+  }
+  const out = {};
+  for (const id of [...buckets.keys()].sort()) {
+    out[id] = summariseRows(buckets.get(id), axisName, labelFor(id));
+  }
+  return out;
+}
+function buildByAxis(cells, axes, aggregateBy) {
+  const rows = flattenRuns(cells);
+  const byName = new Map(axes.map((a) => [a.name, a]));
+  const byAxis = {};
+  for (const name of aggregateBy) {
+    const axis = byName.get(name);
+    const labelFor = (id) => {
+      if (!axis?.label) return id;
+      const found = axis.values.find((v) => v.id === id);
+      if (!found) return id;
+      return axis.label(found.value, id);
+    };
+    byAxis[name] = bucketBy(rows, name, labelFor);
+  }
+  return byAxis;
+}
+
+// src/matrix/runner.ts
+function cartesian(axes) {
+  if (axes.length === 0) return [{ axes: {} }];
+  for (const a of axes) if (a.values.length === 0) return [];
+  const out = [];
+  const idx = new Array(axes.length).fill(0);
+  while (true) {
+    const slot = {};
+    for (let i2 = 0; i2 < axes.length; i2++) {
+      const axis = axes[i2];
+      const v = axis.values[idx[i2]];
+      slot[axis.name] = { id: v.id, value: v.value };
+    }
+    out.push({ axes: slot });
+    let i = 0;
+    while (i < axes.length) {
+      const next = idx[i] + 1;
+      const axis = axes[i];
+      if (next < axis.values.length) {
+        idx[i] = next;
+        break;
+      }
+      idx[i] = 0;
+      i++;
+    }
+    if (i === axes.length) break;
+  }
+  return out;
+}
+function makeMatrixId() {
+  const t = Date.now().toString(36);
+  let r = "";
+  for (let i = 0; i < 8; i++) r += Math.floor(Math.random() * 16).toString(16);
+  return `mtx_${t}_${r}`;
+}
+function makeErrorResult(err) {
+  const e = err;
+  return {
+    output: void 0,
+    verdict: { valid: false, score: 0 },
+    costUsd: 0,
+    durationMs: 0,
+    error: {
+      message: typeof e?.message === "string" ? e.message : String(err),
+      kind: typeof e?.name === "string" ? e.name : "Error"
+    }
+  };
+}
+async function runAgentMatrix(opts) {
+  const startedAt = Date.now();
+  const reps = Math.max(1, opts.reps ?? 1);
+  const maxConcurrency = Math.max(1, opts.maxConcurrency ?? 4);
+  const costCeiling = opts.costCeiling ?? Number.POSITIVE_INFINITY;
+  const aggregateBy = opts.aggregateBy ?? opts.axes.map((a) => a.name);
+  const base = cartesian(opts.axes);
+  const filtered = opts.filter ? base.filter((c) => opts.filter(c)) : base;
+  const filteredOut = base.length - filtered.length;
+  const planned = [];
+  for (let i = 0; i < filtered.length; i++) {
+    for (let r = 0; r < reps; r++) {
+      planned.push({
+        axes: filtered[i].axes,
+        rep: r,
+        ordinal: i * reps + r
+      });
+    }
+  }
+  const cellRecords = [];
+  let cumulativeCost = 0;
+  let costCeilingReached = false;
+  let runsExecuted = 0;
+  let cellsUnscheduled = 0;
+  const aborted = () => opts.signal?.aborted === true;
+  let inFlight = 0;
+  let cursor = 0;
+  let resolveAll;
+  const done = new Promise((res) => {
+    resolveAll = res;
+  });
+  const pump = () => {
+    while (inFlight < maxConcurrency && cursor < planned.length) {
+      if (aborted() || costCeilingReached) {
+        const left = planned.length - cursor;
+        cellsUnscheduled += left;
+        cursor = planned.length;
+        break;
+      }
+      const cell = planned[cursor++];
+      inFlight++;
+      const record = { cell, runs: [] };
+      cellRecords.push(record);
+      const promise = (async () => {
+        try {
+          return await opts.runCell(cell);
+        } catch (err) {
+          return makeErrorResult(err);
+        }
+      })();
+      promise.then((result) => {
+        record.runs.push(result);
+        runsExecuted++;
+        cumulativeCost += result.costUsd;
+        if (cumulativeCost >= costCeiling && !costCeilingReached) {
+          costCeilingReached = true;
+          console.warn("[matrix] cost ceiling reached");
+        }
+        try {
+          opts.onCellComplete?.(cell, result);
+        } catch {
+        }
+        inFlight--;
+        if (cursor < planned.length) {
+          pump();
+        } else if (inFlight === 0) {
+          resolveAll?.();
+        }
+      });
+    }
+    if (cursor >= planned.length && inFlight === 0) resolveAll?.();
+  };
+  const onAbort = () => {
+    if (cursor < planned.length) {
+      cellsUnscheduled += planned.length - cursor;
+      cursor = planned.length;
+    }
+    if (inFlight === 0) resolveAll?.();
+  };
+  if (opts.signal) {
+    if (opts.signal.aborted) {
+      cellsUnscheduled = planned.length;
+      cursor = planned.length;
+      resolveAll?.();
+    } else {
+      opts.signal.addEventListener("abort", onAbort, { once: true });
+    }
+  }
+  if (planned.length === 0) {
+    resolveAll?.();
+  } else {
+    pump();
+  }
+  await done;
+  if (opts.signal) opts.signal.removeEventListener("abort", onAbort);
+  cellRecords.sort((a, b) => a.cell.ordinal - b.cell.ordinal);
+  let pass = 0;
+  let scoreSum = 0;
+  let totalCost = 0;
+  let runCount = 0;
+  for (const { runs } of cellRecords) {
+    for (const r of runs) {
+      runCount++;
+      const errored = r.error !== void 0;
+      if (!errored && r.verdict.valid) pass++;
+      scoreSum += errored ? 0 : r.verdict.score;
+      totalCost += r.costUsd;
+    }
+  }
+  const byAxis = buildByAxis(cellRecords, opts.axes, aggregateBy);
+  return {
+    cells: cellRecords,
+    byAxis,
+    summary: {
+      totalCells: planned.length,
+      runsExecuted,
+      cellsSkipped: cellsUnscheduled + filteredOut * reps,
+      overallPassRate: runCount === 0 ? 0 : pass / runCount,
+      overallMeanScore: runCount === 0 ? 0 : scoreSum / runCount,
+      totalCostUsd: totalCost,
+      durationMs: Date.now() - startedAt
+    },
+    matrixId: makeMatrixId()
+  };
+}
+export {
+  buildByAxis,
+  runAgentMatrix,
+  summariseRows
+};
+//# sourceMappingURL=index.js.map

package/dist/matrix/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/matrix/aggregation.ts","../../src/matrix/runner.ts"],"sourcesContent":["/**\n * Per-axis aggregation of cell runs into `AxisSummary` rows.\n *\n * Pure: consumes the final `cells: [{cell, runs}]` array and returns the\n * `byAxis` table. Error runs contribute 0 to passRate and meanScore. Cost\n * and duration always count — the budget was spent regardless.\n */\n\nimport type { AxisSummary, CellResult, MatrixAxis, MatrixCell, MatrixResult } from './types'\n\ninterface Row<Output> {\n  cell: MatrixCell\n  result: CellResult<Output>\n}\n\nfunction flattenRuns<Output>(cells: MatrixResult<Output>['cells']): Row<Output>[] {\n  const rows: Row<Output>[] = []\n  for (const { cell, runs } of cells) {\n    for (const result of runs) rows.push({ cell, result })\n  }\n  return rows\n}\n\nfunction quantile(sorted: number[], q: number): number {\n  if (sorted.length === 0) return 0\n  if (sorted.length === 1) return sorted[0] as number\n  const pos = (sorted.length - 1) * q\n  const lo = Math.floor(pos)\n  const hi = Math.ceil(pos)\n  if (lo === hi) return sorted[lo] as number\n  const frac = pos - lo\n  return (sorted[lo] as number) * (1 - frac) + (sorted[hi] as number) * frac\n}\n\nexport function summariseRows<Output>(\n  rows: Row<Output>[],\n  axisName: string,\n  axisValue: string,\n): AxisSummary {\n  if (rows.length === 0) {\n    return {\n      axisName,\n      axisValue,\n      cells: 0,\n      passRate: 0,\n      meanScore: 0,\n      p50Score: 0,\n      p90Score: 0,\n      totalCostUsd: 0,\n      meanDurationMs: 0,\n    }\n  }\n  let pass = 0\n  let scoreSum = 0\n  let costSum = 0\n  let durSum = 0\n  const scores: number[] = []\n  for (const { result } of rows) {\n    const errored = result.error !== undefined\n    const score = errored ? 0 : result.verdict.score\n    const valid = !errored && result.verdict.valid\n    if (valid) pass++\n    scoreSum += score\n    scores.push(score)\n    costSum += result.costUsd\n    durSum += result.durationMs\n  }\n  scores.sort((a, b) => a - b)\n  return {\n    axisName,\n    axisValue,\n    cells: rows.length,\n    passRate: pass / rows.length,\n    meanScore: scoreSum / rows.length,\n    p50Score: quantile(scores, 0.5),\n    p90Score: quantile(scores, 0.9),\n    totalCostUsd: costSum,\n    meanDurationMs: durSum / rows.length,\n  }\n}\n\nfunction bucketBy<Output>(\n  rows: Row<Output>[],\n  axisName: string,\n  labelFor: (id: string) => string,\n): Record<string, AxisSummary> {\n  const buckets = new Map<string, Row<Output>[]>()\n  for (const row of rows) {\n    const slot = row.cell.axes[axisName]\n    if (!slot) continue\n    const id = slot.id\n    let arr = buckets.get(id)\n    if (!arr) {\n      arr = []\n      buckets.set(id, arr)\n    }\n    arr.push(row)\n  }\n  const out: Record<string, AxisSummary> = {}\n  // Sorted keys for deterministic JSON serialisation.\n  for (const id of [...buckets.keys()].sort()) {\n    out[id] = summariseRows(buckets.get(id) as Row<Output>[], axisName, labelFor(id))\n  }\n  return out\n}\n\nexport function buildByAxis<Output>(\n  cells: MatrixResult<Output>['cells'],\n  axes: MatrixAxis<unknown>[],\n  aggregateBy: string[],\n): Record<string, Record<string, AxisSummary>> {\n  const rows = flattenRuns(cells)\n  const byName = new Map(axes.map((a) => [a.name, a]))\n  const byAxis: Record<string, Record<string, AxisSummary>> = {}\n  for (const name of aggregateBy) {\n    const axis = byName.get(name)\n    const labelFor = (id: string): string => {\n      if (!axis?.label) return id\n      const found = axis.values.find((v) => v.id === id)\n      if (!found) return id\n      return axis.label(found.value, id)\n    }\n    byAxis[name] = bucketBy(rows, name, labelFor)\n  }\n  return byAxis\n}\n","/**\n * N-axis cartesian runner.\n *\n * Expansion order: cartesian over `axes` in declared order, then `reps` as the\n * inner-most dim → `ordinal = (cartIdx * reps) + rep`. The returned\n * `cells[]` is sorted by `ordinal` so concurrent execution does not reorder\n * the output.\n *\n * Scheduling is a sliding window of in-flight promises capped at\n * `maxConcurrency`. The window stops admitting new cells when the cost\n * ceiling trips or the abort signal fires; in-flight cells finish.\n */\n\nimport { buildByAxis } from './aggregation'\nimport type {\n  CellResult,\n  MatrixAxis,\n  MatrixCell,\n  MatrixResult,\n  RunAgentMatrixOptions,\n} from './types'\n\ninterface BaseCell {\n  axes: Record<string, { id: string; value: unknown }>\n}\n\nfunction cartesian(axes: MatrixAxis<unknown>[]): BaseCell[] {\n  // Empty axes (`values=[]`) collapse the whole product to zero cells. An\n  // empty `axes` array yields a single empty-axes cell — degenerate but\n  // valid (caller is iterating only reps).\n  if (axes.length === 0) return [{ axes: {} }]\n  for (const a of axes) if (a.values.length === 0) return []\n  const out: BaseCell[] = []\n  const idx = new Array(axes.length).fill(0)\n  while (true) {\n    const slot: Record<string, { id: string; value: unknown }> = {}\n    for (let i = 0; i < axes.length; i++) {\n      const axis = axes[i] as MatrixAxis<unknown>\n      const v = axis.values[idx[i] as number] as { id: string; value: unknown }\n      slot[axis.name] = { id: v.id, value: v.value }\n    }\n    out.push({ axes: slot })\n    // Increment like an odometer, left-most axis is fastest.\n    let i = 0\n    while (i < axes.length) {\n      const next = (idx[i] as number) + 1\n      const axis = axes[i] as MatrixAxis<unknown>\n      if (next < axis.values.length) {\n        idx[i] = next\n        break\n      }\n      idx[i] = 0\n      i++\n    }\n    if (i === axes.length) break\n  }\n  return out\n}\n\nfunction makeMatrixId(): string {\n  // Stable id-like string: time + 8 random hex chars. Avoids node:crypto\n  // import to keep the matrix dep-free.\n  const t = Date.now().toString(36)\n  let r = ''\n  for (let i = 0; i < 8; i++) r += Math.floor(Math.random() * 16).toString(16)\n  return `mtx_${t}_${r}`\n}\n\nfunction makeErrorResult<Output>(err: unknown): CellResult<Output> {\n  const e = err as { message?: string; name?: string }\n  return {\n    output: undefined as unknown as Output,\n    verdict: { valid: false, score: 0 },\n    costUsd: 0,\n    durationMs: 0,\n    error: {\n      message: typeof e?.message === 'string' ? e.message : String(err),\n      kind: typeof e?.name === 'string' ? e.name : 'Error',\n    },\n  }\n}\n\nexport async function runAgentMatrix<Output>(\n  opts: RunAgentMatrixOptions<Output>,\n): Promise<MatrixResult<Output>> {\n  const startedAt = Date.now()\n  const reps = Math.max(1, opts.reps ?? 1)\n  const maxConcurrency = Math.max(1, opts.maxConcurrency ?? 4)\n  const costCeiling = opts.costCeiling ?? Number.POSITIVE_INFINITY\n  const aggregateBy = opts.aggregateBy ?? opts.axes.map((a) => a.name)\n\n  const base = cartesian(opts.axes)\n  const filtered = opts.filter\n    ? base.filter((c) => (opts.filter as (b: BaseCell) => boolean)(c))\n    : base\n  const filteredOut = base.length - filtered.length\n\n  const planned: MatrixCell[] = []\n  for (let i = 0; i < filtered.length; i++) {\n    for (let r = 0; r < reps; r++) {\n      planned.push({\n        axes: (filtered[i] as BaseCell).axes,\n        rep: r,\n        ordinal: i * reps + r,\n      })\n    }\n  }\n\n  const cellRecords: Array<{ cell: MatrixCell; runs: CellResult<Output>[] }> = []\n  let cumulativeCost = 0\n  let costCeilingReached = false\n  let runsExecuted = 0\n  let cellsUnscheduled = 0\n\n  const aborted = (): boolean => opts.signal?.aborted === true\n\n  // Per-run abort controller forwards the external signal so cell executors\n  // see cancellation. We don't expose it on `MatrixCell` — the signature on\n  // `runCell` per the public API is `(cell) => Promise<...>`. Executors that\n  // need cancellation use the external signal directly via closure.\n\n  let inFlight = 0\n  let cursor = 0\n  let resolveAll: (() => void) | undefined\n  const done = new Promise<void>((res) => {\n    resolveAll = res\n  })\n\n  const pump = (): void => {\n    while (inFlight < maxConcurrency && cursor < planned.length) {\n      if (aborted() || costCeilingReached) {\n        // Drain remaining as unscheduled.\n        const left = planned.length - cursor\n        cellsUnscheduled += left\n        cursor = planned.length\n        break\n      }\n      const cell = planned[cursor++] as MatrixCell\n      inFlight++\n      // Lazily allocate the record so cells appear in `cells[]` in any\n      // arrival order; we sort by ordinal at the end.\n      const record = { cell, runs: [] as CellResult<Output>[] }\n      cellRecords.push(record)\n      const promise: Promise<CellResult<Output>> = (async () => {\n        try {\n          return await opts.runCell(cell)\n        } catch (err) {\n          return makeErrorResult<Output>(err)\n        }\n      })()\n      promise.then((result) => {\n        record.runs.push(result)\n        runsExecuted++\n        cumulativeCost += result.costUsd\n        if (cumulativeCost >= costCeiling && !costCeilingReached) {\n          costCeilingReached = true\n          // eslint-disable-next-line no-console\n          console.warn('[matrix] cost ceiling reached')\n        }\n        try {\n          opts.onCellComplete?.(cell, result)\n        } catch {\n          // onCellComplete is observational — swallow throws so a noisy\n          // callback can't tank the run.\n        }\n        inFlight--\n        if (cursor < planned.length) {\n          pump()\n        } else if (inFlight === 0) {\n          resolveAll?.()\n        }\n      })\n    }\n    if (cursor >= planned.length && inFlight === 0) resolveAll?.()\n  }\n\n  const onAbort = (): void => {\n    // External abort: stop scheduling. In-flight cells finish; their\n    // executors observe `opts.signal.aborted` directly via closure.\n    if (cursor < planned.length) {\n      cellsUnscheduled += planned.length - cursor\n      cursor = planned.length\n    }\n    if (inFlight === 0) resolveAll?.()\n  }\n  if (opts.signal) {\n    if (opts.signal.aborted) {\n      cellsUnscheduled = planned.length\n      cursor = planned.length\n      resolveAll?.()\n    } else {\n      opts.signal.addEventListener('abort', onAbort, { once: true })\n    }\n  }\n\n  if (planned.length === 0) {\n    resolveAll?.()\n  } else {\n    pump()\n  }\n\n  await done\n  if (opts.signal) opts.signal.removeEventListener('abort', onAbort)\n\n  cellRecords.sort((a, b) => a.cell.ordinal - b.cell.ordinal)\n\n  let pass = 0\n  let scoreSum = 0\n  let totalCost = 0\n  let runCount = 0\n  for (const { runs } of cellRecords) {\n    for (const r of runs) {\n      runCount++\n      const errored = r.error !== undefined\n      if (!errored && r.verdict.valid) pass++\n      scoreSum += errored ? 0 : r.verdict.score\n      totalCost += r.costUsd\n    }\n  }\n\n  const byAxis = buildByAxis(cellRecords, opts.axes, aggregateBy)\n\n  return {\n    cells: cellRecords,\n    byAxis,\n    summary: {\n      totalCells: planned.length,\n      runsExecuted,\n      cellsSkipped: cellsUnscheduled + filteredOut * reps,\n      overallPassRate: runCount === 0 ? 0 : pass / runCount,\n      overallMeanScore: runCount === 0 ? 0 : scoreSum / runCount,\n      totalCostUsd: totalCost,\n      durationMs: Date.now() - startedAt,\n    },\n    matrixId: makeMatrixId(),\n  }\n}\n"],"mappings":";;;AAeA,SAAS,YAAoB,OAAqD;AAChF,QAAM,OAAsB,CAAC;AAC7B,aAAW,EAAE,MAAM,KAAK,KAAK,OAAO;AAClC,eAAW,UAAU,KAAM,MAAK,KAAK,EAAE,MAAM,OAAO,CAAC;AAAA,EACvD;AACA,SAAO;AACT;AAEA,SAAS,SAAS,QAAkB,GAAmB;AACrD,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,MAAI,OAAO,WAAW,EAAG,QAAO,OAAO,CAAC;AACxC,QAAM,OAAO,OAAO,SAAS,KAAK;AAClC,QAAM,KAAK,KAAK,MAAM,GAAG;AACzB,QAAM,KAAK,KAAK,KAAK,GAAG;AACxB,MAAI,OAAO,GAAI,QAAO,OAAO,EAAE;AAC/B,QAAM,OAAO,MAAM;AACnB,SAAQ,OAAO,EAAE,KAAgB,IAAI,QAAS,OAAO,EAAE,IAAe;AACxE;AAEO,SAAS,cACd,MACA,UACA,WACa;AACb,MAAI,KAAK,WAAW,GAAG;AACrB,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO;AAAA,MACP,UAAU;AAAA,MACV,WAAW;AAAA,MACX,UAAU;AAAA,MACV,UAAU;AAAA,MACV,cAAc;AAAA,MACd,gBAAgB;AAAA,IAClB;AAAA,EACF;AACA,MAAI,OAAO;AACX,MAAI,WAAW;AACf,MAAI,UAAU;AACd,MAAI,SAAS;AACb,QAAM,SAAmB,CAAC;AAC1B,aAAW,EAAE,OAAO,KAAK,MAAM;AAC7B,UAAM,UAAU,OAAO,UAAU;AACjC,UAAM,QAAQ,UAAU,IAAI,OAAO,QAAQ;AAC3C,UAAM,QAAQ,CAAC,WAAW,OAAO,QAAQ;AACzC,QAAI,MAAO;AACX,gBAAY;AACZ,WAAO,KAAK,KAAK;AACjB,eAAW,OAAO;AAClB,cAAU,OAAO;AAAA,EACnB;AACA,SAAO,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3B,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,OAAO,KAAK;AAAA,IACZ,UAAU,OAAO,KAAK;AAAA,IACtB,WAAW,WAAW,KAAK;AAAA,IAC3B,UAAU,SAAS,QAAQ,GAAG;AAAA,IAC9B,UAAU,SAAS,QAAQ,GAAG;AAAA,IAC9B,cAAc;AAAA,IACd,gBAAgB,SAAS,KAAK;AAAA,EAChC;AACF;AAEA,SAAS,SACP,MACA,UACA,UAC6B;AAC7B,QAAM,UAAU,oBAAI,IAA2B;AAC/C,aAAW,OAAO,MAAM;AACtB,UAAM,OAAO,IAAI,KAAK,KAAK,QAAQ;AACnC,QAAI,CAAC,KAAM;AACX,UAAM,KAAK,KAAK;AAChB,QAAI,MAAM,QAAQ,IAAI,EAAE;AACxB,QAAI,CAAC,KAAK;AACR,YAAM,CAAC;AACP,cAAQ,IAAI,IAAI,GAAG;AAAA,IACrB;AACA,QAAI,KAAK,GAAG;AAAA,EACd;AACA,QAAM,MAAmC,CAAC;AAE1C,aAAW,MAAM,CAAC,GAAG,QAAQ,KAAK,CAAC,EAAE,KAAK,GAAG;AAC3C,QAAI,EAAE,IAAI,cAAc,QAAQ,IAAI,EAAE,GAAoB,UAAU,SAAS,EAAE,CAAC;AAAA,EAClF;AACA,SAAO;AACT;AAEO,SAAS,YACd,OACA,MACA,aAC6C;AAC7C,QAAM,OAAO,YAAY,KAAK;AAC9B,QAAM,SAAS,IAAI,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AACnD,QAAM,SAAsD,CAAC;AAC7D,aAAW,QAAQ,aAAa;AAC9B,UAAM,OAAO,OAAO,IAAI,IAAI;AAC5B,UAAM,WAAW,CAAC,OAAuB;AACvC,UAAI,CAAC,MAAM,MAAO,QAAO;AACzB,YAAM,QAAQ,KAAK,OAAO,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE;AACjD,UAAI,CAAC,MAAO,QAAO;AACnB,aAAO,KAAK,MAAM,MAAM,OAAO,EAAE;AAAA,IACnC;AACA,WAAO,IAAI,IAAI,SAAS,MAAM,MAAM,QAAQ;AAAA,EAC9C;AACA,SAAO;AACT;;;ACnGA,SAAS,UAAU,MAAyC;AAI1D,MAAI,KAAK,WAAW,EAAG,QAAO,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC;AAC3C,aAAW,KAAK,KAAM,KAAI,EAAE,OAAO,WAAW,EAAG,QAAO,CAAC;AACzD,QAAM,MAAkB,CAAC;AACzB,QAAM,MAAM,IAAI,MAAM,KAAK,MAAM,EAAE,KAAK,CAAC;AACzC,SAAO,MAAM;AACX,UAAM,OAAuD,CAAC;AAC9D,aAASA,KAAI,GAAGA,KAAI,KAAK,QAAQA,MAAK;AACpC,YAAM,OAAO,KAAKA,EAAC;AACnB,YAAM,IAAI,KAAK,OAAO,IAAIA,EAAC,CAAW;AACtC,WAAK,KAAK,IAAI,IAAI,EAAE,IAAI,EAAE,IAAI,OAAO,EAAE,MAAM;AAAA,IAC/C;AACA,QAAI,KAAK,EAAE,MAAM,KAAK,CAAC;AAEvB,QAAI,IAAI;AACR,WAAO,IAAI,KAAK,QAAQ;AACtB,YAAM,OAAQ,IAAI,CAAC,IAAe;AAClC,YAAM,OAAO,KAAK,CAAC;AACnB,UAAI,OAAO,KAAK,OAAO,QAAQ;AAC7B,YAAI,CAAC,IAAI;AACT;AAAA,MACF;AACA,UAAI,CAAC,IAAI;AACT;AAAA,IACF;AACA,QAAI,MAAM,KAAK,OAAQ;AAAA,EACzB;AACA,SAAO;AACT;AAEA,SAAS,eAAuB;AAG9B,QAAM,IAAI,KAAK,IAAI,EAAE,SAAS,EAAE;AAChC,MAAI,IAAI;AACR,WAAS,IAAI,GAAG,IAAI,GAAG,IAAK,MAAK,KAAK,MAAM,KAAK,OAAO,IAAI,EAAE,EAAE,SAAS,EAAE;AAC3E,SAAO,OAAO,CAAC,IAAI,CAAC;AACtB;AAEA,SAAS,gBAAwB,KAAkC;AACjE,QAAM,IAAI;AACV,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,SAAS,EAAE,OAAO,OAAO,OAAO,EAAE;AAAA,IAClC,SAAS;AAAA,IACT,YAAY;AAAA,IACZ,OAAO;AAAA,MACL,SAAS,OAAO,GAAG,YAAY,WAAW,EAAE,UAAU,OAAO,GAAG;AAAA,MAChE,MAAM,OAAO,GAAG,SAAS,WAAW,EAAE,OAAO;AAAA,IAC/C;AAAA,EACF;AACF;AAEA,eAAsB,eACpB,MAC+B;AAC/B,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,OAAO,KAAK,IAAI,GAAG,KAAK,QAAQ,CAAC;AACvC,QAAM,iBAAiB,KAAK,IAAI,GAAG,KAAK,kBAAkB,CAAC;AAC3D,QAAM,cAAc,KAAK,eAAe,OAAO;AAC/C,QAAM,cAAc,KAAK,eAAe,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,IAAI;AAEnE,QAAM,OAAO,UAAU,KAAK,IAAI;AAChC,QAAM,WAAW,KAAK,SAClB,KAAK,OAAO,CAAC,MAAO,KAAK,OAAoC,CAAC,CAAC,IAC/D;AACJ,QAAM,cAAc,KAAK,SAAS,SAAS;AAE3C,QAAM,UAAwB,CAAC;AAC/B,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,aAAS,IAAI,GAAG,IAAI,MAAM,KAAK;AAC7B,cAAQ,KAAK;AAAA,QACX,MAAO,SAAS,CAAC,EAAe;AAAA,QAChC,KAAK;AAAA,QACL,SAAS,IAAI,OAAO;AAAA,MACtB,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,cAAuE,CAAC;AAC9E,MAAI,iBAAiB;AACrB,MAAI,qBAAqB;AACzB,MAAI,eAAe;AACnB,MAAI,mBAAmB;AAEvB,QAAM,UAAU,MAAe,KAAK,QAAQ,YAAY;AAOxD,MAAI,WAAW;AACf,MAAI,SAAS;AACb,MAAI;AACJ,QAAM,OAAO,IAAI,QAAc,CAAC,QAAQ;AACtC,iBAAa;AAAA,EACf,CAAC;AAED,QAAM,OAAO,MAAY;AACvB,WAAO,WAAW,kBAAkB,SAAS,QAAQ,QAAQ;AAC3D,UAAI,QAAQ,KAAK,oBAAoB;AAEnC,cAAM,OAAO,QAAQ,SAAS;AAC9B,4BAAoB;AACpB,iBAAS,QAAQ;AACjB;AAAA,MACF;AACA,YAAM,OAAO,QAAQ,QAAQ;AAC7B;AAGA,YAAM,SAAS,EAAE,MAAM,MAAM,CAAC,EAA0B;AACxD,kBAAY,KAAK,MAAM;AACvB,YAAM,WAAwC,YAAY;AACxD,YAAI;AACF,iBAAO,MAAM,KAAK,QAAQ,IAAI;AAAA,QAChC,SAAS,KAAK;AACZ,iBAAO,gBAAwB,GAAG;AAAA,QACpC;AAAA,MACF,GAAG;AACH,cAAQ,KAAK,CAAC,WAAW;AACvB,eAAO,KAAK,KAAK,MAAM;AACvB;AACA,0BAAkB,OAAO;AACzB,YAAI,kBAAkB,eAAe,CAAC,oBAAoB;AACxD,+BAAqB;AAErB,kBAAQ,KAAK,+BAA+B;AAAA,QAC9C;AACA,YAAI;AACF,eAAK,iBAAiB,MAAM,MAAM;AAAA,QACpC,QAAQ;AAAA,QAGR;AACA;AACA,YAAI,SAAS,QAAQ,QAAQ;AAC3B,eAAK;AAAA,QACP,WAAW,aAAa,GAAG;AACzB,uBAAa;AAAA,QACf;AAAA,MACF,CAAC;AAAA,IACH;AACA,QAAI,UAAU,QAAQ,UAAU,aAAa,EAAG,cAAa;AAAA,EAC/D;AAEA,QAAM,UAAU,MAAY;AAG1B,QAAI,SAAS,QAAQ,QAAQ;AAC3B,0BAAoB,QAAQ,SAAS;AACrC,eAAS,QAAQ;AAAA,IACnB;AACA,QAAI,aAAa,EAAG,cAAa;AAAA,EACnC;AACA,MAAI,KAAK,QAAQ;AACf,QAAI,KAAK,OAAO,SAAS;AACvB,yBAAmB,QAAQ;AAC3B,eAAS,QAAQ;AACjB,mBAAa;AAAA,IACf,OAAO;AACL,WAAK,OAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAAA,IAC/D;AAAA,EACF;AAEA,MAAI,QAAQ,WAAW,GAAG;AACxB,iBAAa;AAAA,EACf,OAAO;AACL,SAAK;AAAA,EACP;AAEA,QAAM;AACN,MAAI,KAAK,OAAQ,MAAK,OAAO,oBAAoB,SAAS,OAAO;AAEjE,cAAY,KAAK,CAAC,GAAG,MAAM,EAAE,KAAK,UAAU,EAAE,KAAK,OAAO;AAE1D,MAAI,OAAO;AACX,MAAI,WAAW;AACf,MAAI,YAAY;AAChB,MAAI,WAAW;AACf,aAAW,EAAE,KAAK,KAAK,aAAa;AAClC,eAAW,KAAK,MAAM;AACpB;AACA,YAAM,UAAU,EAAE,UAAU;AAC5B,UAAI,CAAC,WAAW,EAAE,QAAQ,MAAO;AACjC,kBAAY,UAAU,IAAI,EAAE,QAAQ;AACpC,mBAAa,EAAE;AAAA,IACjB;AAAA,EACF;AAEA,QAAM,SAAS,YAAY,aAAa,KAAK,MAAM,WAAW;AAE9D,SAAO;AAAA,IACL,OAAO;AAAA,IACP;AAAA,IACA,SAAS;AAAA,MACP,YAAY,QAAQ;AAAA,MACpB;AAAA,MACA,cAAc,mBAAmB,cAAc;AAAA,MAC/C,iBAAiB,aAAa,IAAI,IAAI,OAAO;AAAA,MAC7C,kBAAkB,aAAa,IAAI,IAAI,WAAW;AAAA,MAClD,cAAc;AAAA,MACd,YAAY,KAAK,IAAI,IAAI;AAAA,IAC3B;AAAA,IACA,UAAU,aAAa;AAAA,EACzB;AACF;","names":["i"]}

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.35.0",
+    "version": "0.36.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.35.0",
+  "version": "0.36.0",
   "description": "Substrate for self-improving agents: traces, verifiable rewards, preferences, GEPA / reflective mutation, auto-research, replay, sequential anytime-valid stats, and release gates.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -94,6 +94,11 @@
       "import": "./dist/knowledge/index.js",
       "default": "./dist/knowledge/index.js"
     },
+    "./matrix": {
+      "types": "./dist/matrix/index.d.ts",
+      "import": "./dist/matrix/index.js",
+      "default": "./dist/matrix/index.js"
+    },
     "./openapi.json": {
       "default": "./dist/openapi.json"
     }
@@ -128,8 +133,18 @@
     "hono": "^4.12.16",
     "zod": "^4.3.6"
   },
+  "peerDependencies": {
+    "@tangle-network/agent-runtime": "^0.21.0",
+    "@tangle-network/sandbox": "^0.2.1"
+  },
+  "peerDependenciesMeta": {
+    "@tangle-network/agent-runtime": { "optional": true },
+    "@tangle-network/sandbox": { "optional": true }
+  },
   "devDependencies": {
     "@biomejs/biome": "^2.4.15",
+    "@tangle-network/agent-runtime": "^0.21.0",
+    "@tangle-network/sandbox": "^0.2.1",
     "@types/node": "^25.6.0",
     "openapi3-ts": "^4.5.0",
     "tsup": "^8.0.0",
@@ -138,7 +153,7 @@
   },
   "pnpm": {
     "minimumReleaseAge": 4320,
-    "minimumReleaseAgeExclude": [],
+    "minimumReleaseAgeExclude": ["@tangle-network/sandbox", "@tangle-network/agent-runtime"],
     "overrides": {
       "postcss@<8.5.10": "^8.5.10",
       "ws@>=8.0.0 <8.20.1": "^8.20.1"