@tangle-network/agent-eval 0.35.0 → 0.37.0

package/dist/matrix/index.d.ts

@@ -0,0 +1,141 @@
+import { DefaultVerdict } from '@tangle-network/agent-runtime/loops';
+export { DefaultVerdict } from '@tangle-network/agent-runtime/loops';
+
+/**
+ * @experimental
+ *
+ * N-axis cartesian matrix over substrate types — types module.
+ *
+ * The matrix is a runner + aggregator. It iterates the cartesian product of
+ * caller-provided axes (any value type — `AgentProfile` from sandbox, `Driver`
+ * / `Validator` from agent-runtime, rubric records, thinking levels, anything)
+ * and aggregates per-axis pass/score/cost summaries. Substrate types are
+ * imported at the boundary by JSDoc only; the matrix never wraps them.
+ */
+
+/** One axis = one dimension to iterate. `V` is the value type — pass any
+ *  substrate type (AgentProfile, Driver, Validator, rubric record). */
+interface MatrixAxis<V> {
+    /** Axis name. Becomes the key in `MatrixResult.byAxis`. */
+    name: string;
+    /** Stable id per value. Used as the bucket key in aggregation. */
+    values: Array<{
+        id: string;
+        value: V;
+    }>;
+    /** Optional bucket label override. Receives the same `(value, id)` the
+     *  runner stored on the cell; default label is `id`. */
+    label?: (value: V, id: string) => string;
+}
+/** A cell carries one picked value from each axis, keyed by axis name. */
+interface MatrixCell {
+    axes: Record<string, {
+        id: string;
+        value: unknown;
+    }>;
+    /** 0-based replicate index within the same axis combination. */
+    rep: number;
+    /** Stable sort key — preserves cartesian order across concurrent execution. */
+    ordinal: number;
+}
+interface CellResult<Output> {
+    output: Output;
+    verdict: DefaultVerdict;
+    costUsd: number;
+    durationMs: number;
+    runId?: string;
+    /** Populated when `runCell` threw. The cell contributes 0 to passRate AND
+     *  meanScore regardless of `verdict`. */
+    error?: {
+        message: string;
+        kind: string;
+    };
+}
+interface AxisSummary {
+    axisName: string;
+    axisValue: string;
+    cells: number;
+    passRate: number;
+    meanScore: number;
+    p50Score: number;
+    p90Score: number;
+    totalCostUsd: number;
+    meanDurationMs: number;
+}
+interface MatrixResult<Output> {
+    cells: Array<{
+        cell: MatrixCell;
+        runs: CellResult<Output>[];
+    }>;
+    /** `byAxis[axisName][axisValueId] = summary`. Populated only for axes
+     *  named in `aggregateBy` (default = every axis in `axes`). */
+    byAxis: Record<string, Record<string, AxisSummary>>;
+    summary: {
+        totalCells: number;
+        runsExecuted: number;
+        /** Cells removed by `filter` plus cells unscheduled after the cost
+         *  ceiling or abort signal tripped. */
+        cellsSkipped: number;
+        overallPassRate: number;
+        overallMeanScore: number;
+        totalCostUsd: number;
+        durationMs: number;
+    };
+    /** Stable id-like string generated at the end of the run. */
+    matrixId: string;
+}
+interface RunAgentMatrixOptions<Output> {
+    axes: MatrixAxis<unknown>[];
+    /** User-supplied cell executor. May throw; the matrix captures throws as
+     *  `CellResult.error` and continues. */
+    runCell: (cell: MatrixCell) => Promise<CellResult<Output>>;
+    /** Replicates per cell. Default 1. */
+    reps?: number;
+    /** Prune cells from the cartesian BEFORE rep expansion. */
+    filter?: (cell: Omit<MatrixCell, 'rep' | 'ordinal'>) => boolean;
+    /** Axes to aggregate into `byAxis`. Default: every axis in `axes`. */
+    aggregateBy?: string[];
+    /** Max concurrent in-flight `runCell` invocations. Default 4. */
+    maxConcurrency?: number;
+    /** Cumulative-cost abort threshold (USD). When the running sum of
+     *  `result.costUsd` crosses this value, no new cells are scheduled.
+     *  In-flight cells finish. Default `Infinity`. */
+    costCeiling?: number;
+    /** Fires once per executed cell, after its promise settles. */
+    onCellComplete?: (cell: MatrixCell, result: CellResult<Output>) => void;
+    /** External cancellation. Aborts in-flight cells via a forwarded signal
+     *  and suppresses scheduling of new ones. */
+    signal?: AbortSignal;
+}
+
+/**
+ * Per-axis aggregation of cell runs into `AxisSummary` rows.
+ *
+ * Pure: consumes the final `cells: [{cell, runs}]` array and returns the
+ * `byAxis` table. Error runs contribute 0 to passRate and meanScore. Cost
+ * and duration always count — the budget was spent regardless.
+ */
+
+interface Row<Output> {
+    cell: MatrixCell;
+    result: CellResult<Output>;
+}
+declare function summariseRows<Output>(rows: Row<Output>[], axisName: string, axisValue: string): AxisSummary;
+declare function buildByAxis<Output>(cells: MatrixResult<Output>['cells'], axes: MatrixAxis<unknown>[], aggregateBy: string[]): Record<string, Record<string, AxisSummary>>;
+
+/**
+ * N-axis cartesian runner.
+ *
+ * Expansion order: cartesian over `axes` in declared order, then `reps` as the
+ * inner-most dim → `ordinal = (cartIdx * reps) + rep`. The returned
+ * `cells[]` is sorted by `ordinal` so concurrent execution does not reorder
+ * the output.
+ *
+ * Scheduling is a sliding window of in-flight promises capped at
+ * `maxConcurrency`. The window stops admitting new cells when the cost
+ * ceiling trips or the abort signal fires; in-flight cells finish.
+ */
+
+declare function runAgentMatrix<Output>(opts: RunAgentMatrixOptions<Output>): Promise<MatrixResult<Output>>;
+
+export { type AxisSummary, type CellResult, type MatrixAxis, type MatrixCell, type MatrixResult, type RunAgentMatrixOptions, buildByAxis, runAgentMatrix, summariseRows };

package/dist/matrix/index.js

@@ -0,0 +1,277 @@
+import "../chunk-PZ5AY32C.js";
+
+// src/matrix/aggregation.ts
+function flattenRuns(cells) {
+  const rows = [];
+  for (const { cell, runs } of cells) {
+    for (const result of runs) rows.push({ cell, result });
+  }
+  return rows;
+}
+function quantile(sorted, q) {
+  if (sorted.length === 0) return 0;
+  if (sorted.length === 1) return sorted[0];
+  const pos = (sorted.length - 1) * q;
+  const lo = Math.floor(pos);
+  const hi = Math.ceil(pos);
+  if (lo === hi) return sorted[lo];
+  const frac = pos - lo;
+  return sorted[lo] * (1 - frac) + sorted[hi] * frac;
+}
+function summariseRows(rows, axisName, axisValue) {
+  if (rows.length === 0) {
+    return {
+      axisName,
+      axisValue,
+      cells: 0,
+      passRate: 0,
+      meanScore: 0,
+      p50Score: 0,
+      p90Score: 0,
+      totalCostUsd: 0,
+      meanDurationMs: 0
+    };
+  }
+  let pass = 0;
+  let scoreSum = 0;
+  let costSum = 0;
+  let durSum = 0;
+  const scores = [];
+  for (const { result } of rows) {
+    const errored = result.error !== void 0;
+    const score = errored ? 0 : result.verdict.score;
+    const valid = !errored && result.verdict.valid;
+    if (valid) pass++;
+    scoreSum += score;
+    scores.push(score);
+    costSum += result.costUsd;
+    durSum += result.durationMs;
+  }
+  scores.sort((a, b) => a - b);
+  return {
+    axisName,
+    axisValue,
+    cells: rows.length,
+    passRate: pass / rows.length,
+    meanScore: scoreSum / rows.length,
+    p50Score: quantile(scores, 0.5),
+    p90Score: quantile(scores, 0.9),
+    totalCostUsd: costSum,
+    meanDurationMs: durSum / rows.length
+  };
+}
+function bucketBy(rows, axisName, labelFor) {
+  const buckets = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    const slot = row.cell.axes[axisName];
+    if (!slot) continue;
+    const id = slot.id;
+    let arr = buckets.get(id);
+    if (!arr) {
+      arr = [];
+      buckets.set(id, arr);
+    }
+    arr.push(row);
+  }
+  const out = {};
+  for (const id of [...buckets.keys()].sort()) {
+    out[id] = summariseRows(buckets.get(id), axisName, labelFor(id));
+  }
+  return out;
+}
+function buildByAxis(cells, axes, aggregateBy) {
+  const rows = flattenRuns(cells);
+  const byName = new Map(axes.map((a) => [a.name, a]));
+  const byAxis = {};
+  for (const name of aggregateBy) {
+    const axis = byName.get(name);
+    const labelFor = (id) => {
+      if (!axis?.label) return id;
+      const found = axis.values.find((v) => v.id === id);
+      if (!found) return id;
+      return axis.label(found.value, id);
+    };
+    byAxis[name] = bucketBy(rows, name, labelFor);
+  }
+  return byAxis;
+}
+
+// src/matrix/runner.ts
+function cartesian(axes) {
+  if (axes.length === 0) return [{ axes: {} }];
+  for (const a of axes) if (a.values.length === 0) return [];
+  const out = [];
+  const idx = new Array(axes.length).fill(0);
+  while (true) {
+    const slot = {};
+    for (let i2 = 0; i2 < axes.length; i2++) {
+      const axis = axes[i2];
+      const v = axis.values[idx[i2]];
+      slot[axis.name] = { id: v.id, value: v.value };
+    }
+    out.push({ axes: slot });
+    let i = 0;
+    while (i < axes.length) {
+      const next = idx[i] + 1;
+      const axis = axes[i];
+      if (next < axis.values.length) {
+        idx[i] = next;
+        break;
+      }
+      idx[i] = 0;
+      i++;
+    }
+    if (i === axes.length) break;
+  }
+  return out;
+}
+function makeMatrixId() {
+  const t = Date.now().toString(36);
+  let r = "";
+  for (let i = 0; i < 8; i++) r += Math.floor(Math.random() * 16).toString(16);
+  return `mtx_${t}_${r}`;
+}
+function makeErrorResult(err) {
+  const e = err;
+  return {
+    output: void 0,
+    verdict: { valid: false, score: 0 },
+    costUsd: 0,
+    durationMs: 0,
+    error: {
+      message: typeof e?.message === "string" ? e.message : String(err),
+      kind: typeof e?.name === "string" ? e.name : "Error"
+    }
+  };
+}
+async function runAgentMatrix(opts) {
+  const startedAt = Date.now();
+  const reps = Math.max(1, opts.reps ?? 1);
+  const maxConcurrency = Math.max(1, opts.maxConcurrency ?? 4);
+  const costCeiling = opts.costCeiling ?? Number.POSITIVE_INFINITY;
+  const aggregateBy = opts.aggregateBy ?? opts.axes.map((a) => a.name);
+  const base = cartesian(opts.axes);
+  const filtered = opts.filter ? base.filter((c) => opts.filter(c)) : base;
+  const filteredOut = base.length - filtered.length;
+  const planned = [];
+  for (let i = 0; i < filtered.length; i++) {
+    for (let r = 0; r < reps; r++) {
+      planned.push({
+        axes: filtered[i].axes,
+        rep: r,
+        ordinal: i * reps + r
+      });
+    }
+  }
+  const cellRecords = [];
+  let cumulativeCost = 0;
+  let costCeilingReached = false;
+  let runsExecuted = 0;
+  let cellsUnscheduled = 0;
+  const aborted = () => opts.signal?.aborted === true;
+  let inFlight = 0;
+  let cursor = 0;
+  let resolveAll;
+  const done = new Promise((res) => {
+    resolveAll = res;
+  });
+  const pump = () => {
+    while (inFlight < maxConcurrency && cursor < planned.length) {
+      if (aborted() || costCeilingReached) {
+        const left = planned.length - cursor;
+        cellsUnscheduled += left;
+        cursor = planned.length;
+        break;
+      }
+      const cell = planned[cursor++];
+      inFlight++;
+      const record = { cell, runs: [] };
+      cellRecords.push(record);
+      const promise = (async () => {
+        try {
+          return await opts.runCell(cell);
+        } catch (err) {
+          return makeErrorResult(err);
+        }
+      })();
+      promise.then((result) => {
+        record.runs.push(result);
+        runsExecuted++;
+        cumulativeCost += result.costUsd;
+        if (cumulativeCost >= costCeiling && !costCeilingReached) {
+          costCeilingReached = true;
+          console.warn("[matrix] cost ceiling reached");
+        }
+        try {
+          opts.onCellComplete?.(cell, result);
+        } catch {
+        }
+        inFlight--;
+        if (cursor < planned.length) {
+          pump();
+        } else if (inFlight === 0) {
+          resolveAll?.();
+        }
+      });
+    }
+    if (cursor >= planned.length && inFlight === 0) resolveAll?.();
+  };
+  const onAbort = () => {
+    if (cursor < planned.length) {
+      cellsUnscheduled += planned.length - cursor;
+      cursor = planned.length;
+    }
+    if (inFlight === 0) resolveAll?.();
+  };
+  if (opts.signal) {
+    if (opts.signal.aborted) {
+      cellsUnscheduled = planned.length;
+      cursor = planned.length;
+      resolveAll?.();
+    } else {
+      opts.signal.addEventListener("abort", onAbort, { once: true });
+    }
+  }
+  if (planned.length === 0) {
+    resolveAll?.();
+  } else {
+    pump();
+  }
+  await done;
+  if (opts.signal) opts.signal.removeEventListener("abort", onAbort);
+  cellRecords.sort((a, b) => a.cell.ordinal - b.cell.ordinal);
+  let pass = 0;
+  let scoreSum = 0;
+  let totalCost = 0;
+  let runCount = 0;
+  for (const { runs } of cellRecords) {
+    for (const r of runs) {
+      runCount++;
+      const errored = r.error !== void 0;
+      if (!errored && r.verdict.valid) pass++;
+      scoreSum += errored ? 0 : r.verdict.score;
+      totalCost += r.costUsd;
+    }
+  }
+  const byAxis = buildByAxis(cellRecords, opts.axes, aggregateBy);
+  return {
+    cells: cellRecords,
+    byAxis,
+    summary: {
+      totalCells: planned.length,
+      runsExecuted,
+      cellsSkipped: cellsUnscheduled + filteredOut * reps,
+      overallPassRate: runCount === 0 ? 0 : pass / runCount,
+      overallMeanScore: runCount === 0 ? 0 : scoreSum / runCount,
+      totalCostUsd: totalCost,
+      durationMs: Date.now() - startedAt
+    },
+    matrixId: makeMatrixId()
+  };
+}
+export {
+  buildByAxis,
+  runAgentMatrix,
+  summariseRows
+};
+//# sourceMappingURL=index.js.map

package/dist/matrix/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/matrix/aggregation.ts","../../src/matrix/runner.ts"],"sourcesContent":["/**\n * Per-axis aggregation of cell runs into `AxisSummary` rows.\n *\n * Pure: consumes the final `cells: [{cell, runs}]` array and returns the\n * `byAxis` table. Error runs contribute 0 to passRate and meanScore. Cost\n * and duration always count — the budget was spent regardless.\n */\n\nimport type { AxisSummary, CellResult, MatrixAxis, MatrixCell, MatrixResult } from './types'\n\ninterface Row<Output> {\n  cell: MatrixCell\n  result: CellResult<Output>\n}\n\nfunction flattenRuns<Output>(cells: MatrixResult<Output>['cells']): Row<Output>[] {\n  const rows: Row<Output>[] = []\n  for (const { cell, runs } of cells) {\n    for (const result of runs) rows.push({ cell, result })\n  }\n  return rows\n}\n\nfunction quantile(sorted: number[], q: number): number {\n  if (sorted.length === 0) return 0\n  if (sorted.length === 1) return sorted[0] as number\n  const pos = (sorted.length - 1) * q\n  const lo = Math.floor(pos)\n  const hi = Math.ceil(pos)\n  if (lo === hi) return sorted[lo] as number\n  const frac = pos - lo\n  return (sorted[lo] as number) * (1 - frac) + (sorted[hi] as number) * frac\n}\n\nexport function summariseRows<Output>(\n  rows: Row<Output>[],\n  axisName: string,\n  axisValue: string,\n): AxisSummary {\n  if (rows.length === 0) {\n    return {\n      axisName,\n      axisValue,\n      cells: 0,\n      passRate: 0,\n      meanScore: 0,\n      p50Score: 0,\n      p90Score: 0,\n      totalCostUsd: 0,\n      meanDurationMs: 0,\n    }\n  }\n  let pass = 0\n  let scoreSum = 0\n  let costSum = 0\n  let durSum = 0\n  const scores: number[] = []\n  for (const { result } of rows) {\n    const errored = result.error !== undefined\n    const score = errored ? 0 : result.verdict.score\n    const valid = !errored && result.verdict.valid\n    if (valid) pass++\n    scoreSum += score\n    scores.push(score)\n    costSum += result.costUsd\n    durSum += result.durationMs\n  }\n  scores.sort((a, b) => a - b)\n  return {\n    axisName,\n    axisValue,\n    cells: rows.length,\n    passRate: pass / rows.length,\n    meanScore: scoreSum / rows.length,\n    p50Score: quantile(scores, 0.5),\n    p90Score: quantile(scores, 0.9),\n    totalCostUsd: costSum,\n    meanDurationMs: durSum / rows.length,\n  }\n}\n\nfunction bucketBy<Output>(\n  rows: Row<Output>[],\n  axisName: string,\n  labelFor: (id: string) => string,\n): Record<string, AxisSummary> {\n  const buckets = new Map<string, Row<Output>[]>()\n  for (const row of rows) {\n    const slot = row.cell.axes[axisName]\n    if (!slot) continue\n    const id = slot.id\n    let arr = buckets.get(id)\n    if (!arr) {\n      arr = []\n      buckets.set(id, arr)\n    }\n    arr.push(row)\n  }\n  const out: Record<string, AxisSummary> = {}\n  // Sorted keys for deterministic JSON serialisation.\n  for (const id of [...buckets.keys()].sort()) {\n    out[id] = summariseRows(buckets.get(id) as Row<Output>[], axisName, labelFor(id))\n  }\n  return out\n}\n\nexport function buildByAxis<Output>(\n  cells: MatrixResult<Output>['cells'],\n  axes: MatrixAxis<unknown>[],\n  aggregateBy: string[],\n): Record<string, Record<string, AxisSummary>> {\n  const rows = flattenRuns(cells)\n  const byName = new Map(axes.map((a) => [a.name, a]))\n  const byAxis: Record<string, Record<string, AxisSummary>> = {}\n  for (const name of aggregateBy) {\n    const axis = byName.get(name)\n    const labelFor = (id: string): string => {\n      if (!axis?.label) return id\n      const found = axis.values.find((v) => v.id === id)\n      if (!found) return id\n      return axis.label(found.value, id)\n    }\n    byAxis[name] = bucketBy(rows, name, labelFor)\n  }\n  return byAxis\n}\n","/**\n * N-axis cartesian runner.\n *\n * Expansion order: cartesian over `axes` in declared order, then `reps` as the\n * inner-most dim → `ordinal = (cartIdx * reps) + rep`. The returned\n * `cells[]` is sorted by `ordinal` so concurrent execution does not reorder\n * the output.\n *\n * Scheduling is a sliding window of in-flight promises capped at\n * `maxConcurrency`. The window stops admitting new cells when the cost\n * ceiling trips or the abort signal fires; in-flight cells finish.\n */\n\nimport { buildByAxis } from './aggregation'\nimport type {\n  CellResult,\n  MatrixAxis,\n  MatrixCell,\n  MatrixResult,\n  RunAgentMatrixOptions,\n} from './types'\n\ninterface BaseCell {\n  axes: Record<string, { id: string; value: unknown }>\n}\n\nfunction cartesian(axes: MatrixAxis<unknown>[]): BaseCell[] {\n  // Empty axes (`values=[]`) collapse the whole product to zero cells. An\n  // empty `axes` array yields a single empty-axes cell — degenerate but\n  // valid (caller is iterating only reps).\n  if (axes.length === 0) return [{ axes: {} }]\n  for (const a of axes) if (a.values.length === 0) return []\n  const out: BaseCell[] = []\n  const idx = new Array(axes.length).fill(0)\n  while (true) {\n    const slot: Record<string, { id: string; value: unknown }> = {}\n    for (let i = 0; i < axes.length; i++) {\n      const axis = axes[i] as MatrixAxis<unknown>\n      const v = axis.values[idx[i] as number] as { id: string; value: unknown }\n      slot[axis.name] = { id: v.id, value: v.value }\n    }\n    out.push({ axes: slot })\n    // Increment like an odometer, left-most axis is fastest.\n    let i = 0\n    while (i < axes.length) {\n      const next = (idx[i] as number) + 1\n      const axis = axes[i] as MatrixAxis<unknown>\n      if (next < axis.values.length) {\n        idx[i] = next\n        break\n      }\n      idx[i] = 0\n      i++\n    }\n    if (i === axes.length) break\n  }\n  return out\n}\n\nfunction makeMatrixId(): string {\n  // Stable id-like string: time + 8 random hex chars. Avoids node:crypto\n  // import to keep the matrix dep-free.\n  const t = Date.now().toString(36)\n  let r = ''\n  for (let i = 0; i < 8; i++) r += Math.floor(Math.random() * 16).toString(16)\n  return `mtx_${t}_${r}`\n}\n\nfunction makeErrorResult<Output>(err: unknown): CellResult<Output> {\n  const e = err as { message?: string; name?: string }\n  return {\n    output: undefined as unknown as Output,\n    verdict: { valid: false, score: 0 },\n    costUsd: 0,\n    durationMs: 0,\n    error: {\n      message: typeof e?.message === 'string' ? e.message : String(err),\n      kind: typeof e?.name === 'string' ? e.name : 'Error',\n    },\n  }\n}\n\nexport async function runAgentMatrix<Output>(\n  opts: RunAgentMatrixOptions<Output>,\n): Promise<MatrixResult<Output>> {\n  const startedAt = Date.now()\n  const reps = Math.max(1, opts.reps ?? 1)\n  const maxConcurrency = Math.max(1, opts.maxConcurrency ?? 4)\n  const costCeiling = opts.costCeiling ?? Number.POSITIVE_INFINITY\n  const aggregateBy = opts.aggregateBy ?? opts.axes.map((a) => a.name)\n\n  const base = cartesian(opts.axes)\n  const filtered = opts.filter\n    ? base.filter((c) => (opts.filter as (b: BaseCell) => boolean)(c))\n    : base\n  const filteredOut = base.length - filtered.length\n\n  const planned: MatrixCell[] = []\n  for (let i = 0; i < filtered.length; i++) {\n    for (let r = 0; r < reps; r++) {\n      planned.push({\n        axes: (filtered[i] as BaseCell).axes,\n        rep: r,\n        ordinal: i * reps + r,\n      })\n    }\n  }\n\n  const cellRecords: Array<{ cell: MatrixCell; runs: CellResult<Output>[] }> = []\n  let cumulativeCost = 0\n  let costCeilingReached = false\n  let runsExecuted = 0\n  let cellsUnscheduled = 0\n\n  const aborted = (): boolean => opts.signal?.aborted === true\n\n  // Per-run abort controller forwards the external signal so cell executors\n  // see cancellation. We don't expose it on `MatrixCell` — the signature on\n  // `runCell` per the public API is `(cell) => Promise<...>`. Executors that\n  // need cancellation use the external signal directly via closure.\n\n  let inFlight = 0\n  let cursor = 0\n  let resolveAll: (() => void) | undefined\n  const done = new Promise<void>((res) => {\n    resolveAll = res\n  })\n\n  const pump = (): void => {\n    while (inFlight < maxConcurrency && cursor < planned.length) {\n      if (aborted() || costCeilingReached) {\n        // Drain remaining as unscheduled.\n        const left = planned.length - cursor\n        cellsUnscheduled += left\n        cursor = planned.length\n        break\n      }\n      const cell = planned[cursor++] as MatrixCell\n      inFlight++\n      // Lazily allocate the record so cells appear in `cells[]` in any\n      // arrival order; we sort by ordinal at the end.\n      const record = { cell, runs: [] as CellResult<Output>[] }\n      cellRecords.push(record)\n      const promise: Promise<CellResult<Output>> = (async () => {\n        try {\n          return await opts.runCell(cell)\n        } catch (err) {\n          return makeErrorResult<Output>(err)\n        }\n      })()\n      promise.then((result) => {\n        record.runs.push(result)\n        runsExecuted++\n        cumulativeCost += result.costUsd\n        if (cumulativeCost >= costCeiling && !costCeilingReached) {\n          costCeilingReached = true\n          // eslint-disable-next-line no-console\n          console.warn('[matrix] cost ceiling reached')\n        }\n        try {\n          opts.onCellComplete?.(cell, result)\n        } catch {\n          // onCellComplete is observational — swallow throws so a noisy\n          // callback can't tank the run.\n        }\n        inFlight--\n        if (cursor < planned.length) {\n          pump()\n        } else if (inFlight === 0) {\n          resolveAll?.()\n        }\n      })\n    }\n    if (cursor >= planned.length && inFlight === 0) resolveAll?.()\n  }\n\n  const onAbort = (): void => {\n    // External abort: stop scheduling. In-flight cells finish; their\n    // executors observe `opts.signal.aborted` directly via closure.\n    if (cursor < planned.length) {\n      cellsUnscheduled += planned.length - cursor\n      cursor = planned.length\n    }\n    if (inFlight === 0) resolveAll?.()\n  }\n  if (opts.signal) {\n    if (opts.signal.aborted) {\n      cellsUnscheduled = planned.length\n      cursor = planned.length\n      resolveAll?.()\n    } else {\n      opts.signal.addEventListener('abort', onAbort, { once: true })\n    }\n  }\n\n  if (planned.length === 0) {\n    resolveAll?.()\n  } else {\n    pump()\n  }\n\n  await done\n  if (opts.signal) opts.signal.removeEventListener('abort', onAbort)\n\n  cellRecords.sort((a, b) => a.cell.ordinal - b.cell.ordinal)\n\n  let pass = 0\n  let scoreSum = 0\n  let totalCost = 0\n  let runCount = 0\n  for (const { runs } of cellRecords) {\n    for (const r of runs) {\n      runCount++\n      const errored = r.error !== undefined\n      if (!errored && r.verdict.valid) pass++\n      scoreSum += errored ? 0 : r.verdict.score\n      totalCost += r.costUsd\n    }\n  }\n\n  const byAxis = buildByAxis(cellRecords, opts.axes, aggregateBy)\n\n  return {\n    cells: cellRecords,\n    byAxis,\n    summary: {\n      totalCells: planned.length,\n      runsExecuted,\n      cellsSkipped: cellsUnscheduled + filteredOut * reps,\n      overallPassRate: runCount === 0 ? 0 : pass / runCount,\n      overallMeanScore: runCount === 0 ? 0 : scoreSum / runCount,\n      totalCostUsd: totalCost,\n      durationMs: Date.now() - startedAt,\n    },\n    matrixId: makeMatrixId(),\n  }\n}\n"],"mappings":";;;AAeA,SAAS,YAAoB,OAAqD;AAChF,QAAM,OAAsB,CAAC;AAC7B,aAAW,EAAE,MAAM,KAAK,KAAK,OAAO;AAClC,eAAW,UAAU,KAAM,MAAK,KAAK,EAAE,MAAM,OAAO,CAAC;AAAA,EACvD;AACA,SAAO;AACT;AAEA,SAAS,SAAS,QAAkB,GAAmB;AACrD,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,MAAI,OAAO,WAAW,EAAG,QAAO,OAAO,CAAC;AACxC,QAAM,OAAO,OAAO,SAAS,KAAK;AAClC,QAAM,KAAK,KAAK,MAAM,GAAG;AACzB,QAAM,KAAK,KAAK,KAAK,GAAG;AACxB,MAAI,OAAO,GAAI,QAAO,OAAO,EAAE;AAC/B,QAAM,OAAO,MAAM;AACnB,SAAQ,OAAO,EAAE,KAAgB,IAAI,QAAS,OAAO,EAAE,IAAe;AACxE;AAEO,SAAS,cACd,MACA,UACA,WACa;AACb,MAAI,KAAK,WAAW,GAAG;AACrB,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO;AAAA,MACP,UAAU;AAAA,MACV,WAAW;AAAA,MACX,UAAU;AAAA,MACV,UAAU;AAAA,MACV,cAAc;AAAA,MACd,gBAAgB;AAAA,IAClB;AAAA,EACF;AACA,MAAI,OAAO;AACX,MAAI,WAAW;AACf,MAAI,UAAU;AACd,MAAI,SAAS;AACb,QAAM,SAAmB,CAAC;AAC1B,aAAW,EAAE,OAAO,KAAK,MAAM;AAC7B,UAAM,UAAU,OAAO,UAAU;AACjC,UAAM,QAAQ,UAAU,IAAI,OAAO,QAAQ;AAC3C,UAAM,QAAQ,CAAC,WAAW,OAAO,QAAQ;AACzC,QAAI,MAAO;AACX,gBAAY;AACZ,WAAO,KAAK,KAAK;AACjB,eAAW,OAAO;AAClB,cAAU,OAAO;AAAA,EACnB;AACA,SAAO,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3B,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,OAAO,KAAK;AAAA,IACZ,UAAU,OAAO,KAAK;AAAA,IACtB,WAAW,WAAW,KAAK;AAAA,IAC3B,UAAU,SAAS,QAAQ,GAAG;AAAA,IAC9B,UAAU,SAAS,QAAQ,GAAG;AAAA,IAC9B,cAAc;AAAA,IACd,gBAAgB,SAAS,KAAK;AAAA,EAChC;AACF;AAEA,SAAS,SACP,MACA,UACA,UAC6B;AAC7B,QAAM,UAAU,oBAAI,IAA2B;AAC/C,aAAW,OAAO,MAAM;AACtB,UAAM,OAAO,IAAI,KAAK,KAAK,QAAQ;AACnC,QAAI,CAAC,KAAM;AACX,UAAM,KAAK,KAAK;AAChB,QAAI,MAAM,QAAQ,IAAI,EAAE;AACxB,QAAI,CAAC,KAAK;AACR,YAAM,CAAC;AACP,cAAQ,IAAI,IAAI,GAAG;AAAA,IACrB;AACA,QAAI,KAAK,GAAG;AAAA,EACd;AACA,QAAM,MAAmC,CAAC;AAE1C,aAAW,MAAM,CAAC,GAAG,QAAQ,KAAK,CAAC,EAAE,KAAK,GAAG;AAC3C,QAAI,EAAE,IAAI,cAAc,QAAQ,IAAI,EAAE,GAAoB,UAAU,SAAS,EAAE,CAAC;AAAA,EAClF;AACA,SAAO;AACT;AAEO,SAAS,YACd,OACA,MACA,aAC6C;AAC7C,QAAM,OAAO,YAAY,KAAK;AAC9B,QAAM,SAAS,IAAI,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AACnD,QAAM,SAAsD,CAAC;AAC7D,aAAW,QAAQ,aAAa;AAC9B,UAAM,OAAO,OAAO,IAAI,IAAI;AAC5B,UAAM,WAAW,CAAC,OAAuB;AACvC,UAAI,CAAC,MAAM,MAAO,QAAO;AACzB,YAAM,QAAQ,KAAK,OAAO,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE;AACjD,UAAI,CAAC,MAAO,QAAO;AACnB,aAAO,KAAK,MAAM,MAAM,OAAO,EAAE;AAAA,IACnC;AACA,WAAO,IAAI,IAAI,SAAS,MAAM,MAAM,QAAQ;AAAA,EAC9C;AACA,SAAO;AACT;;;ACnGA,SAAS,UAAU,MAAyC;AAI1D,MAAI,KAAK,WAAW,EAAG,QAAO,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC;AAC3C,aAAW,KAAK,KAAM,KAAI,EAAE,OAAO,WAAW,EAAG,QAAO,CAAC;AACzD,QAAM,MAAkB,CAAC;AACzB,QAAM,MAAM,IAAI,MAAM,KAAK,MAAM,EAAE,KAAK,CAAC;AACzC,SAAO,MAAM;AACX,UAAM,OAAuD,CAAC;AAC9D,aAASA,KAAI,GAAGA,KAAI,KAAK,QAAQA,MAAK;AACpC,YAAM,OAAO,KAAKA,EAAC;AACnB,YAAM,IAAI,KAAK,OAAO,IAAIA,EAAC,CAAW;AACtC,WAAK,KAAK,IAAI,IAAI,EAAE,IAAI,EAAE,IAAI,OAAO,EAAE,MAAM;AAAA,IAC/C;AACA,QAAI,KAAK,EAAE,MAAM,KAAK,CAAC;AAEvB,QAAI,IAAI;AACR,WAAO,IAAI,KAAK,QAAQ;AACtB,YAAM,OAAQ,IAAI,CAAC,IAAe;AAClC,YAAM,OAAO,KAAK,CAAC;AACnB,UAAI,OAAO,KAAK,OAAO,QAAQ;AAC7B,YAAI,CAAC,IAAI;AACT;AAAA,MACF;AACA,UAAI,CAAC,IAAI;AACT;AAAA,IACF;AACA,QAAI,MAAM,KAAK,OAAQ;AAAA,EACzB;AACA,SAAO;AACT;AAEA,SAAS,eAAuB;AAG9B,QAAM,IAAI,KAAK,IAAI,EAAE,SAAS,EAAE;AAChC,MAAI,IAAI;AACR,WAAS,IAAI,GAAG,IAAI,GAAG,IAAK,MAAK,KAAK,MAAM,KAAK,OAAO,IAAI,EAAE,EAAE,SAAS,EAAE;AAC3E,SAAO,OAAO,CAAC,IAAI,CAAC;AACtB;AAEA,SAAS,gBAAwB,KAAkC;AACjE,QAAM,IAAI;AACV,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,SAAS,EAAE,OAAO,OAAO,OAAO,EAAE;AAAA,IAClC,SAAS;AAAA,IACT,YAAY;AAAA,IACZ,OAAO;AAAA,MACL,SAAS,OAAO,GAAG,YAAY,WAAW,EAAE,UAAU,OAAO,GAAG;AAAA,MAChE,MAAM,OAAO,GAAG,SAAS,WAAW,EAAE,OAAO;AAAA,IAC/C;AAAA,EACF;AACF;AAEA,eAAsB,eACpB,MAC+B;AAC/B,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,OAAO,KAAK,IAAI,GAAG,KAAK,QAAQ,CAAC;AACvC,QAAM,iBAAiB,KAAK,IAAI,GAAG,KAAK,kBAAkB,CAAC;AAC3D,QAAM,cAAc,KAAK,eAAe,OAAO;AAC/C,QAAM,cAAc,KAAK,eAAe,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,IAAI;AAEnE,QAAM,OAAO,UAAU,KAAK,IAAI;AAChC,QAAM,WAAW,KAAK,SAClB,KAAK,OAAO,CAAC,MAAO,KAAK,OAAoC,CAAC,CAAC,IAC/D;AACJ,QAAM,cAAc,KAAK,SAAS,SAAS;AAE3C,QAAM,UAAwB,CAAC;AAC/B,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,aAAS,IAAI,GAAG,IAAI,MAAM,KAAK;AAC7B,cAAQ,KAAK;AAAA,QACX,MAAO,SAAS,CAAC,EAAe;AAAA,QAChC,KAAK;AAAA,QACL,SAAS,IAAI,OAAO;AAAA,MACtB,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,cAAuE,CAAC;AAC9E,MAAI,iBAAiB;AACrB,MAAI,qBAAqB;AACzB,MAAI,eAAe;AACnB,MAAI,mBAAmB;AAEvB,QAAM,UAAU,MAAe,KAAK,QAAQ,YAAY;AAOxD,MAAI,WAAW;AACf,MAAI,SAAS;AACb,MAAI;AACJ,QAAM,OAAO,IAAI,QAAc,CAAC,QAAQ;AACtC,iBAAa;AAAA,EACf,CAAC;AAED,QAAM,OAAO,MAAY;AACvB,WAAO,WAAW,kBAAkB,SAAS,QAAQ,QAAQ;AAC3D,UAAI,QAAQ,KAAK,oBAAoB;AAEnC,cAAM,OAAO,QAAQ,SAAS;AAC9B,4BAAoB;AACpB,iBAAS,QAAQ;AACjB;AAAA,MACF;AACA,YAAM,OAAO,QAAQ,QAAQ;AAC7B;AAGA,YAAM,SAAS,EAAE,MAAM,MAAM,CAAC,EAA0B;AACxD,kBAAY,KAAK,MAAM;AACvB,YAAM,WAAwC,YAAY;AACxD,YAAI;AACF,iBAAO,MAAM,KAAK,QAAQ,IAAI;AAAA,QAChC,SAAS,KAAK;AACZ,iBAAO,gBAAwB,GAAG;AAAA,QACpC;AAAA,MACF,GAAG;AACH,cAAQ,KAAK,CAAC,WAAW;AACvB,eAAO,KAAK,KAAK,MAAM;AACvB;AACA,0BAAkB,OAAO;AACzB,YAAI,kBAAkB,eAAe,CAAC,oBAAoB;AACxD,+BAAqB;AAErB,kBAAQ,KAAK,+BAA+B;AAAA,QAC9C;AACA,YAAI;AACF,eAAK,iBAAiB,MAAM,MAAM;AAAA,QACpC,QAAQ;AAAA,QAGR;AACA;AACA,YAAI,SAAS,QAAQ,QAAQ;AAC3B,eAAK;AAAA,QACP,WAAW,aAAa,GAAG;AACzB,uBAAa;AAAA,QACf;AAAA,MACF,CAAC;AAAA,IACH;AACA,QAAI,UAAU,QAAQ,UAAU,aAAa,EAAG,cAAa;AAAA,EAC/D;AAEA,QAAM,UAAU,MAAY;AAG1B,QAAI,SAAS,QAAQ,QAAQ;AAC3B,0BAAoB,QAAQ,SAAS;AACrC,eAAS,QAAQ;AAAA,IACnB;AACA,QAAI,aAAa,EAAG,cAAa;AAAA,EACnC;AACA,MAAI,KAAK,QAAQ;AACf,QAAI,KAAK,OAAO,SAAS;AACvB,yBAAmB,QAAQ;AAC3B,eAAS,QAAQ;AACjB,mBAAa;AAAA,IACf,OAAO;AACL,WAAK,OAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAAA,IAC/D;AAAA,EACF;AAEA,MAAI,QAAQ,WAAW,GAAG;AACxB,iBAAa;AAAA,EACf,OAAO;AACL,SAAK;AAAA,EACP;AAEA,QAAM;AACN,MAAI,KAAK,OAAQ,MAAK,OAAO,oBAAoB,SAAS,OAAO;AAEjE,cAAY,KAAK,CAAC,GAAG,MAAM,EAAE,KAAK,UAAU,EAAE,KAAK,OAAO;AAE1D,MAAI,OAAO;AACX,MAAI,WAAW;AACf,MAAI,YAAY;AAChB,MAAI,WAAW;AACf,aAAW,EAAE,KAAK,KAAK,aAAa;AAClC,eAAW,KAAK,MAAM;AACpB;AACA,YAAM,UAAU,EAAE,UAAU;AAC5B,UAAI,CAAC,WAAW,EAAE,QAAQ,MAAO;AACjC,kBAAY,UAAU,IAAI,EAAE,QAAQ;AACpC,mBAAa,EAAE;AAAA,IACjB;AAAA,EACF;AAEA,QAAM,SAAS,YAAY,aAAa,KAAK,MAAM,WAAW;AAE9D,SAAO;AAAA,IACL,OAAO;AAAA,IACP;AAAA,IACA,SAAS;AAAA,MACP,YAAY,QAAQ;AAAA,MACpB;AAAA,MACA,cAAc,mBAAmB,cAAc;AAAA,MAC/C,iBAAiB,aAAa,IAAI,IAAI,OAAO;AAAA,MAC7C,kBAAkB,aAAa,IAAI,IAAI,WAAW;AAAA,MAClD,cAAc;AAAA,MACd,YAAY,KAAK,IAAI,IAAI;AAAA,IAC3B;AAAA,IACA,UAAU,aAAa;AAAA,EACzB;AACF;","names":["i"]}

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.35.0",
+    "version": "0.37.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/traces.d.ts

@@ -1,7 +1,7 @@
 import { N as NotFoundError, R as ReplayError } from './errors-mje_cKOs.js';
 import { R as RawProviderSink, f as RawProviderEvent } from './integrity-DYR5gWlb.js';
 export { F as FileSystemRawProviderSink, c as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, d as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, P as ProviderRedactor, e as RawProviderDirection, g as RawProviderSinkFilter, h as RunIntegrityError, a as RunIntegrityExpectations, i as RunIntegrityIssue, j as RunIntegrityIssueCode, b as RunIntegrityReport, k as assertRunCaptured, l as defaultProviderRedactor, p as providerFromBaseUrl, t as throwIfRunIncomplete } from './integrity-DYR5gWlb.js';
-import { a as RunCompleteHookContext, R as RunCompleteHook } from './emitter-DP_cSSiw.js';
+import { R as RunCompleteHook, a as RunCompleteHookContext } from './emitter-DP_cSSiw.js';
 export { S as SpanHandle, T as TraceEmitter, b as TraceEmitterOptions, l as llmSpanFromProvider } from './emitter-DP_cSSiw.js';
 import { T as TraceStore } from './store-Db2Bv8Cf.js';
 export { A as Artifact, B as BudgetLedgerEntry, g as BudgetSpec, i as EventFilter, E as EventKind, j as FAILURE_CLASSES, F as FailureClass, k as FileSystemTraceStore, l as FileSystemTraceStoreOptions, G as GenericSpan, I as InMemoryTraceStore, J as JudgeSpan, L as LlmSpan, M as Message, e as RetrievalSpan, R as Run, h as RunFilter, m as RunLayer, c as RunOutcome, n as RunStatus, f as SandboxSpan, S as Span, o as SpanBase, p as SpanFilter, d as SpanKind, q as SpanStatus, r as TRACE_SCHEMA_VERSION, a as ToolSpan, b as TraceEvent, s as isJudgeSpan, t as isLlmSpan, u as isRetrievalSpan, v as isSandboxSpan, w as isToolSpan } from './store-Db2Bv8Cf.js';
@@ -65,6 +65,85 @@ interface OtlpExport {
 /** Export a single run's spans + events in OTLP/JSON. */
 declare function exportRunAsOtlp(store: TraceStore, runId: string, resourceAttrs?: Record<string, string | number | boolean>): Promise<OtlpExport>;
 
+/**
+ * OTEL span exporter — streams spans to an OTLP/HTTP collector.
+ *
+ * Reads OTEL_EXPORTER_OTLP_ENDPOINT + OTEL_EXPORTER_OTLP_HEADERS from env
+ * when no explicit config is given. Batches spans and flushes periodically
+ * or when the batch fills. No @opentelemetry SDK dependency — minimal
+ * OTLP/JSON serializer (~120 LOC) using the existing otel.ts helpers.
+ */
+interface OtelExportConfig {
+    /** OTLP endpoint. Reads OTEL_EXPORTER_OTLP_ENDPOINT env by default. */
+    endpoint?: string;
+    /** OTLP headers. Reads OTEL_EXPORTER_OTLP_HEADERS env by default. */
+    headers?: Record<string, string>;
+    /** Batch size before flush. Default 64. */
+    batchSize?: number;
+    /** Flush interval ms. Default 5000. */
+    flushIntervalMs?: number;
+    /** Resource attributes stamped on every export. */
+    resourceAttributes?: Record<string, string | number | boolean>;
+    /** Service name. Default 'agent-eval'. */
+    serviceName?: string;
+}
+interface OtelExporter {
+    /** Called by the TraceEmitter on every span close. */
+    exportSpan(span: ExportableSpan): void;
+    /** Force flush pending spans. */
+    flush(): Promise<void>;
+    /** Shutdown cleanly — flushes remaining spans and stops the timer. */
+    shutdown(): Promise<void>;
+}
+interface ExportableSpan {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    kind: string;
+    startedAt: number;
+    endedAt?: number;
+    status?: string;
+    error?: string;
+    model?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+    costUsd?: number;
+    attributes?: Record<string, unknown>;
+}
+/**
+ * Create an OTEL exporter. Returns undefined when no endpoint is configured
+ * (neither via config nor env) — callers should check before attaching.
+ */
+declare function createOtelExporter(config?: OtelExportConfig): OtelExporter | undefined;
+
+/**
+ * OTEL bridge — connects TraceEmitter span lifecycle to the OtelExporter.
+ *
+ * When an OtelExporter is active, every span that closes through the
+ * TraceEmitter is also pushed to the exporter for real-time streaming to
+ * the user's OTEL collector.
+ *
+ * The bridge is opt-in: attach via `otelRunCompleteHook(exporter)` as a
+ * RunCompleteHook, or wrap the store with `createOtelTracingStore` for
+ * real-time per-span export.
+ */
+
+/**
+ * Create a RunCompleteHook that exports all spans from the completed run
+ * to the OTEL exporter, then flushes.
+ */
+declare function otelRunCompleteHook(exporter: OtelExporter): RunCompleteHook;
+/**
+ * Create an auto-exporting TraceStore wrapper that intercepts updateSpan
+ * calls. When a span gets an endedAt, it's exported immediately. This
+ * gives real-time streaming instead of batch-at-end.
+ *
+ * This is the preferred integration path: wrap the store before
+ * constructing the TraceEmitter.
+ */
+declare function createOtelTracingStore(inner: TraceStore, exporter: OtelExporter, traceId: string): TraceStore;
+
 /**
  * Redaction — remove PII / secrets from trace payloads before persist.
  *
@@ -807,4 +886,4 @@ declare function iterateRawCalls(sink: RawProviderSink, filter?: {
     spanId?: string;
 }): AsyncGenerator<ReplayCacheEntry>;
 
-export { type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, type AnalyzeTracesTurnSnapshot, DEFAULT_REDACTION_RULES, DEFAULT_TRACE_ANALYST_BUDGETS, type DatasetOverview, OTEL_AGENT_EVAL_SCOPE, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpResourceSpans, type OtlpSpan, type QueryTracesPage, REDACTION_VERSION, RawProviderEvent, RawProviderSink, type RedactionReport, type RedactionRule, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, RunCompleteHook, RunCompleteHookContext, type SearchSpanResult, type SearchTraceResult, type SpanMatchRecord, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, type TraceAnalysisStore, type TraceAnalystByteBudgets, type TraceAnalystFilters, type TraceAnalystHookOptions, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, analyzeTraces, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, inferDomainKeywords, iterateRawCalls, planTraceInsightQuestions, redactString, redactValue, scoreTraceInsightReadiness, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete };
+export { type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, type AnalyzeTracesTurnSnapshot, DEFAULT_REDACTION_RULES, DEFAULT_TRACE_ANALYST_BUDGETS, type DatasetOverview, type ExportableSpan, OTEL_AGENT_EVAL_SCOPE, type OtelExportConfig, type OtelExporter, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpResourceSpans, type OtlpSpan, type QueryTracesPage, REDACTION_VERSION, RawProviderEvent, RawProviderSink, type RedactionReport, type RedactionRule, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, RunCompleteHook, RunCompleteHookContext, type SearchSpanResult, type SearchTraceResult, type SpanMatchRecord, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, type TraceAnalysisStore, type TraceAnalystByteBudgets, type TraceAnalystFilters, type TraceAnalystHookOptions, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, analyzeTraces, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, inferDomainKeywords, iterateRawCalls, otelRunCompleteHook, planTraceInsightQuestions, redactString, redactValue, scoreTraceInsightReadiness, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete };

package/dist/traces.js

@@ -19,6 +19,8 @@ import {
   buildTraceAnalystTools,
   buildTraceInsightContext,
   buildTraceInsightPrompt,
+  createOtelExporter,
+  createOtelTracingStore,
   createReplayFetch,
   defaultTraceInsightPanel,
   describeTraceInsightScope,
@@ -26,6 +28,7 @@ import {
   exportRunAsOtlp,
   inferDomainKeywords,
   iterateRawCalls,
+  otelRunCompleteHook,
   planTraceInsightQuestions,
   redactString,
   redactValue,
@@ -33,7 +36,7 @@ import {
   tokenizeDomainWords,
   traceAnalystFunctionGroup,
   traceAnalystOnRunComplete
-} from "./chunk-HIO4UIS5.js";
+} from "./chunk-L5UNCDAJ.js";
 import {
   aggregateLlm,
   argHash,
@@ -103,6 +106,8 @@ export {
   buildTraceAnalystTools,
   buildTraceInsightContext,
   buildTraceInsightPrompt,
+  createOtelExporter,
+  createOtelTracingStore,
   createReplayFetch,
   defaultProviderRedactor,
   defaultTraceInsightPanel,
@@ -120,6 +125,7 @@ export {
   judgeSpans,
   llmSpanFromProvider,
   llmSpans,
+  otelRunCompleteHook,
   planTraceInsightQuestions,
   providerFromBaseUrl,
   redactString,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.35.0",
+  "version": "0.37.0",
   "description": "Substrate for self-improving agents: traces, verifiable rewards, preferences, GEPA / reflective mutation, auto-research, replay, sequential anytime-valid stats, and release gates.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -94,6 +94,11 @@
       "import": "./dist/knowledge/index.js",
       "default": "./dist/knowledge/index.js"
     },
+    "./matrix": {
+      "types": "./dist/matrix/index.d.ts",
+      "import": "./dist/matrix/index.js",
+      "default": "./dist/matrix/index.js"
+    },
     "./openapi.json": {
       "default": "./dist/openapi.json"
     }
@@ -128,8 +133,18 @@
     "hono": "^4.12.16",
     "zod": "^4.3.6"
   },
+  "peerDependencies": {
+    "@tangle-network/agent-runtime": "^0.21.0",
+    "@tangle-network/sandbox": "^0.2.1"
+  },
+  "peerDependenciesMeta": {
+    "@tangle-network/agent-runtime": { "optional": true },
+    "@tangle-network/sandbox": { "optional": true }
+  },
   "devDependencies": {
     "@biomejs/biome": "^2.4.15",
+    "@tangle-network/agent-runtime": "^0.21.0",
+    "@tangle-network/sandbox": "^0.2.1",
     "@types/node": "^25.6.0",
     "openapi3-ts": "^4.5.0",
     "tsup": "^8.0.0",
@@ -138,7 +153,7 @@
   },
   "pnpm": {
     "minimumReleaseAge": 4320,
-    "minimumReleaseAgeExclude": [],
+    "minimumReleaseAgeExclude": ["@tangle-network/sandbox", "@tangle-network/agent-runtime"],
     "overrides": {
       "postcss@<8.5.10": "^8.5.10",
       "ws@>=8.0.0 <8.20.1": "^8.20.1"