@tangle-network/agent-eval 0.37.0 → 0.38.0

package/dist/types-DHqkLwEU.d.ts

@@ -0,0 +1,110 @@
+import { DefaultVerdict } from '@tangle-network/agent-runtime/loops';
+
+/**
+ * @experimental
+ *
+ * N-axis cartesian matrix over substrate types — types module.
+ *
+ * The matrix is a runner + aggregator. It iterates the cartesian product of
+ * caller-provided axes (any value type — `AgentProfile` from sandbox, `Driver`
+ * / `Validator` from agent-runtime, rubric records, thinking levels, anything)
+ * and aggregates per-axis pass/score/cost summaries. Substrate types are
+ * imported at the boundary by JSDoc only; the matrix never wraps them.
+ */
+
+/** One axis = one dimension to iterate. `V` is the value type — pass any
+ *  substrate type (AgentProfile, Driver, Validator, rubric record). */
+interface MatrixAxis<V> {
+    /** Axis name. Becomes the key in `MatrixResult.byAxis`. */
+    name: string;
+    /** Stable id per value. Used as the bucket key in aggregation. */
+    values: Array<{
+        id: string;
+        value: V;
+    }>;
+    /** Optional bucket label override. Receives the same `(value, id)` the
+     *  runner stored on the cell; default label is `id`. */
+    label?: (value: V, id: string) => string;
+}
+/** A cell carries one picked value from each axis, keyed by axis name. */
+interface MatrixCell {
+    axes: Record<string, {
+        id: string;
+        value: unknown;
+    }>;
+    /** 0-based replicate index within the same axis combination. */
+    rep: number;
+    /** Stable sort key — preserves cartesian order across concurrent execution. */
+    ordinal: number;
+}
+interface CellResult<Output> {
+    output: Output;
+    verdict: DefaultVerdict;
+    costUsd: number;
+    durationMs: number;
+    runId?: string;
+    /** Populated when `runCell` threw. The cell contributes 0 to passRate AND
+     *  meanScore regardless of `verdict`. */
+    error?: {
+        message: string;
+        kind: string;
+    };
+}
+interface AxisSummary {
+    axisName: string;
+    axisValue: string;
+    cells: number;
+    passRate: number;
+    meanScore: number;
+    p50Score: number;
+    p90Score: number;
+    totalCostUsd: number;
+    meanDurationMs: number;
+}
+interface MatrixResult<Output> {
+    cells: Array<{
+        cell: MatrixCell;
+        runs: CellResult<Output>[];
+    }>;
+    /** `byAxis[axisName][axisValueId] = summary`. Populated only for axes
+     *  named in `aggregateBy` (default = every axis in `axes`). */
+    byAxis: Record<string, Record<string, AxisSummary>>;
+    summary: {
+        totalCells: number;
+        runsExecuted: number;
+        /** Cells removed by `filter` plus cells unscheduled after the cost
+         *  ceiling or abort signal tripped. */
+        cellsSkipped: number;
+        overallPassRate: number;
+        overallMeanScore: number;
+        totalCostUsd: number;
+        durationMs: number;
+    };
+    /** Stable id-like string generated at the end of the run. */
+    matrixId: string;
+}
+interface RunAgentMatrixOptions<Output> {
+    axes: MatrixAxis<unknown>[];
+    /** User-supplied cell executor. May throw; the matrix captures throws as
+     *  `CellResult.error` and continues. */
+    runCell: (cell: MatrixCell) => Promise<CellResult<Output>>;
+    /** Replicates per cell. Default 1. */
+    reps?: number;
+    /** Prune cells from the cartesian BEFORE rep expansion. */
+    filter?: (cell: Omit<MatrixCell, 'rep' | 'ordinal'>) => boolean;
+    /** Axes to aggregate into `byAxis`. Default: every axis in `axes`. */
+    aggregateBy?: string[];
+    /** Max concurrent in-flight `runCell` invocations. Default 4. */
+    maxConcurrency?: number;
+    /** Cumulative-cost abort threshold (USD). When the running sum of
+     *  `result.costUsd` crosses this value, no new cells are scheduled.
+     *  In-flight cells finish. Default `Infinity`. */
+    costCeiling?: number;
+    /** Fires once per executed cell, after its promise settles. */
+    onCellComplete?: (cell: MatrixCell, result: CellResult<Output>) => void;
+    /** External cancellation. Aborts in-flight cells via a forwarded signal
+     *  and suppresses scheduling of new ones. */
+    signal?: AbortSignal;
+}
+
+export type { AxisSummary as A, CellResult as C, MatrixResult as M, RunAgentMatrixOptions as R, MatrixAxis as a, MatrixCell as b };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.37.0",
+  "version": "0.38.0",
   "description": "Substrate for self-improving agents: traces, verifiable rewards, preferences, GEPA / reflective mutation, auto-research, replay, sequential anytime-valid stats, and release gates.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -99,6 +99,11 @@
       "import": "./dist/matrix/index.js",
       "default": "./dist/matrix/index.js"
     },
+    "./multishot": {
+      "types": "./dist/multishot/index.d.ts",
+      "import": "./dist/multishot/index.js",
+      "default": "./dist/multishot/index.js"
+    },
     "./openapi.json": {
       "default": "./dist/openapi.json"
     }