@tangle-network/agent-eval 0.37.0 → 0.38.0

package/dist/matrix/index.js

@@ -1,274 +1,9 @@
+import {
+  buildByAxis,
+  runAgentMatrix,
+  summariseRows
+} from "../chunk-QWV226SL.js";
 import "../chunk-PZ5AY32C.js";
-
-// src/matrix/aggregation.ts
-function flattenRuns(cells) {
-  const rows = [];
-  for (const { cell, runs } of cells) {
-    for (const result of runs) rows.push({ cell, result });
-  }
-  return rows;
-}
-function quantile(sorted, q) {
-  if (sorted.length === 0) return 0;
-  if (sorted.length === 1) return sorted[0];
-  const pos = (sorted.length - 1) * q;
-  const lo = Math.floor(pos);
-  const hi = Math.ceil(pos);
-  if (lo === hi) return sorted[lo];
-  const frac = pos - lo;
-  return sorted[lo] * (1 - frac) + sorted[hi] * frac;
-}
-function summariseRows(rows, axisName, axisValue) {
-  if (rows.length === 0) {
-    return {
-      axisName,
-      axisValue,
-      cells: 0,
-      passRate: 0,
-      meanScore: 0,
-      p50Score: 0,
-      p90Score: 0,
-      totalCostUsd: 0,
-      meanDurationMs: 0
-    };
-  }
-  let pass = 0;
-  let scoreSum = 0;
-  let costSum = 0;
-  let durSum = 0;
-  const scores = [];
-  for (const { result } of rows) {
-    const errored = result.error !== void 0;
-    const score = errored ? 0 : result.verdict.score;
-    const valid = !errored && result.verdict.valid;
-    if (valid) pass++;
-    scoreSum += score;
-    scores.push(score);
-    costSum += result.costUsd;
-    durSum += result.durationMs;
-  }
-  scores.sort((a, b) => a - b);
-  return {
-    axisName,
-    axisValue,
-    cells: rows.length,
-    passRate: pass / rows.length,
-    meanScore: scoreSum / rows.length,
-    p50Score: quantile(scores, 0.5),
-    p90Score: quantile(scores, 0.9),
-    totalCostUsd: costSum,
-    meanDurationMs: durSum / rows.length
-  };
-}
-function bucketBy(rows, axisName, labelFor) {
-  const buckets = /* @__PURE__ */ new Map();
-  for (const row of rows) {
-    const slot = row.cell.axes[axisName];
-    if (!slot) continue;
-    const id = slot.id;
-    let arr = buckets.get(id);
-    if (!arr) {
-      arr = [];
-      buckets.set(id, arr);
-    }
-    arr.push(row);
-  }
-  const out = {};
-  for (const id of [...buckets.keys()].sort()) {
-    out[id] = summariseRows(buckets.get(id), axisName, labelFor(id));
-  }
-  return out;
-}
-function buildByAxis(cells, axes, aggregateBy) {
-  const rows = flattenRuns(cells);
-  const byName = new Map(axes.map((a) => [a.name, a]));
-  const byAxis = {};
-  for (const name of aggregateBy) {
-    const axis = byName.get(name);
-    const labelFor = (id) => {
-      if (!axis?.label) return id;
-      const found = axis.values.find((v) => v.id === id);
-      if (!found) return id;
-      return axis.label(found.value, id);
-    };
-    byAxis[name] = bucketBy(rows, name, labelFor);
-  }
-  return byAxis;
-}
-
-// src/matrix/runner.ts
-function cartesian(axes) {
-  if (axes.length === 0) return [{ axes: {} }];
-  for (const a of axes) if (a.values.length === 0) return [];
-  const out = [];
-  const idx = new Array(axes.length).fill(0);
-  while (true) {
-    const slot = {};
-    for (let i2 = 0; i2 < axes.length; i2++) {
-      const axis = axes[i2];
-      const v = axis.values[idx[i2]];
-      slot[axis.name] = { id: v.id, value: v.value };
-    }
-    out.push({ axes: slot });
-    let i = 0;
-    while (i < axes.length) {
-      const next = idx[i] + 1;
-      const axis = axes[i];
-      if (next < axis.values.length) {
-        idx[i] = next;
-        break;
-      }
-      idx[i] = 0;
-      i++;
-    }
-    if (i === axes.length) break;
-  }
-  return out;
-}
-function makeMatrixId() {
-  const t = Date.now().toString(36);
-  let r = "";
-  for (let i = 0; i < 8; i++) r += Math.floor(Math.random() * 16).toString(16);
-  return `mtx_${t}_${r}`;
-}
-function makeErrorResult(err) {
-  const e = err;
-  return {
-    output: void 0,
-    verdict: { valid: false, score: 0 },
-    costUsd: 0,
-    durationMs: 0,
-    error: {
-      message: typeof e?.message === "string" ? e.message : String(err),
-      kind: typeof e?.name === "string" ? e.name : "Error"
-    }
-  };
-}
-async function runAgentMatrix(opts) {
-  const startedAt = Date.now();
-  const reps = Math.max(1, opts.reps ?? 1);
-  const maxConcurrency = Math.max(1, opts.maxConcurrency ?? 4);
-  const costCeiling = opts.costCeiling ?? Number.POSITIVE_INFINITY;
-  const aggregateBy = opts.aggregateBy ?? opts.axes.map((a) => a.name);
-  const base = cartesian(opts.axes);
-  const filtered = opts.filter ? base.filter((c) => opts.filter(c)) : base;
-  const filteredOut = base.length - filtered.length;
-  const planned = [];
-  for (let i = 0; i < filtered.length; i++) {
-    for (let r = 0; r < reps; r++) {
-      planned.push({
-        axes: filtered[i].axes,
-        rep: r,
-        ordinal: i * reps + r
-      });
-    }
-  }
-  const cellRecords = [];
-  let cumulativeCost = 0;
-  let costCeilingReached = false;
-  let runsExecuted = 0;
-  let cellsUnscheduled = 0;
-  const aborted = () => opts.signal?.aborted === true;
-  let inFlight = 0;
-  let cursor = 0;
-  let resolveAll;
-  const done = new Promise((res) => {
-    resolveAll = res;
-  });
-  const pump = () => {
-    while (inFlight < maxConcurrency && cursor < planned.length) {
-      if (aborted() || costCeilingReached) {
-        const left = planned.length - cursor;
-        cellsUnscheduled += left;
-        cursor = planned.length;
-        break;
-      }
-      const cell = planned[cursor++];
-      inFlight++;
-      const record = { cell, runs: [] };
-      cellRecords.push(record);
-      const promise = (async () => {
-        try {
-          return await opts.runCell(cell);
-        } catch (err) {
-          return makeErrorResult(err);
-        }
-      })();
-      promise.then((result) => {
-        record.runs.push(result);
-        runsExecuted++;
-        cumulativeCost += result.costUsd;
-        if (cumulativeCost >= costCeiling && !costCeilingReached) {
-          costCeilingReached = true;
-          console.warn("[matrix] cost ceiling reached");
-        }
-        try {
-          opts.onCellComplete?.(cell, result);
-        } catch {
-        }
-        inFlight--;
-        if (cursor < planned.length) {
-          pump();
-        } else if (inFlight === 0) {
-          resolveAll?.();
-        }
-      });
-    }
-    if (cursor >= planned.length && inFlight === 0) resolveAll?.();
-  };
-  const onAbort = () => {
-    if (cursor < planned.length) {
-      cellsUnscheduled += planned.length - cursor;
-      cursor = planned.length;
-    }
-    if (inFlight === 0) resolveAll?.();
-  };
-  if (opts.signal) {
-    if (opts.signal.aborted) {
-      cellsUnscheduled = planned.length;
-      cursor = planned.length;
-      resolveAll?.();
-    } else {
-      opts.signal.addEventListener("abort", onAbort, { once: true });
-    }
-  }
-  if (planned.length === 0) {
-    resolveAll?.();
-  } else {
-    pump();
-  }
-  await done;
-  if (opts.signal) opts.signal.removeEventListener("abort", onAbort);
-  cellRecords.sort((a, b) => a.cell.ordinal - b.cell.ordinal);
-  let pass = 0;
-  let scoreSum = 0;
-  let totalCost = 0;
-  let runCount = 0;
-  for (const { runs } of cellRecords) {
-    for (const r of runs) {
-      runCount++;
-      const errored = r.error !== void 0;
-      if (!errored && r.verdict.valid) pass++;
-      scoreSum += errored ? 0 : r.verdict.score;
-      totalCost += r.costUsd;
-    }
-  }
-  const byAxis = buildByAxis(cellRecords, opts.axes, aggregateBy);
-  return {
-    cells: cellRecords,
-    byAxis,
-    summary: {
-      totalCells: planned.length,
-      runsExecuted,
-      cellsSkipped: cellsUnscheduled + filteredOut * reps,
-      overallPassRate: runCount === 0 ? 0 : pass / runCount,
-      overallMeanScore: runCount === 0 ? 0 : scoreSum / runCount,
-      totalCostUsd: totalCost,
-      durationMs: Date.now() - startedAt
-    },
-    matrixId: makeMatrixId()
-  };
-}
 export {
   buildByAxis,
   runAgentMatrix,

package/dist/matrix/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/matrix/aggregation.ts","../../src/matrix/runner.ts"],"sourcesContent":["/**\n * Per-axis aggregation of cell runs into `AxisSummary` rows.\n *\n * Pure: consumes the final `cells: [{cell, runs}]` array and returns the\n * `byAxis` table. Error runs contribute 0 to passRate and meanScore. Cost\n * and duration always count — the budget was spent regardless.\n */\n\nimport type { AxisSummary, CellResult, MatrixAxis, MatrixCell, MatrixResult } from './types'\n\ninterface Row<Output> {\n  cell: MatrixCell\n  result: CellResult<Output>\n}\n\nfunction flattenRuns<Output>(cells: MatrixResult<Output>['cells']): Row<Output>[] {\n  const rows: Row<Output>[] = []\n  for (const { cell, runs } of cells) {\n    for (const result of runs) rows.push({ cell, result })\n  }\n  return rows\n}\n\nfunction quantile(sorted: number[], q: number): number {\n  if (sorted.length === 0) return 0\n  if (sorted.length === 1) return sorted[0] as number\n  const pos = (sorted.length - 1) * q\n  const lo = Math.floor(pos)\n  const hi = Math.ceil(pos)\n  if (lo === hi) return sorted[lo] as number\n  const frac = pos - lo\n  return (sorted[lo] as number) * (1 - frac) + (sorted[hi] as number) * frac\n}\n\nexport function summariseRows<Output>(\n  rows: Row<Output>[],\n  axisName: string,\n  axisValue: string,\n): AxisSummary {\n  if (rows.length === 0) {\n    return {\n      axisName,\n      axisValue,\n      cells: 0,\n      passRate: 0,\n      meanScore: 0,\n      p50Score: 0,\n      p90Score: 0,\n      totalCostUsd: 0,\n      meanDurationMs: 0,\n    }\n  }\n  let pass = 0\n  let scoreSum = 0\n  let costSum = 0\n  let durSum = 0\n  const scores: number[] = []\n  for (const { result } of rows) {\n    const errored = result.error !== undefined\n    const score = errored ? 0 : result.verdict.score\n    const valid = !errored && result.verdict.valid\n    if (valid) pass++\n    scoreSum += score\n    scores.push(score)\n    costSum += result.costUsd\n    durSum += result.durationMs\n  }\n  scores.sort((a, b) => a - b)\n  return {\n    axisName,\n    axisValue,\n    cells: rows.length,\n    passRate: pass / rows.length,\n    meanScore: scoreSum / rows.length,\n    p50Score: quantile(scores, 0.5),\n    p90Score: quantile(scores, 0.9),\n    totalCostUsd: costSum,\n    meanDurationMs: durSum / rows.length,\n  }\n}\n\nfunction bucketBy<Output>(\n  rows: Row<Output>[],\n  axisName: string,\n  labelFor: (id: string) => string,\n): Record<string, AxisSummary> {\n  const buckets = new Map<string, Row<Output>[]>()\n  for (const row of rows) {\n    const slot = row.cell.axes[axisName]\n    if (!slot) continue\n    const id = slot.id\n    let arr = buckets.get(id)\n    if (!arr) {\n      arr = []\n      buckets.set(id, arr)\n    }\n    arr.push(row)\n  }\n  const out: Record<string, AxisSummary> = {}\n  // Sorted keys for deterministic JSON serialisation.\n  for (const id of [...buckets.keys()].sort()) {\n    out[id] = summariseRows(buckets.get(id) as Row<Output>[], axisName, labelFor(id))\n  }\n  return out\n}\n\nexport function buildByAxis<Output>(\n  cells: MatrixResult<Output>['cells'],\n  axes: MatrixAxis<unknown>[],\n  aggregateBy: string[],\n): Record<string, Record<string, AxisSummary>> {\n  const rows = flattenRuns(cells)\n  const byName = new Map(axes.map((a) => [a.name, a]))\n  const byAxis: Record<string, Record<string, AxisSummary>> = {}\n  for (const name of aggregateBy) {\n    const axis = byName.get(name)\n    const labelFor = (id: string): string => {\n      if (!axis?.label) return id\n      const found = axis.values.find((v) => v.id === id)\n      if (!found) return id\n      return axis.label(found.value, id)\n    }\n    byAxis[name] = bucketBy(rows, name, labelFor)\n  }\n  return byAxis\n}\n","/**\n * N-axis cartesian runner.\n *\n * Expansion order: cartesian over `axes` in declared order, then `reps` as the\n * inner-most dim → `ordinal = (cartIdx * reps) + rep`. The returned\n * `cells[]` is sorted by `ordinal` so concurrent execution does not reorder\n * the output.\n *\n * Scheduling is a sliding window of in-flight promises capped at\n * `maxConcurrency`. The window stops admitting new cells when the cost\n * ceiling trips or the abort signal fires; in-flight cells finish.\n */\n\nimport { buildByAxis } from './aggregation'\nimport type {\n  CellResult,\n  MatrixAxis,\n  MatrixCell,\n  MatrixResult,\n  RunAgentMatrixOptions,\n} from './types'\n\ninterface BaseCell {\n  axes: Record<string, { id: string; value: unknown }>\n}\n\nfunction cartesian(axes: MatrixAxis<unknown>[]): BaseCell[] {\n  // Empty axes (`values=[]`) collapse the whole product to zero cells. An\n  // empty `axes` array yields a single empty-axes cell — degenerate but\n  // valid (caller is iterating only reps).\n  if (axes.length === 0) return [{ axes: {} }]\n  for (const a of axes) if (a.values.length === 0) return []\n  const out: BaseCell[] = []\n  const idx = new Array(axes.length).fill(0)\n  while (true) {\n    const slot: Record<string, { id: string; value: unknown }> = {}\n    for (let i = 0; i < axes.length; i++) {\n      const axis = axes[i] as MatrixAxis<unknown>\n      const v = axis.values[idx[i] as number] as { id: string; value: unknown }\n      slot[axis.name] = { id: v.id, value: v.value }\n    }\n    out.push({ axes: slot })\n    // Increment like an odometer, left-most axis is fastest.\n    let i = 0\n    while (i < axes.length) {\n      const next = (idx[i] as number) + 1\n      const axis = axes[i] as MatrixAxis<unknown>\n      if (next < axis.values.length) {\n        idx[i] = next\n        break\n      }\n      idx[i] = 0\n      i++\n    }\n    if (i === axes.length) break\n  }\n  return out\n}\n\nfunction makeMatrixId(): string {\n  // Stable id-like string: time + 8 random hex chars. Avoids node:crypto\n  // import to keep the matrix dep-free.\n  const t = Date.now().toString(36)\n  let r = ''\n  for (let i = 0; i < 8; i++) r += Math.floor(Math.random() * 16).toString(16)\n  return `mtx_${t}_${r}`\n}\n\nfunction makeErrorResult<Output>(err: unknown): CellResult<Output> {\n  const e = err as { message?: string; name?: string }\n  return {\n    output: undefined as unknown as Output,\n    verdict: { valid: false, score: 0 },\n    costUsd: 0,\n    durationMs: 0,\n    error: {\n      message: typeof e?.message === 'string' ? e.message : String(err),\n      kind: typeof e?.name === 'string' ? e.name : 'Error',\n    },\n  }\n}\n\nexport async function runAgentMatrix<Output>(\n  opts: RunAgentMatrixOptions<Output>,\n): Promise<MatrixResult<Output>> {\n  const startedAt = Date.now()\n  const reps = Math.max(1, opts.reps ?? 1)\n  const maxConcurrency = Math.max(1, opts.maxConcurrency ?? 4)\n  const costCeiling = opts.costCeiling ?? Number.POSITIVE_INFINITY\n  const aggregateBy = opts.aggregateBy ?? opts.axes.map((a) => a.name)\n\n  const base = cartesian(opts.axes)\n  const filtered = opts.filter\n    ? base.filter((c) => (opts.filter as (b: BaseCell) => boolean)(c))\n    : base\n  const filteredOut = base.length - filtered.length\n\n  const planned: MatrixCell[] = []\n  for (let i = 0; i < filtered.length; i++) {\n    for (let r = 0; r < reps; r++) {\n      planned.push({\n        axes: (filtered[i] as BaseCell).axes,\n        rep: r,\n        ordinal: i * reps + r,\n      })\n    }\n  }\n\n  const cellRecords: Array<{ cell: MatrixCell; runs: CellResult<Output>[] }> = []\n  let cumulativeCost = 0\n  let costCeilingReached = false\n  let runsExecuted = 0\n  let cellsUnscheduled = 0\n\n  const aborted = (): boolean => opts.signal?.aborted === true\n\n  // Per-run abort controller forwards the external signal so cell executors\n  // see cancellation. We don't expose it on `MatrixCell` — the signature on\n  // `runCell` per the public API is `(cell) => Promise<...>`. Executors that\n  // need cancellation use the external signal directly via closure.\n\n  let inFlight = 0\n  let cursor = 0\n  let resolveAll: (() => void) | undefined\n  const done = new Promise<void>((res) => {\n    resolveAll = res\n  })\n\n  const pump = (): void => {\n    while (inFlight < maxConcurrency && cursor < planned.length) {\n      if (aborted() || costCeilingReached) {\n        // Drain remaining as unscheduled.\n        const left = planned.length - cursor\n        cellsUnscheduled += left\n        cursor = planned.length\n        break\n      }\n      const cell = planned[cursor++] as MatrixCell\n      inFlight++\n      // Lazily allocate the record so cells appear in `cells[]` in any\n      // arrival order; we sort by ordinal at the end.\n      const record = { cell, runs: [] as CellResult<Output>[] }\n      cellRecords.push(record)\n      const promise: Promise<CellResult<Output>> = (async () => {\n        try {\n          return await opts.runCell(cell)\n        } catch (err) {\n          return makeErrorResult<Output>(err)\n        }\n      })()\n      promise.then((result) => {\n        record.runs.push(result)\n        runsExecuted++\n        cumulativeCost += result.costUsd\n        if (cumulativeCost >= costCeiling && !costCeilingReached) {\n          costCeilingReached = true\n          // eslint-disable-next-line no-console\n          console.warn('[matrix] cost ceiling reached')\n        }\n        try {\n          opts.onCellComplete?.(cell, result)\n        } catch {\n          // onCellComplete is observational — swallow throws so a noisy\n          // callback can't tank the run.\n        }\n        inFlight--\n        if (cursor < planned.length) {\n          pump()\n        } else if (inFlight === 0) {\n          resolveAll?.()\n        }\n      })\n    }\n    if (cursor >= planned.length && inFlight === 0) resolveAll?.()\n  }\n\n  const onAbort = (): void => {\n    // External abort: stop scheduling. In-flight cells finish; their\n    // executors observe `opts.signal.aborted` directly via closure.\n    if (cursor < planned.length) {\n      cellsUnscheduled += planned.length - cursor\n      cursor = planned.length\n    }\n    if (inFlight === 0) resolveAll?.()\n  }\n  if (opts.signal) {\n    if (opts.signal.aborted) {\n      cellsUnscheduled = planned.length\n      cursor = planned.length\n      resolveAll?.()\n    } else {\n      opts.signal.addEventListener('abort', onAbort, { once: true })\n    }\n  }\n\n  if (planned.length === 0) {\n    resolveAll?.()\n  } else {\n    pump()\n  }\n\n  await done\n  if (opts.signal) opts.signal.removeEventListener('abort', onAbort)\n\n  cellRecords.sort((a, b) => a.cell.ordinal - b.cell.ordinal)\n\n  let pass = 0\n  let scoreSum = 0\n  let totalCost = 0\n  let runCount = 0\n  for (const { runs } of cellRecords) {\n    for (const r of runs) {\n      runCount++\n      const errored = r.error !== undefined\n      if (!errored && r.verdict.valid) pass++\n      scoreSum += errored ? 0 : r.verdict.score\n      totalCost += r.costUsd\n    }\n  }\n\n  const byAxis = buildByAxis(cellRecords, opts.axes, aggregateBy)\n\n  return {\n    cells: cellRecords,\n    byAxis,\n    summary: {\n      totalCells: planned.length,\n      runsExecuted,\n      cellsSkipped: cellsUnscheduled + filteredOut * reps,\n      overallPassRate: runCount === 0 ? 0 : pass / runCount,\n      overallMeanScore: runCount === 0 ? 0 : scoreSum / runCount,\n      totalCostUsd: totalCost,\n      durationMs: Date.now() - startedAt,\n    },\n    matrixId: makeMatrixId(),\n  }\n}\n"],"mappings":";;;AAeA,SAAS,YAAoB,OAAqD;AAChF,QAAM,OAAsB,CAAC;AAC7B,aAAW,EAAE,MAAM,KAAK,KAAK,OAAO;AAClC,eAAW,UAAU,KAAM,MAAK,KAAK,EAAE,MAAM,OAAO,CAAC;AAAA,EACvD;AACA,SAAO;AACT;AAEA,SAAS,SAAS,QAAkB,GAAmB;AACrD,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,MAAI,OAAO,WAAW,EAAG,QAAO,OAAO,CAAC;AACxC,QAAM,OAAO,OAAO,SAAS,KAAK;AAClC,QAAM,KAAK,KAAK,MAAM,GAAG;AACzB,QAAM,KAAK,KAAK,KAAK,GAAG;AACxB,MAAI,OAAO,GAAI,QAAO,OAAO,EAAE;AAC/B,QAAM,OAAO,MAAM;AACnB,SAAQ,OAAO,EAAE,KAAgB,IAAI,QAAS,OAAO,EAAE,IAAe;AACxE;AAEO,SAAS,cACd,MACA,UACA,WACa;AACb,MAAI,KAAK,WAAW,GAAG;AACrB,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO;AAAA,MACP,UAAU;AAAA,MACV,WAAW;AAAA,MACX,UAAU;AAAA,MACV,UAAU;AAAA,MACV,cAAc;AAAA,MACd,gBAAgB;AAAA,IAClB;AAAA,EACF;AACA,MAAI,OAAO;AACX,MAAI,WAAW;AACf,MAAI,UAAU;AACd,MAAI,SAAS;AACb,QAAM,SAAmB,CAAC;AAC1B,aAAW,EAAE,OAAO,KAAK,MAAM;AAC7B,UAAM,UAAU,OAAO,UAAU;AACjC,UAAM,QAAQ,UAAU,IAAI,OAAO,QAAQ;AAC3C,UAAM,QAAQ,CAAC,WAAW,OAAO,QAAQ;AACzC,QAAI,MAAO;AACX,gBAAY;AACZ,WAAO,KAAK,KAAK;AACjB,eAAW,OAAO;AAClB,cAAU,OAAO;AAAA,EACnB;AACA,SAAO,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3B,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,OAAO,KAAK;AAAA,IACZ,UAAU,OAAO,KAAK;AAAA,IACtB,WAAW,WAAW,KAAK;AAAA,IAC3B,UAAU,SAAS,QAAQ,GAAG;AAAA,IAC9B,UAAU,SAAS,QAAQ,GAAG;AAAA,IAC9B,cAAc;AAAA,IACd,gBAAgB,SAAS,KAAK;AAAA,EAChC;AACF;AAEA,SAAS,SACP,MACA,UACA,UAC6B;AAC7B,QAAM,UAAU,oBAAI,IAA2B;AAC/C,aAAW,OAAO,MAAM;AACtB,UAAM,OAAO,IAAI,KAAK,KAAK,QAAQ;AACnC,QAAI,CAAC,KAAM;AACX,UAAM,KAAK,KAAK;AAChB,QAAI,MAAM,QAAQ,IAAI,EAAE;AACxB,QAAI,CAAC,KAAK;AACR,YAAM,CAAC;AACP,cAAQ,IAAI,IAAI,GAAG;AAAA,IACrB;AACA,QAAI,KAAK,GAAG;AAAA,EACd;AACA,QAAM,MAAmC,CAAC;AAE1C,aAAW,MAAM,CAAC,GAAG,QAAQ,KAAK,CAAC,EAAE,KAAK,GAAG;AAC3C,QAAI,EAAE,IAAI,cAAc,QAAQ,IAAI,EAAE,GAAoB,UAAU,SAAS,EAAE,CAAC;AAAA,EAClF;AACA,SAAO;AACT;AAEO,SAAS,YACd,OACA,MACA,aAC6C;AAC7C,QAAM,OAAO,YAAY,KAAK;AAC9B,QAAM,SAAS,IAAI,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AACnD,QAAM,SAAsD,CAAC;AAC7D,aAAW,QAAQ,aAAa;AAC9B,UAAM,OAAO,OAAO,IAAI,IAAI;AAC5B,UAAM,WAAW,CAAC,OAAuB;AACvC,UAAI,CAAC,MAAM,MAAO,QAAO;AACzB,YAAM,QAAQ,KAAK,OAAO,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE;AACjD,UAAI,CAAC,MAAO,QAAO;AACnB,aAAO,KAAK,MAAM,MAAM,OAAO,EAAE;AAAA,IACnC;AACA,WAAO,IAAI,IAAI,SAAS,MAAM,MAAM,QAAQ;AAAA,EAC9C;AACA,SAAO;AACT;;;ACnGA,SAAS,UAAU,MAAyC;AAI1D,MAAI,KAAK,WAAW,EAAG,QAAO,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC;AAC3C,aAAW,KAAK,KAAM,KAAI,EAAE,OAAO,WAAW,EAAG,QAAO,CAAC;AACzD,QAAM,MAAkB,CAAC;AACzB,QAAM,MAAM,IAAI,MAAM,KAAK,MAAM,EAAE,KAAK,CAAC;AACzC,SAAO,MAAM;AACX,UAAM,OAAuD,CAAC;AAC9D,aAASA,KAAI,GAAGA,KAAI,KAAK,QAAQA,MAAK;AACpC,YAAM,OAAO,KAAKA,EAAC;AACnB,YAAM,IAAI,KAAK,OAAO,IAAIA,EAAC,CAAW;AACtC,WAAK,KAAK,IAAI,IAAI,EAAE,IAAI,EAAE,IAAI,OAAO,EAAE,MAAM;AAAA,IAC/C;AACA,QAAI,KAAK,EAAE,MAAM,KAAK,CAAC;AAEvB,QAAI,IAAI;AACR,WAAO,IAAI,KAAK,QAAQ;AACtB,YAAM,OAAQ,IAAI,CAAC,IAAe;AAClC,YAAM,OAAO,KAAK,CAAC;AACnB,UAAI,OAAO,KAAK,OAAO,QAAQ;AAC7B,YAAI,CAAC,IAAI;AACT;AAAA,MACF;AACA,UAAI,CAAC,IAAI;AACT;AAAA,IACF;AACA,QAAI,MAAM,KAAK,OAAQ;AAAA,EACzB;AACA,SAAO;AACT;AAEA,SAAS,eAAuB;AAG9B,QAAM,IAAI,KAAK,IAAI,EAAE,SAAS,EAAE;AAChC,MAAI,IAAI;AACR,WAAS,IAAI,GAAG,IAAI,GAAG,IAAK,MAAK,KAAK,MAAM,KAAK,OAAO,IAAI,EAAE,EAAE,SAAS,EAAE;AAC3E,SAAO,OAAO,CAAC,IAAI,CAAC;AACtB;AAEA,SAAS,gBAAwB,KAAkC;AACjE,QAAM,IAAI;AACV,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,SAAS,EAAE,OAAO,OAAO,OAAO,EAAE;AAAA,IAClC,SAAS;AAAA,IACT,YAAY;AAAA,IACZ,OAAO;AAAA,MACL,SAAS,OAAO,GAAG,YAAY,WAAW,EAAE,UAAU,OAAO,GAAG;AAAA,MAChE,MAAM,OAAO,GAAG,SAAS,WAAW,EAAE,OAAO;AAAA,IAC/C;AAAA,EACF;AACF;AAEA,eAAsB,eACpB,MAC+B;AAC/B,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,OAAO,KAAK,IAAI,GAAG,KAAK,QAAQ,CAAC;AACvC,QAAM,iBAAiB,KAAK,IAAI,GAAG,KAAK,kBAAkB,CAAC;AAC3D,QAAM,cAAc,KAAK,eAAe,OAAO;AAC/C,QAAM,cAAc,KAAK,eAAe,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,IAAI;AAEnE,QAAM,OAAO,UAAU,KAAK,IAAI;AAChC,QAAM,WAAW,KAAK,SAClB,KAAK,OAAO,CAAC,MAAO,KAAK,OAAoC,CAAC,CAAC,IAC/D;AACJ,QAAM,cAAc,KAAK,SAAS,SAAS;AAE3C,QAAM,UAAwB,CAAC;AAC/B,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,aAAS,IAAI,GAAG,IAAI,MAAM,KAAK;AAC7B,cAAQ,KAAK;AAAA,QACX,MAAO,SAAS,CAAC,EAAe;AAAA,QAChC,KAAK;AAAA,QACL,SAAS,IAAI,OAAO;AAAA,MACtB,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,cAAuE,CAAC;AAC9E,MAAI,iBAAiB;AACrB,MAAI,qBAAqB;AACzB,MAAI,eAAe;AACnB,MAAI,mBAAmB;AAEvB,QAAM,UAAU,MAAe,KAAK,QAAQ,YAAY;AAOxD,MAAI,WAAW;AACf,MAAI,SAAS;AACb,MAAI;AACJ,QAAM,OAAO,IAAI,QAAc,CAAC,QAAQ;AACtC,iBAAa;AAAA,EACf,CAAC;AAED,QAAM,OAAO,MAAY;AACvB,WAAO,WAAW,kBAAkB,SAAS,QAAQ,QAAQ;AAC3D,UAAI,QAAQ,KAAK,oBAAoB;AAEnC,cAAM,OAAO,QAAQ,SAAS;AAC9B,4BAAoB;AACpB,iBAAS,QAAQ;AACjB;AAAA,MACF;AACA,YAAM,OAAO,QAAQ,QAAQ;AAC7B;AAGA,YAAM,SAAS,EAAE,MAAM,MAAM,CAAC,EAA0B;AACxD,kBAAY,KAAK,MAAM;AACvB,YAAM,WAAwC,YAAY;AACxD,YAAI;AACF,iBAAO,MAAM,KAAK,QAAQ,IAAI;AAAA,QAChC,SAAS,KAAK;AACZ,iBAAO,gBAAwB,GAAG;AAAA,QACpC;AAAA,MACF,GAAG;AACH,cAAQ,KAAK,CAAC,WAAW;AACvB,eAAO,KAAK,KAAK,MAAM;AACvB;AACA,0BAAkB,OAAO;AACzB,YAAI,kBAAkB,eAAe,CAAC,oBAAoB;AACxD,+BAAqB;AAErB,kBAAQ,KAAK,+BAA+B;AAAA,QAC9C;AACA,YAAI;AACF,eAAK,iBAAiB,MAAM,MAAM;AAAA,QACpC,QAAQ;AAAA,QAGR;AACA;AACA,YAAI,SAAS,QAAQ,QAAQ;AAC3B,eAAK;AAAA,QACP,WAAW,aAAa,GAAG;AACzB,uBAAa;AAAA,QACf;AAAA,MACF,CAAC;AAAA,IACH;AACA,QAAI,UAAU,QAAQ,UAAU,aAAa,EAAG,cAAa;AAAA,EAC/D;AAEA,QAAM,UAAU,MAAY;AAG1B,QAAI,SAAS,QAAQ,QAAQ;AAC3B,0BAAoB,QAAQ,SAAS;AACrC,eAAS,QAAQ;AAAA,IACnB;AACA,QAAI,aAAa,EAAG,cAAa;AAAA,EACnC;AACA,MAAI,KAAK,QAAQ;AACf,QAAI,KAAK,OAAO,SAAS;AACvB,yBAAmB,QAAQ;AAC3B,eAAS,QAAQ;AACjB,mBAAa;AAAA,IACf,OAAO;AACL,WAAK,OAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAAA,IAC/D;AAAA,EACF;AAEA,MAAI,QAAQ,WAAW,GAAG;AACxB,iBAAa;AAAA,EACf,OAAO;AACL,SAAK;AAAA,EACP;AAEA,QAAM;AACN,MAAI,KAAK,OAAQ,MAAK,OAAO,oBAAoB,SAAS,OAAO;AAEjE,cAAY,KAAK,CAAC,GAAG,MAAM,EAAE,KAAK,UAAU,EAAE,KAAK,OAAO;AAE1D,MAAI,OAAO;AACX,MAAI,WAAW;AACf,MAAI,YAAY;AAChB,MAAI,WAAW;AACf,aAAW,EAAE,KAAK,KAAK,aAAa;AAClC,eAAW,KAAK,MAAM;AACpB;AACA,YAAM,UAAU,EAAE,UAAU;AAC5B,UAAI,CAAC,WAAW,EAAE,QAAQ,MAAO;AACjC,kBAAY,UAAU,IAAI,EAAE,QAAQ;AACpC,mBAAa,EAAE;AAAA,IACjB;AAAA,EACF;AAEA,QAAM,SAAS,YAAY,aAAa,KAAK,MAAM,WAAW;AAE9D,SAAO;AAAA,IACL,OAAO;AAAA,IACP;AAAA,IACA,SAAS;AAAA,MACP,YAAY,QAAQ;AAAA,MACpB;AAAA,MACA,cAAc,mBAAmB,cAAc;AAAA,MAC/C,iBAAiB,aAAa,IAAI,IAAI,OAAO;AAAA,MAC7C,kBAAkB,aAAa,IAAI,IAAI,WAAW;AAAA,MAClD,cAAc;AAAA,MACd,YAAY,KAAK,IAAI,IAAI;AAAA,IAC3B;AAAA,IACA,UAAU,aAAa;AAAA,EACzB;AACF;","names":["i"]}
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/multishot/index.d.ts

@@ -0,0 +1,276 @@
+import { AgentProfile } from '@tangle-network/sandbox';
+import { M as MatrixResult } from '../types-DHqkLwEU.js';
+import '@tangle-network/agent-runtime/loops';
+
+interface MultishotMessage {
+    role: 'user' | 'assistant' | 'tool';
+    content: string;
+    toolCallId?: string;
+    toolCalls?: Array<{
+        id: string;
+        name: string;
+        args: Record<string, unknown>;
+    }>;
+}
+interface MultishotArtifact {
+    type: string;
+    turn: number;
+    invocation: {
+        name: string;
+        args: Record<string, unknown>;
+    };
+    content: string;
+}
+interface MultishotResult {
+    transcript: MultishotMessage[];
+    artifacts: MultishotArtifact[];
+    toolCalls: number;
+    durationMs: number;
+    costUsd: number;
+}
+interface MultishotToolDefinition {
+    type: 'function';
+    function: {
+        name: string;
+        description: string;
+        parameters: Record<string, unknown>;
+    };
+}
+type MultishotToolExecutor = (args: Record<string, unknown>, ctx: {
+    apiKey: string;
+    baseUrl: string;
+    signal?: AbortSignal;
+}) => Promise<{
+    content: string;
+    costUsd: number;
+}>;
+interface MultishotPersona {
+    /** Stable identifier — used for per-cell artifact paths + matrix axis keys. */
+    id: string;
+    /** Per-domain payload (income/profile/voice/etc.) shaped by the consumer. */
+    [k: string]: unknown;
+}
+interface MultishotShape<TPersona extends MultishotPersona> {
+    /** Opening user message (turn 0) — the persona's first ask. */
+    buildOpener: (persona: TPersona) => string;
+    /** System prompt the driver LLM uses to roleplay the persona. Should set
+     *  voice, goals, constraints, time-pressure, and the "never go silent" rule. */
+    buildDriverSystemPrompt: (persona: TPersona) => string;
+}
+declare class MultishotDriverEmptyError extends Error {
+    readonly turn: number;
+    constructor(turn: number);
+}
+
+interface RouterCompletionRequest {
+    apiKey: string;
+    baseUrl: string;
+    model: string;
+    messages: Array<Record<string, unknown>>;
+    tools?: MultishotToolDefinition[];
+    temperature?: number;
+    maxTokens?: number;
+    signal?: AbortSignal;
+}
+interface RouterToolCall {
+    id: string;
+    type: 'function';
+    function: {
+        name: string;
+        arguments: string;
+    };
+}
+interface RouterCompletionResponse {
+    message: {
+        content?: string | null;
+        tool_calls?: RouterToolCall[];
+    };
+    usage?: {
+        prompt_tokens?: number;
+        completion_tokens?: number;
+    };
+}
+declare function routerCompletion(req: RouterCompletionRequest): Promise<RouterCompletionResponse>;
+declare function estimateRouterCost(model: string, usage?: {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+}): number;
+declare function defaultRouterBaseUrl(): string;
+declare function requireRouterApiKey(): string;
+
+declare const DEFAULT_RESEARCHER_MODEL = "openai/gpt-4o-mini";
+declare const DEFAULT_CODER_MODEL = "openai/gpt-4o-mini";
+interface DefaultResearcherConfig {
+    /** Replace the system prompt to bias the researcher toward a domain's
+     *  citation style. Defaults to a generic "cite sources by name" prompt. */
+    systemPrompt?: string;
+    model?: string;
+}
+interface DefaultCoderConfig {
+    /** Replace the system prompt to bias the coder toward a language /
+     *  framework / artifact style. */
+    systemPrompt?: string;
+    model?: string;
+}
+declare const DEFAULT_DELEGATE_RESEARCH_TOOL: MultishotToolDefinition;
+declare const DEFAULT_DELEGATE_CODE_TOOL: MultishotToolDefinition;
+declare function createResearchExecutor(config?: DefaultResearcherConfig): MultishotToolExecutor;
+declare function createCodeExecutor(config?: DefaultCoderConfig): MultishotToolExecutor;
+interface DefaultToolsConfig {
+    research?: DefaultResearcherConfig;
+    code?: DefaultCoderConfig;
+    /** When true (default), each tool result is recorded as a typed artifact:
+     *  research → type='research', code → type='code'. */
+    recordArtifacts?: boolean;
+}
+interface DefaultToolsBundle {
+    tools: MultishotToolDefinition[];
+    executors: Record<string, MultishotToolExecutor>;
+    artifactTypeFor: (toolName: string) => string | undefined;
+}
+declare function defaultDelegationTools(config?: DefaultToolsConfig): DefaultToolsBundle;
+
+declare const DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
+interface JudgeDimension {
+    /** JSON field name + score key. */
+    key: string;
+    /** Description shown in the judge's user prompt. */
+    description: string;
+}
+interface JudgeConfig<TInput> {
+    /** Display name (for trace + log). */
+    name: string;
+    /** Model used for this judge. */
+    model?: string;
+    /** 0-10 scored dimensions. */
+    dimensions: JudgeDimension[];
+    /** Judge system prompt — sets persona + JSON-only constraint. */
+    systemPrompt: string;
+    /** Build the user prompt from the typed input. Must include "Respond with
+     *  ONLY this JSON: { ... }" listing each dimension key. */
+    buildPrompt: (input: TInput) => string;
+    /** Optional model + api overrides. */
+    apiKey?: string;
+    baseUrl?: string;
+}
+interface JudgeScore {
+    /** Per-dimension 0-10 score. Missing dims default to 0. */
+    dimensions: Record<string, number>;
+    /** Mean across dimensions. */
+    composite: number;
+    /** Free-form 1-2 sentence critique from the judge (when provided). */
+    notes: string;
+}
+declare function runJudge<TInput>(judge: JudgeConfig<TInput>, input: TInput): Promise<JudgeScore>;
+/** Convenience: stringified dimension list for inclusion in a judge prompt.
+ *  Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */
+declare function renderDimensions(dims: readonly JudgeDimension[]): string;
+/** Convenience: build the "Respond with ONLY this JSON" footer for a judge prompt. */
+declare function renderJsonFooter(dims: readonly JudgeDimension[]): string;
+
+interface RunMultishotOptions<TPersona extends MultishotPersona> {
+    profile: AgentProfile;
+    persona: TPersona;
+    shape: MultishotShape<TPersona>;
+    /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
+    tools?: MultishotToolDefinition[];
+    /** Map from tool name → executor invoked inline when the agent emits a tool_call. */
+    toolExecutors?: Record<string, MultishotToolExecutor>;
+    /** Map from tool name → artifact type label written into MultishotArtifact.type.
+     *  Tools without a mapping still execute, but their results aren't surfaced as
+     *  typed artifacts (only as tool messages in the transcript). */
+    artifactTypeFor?: (toolName: string) => string | undefined;
+    maxTurns?: number;
+    agentModel?: string;
+    driverModel?: string;
+    apiKey?: string;
+    baseUrl?: string;
+    signal?: AbortSignal;
+}
+declare function runMultishot<TPersona extends MultishotPersona>(opts: RunMultishotOptions<TPersona>): Promise<MultishotResult>;
+
+interface ConversationJudgeInput<TPersona extends MultishotPersona> {
+    transcript: MultishotMessage[];
+    persona: TPersona;
+}
+interface ArtifactJudgeInput<TPersona extends MultishotPersona> {
+    artifact: MultishotArtifact;
+    persona: TPersona;
+}
+interface MultishotJudges<TPersona extends MultishotPersona> {
+    /** Scores the full transcript end-to-end (always runs). */
+    conversation: JudgeConfig<ConversationJudgeInput<TPersona>>;
+    /** Scores each code-type artifact. Optional — omit when domain has no code artifacts. */
+    codeReview?: JudgeConfig<ArtifactJudgeInput<TPersona>>;
+    /** Scores each non-code (research/content/template) artifact. Optional. */
+    contentQuality?: JudgeConfig<ArtifactJudgeInput<TPersona>>;
+    /** Which artifact types route to codeReview. Defaults to ['code']. */
+    codeArtifactTypes?: string[];
+    /** Which artifact types route to contentQuality. Defaults to ['research']. */
+    contentArtifactTypes?: string[];
+}
+interface CellCompositeScore {
+    composite: number;
+    conversation: JudgeScore;
+    codeReview?: {
+        perArtifact: Array<JudgeScore & {
+            turn: number;
+            type: string;
+        }>;
+        composite: number;
+    };
+    contentQuality?: {
+        perArtifact: Array<JudgeScore & {
+            turn: number;
+            type: string;
+        }>;
+        composite: number;
+    };
+}
+interface RunMultishotMatrixOptions<TPersona extends MultishotPersona> {
+    /** AgentProfile axis (matrix primary). */
+    profiles: Array<{
+        id: string;
+        value: AgentProfile;
+    }>;
+    /** Persona axis. */
+    personas: TPersona[];
+    /** Persona-shaping callbacks. */
+    shape: MultishotShape<TPersona>;
+    /** Judge configurations. */
+    judges: MultishotJudges<TPersona>;
+    /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
+    tools?: MultishotToolDefinition[];
+    /** Map from tool name → inline executor. Must align with `tools`. */
+    toolExecutors?: Record<string, MultishotToolExecutor>;
+    /** Tool name → artifact type label. Defaults to research/code mapping. */
+    artifactTypeFor?: (toolName: string) => string | undefined;
+    /** Where per-cell artifacts land. Cells write to `<runDir>/<profileId>/<personaId>/rep-N/`. */
+    runDir: string;
+    /** Replicates per (profile, persona) cell. */
+    reps?: number;
+    /** Max conversation turns per cell. */
+    maxTurns?: number;
+    /** Max concurrent cells. */
+    maxConcurrency?: number;
+    /** Total $ ceiling across the matrix; cells aborted past this. */
+    costCeiling?: number;
+    /** Agent model. */
+    agentModel?: string;
+    /** Driver model. */
+    driverModel?: string;
+    /** Pass-thru fields. */
+    apiKey?: string;
+    baseUrl?: string;
+}
+interface CellOutput {
+    turns: number;
+    toolCalls: number;
+    artifactCount: number;
+}
+interface RunMultishotMatrixResult {
+    matrix: MatrixResult<CellOutput>;
+}
+declare function runMultishotMatrix<TPersona extends MultishotPersona>(opts: RunMultishotMatrixOptions<TPersona>): Promise<RunMultishotMatrixResult>;
+
+export { type ArtifactJudgeInput, type CellCompositeScore, type ConversationJudgeInput, DEFAULT_CODER_MODEL, DEFAULT_DELEGATE_CODE_TOOL, DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_JUDGE_MODEL, DEFAULT_RESEARCHER_MODEL, type DefaultCoderConfig, type DefaultResearcherConfig, type DefaultToolsBundle, type DefaultToolsConfig, type JudgeConfig, type JudgeDimension, type JudgeScore, type MultishotArtifact, MultishotDriverEmptyError, type MultishotJudges, type MultishotMessage, type MultishotPersona, type MultishotResult, type MultishotShape, type MultishotToolDefinition, type MultishotToolExecutor, type RouterCompletionRequest, type RouterCompletionResponse, type RouterToolCall, type RunMultishotMatrixOptions, type RunMultishotMatrixResult, type RunMultishotOptions, createCodeExecutor, createResearchExecutor, defaultDelegationTools, defaultRouterBaseUrl, estimateRouterCost, renderDimensions, renderJsonFooter, requireRouterApiKey, routerCompletion, runJudge, runMultishot, runMultishotMatrix };