@tangle-network/agent-eval 0.23.0 → 0.24.0

package/dist/pipelines/index.js

@@ -0,0 +1,409 @@
+import {
+  DEFAULT_RULES,
+  classifyFailure,
+  compareToBaseline,
+  computeToolUseMetrics
+} from "../chunk-OHEPNJQN.js";
+import {
+  buildTrajectory
+} from "../chunk-RZTMDUO7.js";
+import {
+  interRaterReliability
+} from "../chunk-I4MBDTY5.js";
+import {
+  aggregateLlm,
+  argHash,
+  llmSpans,
+  runFailureClass,
+  toolSpans
+} from "../chunk-47X6LRCE.js";
+import "../chunk-5BKGXME7.js";
+import "../chunk-NG236HPC.js";
+import "../chunk-PZ5AY32C.js";
+
+// src/pipelines/budget-breach.ts
+async function budgetBreachView(store, options = {}) {
+  const runs = await store.listRuns({
+    scenarioId: options.scenarioId,
+    variantId: options.variantId
+  });
+  const findings = [];
+  const byDimension = {};
+  const byScenario = {};
+  const byVariant = {};
+  for (const run of runs) {
+    const entries = await store.budget(run.runId);
+    for (const e of entries) {
+      if (!e.breached) continue;
+      const excessRatio = e.limit > 0 ? e.consumed / e.limit : Infinity;
+      findings.push({
+        runId: run.runId,
+        scenarioId: run.scenarioId,
+        variantId: run.variantId,
+        dimension: e.dimension,
+        limit: e.limit,
+        consumed: e.consumed,
+        excessRatio,
+        timestamp: e.timestamp
+      });
+      byDimension[e.dimension] = (byDimension[e.dimension] ?? 0) + 1;
+      byScenario[run.scenarioId] = (byScenario[run.scenarioId] ?? 0) + 1;
+      if (run.variantId) byVariant[run.variantId] = (byVariant[run.variantId] ?? 0) + 1;
+    }
+  }
+  const breachedRuns = new Set(findings.map((f) => f.runId));
+  return {
+    findings,
+    byDimension,
+    byScenario,
+    byVariant,
+    totalRuns: runs.length,
+    breachedRunRatio: runs.length > 0 ? breachedRuns.size / runs.length : 0
+  };
+}
+
+// src/pipelines/failure-cluster.ts
+async function failureClusterView(store, options = {}) {
+  const rules = options.rules ?? DEFAULT_RULES;
+  const minSize = options.minClusterSize ?? 1;
+  const runs = await store.listRuns();
+  const clusters = /* @__PURE__ */ new Map();
+  let totalFailures = 0;
+  for (const run of runs) {
+    if (run.status === "completed" && run.outcome?.pass !== false) continue;
+    totalFailures++;
+    const spans = await store.spans({ runId: run.runId });
+    const events = await store.events({ runId: run.runId });
+    const cls = classifyFailure({ run, spans, events }, rules);
+    let toolName;
+    let argPrefix;
+    let dimension;
+    if (cls.triggerSpanId) {
+      const trig = spans.find((s) => s.spanId === cls.triggerSpanId);
+      if (trig?.kind === "tool") {
+        toolName = trig.toolName;
+        argPrefix = argHash(trig.args).slice(0, 16);
+      } else if (trig?.kind === "judge") {
+        dimension = trig.dimension;
+      }
+    }
+    if (!toolName) {
+      const ts = await toolSpans(store, run.runId);
+      const errored = ts.filter((t) => t.status === "error").pop();
+      if (errored) {
+        toolName = errored.toolName;
+        argPrefix = argHash(errored.args).slice(0, 16);
+      }
+    }
+    if (!dimension) {
+      const judge = spans.find((s) => s.kind === "judge" && typeof s.dimension === "string");
+      if (judge?.kind === "judge") dimension = judge.dimension;
+    }
+    const key = `${cls.failureClass}|${toolName ?? ""}|${argPrefix ?? ""}|${dimension ?? ""}`;
+    let cluster = clusters.get(key);
+    if (!cluster) {
+      cluster = {
+        failureClass: cls.failureClass,
+        toolName,
+        argPrefix,
+        dimension,
+        runCount: 0,
+        scenarioIds: [],
+        exampleRunId: run.runId,
+        exampleError: firstErrorMessage(spans) ?? cls.reason
+      };
+      clusters.set(key, cluster);
+    }
+    cluster.runCount++;
+    if (!cluster.scenarioIds.includes(run.scenarioId)) cluster.scenarioIds.push(run.scenarioId);
+  }
+  const arr = [...clusters.values()].filter((c) => c.runCount >= minSize).sort((a, b) => b.runCount - a.runCount);
+  return { clusters: arr, totalFailures, totalRuns: runs.length };
+}
+function firstErrorMessage(spans) {
+  const errored = spans.find((s) => s.status === "error");
+  return errored?.error;
+}
+
+// src/pipelines/first-divergence.ts
+async function firstDivergenceView(store, runA, runB, options = {}) {
+  const [a, b] = await Promise.all([buildTrajectory(store, runA), buildTrajectory(store, runB)]);
+  const eq = options.stepEquals ?? defaultStepEquals;
+  const minLen = Math.min(a.steps.length, b.steps.length);
+  for (let i = 0; i < minLen; i++) {
+    const aStep = a.steps[i];
+    const bStep = b.steps[i];
+    if (!eq(aStep, bStep)) {
+      return {
+        runA,
+        runB,
+        firstDivergenceIndex: i,
+        aStep,
+        bStep,
+        reason: describeDifference(aStep, bStep),
+        commonPrefixLen: i
+      };
+    }
+  }
+  if (a.steps.length === b.steps.length) {
+    return { runA, runB, firstDivergenceIndex: null, commonPrefixLen: minLen };
+  }
+  const longer = a.steps.length > b.steps.length ? a : b;
+  return {
+    runA,
+    runB,
+    firstDivergenceIndex: minLen,
+    aStep: a.steps[minLen],
+    bStep: b.steps[minLen],
+    reason: `one trajectory has ${longer.steps.length - minLen} more step(s) after index ${minLen - 1}`,
+    commonPrefixLen: minLen
+  };
+}
+function defaultStepEquals(a, b) {
+  if (a.span.kind !== b.span.kind) return false;
+  if (a.span.kind === "tool" && b.span.kind === "tool") return a.span.toolName === b.span.toolName;
+  if (a.span.kind === "llm" && b.span.kind === "llm") return a.span.model === b.span.model;
+  if (a.span.kind === "judge" && b.span.kind === "judge")
+    return a.span.dimension === b.span.dimension;
+  return a.span.name === b.span.name;
+}
+function describeDifference(a, b) {
+  if (a.span.kind !== b.span.kind) return `kind ${a.span.kind} vs ${b.span.kind}`;
+  if (a.span.kind === "tool" && b.span.kind === "tool" && a.span.toolName !== b.span.toolName) {
+    return `tool ${a.span.toolName} vs ${b.span.toolName}`;
+  }
+  if (a.span.kind === "llm" && b.span.kind === "llm" && a.span.model !== b.span.model) {
+    return `model ${a.span.model} vs ${b.span.model}`;
+  }
+  return `name "${a.span.name}" vs "${b.span.name}"`;
+}
+
+// src/pipelines/judge-agreement.ts
+async function judgeAgreementView(store) {
+  const all = (await store.spans({ kind: "judge" })).filter(
+    (s) => s.kind === "judge"
+  );
+  if (all.length === 0) return { pairs: [], dimensions: [], judgeIds: [] };
+  const byDimension = /* @__PURE__ */ new Map();
+  for (const s of all) {
+    const arr = byDimension.get(s.dimension) ?? [];
+    arr.push(s);
+    byDimension.set(s.dimension, arr);
+  }
+  const judgeIds = [...new Set(all.map((s) => s.judgeId))].sort();
+  const pairs = [];
+  for (const [dim, spans] of byDimension) {
+    const byJudge = /* @__PURE__ */ new Map();
+    for (const s of spans) {
+      const m = byJudge.get(s.judgeId) ?? /* @__PURE__ */ new Map();
+      m.set(s.targetSpanId, s.score);
+      byJudge.set(s.judgeId, m);
+    }
+    const judgesHere = [...byJudge.keys()];
+    for (let i = 0; i < judgesHere.length; i++) {
+      for (let j = i + 1; j < judgesHere.length; j++) {
+        const judgeI = judgesHere[i];
+        const judgeJ = judgesHere[j];
+        const a = byJudge.get(judgeI);
+        const b = byJudge.get(judgeJ);
+        const common = [];
+        for (const [target, scoreA] of a) {
+          const scoreB = b.get(target);
+          if (scoreB !== void 0) common.push([scoreA, scoreB]);
+        }
+        if (common.length < 2) continue;
+        const judgeScores = common.map(
+          ([scoreA, scoreB]) => [
+            { judgeName: judgeI, dimension: dim, score: scoreA, reasoning: "" },
+            { judgeName: judgeJ, dimension: dim, score: scoreB, reasoning: "" }
+          ]
+        );
+        const k = interRaterReliability(
+          judgeScores[0].map((_, k2) => judgeScores.map((pair) => pair[k2]))
+        );
+        pairs.push({
+          judgeA: judgeI,
+          judgeB: judgeJ,
+          dimension: dim,
+          commonItems: common.length,
+          pearson: pearson(
+            common.map((c) => c[0]),
+            common.map((c) => c[1])
+          ),
+          krippendorff: k
+        });
+      }
+    }
+  }
+  return {
+    pairs: pairs.sort((a, b) => b.commonItems - a.commonItems),
+    dimensions: [...byDimension.keys()].sort(),
+    judgeIds
+  };
+}
+function pearson(a, b) {
+  if (a.length !== b.length || a.length < 2) return NaN;
+  const mA = a.reduce((s, v) => s + v, 0) / a.length;
+  const mB = b.reduce((s, v) => s + v, 0) / b.length;
+  let num = 0, denA = 0, denB = 0;
+  for (let i = 0; i < a.length; i++) {
+    const dA = a[i] - mA;
+    const dB = b[i] - mB;
+    num += dA * dB;
+    denA += dA * dA;
+    denB += dB * dB;
+  }
+  if (denA === 0 || denB === 0) return denA === 0 && denB === 0 ? 1 : 0;
+  return num / Math.sqrt(denA * denB);
+}
+
+// src/pipelines/regression.ts
+async function regressionView(store, metrics, options) {
+  const baselineRuns = await store.listRuns(options.baseline);
+  const candidateRuns = await store.listRuns(options.candidate);
+  const samples = await Promise.all(
+    metrics.map(async (m) => {
+      const extract = m.extract ?? defaultExtract(m.metric);
+      const baseline = await extractAll(baselineRuns, extract, store);
+      const candidate = await extractAll(candidateRuns, extract, store);
+      return { metric: m.metric, higherIsBetter: m.higherIsBetter, baseline, candidate };
+    })
+  );
+  return compareToBaseline(samples, options);
+}
+async function extractAll(runs, extract, store) {
+  const out = [];
+  for (const r of runs) {
+    const v = await extract(r, store);
+    if (v !== null && Number.isFinite(v)) out.push(v);
+  }
+  return out;
+}
+function defaultExtract(metric) {
+  return async (run, store) => {
+    switch (metric) {
+      case "score":
+      case "overallScore":
+        return run.outcome?.score ?? null;
+      case "pass":
+        return run.outcome?.pass === true ? 1 : 0;
+      case "durationMs":
+        return run.endedAt && run.startedAt ? run.endedAt - run.startedAt : null;
+      case "costUsd": {
+        const llm = await llmSpans(store, run.runId);
+        return aggregateLlm(llm).costUsd;
+      }
+      case "inputTokens": {
+        const llm = await llmSpans(store, run.runId);
+        return aggregateLlm(llm).inputTokens;
+      }
+      case "outputTokens": {
+        const llm = await llmSpans(store, run.runId);
+        return aggregateLlm(llm).outputTokens;
+      }
+      case "failureClass": {
+        return runFailureClass(run) === "success" ? 1 : 0;
+      }
+      default:
+        return null;
+    }
+  };
+}
+
+// src/pipelines/stuck-loop.ts
+async function stuckLoopView(store, options = {}) {
+  const minOccurrences = options.minOccurrences ?? 3;
+  const runs = options.runId ? [{ runId: options.runId }] : (await store.listRuns()).map((r) => ({ runId: r.runId }));
+  const findings = [];
+  for (const { runId } of runs) {
+    const tools = await toolSpans(store, runId);
+    const byKey = /* @__PURE__ */ new Map();
+    for (const t of tools) {
+      const h = argHash(t.args);
+      const key = `${t.toolName}|${h}`;
+      const bucket = byKey.get(key) ?? { spans: [], argHash: h };
+      bucket.spans.push(t);
+      byKey.set(key, bucket);
+    }
+    for (const [key, { spans, argHash: h }] of byKey) {
+      if (spans.length < minOccurrences) continue;
+      const sorted = [...spans].sort((a, b) => a.startedAt - b.startedAt);
+      const first = sorted[0].startedAt;
+      const last = sorted[sorted.length - 1].startedAt;
+      findings.push({
+        runId,
+        toolName: key.split("|")[0],
+        argHash: h,
+        occurrences: sorted.length,
+        spanIds: sorted.map((s) => s.spanId),
+        windowMs: last - first
+      });
+    }
+  }
+  const affectedRuns = new Set(findings.map((f) => f.runId));
+  return {
+    findings,
+    affectedRunRatio: runs.length > 0 ? affectedRuns.size / runs.length : 0,
+    totalRuns: runs.length
+  };
+}
+
+// src/pipelines/tool-waste.ts
+async function toolWasteView(store, options = {}) {
+  const runs = options.runId ? [options.runId] : (await store.listRuns()).map((r) => r.runId);
+  const byRun = [];
+  let totalCalls = 0;
+  let totalWasted = 0;
+  for (const runId of runs) {
+    const tools = await toolSpans(store, runId);
+    if (tools.length === 0) {
+      byRun.push({ runId, wastedCalls: 0, totalCalls: 0, wasteRate: 0 });
+      continue;
+    }
+    const llms = await llmSpans(store, runId);
+    let wasted = 0;
+    for (const t of tools) {
+      if (t.status === "error") {
+        wasted++;
+        continue;
+      }
+      const laterLlm = llms.filter((l) => l.startedAt > t.startedAt);
+      if (options.usageOracle) {
+        if (!options.usageOracle(t, { llm: laterLlm })) wasted++;
+      } else {
+        const resultStr = stringify(t.result);
+        const used = laterLlm.some(
+          (l) => l.messages.some(
+            (m) => typeof m.content === "string" && resultStr && m.content.includes(resultStr.slice(0, 120))
+          )
+        );
+        if (!used) wasted++;
+      }
+    }
+    const wasteRate = wasted / tools.length;
+    byRun.push({ runId, wastedCalls: wasted, totalCalls: tools.length, wasteRate });
+    totalCalls += tools.length;
+    totalWasted += wasted;
+  }
+  return { byRun, overallWasteRate: totalCalls > 0 ? totalWasted / totalCalls : 0 };
+}
+function stringify(v) {
+  if (v === null || v === void 0) return "";
+  if (typeof v === "string") return v;
+  try {
+    return JSON.stringify(v);
+  } catch {
+    return String(v);
+  }
+}
+export {
+  budgetBreachView,
+  computeToolUseMetrics,
+  failureClusterView,
+  firstDivergenceView,
+  judgeAgreementView,
+  regressionView,
+  stuckLoopView,
+  toolWasteView
+};
+//# sourceMappingURL=index.js.map

package/dist/pipelines/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/pipelines/budget-breach.ts","../../src/pipelines/failure-cluster.ts","../../src/pipelines/first-divergence.ts","../../src/pipelines/judge-agreement.ts","../../src/pipelines/regression.ts","../../src/pipelines/stuck-loop.ts","../../src/pipelines/tool-waste.ts"],"sourcesContent":["/**\n * BudgetBreachView — aggregates breach events across the corpus.\n *\n * Answers: which dimensions get hit most often? Which scenarios are\n * underbudgeted? Which variants trigger the most breaches?\n */\n\nimport type { BudgetSpec } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface BudgetBreachFinding {\n  runId: string\n  scenarioId: string\n  variantId?: string\n  dimension: keyof BudgetSpec\n  limit: number\n  consumed: number\n  excessRatio: number\n  timestamp: number\n}\n\nexport interface BudgetBreachReport {\n  findings: BudgetBreachFinding[]\n  byDimension: Record<string, number>\n  byScenario: Record<string, number>\n  byVariant: Record<string, number>\n  totalRuns: number\n  breachedRunRatio: number\n}\n\nexport async function budgetBreachView(\n  store: TraceStore,\n  options: { scenarioId?: string; variantId?: string } = {},\n): Promise<BudgetBreachReport> {\n  const runs = await store.listRuns({\n    scenarioId: options.scenarioId,\n    variantId: options.variantId,\n  })\n  const findings: BudgetBreachFinding[] = []\n  const byDimension: Record<string, number> = {}\n  const byScenario: Record<string, number> = {}\n  const byVariant: Record<string, number> = {}\n\n  for (const run of runs) {\n    const entries = await store.budget(run.runId)\n    for (const e of entries) {\n      if (!e.breached) continue\n      const excessRatio = e.limit > 0 ? e.consumed / e.limit : Infinity\n      findings.push({\n        runId: run.runId,\n        scenarioId: run.scenarioId,\n        variantId: run.variantId,\n        dimension: e.dimension,\n        limit: e.limit,\n        consumed: e.consumed,\n        excessRatio,\n        timestamp: e.timestamp,\n      })\n      byDimension[e.dimension] = (byDimension[e.dimension] ?? 0) + 1\n      byScenario[run.scenarioId] = (byScenario[run.scenarioId] ?? 0) + 1\n      if (run.variantId) byVariant[run.variantId] = (byVariant[run.variantId] ?? 0) + 1\n    }\n  }\n\n  const breachedRuns = new Set(findings.map((f) => f.runId))\n  return {\n    findings,\n    byDimension,\n    byScenario,\n    byVariant,\n    totalRuns: runs.length,\n    breachedRunRatio: runs.length > 0 ? breachedRuns.size / runs.length : 0,\n  }\n}\n","/**\n * FailureClusterView — groups failed runs by (failureClass, triggerTool,\n * argHash-prefix) so weekly reviews can prioritize the top-N clusters.\n *\n * Each cluster includes: N runs, scenarios affected, representative\n * error message, a proposed mitigation hint (rule → action table).\n */\n\nimport { classifyFailure, DEFAULT_RULES, type FailureRule } from '../failure-taxonomy'\nimport { argHash, toolSpans } from '../trace/query'\nimport type { FailureClass, Span } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface FailureCluster {\n  failureClass: FailureClass\n  /** Tool name when the trigger was a tool span, else undefined. */\n  toolName?: string\n  /** First 16 chars of argHash — clusters similar args. */\n  argPrefix?: string\n  /**\n   * Source dimension when the trigger was a judge span (e.g. `'format'`,\n   * `'safety'`, `'correctness'`). Lets cross-template aggregators\n   * group failures by the dimension that fired without overloading\n   * `argPrefix`. Optional — legacy clusters without this field\n   * deserialize cleanly.\n   */\n  dimension?: string\n  runCount: number\n  scenarioIds: string[]\n  exampleError?: string\n  exampleRunId: string\n}\n\nexport interface FailureClusterReport {\n  clusters: FailureCluster[]\n  totalFailures: number\n  totalRuns: number\n}\n\nexport async function failureClusterView(\n  store: TraceStore,\n  options: { rules?: FailureRule[]; minClusterSize?: number } = {},\n): Promise<FailureClusterReport> {\n  const rules = options.rules ?? DEFAULT_RULES\n  const minSize = options.minClusterSize ?? 1\n  const runs = await store.listRuns()\n\n  type Key = string\n  const clusters = new Map<Key, FailureCluster>()\n  let totalFailures = 0\n\n  for (const run of runs) {\n    if (run.status === 'completed' && run.outcome?.pass !== false) continue\n    totalFailures++\n    const spans = await store.spans({ runId: run.runId })\n    const events = await store.events({ runId: run.runId })\n    const cls = classifyFailure({ run, spans, events }, rules)\n\n    let toolName: string | undefined\n    let argPrefix: string | undefined\n    let dimension: string | undefined\n    if (cls.triggerSpanId) {\n      const trig = spans.find((s) => s.spanId === cls.triggerSpanId)\n      if (trig?.kind === 'tool') {\n        toolName = trig.toolName\n        argPrefix = argHash(trig.args).slice(0, 16)\n      } else if (trig?.kind === 'judge') {\n        dimension = trig.dimension\n      }\n    }\n    // Fallback: look at the last errored tool span\n    if (!toolName) {\n      const ts = await toolSpans(store, run.runId)\n      const errored = ts.filter((t) => t.status === 'error').pop()\n      if (errored) {\n        toolName = errored.toolName\n        argPrefix = argHash(errored.args).slice(0, 16)\n      }\n    }\n    // Secondary signal: any judge span on the failed run carries a\n    // dimension. Useful when the rule classified by judge score but\n    // didn't surface the trigger span (or surfaced a non-judge span).\n    if (!dimension) {\n      const judge = spans.find((s) => s.kind === 'judge' && typeof s.dimension === 'string')\n      if (judge?.kind === 'judge') dimension = judge.dimension\n    }\n\n    const key = `${cls.failureClass}|${toolName ?? ''}|${argPrefix ?? ''}|${dimension ?? ''}`\n    let cluster = clusters.get(key)\n    if (!cluster) {\n      cluster = {\n        failureClass: cls.failureClass,\n        toolName,\n        argPrefix,\n        dimension,\n        runCount: 0,\n        scenarioIds: [],\n        exampleRunId: run.runId,\n        exampleError: firstErrorMessage(spans) ?? cls.reason,\n      }\n      clusters.set(key, cluster)\n    }\n    cluster.runCount++\n    if (!cluster.scenarioIds.includes(run.scenarioId)) cluster.scenarioIds.push(run.scenarioId)\n  }\n\n  const arr = [...clusters.values()]\n    .filter((c) => c.runCount >= minSize)\n    .sort((a, b) => b.runCount - a.runCount)\n\n  return { clusters: arr, totalFailures, totalRuns: runs.length }\n}\n\nfunction firstErrorMessage(spans: Span[]): string | undefined {\n  const errored = spans.find((s) => s.status === 'error')\n  return errored?.error\n}\n","/**\n * FirstDivergenceView — aligns two trajectories by step index, reports\n * the first step where they differ.\n *\n * \"Differ\" is configurable — default is (kind, toolName if tool, model\n * if llm). Use this view to attribute \"why is variant B better?\" to a\n * specific step rather than an aggregate mean delta.\n */\n\nimport type { TraceStore } from '../trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from '../trajectory'\n\nexport interface DivergenceReport {\n  runA: string\n  runB: string\n  firstDivergenceIndex: number | null\n  aStep?: TrajectoryStep\n  bStep?: TrajectoryStep\n  reason?: string\n  /** Common prefix length (steps that matched). */\n  commonPrefixLen: number\n}\n\nexport interface DivergenceOptions {\n  /** Returns true if two steps are considered equal. Default: kind + tool/model match. */\n  stepEquals?: (a: TrajectoryStep, b: TrajectoryStep) => boolean\n}\n\nexport async function firstDivergenceView(\n  store: TraceStore,\n  runA: string,\n  runB: string,\n  options: DivergenceOptions = {},\n): Promise<DivergenceReport> {\n  const [a, b] = await Promise.all([buildTrajectory(store, runA), buildTrajectory(store, runB)])\n  const eq = options.stepEquals ?? defaultStepEquals\n  const minLen = Math.min(a.steps.length, b.steps.length)\n  for (let i = 0; i < minLen; i++) {\n    const aStep = a.steps[i]!\n    const bStep = b.steps[i]!\n    if (!eq(aStep, bStep)) {\n      return {\n        runA,\n        runB,\n        firstDivergenceIndex: i,\n        aStep,\n        bStep,\n        reason: describeDifference(aStep, bStep),\n        commonPrefixLen: i,\n      }\n    }\n  }\n  if (a.steps.length === b.steps.length) {\n    return { runA, runB, firstDivergenceIndex: null, commonPrefixLen: minLen }\n  }\n  const longer: Trajectory = a.steps.length > b.steps.length ? a : b\n  return {\n    runA,\n    runB,\n    firstDivergenceIndex: minLen,\n    aStep: a.steps[minLen],\n    bStep: b.steps[minLen],\n    reason: `one trajectory has ${longer.steps.length - minLen} more step(s) after index ${minLen - 1}`,\n    commonPrefixLen: minLen,\n  }\n}\n\nfunction defaultStepEquals(a: TrajectoryStep, b: TrajectoryStep): boolean {\n  if (a.span.kind !== b.span.kind) return false\n  if (a.span.kind === 'tool' && b.span.kind === 'tool') return a.span.toolName === b.span.toolName\n  if (a.span.kind === 'llm' && b.span.kind === 'llm') return a.span.model === b.span.model\n  if (a.span.kind === 'judge' && b.span.kind === 'judge')\n    return a.span.dimension === b.span.dimension\n  return a.span.name === b.span.name\n}\n\nfunction describeDifference(a: TrajectoryStep, b: TrajectoryStep): string {\n  if (a.span.kind !== b.span.kind) return `kind ${a.span.kind} vs ${b.span.kind}`\n  if (a.span.kind === 'tool' && b.span.kind === 'tool' && a.span.toolName !== b.span.toolName) {\n    return `tool ${a.span.toolName} vs ${b.span.toolName}`\n  }\n  if (a.span.kind === 'llm' && b.span.kind === 'llm' && a.span.model !== b.span.model) {\n    return `model ${a.span.model} vs ${b.span.model}`\n  }\n  return `name \"${a.span.name}\" vs \"${b.span.name}\"`\n}\n","/**\n * JudgeAgreementView — pairwise agreement between judges across the\n * corpus, grouped by dimension.\n *\n * Output drives two workflows:\n *   - Judge robustness audit: \"does Claude agree with GPT at κ ≥ 0.6?\"\n *   - Calibration tracking: κ vs golden human labels over time (by\n *     providing a `humanGoldenJudgeId`).\n */\n\nimport { interRaterReliability } from '../statistics'\nimport type { JudgeSpan } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface JudgePair {\n  judgeA: string\n  judgeB: string\n  dimension: string\n  /** Number of (targetSpanId, dimension) tuples both judges scored. */\n  commonItems: number\n  pearson: number\n  krippendorff: number\n}\n\nexport interface JudgeAgreementReport {\n  pairs: JudgePair[]\n  dimensions: string[]\n  judgeIds: string[]\n}\n\nexport async function judgeAgreementView(store: TraceStore): Promise<JudgeAgreementReport> {\n  const all = (await store.spans({ kind: 'judge' })).filter(\n    (s): s is JudgeSpan => s.kind === 'judge',\n  )\n  if (all.length === 0) return { pairs: [], dimensions: [], judgeIds: [] }\n\n  const byDimension = new Map<string, JudgeSpan[]>()\n  for (const s of all) {\n    const arr = byDimension.get(s.dimension) ?? []\n    arr.push(s)\n    byDimension.set(s.dimension, arr)\n  }\n\n  const judgeIds = [...new Set(all.map((s) => s.judgeId))].sort()\n  const pairs: JudgePair[] = []\n  for (const [dim, spans] of byDimension) {\n    const byJudge = new Map<string, Map<string, number>>()\n    for (const s of spans) {\n      const m = byJudge.get(s.judgeId) ?? new Map<string, number>()\n      m.set(s.targetSpanId, s.score)\n      byJudge.set(s.judgeId, m)\n    }\n    const judgesHere = [...byJudge.keys()]\n    for (let i = 0; i < judgesHere.length; i++) {\n      for (let j = i + 1; j < judgesHere.length; j++) {\n        const judgeI = judgesHere[i]!\n        const judgeJ = judgesHere[j]!\n        const a = byJudge.get(judgeI)!\n        const b = byJudge.get(judgeJ)!\n        const common: Array<[number, number]> = []\n        for (const [target, scoreA] of a) {\n          const scoreB = b.get(target)\n          if (scoreB !== undefined) common.push([scoreA, scoreB])\n        }\n        if (common.length < 2) continue\n        const judgeScores = common.map(\n          ([scoreA, scoreB]) =>\n            [\n              { judgeName: judgeI, dimension: dim, score: scoreA, reasoning: '' },\n              { judgeName: judgeJ, dimension: dim, score: scoreB, reasoning: '' },\n            ] as const,\n        )\n        const k = interRaterReliability(\n          judgeScores[0]!.map((_, k2) => judgeScores.map((pair) => pair[k2]!)),\n        )\n        pairs.push({\n          judgeA: judgeI,\n          judgeB: judgeJ,\n          dimension: dim,\n          commonItems: common.length,\n          pearson: pearson(\n            common.map((c) => c[0]),\n            common.map((c) => c[1]),\n          ),\n          krippendorff: k,\n        })\n      }\n    }\n  }\n\n  return {\n    pairs: pairs.sort((a, b) => b.commonItems - a.commonItems),\n    dimensions: [...byDimension.keys()].sort(),\n    judgeIds,\n  }\n}\n\nfunction pearson(a: number[], b: number[]): number {\n  if (a.length !== b.length || a.length < 2) return NaN\n  const mA = a.reduce((s, v) => s + v, 0) / a.length\n  const mB = b.reduce((s, v) => s + v, 0) / b.length\n  let num = 0,\n    denA = 0,\n    denB = 0\n  for (let i = 0; i < a.length; i++) {\n    const dA = a[i]! - mA\n    const dB = b[i]! - mB\n    num += dA * dB\n    denA += dA * dA\n    denB += dB * dB\n  }\n  if (denA === 0 || denB === 0) return denA === 0 && denB === 0 ? 1 : 0\n  return num / Math.sqrt(denA * denB)\n}\n","/**\n * RegressionView — compares a candidate slice to a baseline slice on a\n * named metric. Delegates the statistics (Welch's t-test, Cohen's d,\n * IQR stability) to `baseline.ts`.\n *\n * This is the entry point for CI regression gates: \"given runs tagged\n * release=A and release=B, did any metric regress?\"\n */\n\nimport { type BaselineOptions, type BaselineReport, compareToBaseline } from '../baseline'\nimport { aggregateLlm, llmSpans, runFailureClass } from '../trace/query'\nimport type { Run } from '../trace/schema'\nimport type { RunFilter, TraceStore } from '../trace/store'\n\nexport interface RegressionSpec {\n  metric: string\n  higherIsBetter: boolean\n  /** Extract a scalar from a run. Default extractors handle common metrics. */\n  extract?: (run: Run, store: TraceStore) => Promise<number | null>\n}\n\nexport interface RegressionOptions extends BaselineOptions {\n  baseline: RunFilter\n  candidate: RunFilter\n}\n\nexport async function regressionView(\n  store: TraceStore,\n  metrics: RegressionSpec[],\n  options: RegressionOptions,\n): Promise<BaselineReport> {\n  const baselineRuns = await store.listRuns(options.baseline)\n  const candidateRuns = await store.listRuns(options.candidate)\n  const samples = await Promise.all(\n    metrics.map(async (m) => {\n      const extract = m.extract ?? defaultExtract(m.metric)\n      const baseline = await extractAll(baselineRuns, extract, store)\n      const candidate = await extractAll(candidateRuns, extract, store)\n      return { metric: m.metric, higherIsBetter: m.higherIsBetter, baseline, candidate }\n    }),\n  )\n  return compareToBaseline(samples, options)\n}\n\nasync function extractAll(\n  runs: Run[],\n  extract: (r: Run, s: TraceStore) => Promise<number | null>,\n  store: TraceStore,\n): Promise<number[]> {\n  const out: number[] = []\n  for (const r of runs) {\n    const v = await extract(r, store)\n    if (v !== null && Number.isFinite(v)) out.push(v)\n  }\n  return out\n}\n\nfunction defaultExtract(metric: string): (run: Run, store: TraceStore) => Promise<number | null> {\n  return async (run, store) => {\n    switch (metric) {\n      case 'score':\n      case 'overallScore':\n        return run.outcome?.score ?? null\n      case 'pass':\n        return run.outcome?.pass === true ? 1 : 0\n      case 'durationMs':\n        return run.endedAt && run.startedAt ? run.endedAt - run.startedAt : null\n      case 'costUsd': {\n        const llm = await llmSpans(store, run.runId)\n        return aggregateLlm(llm).costUsd\n      }\n      case 'inputTokens': {\n        const llm = await llmSpans(store, run.runId)\n        return aggregateLlm(llm).inputTokens\n      }\n      case 'outputTokens': {\n        const llm = await llmSpans(store, run.runId)\n        return aggregateLlm(llm).outputTokens\n      }\n      case 'failureClass': {\n        return runFailureClass(run) === 'success' ? 1 : 0\n      }\n      default:\n        return null\n    }\n  }\n}\n","/**\n * StuckLoopView — detects when an agent calls the same tool with the\n * same (or structurally similar) arguments ≥ N times in a short window.\n *\n * Rationale: agents that loop are the number-one production failure\n * mode on long-horizon flows. The view returns (runId, toolName,\n * argHash, occurrences, windowMs) for each detected loop plus a\n * fraction of runs affected.\n */\n\nimport { argHash, toolSpans } from '../trace/query'\nimport type { TraceStore } from '../trace/store'\n\nexport interface StuckLoopFinding {\n  runId: string\n  toolName: string\n  argHash: string\n  occurrences: number\n  spanIds: string[]\n  /** Milliseconds between first and last call in the loop. */\n  windowMs: number\n}\n\nexport interface StuckLoopReport {\n  findings: StuckLoopFinding[]\n  affectedRunRatio: number\n  totalRuns: number\n}\n\nexport interface StuckLoopOptions {\n  /** Minimum call count to flag a loop (default 3). */\n  minOccurrences?: number\n  /** Filter to a specific runId; omit to scan the entire corpus. */\n  runId?: string\n}\n\nexport async function stuckLoopView(\n  store: TraceStore,\n  options: StuckLoopOptions = {},\n): Promise<StuckLoopReport> {\n  const minOccurrences = options.minOccurrences ?? 3\n  const runs = options.runId\n    ? [{ runId: options.runId }]\n    : (await store.listRuns()).map((r) => ({ runId: r.runId }))\n\n  const findings: StuckLoopFinding[] = []\n  for (const { runId } of runs) {\n    const tools = await toolSpans(store, runId)\n    const byKey = new Map<string, { spans: typeof tools; argHash: string }>()\n    for (const t of tools) {\n      const h = argHash(t.args)\n      const key = `${t.toolName}|${h}`\n      const bucket = byKey.get(key) ?? { spans: [], argHash: h }\n      bucket.spans.push(t)\n      byKey.set(key, bucket)\n    }\n    for (const [key, { spans, argHash: h }] of byKey) {\n      if (spans.length < minOccurrences) continue\n      const sorted = [...spans].sort((a, b) => a.startedAt - b.startedAt)\n      const first = sorted[0]!.startedAt\n      const last = sorted[sorted.length - 1]!.startedAt\n      findings.push({\n        runId,\n        toolName: key.split('|')[0]!,\n        argHash: h,\n        occurrences: sorted.length,\n        spanIds: sorted.map((s) => s.spanId),\n        windowMs: last - first,\n      })\n    }\n  }\n\n  const affectedRuns = new Set(findings.map((f) => f.runId))\n  return {\n    findings,\n    affectedRunRatio: runs.length > 0 ? affectedRuns.size / runs.length : 0,\n    totalRuns: runs.length,\n  }\n}\n","/**\n * ToolWasteView — fraction of tool calls whose results weren't used\n * downstream. Without a \"used\" signal we fall back to structural\n * proxies: error calls, duplicate calls, and tool calls followed by\n * zero subsequent LLM spans are all considered waste.\n *\n * Consumers can pass a `usageOracle` that inspects a tool span and\n * returns true iff the tool's result appears in a later LLM message,\n * artifact, or state mutation — that's the canonical definition; the\n * default heuristic is a reasonable fallback.\n */\n\nimport { computeToolUseMetrics } from '../tool-use-metrics'\nimport { llmSpans, toolSpans } from '../trace/query'\nimport type { ToolSpan } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface ToolWasteFinding {\n  runId: string\n  wastedCalls: number\n  totalCalls: number\n  wasteRate: number\n}\n\nexport interface ToolWasteReport {\n  byRun: ToolWasteFinding[]\n  overallWasteRate: number\n}\n\nexport interface ToolWasteOptions {\n  runId?: string\n  usageOracle?: (tool: ToolSpan, later: { llm: Awaited<ReturnType<typeof llmSpans>> }) => boolean\n}\n\nexport async function toolWasteView(\n  store: TraceStore,\n  options: ToolWasteOptions = {},\n): Promise<ToolWasteReport> {\n  const runs = options.runId ? [options.runId] : (await store.listRuns()).map((r) => r.runId)\n\n  const byRun: ToolWasteFinding[] = []\n  let totalCalls = 0\n  let totalWasted = 0\n  for (const runId of runs) {\n    const tools = await toolSpans(store, runId)\n    if (tools.length === 0) {\n      byRun.push({ runId, wastedCalls: 0, totalCalls: 0, wasteRate: 0 })\n      continue\n    }\n    const llms = await llmSpans(store, runId)\n    let wasted = 0\n    for (const t of tools) {\n      if (t.status === 'error') {\n        wasted++\n        continue\n      }\n      const laterLlm = llms.filter((l) => l.startedAt > t.startedAt)\n      if (options.usageOracle) {\n        if (!options.usageOracle(t, { llm: laterLlm })) wasted++\n      } else {\n        // Default heuristic: a tool whose result is NOT mentioned in any\n        // later LLM input message is likely wasted.\n        const resultStr = stringify(t.result)\n        const used = laterLlm.some((l) =>\n          l.messages.some(\n            (m) =>\n              typeof m.content === 'string' &&\n              resultStr &&\n              m.content.includes(resultStr.slice(0, 120)),\n          ),\n        )\n        if (!used) wasted++\n      }\n    }\n    const wasteRate = wasted / tools.length\n    byRun.push({ runId, wastedCalls: wasted, totalCalls: tools.length, wasteRate })\n    totalCalls += tools.length\n    totalWasted += wasted\n  }\n  return { byRun, overallWasteRate: totalCalls > 0 ? totalWasted / totalCalls : 0 }\n}\n\nfunction stringify(v: unknown): string {\n  if (v === null || v === undefined) return ''\n  if (typeof v === 'string') return v\n  try {\n    return JSON.stringify(v)\n  } catch {\n    return String(v)\n  }\n}\n\n// Re-export for convenience in consumers that want both descriptive and usage metrics.\nexport { computeToolUseMetrics }\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA8BA,eAAsB,iBACpB,OACA,UAAuD,CAAC,GAC3B;AAC7B,QAAM,OAAO,MAAM,MAAM,SAAS;AAAA,IAChC,YAAY,QAAQ;AAAA,IACpB,WAAW,QAAQ;AAAA,EACrB,CAAC;AACD,QAAM,WAAkC,CAAC;AACzC,QAAM,cAAsC,CAAC;AAC7C,QAAM,aAAqC,CAAC;AAC5C,QAAM,YAAoC,CAAC;AAE3C,aAAW,OAAO,MAAM;AACtB,UAAM,UAAU,MAAM,MAAM,OAAO,IAAI,KAAK;AAC5C,eAAW,KAAK,SAAS;AACvB,UAAI,CAAC,EAAE,SAAU;AACjB,YAAM,cAAc,EAAE,QAAQ,IAAI,EAAE,WAAW,EAAE,QAAQ;AACzD,eAAS,KAAK;AAAA,QACZ,OAAO,IAAI;AAAA,QACX,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,WAAW,EAAE;AAAA,QACb,OAAO,EAAE;AAAA,QACT,UAAU,EAAE;AAAA,QACZ;AAAA,QACA,WAAW,EAAE;AAAA,MACf,CAAC;AACD,kBAAY,EAAE,SAAS,KAAK,YAAY,EAAE,SAAS,KAAK,KAAK;AAC7D,iBAAW,IAAI,UAAU,KAAK,WAAW,IAAI,UAAU,KAAK,KAAK;AACjE,UAAI,IAAI,UAAW,WAAU,IAAI,SAAS,KAAK,UAAU,IAAI,SAAS,KAAK,KAAK;AAAA,IAClF;AAAA,EACF;AAEA,QAAM,eAAe,IAAI,IAAI,SAAS,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC;AACzD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW,KAAK;AAAA,IAChB,kBAAkB,KAAK,SAAS,IAAI,aAAa,OAAO,KAAK,SAAS;AAAA,EACxE;AACF;;;AClCA,eAAsB,mBACpB,OACA,UAA8D,CAAC,GAChC;AAC/B,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,UAAU,QAAQ,kBAAkB;AAC1C,QAAM,OAAO,MAAM,MAAM,SAAS;AAGlC,QAAM,WAAW,oBAAI,IAAyB;AAC9C,MAAI,gBAAgB;AAEpB,aAAW,OAAO,MAAM;AACtB,QAAI,IAAI,WAAW,eAAe,IAAI,SAAS,SAAS,MAAO;AAC/D;AACA,UAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,IAAI,MAAM,CAAC;AACpD,UAAM,SAAS,MAAM,MAAM,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC;AACtD,UAAM,MAAM,gBAAgB,EAAE,KAAK,OAAO,OAAO,GAAG,KAAK;AAEzD,QAAI;AACJ,QAAI;AACJ,QAAI;AACJ,QAAI,IAAI,eAAe;AACrB,YAAM,OAAO,MAAM,KAAK,CAAC,MAAM,EAAE,WAAW,IAAI,aAAa;AAC7D,UAAI,MAAM,SAAS,QAAQ;AACzB,mBAAW,KAAK;AAChB,oBAAY,QAAQ,KAAK,IAAI,EAAE,MAAM,GAAG,EAAE;AAAA,MAC5C,WAAW,MAAM,SAAS,SAAS;AACjC,oBAAY,KAAK;AAAA,MACnB;AAAA,IACF;AAEA,QAAI,CAAC,UAAU;AACb,YAAM,KAAK,MAAM,UAAU,OAAO,IAAI,KAAK;AAC3C,YAAM,UAAU,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,OAAO,EAAE,IAAI;AAC3D,UAAI,SAAS;AACX,mBAAW,QAAQ;AACnB,oBAAY,QAAQ,QAAQ,IAAI,EAAE,MAAM,GAAG,EAAE;AAAA,MAC/C;AAAA,IACF;AAIA,QAAI,CAAC,WAAW;AACd,YAAM,QAAQ,MAAM,KAAK,CAAC,MAAM,EAAE,SAAS,WAAW,OAAO,EAAE,cAAc,QAAQ;AACrF,UAAI,OAAO,SAAS,QAAS,aAAY,MAAM;AAAA,IACjD;AAEA,UAAM,MAAM,GAAG,IAAI,YAAY,IAAI,YAAY,EAAE,IAAI,aAAa,EAAE,IAAI,aAAa,EAAE;AACvF,QAAI,UAAU,SAAS,IAAI,GAAG;AAC9B,QAAI,CAAC,SAAS;AACZ,gBAAU;AAAA,QACR,cAAc,IAAI;AAAA,QAClB;AAAA,QACA;AAAA,QACA;AAAA,QACA,UAAU;AAAA,QACV,aAAa,CAAC;AAAA,QACd,cAAc,IAAI;AAAA,QAClB,cAAc,kBAAkB,KAAK,KAAK,IAAI;AAAA,MAChD;AACA,eAAS,IAAI,KAAK,OAAO;AAAA,IAC3B;AACA,YAAQ;AACR,QAAI,CAAC,QAAQ,YAAY,SAAS,IAAI,UAAU,EAAG,SAAQ,YAAY,KAAK,IAAI,UAAU;AAAA,EAC5F;AAEA,QAAM,MAAM,CAAC,GAAG,SAAS,OAAO,CAAC,EAC9B,OAAO,CAAC,MAAM,EAAE,YAAY,OAAO,EACnC,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAEzC,SAAO,EAAE,UAAU,KAAK,eAAe,WAAW,KAAK,OAAO;AAChE;AAEA,SAAS,kBAAkB,OAAmC;AAC5D,QAAM,UAAU,MAAM,KAAK,CAAC,MAAM,EAAE,WAAW,OAAO;AACtD,SAAO,SAAS;AAClB;;;ACxFA,eAAsB,oBACpB,OACA,MACA,MACA,UAA6B,CAAC,GACH;AAC3B,QAAM,CAAC,GAAG,CAAC,IAAI,MAAM,QAAQ,IAAI,CAAC,gBAAgB,OAAO,IAAI,GAAG,gBAAgB,OAAO,IAAI,CAAC,CAAC;AAC7F,QAAM,KAAK,QAAQ,cAAc;AACjC,QAAM,SAAS,KAAK,IAAI,EAAE,MAAM,QAAQ,EAAE,MAAM,MAAM;AACtD,WAAS,IAAI,GAAG,IAAI,QAAQ,KAAK;AAC/B,UAAM,QAAQ,EAAE,MAAM,CAAC;AACvB,UAAM,QAAQ,EAAE,MAAM,CAAC;AACvB,QAAI,CAAC,GAAG,OAAO,KAAK,GAAG;AACrB,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,sBAAsB;AAAA,QACtB;AAAA,QACA;AAAA,QACA,QAAQ,mBAAmB,OAAO,KAAK;AAAA,QACvC,iBAAiB;AAAA,MACnB;AAAA,IACF;AAAA,EACF;AACA,MAAI,EAAE,MAAM,WAAW,EAAE,MAAM,QAAQ;AACrC,WAAO,EAAE,MAAM,MAAM,sBAAsB,MAAM,iBAAiB,OAAO;AAAA,EAC3E;AACA,QAAM,SAAqB,EAAE,MAAM,SAAS,EAAE,MAAM,SAAS,IAAI;AACjE,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,sBAAsB;AAAA,IACtB,OAAO,EAAE,MAAM,MAAM;AAAA,IACrB,OAAO,EAAE,MAAM,MAAM;AAAA,IACrB,QAAQ,sBAAsB,OAAO,MAAM,SAAS,MAAM,6BAA6B,SAAS,CAAC;AAAA,IACjG,iBAAiB;AAAA,EACnB;AACF;AAEA,SAAS,kBAAkB,GAAmB,GAA4B;AACxE,MAAI,EAAE,KAAK,SAAS,EAAE,KAAK,KAAM,QAAO;AACxC,MAAI,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,SAAS,OAAQ,QAAO,EAAE,KAAK,aAAa,EAAE,KAAK;AACxF,MAAI,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,SAAS,MAAO,QAAO,EAAE,KAAK,UAAU,EAAE,KAAK;AACnF,MAAI,EAAE,KAAK,SAAS,WAAW,EAAE,KAAK,SAAS;AAC7C,WAAO,EAAE,KAAK,cAAc,EAAE,KAAK;AACrC,SAAO,EAAE,KAAK,SAAS,EAAE,KAAK;AAChC;AAEA,SAAS,mBAAmB,GAAmB,GAA2B;AACxE,MAAI,EAAE,KAAK,SAAS,EAAE,KAAK,KAAM,QAAO,QAAQ,EAAE,KAAK,IAAI,OAAO,EAAE,KAAK,IAAI;AAC7E,MAAI,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,aAAa,EAAE,KAAK,UAAU;AAC3F,WAAO,QAAQ,EAAE,KAAK,QAAQ,OAAO,EAAE,KAAK,QAAQ;AAAA,EACtD;AACA,MAAI,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,OAAO;AACnF,WAAO,SAAS,EAAE,KAAK,KAAK,OAAO,EAAE,KAAK,KAAK;AAAA,EACjD;AACA,SAAO,SAAS,EAAE,KAAK,IAAI,SAAS,EAAE,KAAK,IAAI;AACjD;;;ACvDA,eAAsB,mBAAmB,OAAkD;AACzF,QAAM,OAAO,MAAM,MAAM,MAAM,EAAE,MAAM,QAAQ,CAAC,GAAG;AAAA,IACjD,CAAC,MAAsB,EAAE,SAAS;AAAA,EACpC;AACA,MAAI,IAAI,WAAW,EAAG,QAAO,EAAE,OAAO,CAAC,GAAG,YAAY,CAAC,GAAG,UAAU,CAAC,EAAE;AAEvE,QAAM,cAAc,oBAAI,IAAyB;AACjD,aAAW,KAAK,KAAK;AACnB,UAAM,MAAM,YAAY,IAAI,EAAE,SAAS,KAAK,CAAC;AAC7C,QAAI,KAAK,CAAC;AACV,gBAAY,IAAI,EAAE,WAAW,GAAG;AAAA,EAClC;AAEA,QAAM,WAAW,CAAC,GAAG,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,KAAK;AAC9D,QAAM,QAAqB,CAAC;AAC5B,aAAW,CAAC,KAAK,KAAK,KAAK,aAAa;AACtC,UAAM,UAAU,oBAAI,IAAiC;AACrD,eAAW,KAAK,OAAO;AACrB,YAAM,IAAI,QAAQ,IAAI,EAAE,OAAO,KAAK,oBAAI,IAAoB;AAC5D,QAAE,IAAI,EAAE,cAAc,EAAE,KAAK;AAC7B,cAAQ,IAAI,EAAE,SAAS,CAAC;AAAA,IAC1B;AACA,UAAM,aAAa,CAAC,GAAG,QAAQ,KAAK,CAAC;AACrC,aAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,eAAS,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC9C,cAAM,SAAS,WAAW,CAAC;AAC3B,cAAM,SAAS,WAAW,CAAC;AAC3B,cAAM,IAAI,QAAQ,IAAI,MAAM;AAC5B,cAAM,IAAI,QAAQ,IAAI,MAAM;AAC5B,cAAM,SAAkC,CAAC;AACzC,mBAAW,CAAC,QAAQ,MAAM,KAAK,GAAG;AAChC,gBAAM,SAAS,EAAE,IAAI,MAAM;AAC3B,cAAI,WAAW,OAAW,QAAO,KAAK,CAAC,QAAQ,MAAM,CAAC;AAAA,QACxD;AACA,YAAI,OAAO,SAAS,EAAG;AACvB,cAAM,cAAc,OAAO;AAAA,UACzB,CAAC,CAAC,QAAQ,MAAM,MACd;AAAA,YACE,EAAE,WAAW,QAAQ,WAAW,KAAK,OAAO,QAAQ,WAAW,GAAG;AAAA,YAClE,EAAE,WAAW,QAAQ,WAAW,KAAK,OAAO,QAAQ,WAAW,GAAG;AAAA,UACpE;AAAA,QACJ;AACA,cAAM,IAAI;AAAA,UACR,YAAY,CAAC,EAAG,IAAI,CAAC,GAAG,OAAO,YAAY,IAAI,CAAC,SAAS,KAAK,EAAE,CAAE,CAAC;AAAA,QACrE;AACA,cAAM,KAAK;AAAA,UACT,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR,WAAW;AAAA,UACX,aAAa,OAAO;AAAA,UACpB,SAAS;AAAA,YACP,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAAA,YACtB,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAAA,UACxB;AAAA,UACA,cAAc;AAAA,QAChB,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,OAAO,MAAM,KAAK,CAAC,GAAG,MAAM,EAAE,cAAc,EAAE,WAAW;AAAA,IACzD,YAAY,CAAC,GAAG,YAAY,KAAK,CAAC,EAAE,KAAK;AAAA,IACzC;AAAA,EACF;AACF;AAEA,SAAS,QAAQ,GAAa,GAAqB;AACjD,MAAI,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAG,QAAO;AAClD,QAAM,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE;AAC5C,QAAM,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE;AAC5C,MAAI,MAAM,GACR,OAAO,GACP,OAAO;AACT,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,UAAM,KAAK,EAAE,CAAC,IAAK;AACnB,UAAM,KAAK,EAAE,CAAC,IAAK;AACnB,WAAO,KAAK;AACZ,YAAQ,KAAK;AACb,YAAQ,KAAK;AAAA,EACf;AACA,MAAI,SAAS,KAAK,SAAS,EAAG,QAAO,SAAS,KAAK,SAAS,IAAI,IAAI;AACpE,SAAO,MAAM,KAAK,KAAK,OAAO,IAAI;AACpC;;;ACvFA,eAAsB,eACpB,OACA,SACA,SACyB;AACzB,QAAM,eAAe,MAAM,MAAM,SAAS,QAAQ,QAAQ;AAC1D,QAAM,gBAAgB,MAAM,MAAM,SAAS,QAAQ,SAAS;AAC5D,QAAM,UAAU,MAAM,QAAQ;AAAA,IAC5B,QAAQ,IAAI,OAAO,MAAM;AACvB,YAAM,UAAU,EAAE,WAAW,eAAe,EAAE,MAAM;AACpD,YAAM,WAAW,MAAM,WAAW,cAAc,SAAS,KAAK;AAC9D,YAAM,YAAY,MAAM,WAAW,eAAe,SAAS,KAAK;AAChE,aAAO,EAAE,QAAQ,EAAE,QAAQ,gBAAgB,EAAE,gBAAgB,UAAU,UAAU;AAAA,IACnF,CAAC;AAAA,EACH;AACA,SAAO,kBAAkB,SAAS,OAAO;AAC3C;AAEA,eAAe,WACb,MACA,SACA,OACmB;AACnB,QAAM,MAAgB,CAAC;AACvB,aAAW,KAAK,MAAM;AACpB,UAAM,IAAI,MAAM,QAAQ,GAAG,KAAK;AAChC,QAAI,MAAM,QAAQ,OAAO,SAAS,CAAC,EAAG,KAAI,KAAK,CAAC;AAAA,EAClD;AACA,SAAO;AACT;AAEA,SAAS,eAAe,QAAyE;AAC/F,SAAO,OAAO,KAAK,UAAU;AAC3B,YAAQ,QAAQ;AAAA,MACd,KAAK;AAAA,MACL,KAAK;AACH,eAAO,IAAI,SAAS,SAAS;AAAA,MAC/B,KAAK;AACH,eAAO,IAAI,SAAS,SAAS,OAAO,IAAI;AAAA,MAC1C,KAAK;AACH,eAAO,IAAI,WAAW,IAAI,YAAY,IAAI,UAAU,IAAI,YAAY;AAAA,MACtE,KAAK,WAAW;AACd,cAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;AAC3C,eAAO,aAAa,GAAG,EAAE;AAAA,MAC3B;AAAA,MACA,KAAK,eAAe;AAClB,cAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;AAC3C,eAAO,aAAa,GAAG,EAAE;AAAA,MAC3B;AAAA,MACA,KAAK,gBAAgB;AACnB,cAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;AAC3C,eAAO,aAAa,GAAG,EAAE;AAAA,MAC3B;AAAA,MACA,KAAK,gBAAgB;AACnB,eAAO,gBAAgB,GAAG,MAAM,YAAY,IAAI;AAAA,MAClD;AAAA,MACA;AACE,eAAO;AAAA,IACX;AAAA,EACF;AACF;;;AClDA,eAAsB,cACpB,OACA,UAA4B,CAAC,GACH;AAC1B,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,OAAO,QAAQ,QACjB,CAAC,EAAE,OAAO,QAAQ,MAAM,CAAC,KACxB,MAAM,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE;AAE5D,QAAM,WAA+B,CAAC;AACtC,aAAW,EAAE,MAAM,KAAK,MAAM;AAC5B,UAAM,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC1C,UAAM,QAAQ,oBAAI,IAAsD;AACxE,eAAW,KAAK,OAAO;AACrB,YAAM,IAAI,QAAQ,EAAE,IAAI;AACxB,YAAM,MAAM,GAAG,EAAE,QAAQ,IAAI,CAAC;AAC9B,YAAM,SAAS,MAAM,IAAI,GAAG,KAAK,EAAE,OAAO,CAAC,GAAG,SAAS,EAAE;AACzD,aAAO,MAAM,KAAK,CAAC;AACnB,YAAM,IAAI,KAAK,MAAM;AAAA,IACvB;AACA,eAAW,CAAC,KAAK,EAAE,OAAO,SAAS,EAAE,CAAC,KAAK,OAAO;AAChD,UAAI,MAAM,SAAS,eAAgB;AACnC,YAAM,SAAS,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAClE,YAAM,QAAQ,OAAO,CAAC,EAAG;AACzB,YAAM,OAAO,OAAO,OAAO,SAAS,CAAC,EAAG;AACxC,eAAS,KAAK;AAAA,QACZ;AAAA,QACA,UAAU,IAAI,MAAM,GAAG,EAAE,CAAC;AAAA,QAC1B,SAAS;AAAA,QACT,aAAa,OAAO;AAAA,QACpB,SAAS,OAAO,IAAI,CAAC,MAAM,EAAE,MAAM;AAAA,QACnC,UAAU,OAAO;AAAA,MACnB,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,eAAe,IAAI,IAAI,SAAS,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC;AACzD,SAAO;AAAA,IACL;AAAA,IACA,kBAAkB,KAAK,SAAS,IAAI,aAAa,OAAO,KAAK,SAAS;AAAA,IACtE,WAAW,KAAK;AAAA,EAClB;AACF;;;AC5CA,eAAsB,cACpB,OACA,UAA4B,CAAC,GACH;AAC1B,QAAM,OAAO,QAAQ,QAAQ,CAAC,QAAQ,KAAK,KAAK,MAAM,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK;AAE1F,QAAM,QAA4B,CAAC;AACnC,MAAI,aAAa;AACjB,MAAI,cAAc;AAClB,aAAW,SAAS,MAAM;AACxB,UAAM,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC1C,QAAI,MAAM,WAAW,GAAG;AACtB,YAAM,KAAK,EAAE,OAAO,aAAa,GAAG,YAAY,GAAG,WAAW,EAAE,CAAC;AACjE;AAAA,IACF;AACA,UAAM,OAAO,MAAM,SAAS,OAAO,KAAK;AACxC,QAAI,SAAS;AACb,eAAW,KAAK,OAAO;AACrB,UAAI,EAAE,WAAW,SAAS;AACxB;AACA;AAAA,MACF;AACA,YAAM,WAAW,KAAK,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,SAAS;AAC7D,UAAI,QAAQ,aAAa;AACvB,YAAI,CAAC,QAAQ,YAAY,GAAG,EAAE,KAAK,SAAS,CAAC,EAAG;AAAA,MAClD,OAAO;AAGL,cAAM,YAAY,UAAU,EAAE,MAAM;AACpC,cAAM,OAAO,SAAS;AAAA,UAAK,CAAC,MAC1B,EAAE,SAAS;AAAA,YACT,CAAC,MACC,OAAO,EAAE,YAAY,YACrB,aACA,EAAE,QAAQ,SAAS,UAAU,MAAM,GAAG,GAAG,CAAC;AAAA,UAC9C;AAAA,QACF;AACA,YAAI,CAAC,KAAM;AAAA,MACb;AAAA,IACF;AACA,UAAM,YAAY,SAAS,MAAM;AACjC,UAAM,KAAK,EAAE,OAAO,aAAa,QAAQ,YAAY,MAAM,QAAQ,UAAU,CAAC;AAC9E,kBAAc,MAAM;AACpB,mBAAe;AAAA,EACjB;AACA,SAAO,EAAE,OAAO,kBAAkB,aAAa,IAAI,cAAc,aAAa,EAAE;AAClF;AAEA,SAAS,UAAU,GAAoB;AACrC,MAAI,MAAM,QAAQ,MAAM,OAAW,QAAO;AAC1C,MAAI,OAAO,MAAM,SAAU,QAAO;AAClC,MAAI;AACF,WAAO,KAAK,UAAU,CAAC;AAAA,EACzB,QAAQ;AACN,WAAO,OAAO,CAAC;AAAA,EACjB;AACF;","names":[]}

package/dist/prm/index.d.ts

@@ -0,0 +1,99 @@
+import { P as PrmGradedTrace, S as StepRubric, a as PrmGrader } from '../rubric-D5tjHNJQ.js';
+export { G as GradedStep, b as StepContext, i as isPrmVerdict } from '../rubric-D5tjHNJQ.js';
+import { S as Span, T as TraceStore } from '../store-Db2Bv8Cf.js';
+import '../trajectory-CnoBo-JY.js';
+
+/**
+ * Export PRM-graded traces as training data for downstream reward-model
+ * fine-tuning. Canonical format is NDJSON of
+ * `{ trajectory_text, step_index, rubric, score }` so a small model can
+ * learn to predict step rewards from step context.
+ *
+ * The framework doesn't train the model — we emit the data; callers
+ * plug it into their preferred trainer (TRL, Unsloth, custom).
+ */
+
+interface PrmTrainingSample {
+    runId: string;
+    spanId: string;
+    rubricId: string;
+    score: number;
+    /** Serialized step context — step + surrounding conversation. */
+    context: {
+        priorTurns: Array<{
+            role: string;
+            content: string;
+        }>;
+        step: {
+            kind: Span['kind'];
+            text: string;
+        };
+    };
+    /** Optional evidence + rationale for auditability. */
+    rationale?: string;
+    evidence?: string;
+}
+declare function exportTrainingData(store: TraceStore, graded: PrmGradedTrace[], options?: {
+    contextWindow?: number;
+}): Promise<PrmTrainingSample[]>;
+/** NDJSON serialization — write to file or stream directly to a trainer. */
+declare function toNdjson(samples: PrmTrainingSample[]): string;
+
+/**
+ * Built-in reference rubrics. Consumers combine these with domain
+ * rubrics. All are deterministic, rule-based — cheap to run + easy
+ * to unit-test. LLM-based rubrics are trivially authored by
+ * following the StepRubric contract.
+ */
+
+/** Penalize very short or very long assistant outputs. */
+declare function outputLengthRubric(args?: {
+    minChars?: number;
+    maxChars?: number;
+    weight?: number;
+}): StepRubric;
+/** Reward tool calls that succeeded (status='ok') with an informative result. */
+declare function toolSuccessRubric(args?: {
+    weight?: number;
+}): StepRubric;
+/** Penalize tool calls that duplicate a prior call with identical args. */
+declare function toolNonRedundantRubric(args?: {
+    weight?: number;
+}): StepRubric;
+/** Penalize LLM outputs that contain common refusal markers when a refusal
+ *  is NOT expected (caller inverts weight for scenarios where refusal IS expected). */
+declare function nonRefusalRubric(args?: {
+    markers?: RegExp[];
+    weight?: number;
+}): StepRubric;
+/** Reward outputs that invoke the next-step tool the trajectory actually uses
+ *  (i.e. the LLM span announced "I will call X" and the following tool span IS X). */
+declare function toolIntentAlignmentRubric(args?: {
+    weight?: number;
+}): StepRubric;
+
+/**
+ * Inference-time PRM scoring — pick the best of N candidate trajectories
+ * using a trained reward model (or a rule-based PRM as a proxy).
+ *
+ * The canonical Best-of-N pattern: generate N completions, score each
+ * with a PRM, pick the winner. Here the scoring loop is framework-agnostic
+ * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.
+ */
+
+interface BestOfNResult {
+    winner: PrmGradedTrace;
+    ranked: PrmGradedTrace[];
+    /** Standard deviation of aggregate scores — small = candidates were homogenous. */
+    stdDev: number;
+}
+declare function prmBestOfN(store: TraceStore, grader: PrmGrader, runIds: string[]): Promise<BestOfNResult>;
+/**
+ * Weighted vote across multiple graders — use when you want a PRM ensemble
+ * (e.g. rule-based + LLM-based + trained model). Each grader produces its
+ * own ranking; we aggregate via rank-sum (Borda count) so no single grader
+ * dominates via a different score scale.
+ */
+declare function prmEnsembleBestOfN(store: TraceStore, graders: PrmGrader[], runIds: string[]): Promise<BestOfNResult>;
+
+export { type BestOfNResult, PrmGradedTrace, PrmGrader, type PrmTrainingSample, StepRubric, exportTrainingData, nonRefusalRubric, outputLengthRubric, prmBestOfN, prmEnsembleBestOfN, toNdjson, toolIntentAlignmentRubric, toolNonRedundantRubric, toolSuccessRubric };