npm - @tangle-network/agent-eval - Versions diffs - 0.93.0 → 0.94.0 - Mend

@tangle-network/agent-eval 0.93.0 → 0.94.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (121) hide show

package/dist/{insight-report-BBwvOh6x.d.ts → insight-report-DWl3z9tl.d.ts} RENAMED Viewed

@@ -1,4 +1,4 @@
-import { G as GainDistributionBin, P as ParetoFigureSpec } from './summary-report-DGmUucwQ.js';
+import { G as GainDistributionBin, P as ParetoFigureSpec } from './summary-report-BDOFevaT.js';
 import { C as ContinuousAgreement } from './judge-calibration-0p2QcWNE.js';
 /**

package/dist/{integrity-VJ9A7aST.d.ts → integrity-D2t12mMw.d.ts} RENAMED Viewed

@@ -1,6 +1,6 @@
 import { C as CaptureIntegrityError } from './errors-CzMUYo7b.js';
 import { R as RawProviderSink } from './raw-provider-sink-C46HDghv.js';
-import { T as TraceStore } from './store-CKUAgsJz.js';
+import { T as TraceStore } from './store-BcFXE6LG.js';
 /**
  * Run-completion integrity check — at end of run, verify the expected event

package/dist/{kind-factory-5b7xXXOr.d.ts → kind-factory-0BhLSI27.d.ts} RENAMED Viewed

@@ -1,7 +1,7 @@
 import { AxAIService, AxFunction } from '@ax-llm/ax';
 import { T as TraceAnalysisStore } from './store-C1YxJDEK.js';
 import { z } from 'zod';
-import { g as AnalystCost, a as AnalystContext, A as Analyst } from './types-2VVIL04s.js';
+import { g as AnalystCost, a as AnalystContext, A as Analyst } from './types-Ce17tDlG.js';
 /**
  * Typed Ax output for analyst findings.

package/dist/knowledge/index.d.ts CHANGED Viewed

@@ -1,7 +1,7 @@
-import { j as ControlSeverity, C as ControlEvalResult } from '../control-runtime-DuFBYg7A.js';
-import { T as TraceEmitter } from '../emitter-DEZwY14K.js';
+import { j as ControlSeverity, C as ControlEvalResult } from '../control-runtime-Acf9CGhw.js';
+import { T as TraceEmitter } from '../emitter-C2rqGH_l.js';
 import '../schema-m0gsnbt3.js';
-import '../store-CKUAgsJz.js';
+import '../store-BcFXE6LG.js';
 type KnowledgeRequirementCategory = 'user_specific' | 'company_specific' | 'domain_specific' | 'codebase_specific' | 'market_specific' | 'regulatory' | 'tool_api' | 'credential_or_secret' | 'runtime_environment' | 'preference' | 'historical_context';
 type KnowledgeAcquisitionMode = 'ask_user' | 'search_web' | 'query_connector' | 'inspect_repo' | 'run_command' | 'infer_low_confidence' | 'not_available';

package/dist/{llm-client-BeEcAokY.d.ts → llm-client-Bj7g0rqu.d.ts} RENAMED Viewed

@@ -75,6 +75,22 @@ interface LlmCallResult {
     model: string;
     /** Wall-clock duration of the HTTP call (last attempt, if retried). */
     durationMs: number;
+    /**
+     * `finish_reason` echoed from the first choice (`stop`, `length`,
+     * `content_filter`, `tool_calls`, ...). `null` when the provider omits it.
+     * Exposed so a free-form `callLlm` caller CAN detect a truncated answer
+     * (`length`) instead of treating a cut-off completion as complete. Note:
+     * `callLlm` does not itself reject on it — acting on this signal is the
+     * caller's responsibility (in-repo free-form drivers do not yet enforce it).
+     */
+    finishReason?: string | null;
+    /**
+     * True when `content.trim()` is empty. An empty completion is a silent zero
+     * for free-form `callLlm` callers; this flag is the signal a caller can
+     * inspect to fail loud rather than proceed on an empty string. `callLlm`
+     * surfaces it but does not throw on it.
+     */
+    contentEmpty?: boolean;
     /** Raw response body. */
     raw: Record<string, unknown>;
 }
@@ -97,6 +113,21 @@ interface LlmClientOptions {
     };
     /** Default timeout in ms. Per-call can override. */
     defaultTimeoutMs?: number;
+    /**
+     * Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to
+     * each attempt's per-attempt timeout controller, so aborting it cancels
+     * the in-flight fetch. A caller abort is FATAL: it is not retried even
+     * though an AbortError otherwise matches the transient patterns.
+     */
+    signal?: AbortSignal;
+    /**
+     * Cross-attempt wall-clock budget in ms, measured from the first attempt.
+     * Before launching each attempt the loop checks the remaining budget and
+     * stops retrying once it is exhausted, rather than waiting the full
+     * per-attempt timeout on every retry. Bounds total time independent of
+     * `maxRetries` × `timeoutMs`.
+     */
+    deadlineMs?: number;
     /** Max retry attempts on retriable errors. Default 3 (1 initial + 2 retries). */
     maxRetries?: number;
     /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */

package/dist/meta-eval/index.d.ts CHANGED Viewed

@@ -1,14 +1,14 @@
-export { C as CalibrationBin, a as CalibrationOptions, b as CalibrationPair, c as CalibrationReport, d as CorrelationResult, e as CorrelationStudyOptions, f as CorrelationStudyResult, E as EvalMetricSpec, O as OutcomePair, g as calibrationCurve, h as calibrationFromPairs, i as correlationStudy } from '../calibration-Cpr3WaX3.js';
+export { C as CalibrationBin, a as CalibrationOptions, b as CalibrationPair, c as CalibrationReport, d as CorrelationResult, e as CorrelationStudyOptions, f as CorrelationStudyResult, E as EvalMetricSpec, O as OutcomePair, g as calibrationCurve, h as calibrationFromPairs, i as correlationStudy } from '../calibration-BPmzuVPk.js';
 export { D as DeploymentOutcome, F as FileSystemOutcomeStore, a as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore, O as OutcomeFilter, b as OutcomeStore } from '../outcome-store-rnXLEqSn.js';
 export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from '../rubric-predictive-validity-Cy_W-hWZ.js';
 import { C as ContinuousAgreement, a as CalibrationResult, b as ContinuousCalibrationResult, c as CandidateScore, G as GoldenItem } from '../judge-calibration-0p2QcWNE.js';
 import { S as SeriesConvergenceOptions, a as SeriesConvergenceResult } from '../series-convergence-D5OWMBg6.js';
-import { C as CorpusAgreementReport } from '../statistics-C7PozGrZ.js';
-import '../store-CKUAgsJz.js';
+import { C as CorpusAgreementReport } from '../statistics-CCJpTGOS.js';
+import '../store-BcFXE6LG.js';
 import '../schema-m0gsnbt3.js';
 import '../run-record-e7vj1uZQ.js';
 import '../errors-CzMUYo7b.js';
-import '../types-Croy5h7V.js';
+import '../types-C7DGg5ex.js';
 import '@tangle-network/tcloud';
 /**

package/dist/meta-eval/index.js CHANGED Viewed

@@ -15,7 +15,7 @@ import {
 import {
   aggregateLlm,
   llmSpans
-} from "../chunk-47X6LRCE.js";
+} from "../chunk-UFSG7ACU.js";
 import "../chunk-5BKGXME7.js";
 import {
   ValidationError

package/dist/openapi.json CHANGED Viewed

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.93.0",
+    "version": "0.94.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/pipelines/index.d.ts CHANGED Viewed

@@ -1,10 +1,10 @@
 import { h as BudgetSpec, R as Run, T as ToolSpan } from '../schema-m0gsnbt3.js';
-import { T as TraceStore, R as RunFilter } from '../store-CKUAgsJz.js';
-export { a as FailureCluster, F as FailureClusterReport, f as failureClusterView } from '../failure-cluster-CL7IVgkJ.js';
-import { a as TrajectoryStep } from '../trajectory-GEdXJCL5.js';
-import { B as BaselineOptions, a as BaselineReport } from '../baseline-DE36-Np7.js';
-export { c as computeToolUseMetrics } from '../baseline-DE36-Np7.js';
-import { l as llmSpans } from '../query-CqTxMwDw.js';
+import { T as TraceStore, R as RunFilter } from '../store-BcFXE6LG.js';
+export { a as FailureCluster, F as FailureClusterReport, f as failureClusterView } from '../failure-cluster-DH9Flgcf.js';
+import { a as TrajectoryStep } from '../trajectory-2TkpSEVh.js';
+import { B as BaselineOptions, a as BaselineReport } from '../baseline-Bbid3WoO.js';
+export { c as computeToolUseMetrics } from '../baseline-Bbid3WoO.js';
+import { l as llmSpans } from '../query-B7GGjRox.js';
 /**
  * BudgetBreachView — aggregates breach events across the corpus.

package/dist/pipelines/index.js CHANGED Viewed

@@ -3,20 +3,20 @@ import {
   classifyFailure,
   compareToBaseline,
   computeToolUseMetrics
-} from "../chunk-L3JOU6XM.js";
+} from "../chunk-NACM5RS7.js";
 import {
   buildTrajectory
 } from "../chunk-RZTMDUO7.js";
 import {
   interRaterReliability
-} from "../chunk-LMZQ2Z4U.js";
+} from "../chunk-56GC6VYO.js";
 import {
   aggregateLlm,
   argHash,
   llmSpans,
   runFailureClass,
   toolSpans
-} from "../chunk-47X6LRCE.js";
+} from "../chunk-UFSG7ACU.js";
 import "../chunk-5BKGXME7.js";
 import "../chunk-3BFEG2F6.js";
 import "../chunk-PZ5AY32C.js";
@@ -149,13 +149,15 @@ async function firstDivergenceView(store, runA, runB, options = {}) {
     return { runA, runB, firstDivergenceIndex: null, commonPrefixLen: minLen };
   }
   const longer = a.steps.length > b.steps.length ? a : b;
+  const extra = longer.steps.length - minLen;
+  const reason = minLen === 0 ? `one trajectory is empty; the other has ${extra} step(s) starting at index 0` : `one trajectory has ${extra} more step(s) after index ${minLen - 1}`;
   return {
     runA,
     runB,
     firstDivergenceIndex: minLen,
     aStep: a.steps[minLen],
     bStep: b.steps[minLen],
-    reason: `one trajectory has ${longer.steps.length - minLen} more step(s) after index ${minLen - 1}`,
+    reason,
     commonPrefixLen: minLen
   };
 }
@@ -320,19 +322,19 @@ async function stuckLoopView(store, options = {}) {
     const byKey = /* @__PURE__ */ new Map();
     for (const t of tools) {
       const h = argHash(t.args);
-      const key = `${t.toolName}|${h}`;
-      const bucket = byKey.get(key) ?? { spans: [], argHash: h };
+      const key = `${t.toolName}\0${h}`;
+      const bucket = byKey.get(key) ?? { spans: [], argHash: h, toolName: t.toolName };
       bucket.spans.push(t);
       byKey.set(key, bucket);
     }
-    for (const [key, { spans, argHash: h }] of byKey) {
+    for (const { spans, argHash: h, toolName } of byKey.values()) {
       if (spans.length < minOccurrences) continue;
       const sorted = [...spans].sort((a, b) => a.startedAt - b.startedAt);
       const first = sorted[0].startedAt;
       const last = sorted[sorted.length - 1].startedAt;
       findings.push({
         runId,
-        toolName: key.split("|")[0],
+        toolName,
         argHash: h,
         occurrences: sorted.length,
         spanIds: sorted.map((s) => s.spanId),
@@ -361,22 +363,23 @@ async function toolWasteView(store, options = {}) {
       continue;
     }
     const llms = await llmSpans(store, runId);
+    const sortedLlm = [...llms].sort((a, b) => a.startedAt - b.startedAt);
+    const startTimes = sortedLlm.map((l) => l.startedAt);
+    const suffixText = buildSuffixText(sortedLlm);
     let wasted = 0;
     for (const t of tools) {
       if (t.status === "error") {
         wasted++;
         continue;
       }
-      const laterLlm = llms.filter((l) => l.startedAt > t.startedAt);
+      const cutoff = upperBound(startTimes, t.startedAt);
       if (options.usageOracle) {
-        if (!options.usageOracle(t, { llm: laterLlm })) wasted++;
+        if (!options.usageOracle(t, { llm: sortedLlm.slice(cutoff) })) wasted++;
       } else {
         const resultStr = stringify(t.result);
-        const used = laterLlm.some(
-          (l) => l.messages.some(
-            (m) => typeof m.content === "string" && resultStr && m.content.includes(resultStr.slice(0, 120))
-          )
-        );
+        if (resultStr === "") continue;
+        const haystack = suffixText[cutoff] ?? "";
+        const used = haystack.includes(resultStr.slice(0, 120));
         if (!used) wasted++;
       }
     }
@@ -387,6 +390,28 @@ async function toolWasteView(store, options = {}) {
   }
   return { byRun, overallWasteRate: totalCalls > 0 ? totalWasted / totalCalls : 0 };
 }
+function buildSuffixText(spans) {
+  const result = new Array(spans.length + 1);
+  result[spans.length] = "";
+  for (let i = spans.length - 1; i >= 0; i--) {
+    const own = spans[i].messages.map(
+      (m) => typeof m.content === "string" ? m.content : ""
+    ).join("\n");
+    result[i] = `${own}
+${result[i + 1]}`;
+  }
+  return result;
+}
+function upperBound(sorted, target) {
+  let lo = 0;
+  let hi = sorted.length;
+  while (lo < hi) {
+    const mid = lo + hi >>> 1;
+    if (sorted[mid] <= target) lo = mid + 1;
+    else hi = mid;
+  }
+  return lo;
+}
 function stringify(v) {
   if (v === null || v === void 0) return "";
   if (typeof v === "string") return v;

package/dist/pipelines/index.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../../src/pipelines/budget-breach.ts","../../src/pipelines/failure-cluster.ts","../../src/pipelines/first-divergence.ts","../../src/pipelines/judge-agreement.ts","../../src/pipelines/regression.ts","../../src/pipelines/stuck-loop.ts","../../src/pipelines/tool-waste.ts"],"sourcesContent":["/*\n BudgetBreachView — aggregates breach events across the corpus.\n \n Answers: which dimensions get hit most often? Which scenarios are\n * underbudgeted? Which variants trigger the most breaches?\n /\n\nimport type { BudgetSpec } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface BudgetBreachFinding {\n runId: string\n scenarioId: string\n variantId?: string\n dimension: keyof BudgetSpec\n limit: number\n consumed: number\n excessRatio: number\n timestamp: number\n}\n\nexport interface BudgetBreachReport {\n findings: BudgetBreachFinding[]\n byDimension: Record<string, number>\n byScenario: Record<string, number>\n byVariant: Record<string, number>\n totalRuns: number\n breachedRunRatio: number\n}\n\nexport async function budgetBreachView(\n store: TraceStore,\n options: { scenarioId?: string; variantId?: string } = {},\n): Promise<BudgetBreachReport> {\n const runs = await store.listRuns({\n scenarioId: options.scenarioId,\n variantId: options.variantId,\n })\n const findings: BudgetBreachFinding[] = []\n const byDimension: Record<string, number> = {}\n const byScenario: Record<string, number> = {}\n const byVariant: Record<string, number> = {}\n\n for (const run of runs) {\n const entries = await store.budget(run.runId)\n for (const e of entries) {\n if (!e.breached) continue\n const excessRatio = e.limit > 0 ? e.consumed / e.limit : Infinity\n findings.push({\n runId: run.runId,\n scenarioId: run.scenarioId,\n variantId: run.variantId,\n dimension: e.dimension,\n limit: e.limit,\n consumed: e.consumed,\n excessRatio,\n timestamp: e.timestamp,\n })\n byDimension[e.dimension] = (byDimension[e.dimension] ?? 0) + 1\n byScenario[run.scenarioId] = (byScenario[run.scenarioId] ?? 0) + 1\n if (run.variantId) byVariant[run.variantId] = (byVariant[run.variantId] ?? 0) + 1\n }\n }\n\n const breachedRuns = new Set(findings.map((f) => f.runId))\n return {\n findings,\n byDimension,\n byScenario,\n byVariant,\n totalRuns: runs.length,\n breachedRunRatio: runs.length > 0 ? breachedRuns.size / runs.length : 0,\n }\n}\n","/\n FailureClusterView — groups failed runs by (failureClass, triggerTool,\n * argHash-prefix) so weekly reviews can prioritize the top-N clusters.\n \n Each cluster includes: N runs, scenarios affected, representative\n * error message, a proposed mitigation hint (rule → action table).\n /\n\nimport { classifyFailure, DEFAULT_RULES, type FailureRule } from '../failure-taxonomy'\nimport { argHash, toolSpans } from '../trace/query'\nimport type { FailureClass, Span } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface FailureCluster {\n failureClass: FailureClass\n /* Tool name when the trigger was a tool span, else undefined. /\n toolName?: string\n /* First 16 chars of argHash — clusters similar args. /\n argPrefix?: string\n /\n Source dimension when the trigger was a judge span (e.g. `'format'`,\n * `'safety'`, `'correctness'`). Lets cross-template aggregators\n * group failures by the dimension that fired without overloading\n * `argPrefix`. Optional — clusters without this field deserialize cleanly.\n /\n dimension?: string\n runCount: number\n scenarioIds: string[]\n exampleError?: string\n exampleRunId: string\n}\n\nexport interface FailureClusterReport {\n clusters: FailureCluster[]\n totalFailures: number\n totalRuns: number\n}\n\nexport async function failureClusterView(\n store: TraceStore,\n options: { rules?: FailureRule[]; minClusterSize?: number } = {},\n): Promise<FailureClusterReport> {\n const rules = options.rules ?? DEFAULT_RULES\n const minSize = options.minClusterSize ?? 1\n const runs = await store.listRuns()\n\n type Key = string\n const clusters = new Map<Key, FailureCluster>()\n let totalFailures = 0\n\n for (const run of runs) {\n if (run.status === 'completed' && run.outcome?.pass !== false) continue\n totalFailures++\n const spans = await store.spans({ runId: run.runId })\n const events = await store.events({ runId: run.runId })\n const cls = classifyFailure({ run, spans, events }, rules)\n\n let toolName: string \| undefined\n let argPrefix: string \| undefined\n let dimension: string \| undefined\n if (cls.triggerSpanId) {\n const trig = spans.find((s) => s.spanId === cls.triggerSpanId)\n if (trig?.kind === 'tool') {\n toolName = trig.toolName\n argPrefix = argHash(trig.args).slice(0, 16)\n } else if (trig?.kind === 'judge') {\n dimension = trig.dimension\n }\n }\n // Fallback: look at the last errored tool span\n if (!toolName) {\n const ts = await toolSpans(store, run.runId)\n const errored = ts.filter((t) => t.status === 'error').pop()\n if (errored) {\n toolName = errored.toolName\n argPrefix = argHash(errored.args).slice(0, 16)\n }\n }\n // Secondary signal: any judge span on the failed run carries a\n // dimension. Useful when the rule classified by judge score but\n // didn't surface the trigger span (or surfaced a non-judge span).\n if (!dimension) {\n const judge = spans.find((s) => s.kind === 'judge' && typeof s.dimension === 'string')\n if (judge?.kind === 'judge') dimension = judge.dimension\n }\n\n const key = `${cls.failureClass}\|${toolName ?? ''}\|${argPrefix ?? ''}\|${dimension ?? ''}`\n let cluster = clusters.get(key)\n if (!cluster) {\n cluster = {\n failureClass: cls.failureClass,\n toolName,\n argPrefix,\n dimension,\n runCount: 0,\n scenarioIds: [],\n exampleRunId: run.runId,\n exampleError: firstErrorMessage(spans) ?? cls.reason,\n }\n clusters.set(key, cluster)\n }\n cluster.runCount++\n if (!cluster.scenarioIds.includes(run.scenarioId)) cluster.scenarioIds.push(run.scenarioId)\n }\n\n const arr = [...clusters.values()]\n .filter((c) => c.runCount >= minSize)\n .sort((a, b) => b.runCount - a.runCount)\n\n return { clusters: arr, totalFailures, totalRuns: runs.length }\n}\n\nfunction firstErrorMessage(spans: Span[]): string \| undefined {\n const errored = spans.find((s) => s.status === 'error')\n return errored?.error\n}\n","/\n FirstDivergenceView — aligns two trajectories by step index, reports\n * the first step where they differ.\n \n \"Differ\" is configurable — default is (kind, toolName if tool, model\n * if llm). Use this view to attribute \"why is variant B better?\" to a\n * specific step rather than an aggregate mean delta.\n /\n\nimport type { TraceStore } from '../trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from '../trajectory'\n\nexport interface DivergenceReport {\n runA: string\n runB: string\n firstDivergenceIndex: number \| null\n aStep?: TrajectoryStep\n bStep?: TrajectoryStep\n reason?: string\n /* Common prefix length (steps that matched). /\n commonPrefixLen: number\n}\n\nexport interface DivergenceOptions {\n /* Returns true if two steps are considered equal. Default: kind + tool/model match. /\n stepEquals?: (a: TrajectoryStep, b: TrajectoryStep) => boolean\n}\n\nexport async function firstDivergenceView(\n store: TraceStore,\n runA: string,\n runB: string,\n options: DivergenceOptions = {},\n): Promise<DivergenceReport> {\n const [a, b] = await Promise.all([buildTrajectory(store, runA), buildTrajectory(store, runB)])\n const eq = options.stepEquals ?? defaultStepEquals\n const minLen = Math.min(a.steps.length, b.steps.length)\n for (let i = 0; i < minLen; i++) {\n const aStep = a.steps[i]!\n const bStep = b.steps[i]!\n if (!eq(aStep, bStep)) {\n return {\n runA,\n runB,\n firstDivergenceIndex: i,\n aStep,\n bStep,\n reason: describeDifference(aStep, bStep),\n commonPrefixLen: i,\n }\n }\n }\n if (a.steps.length === b.steps.length) {\n return { runA, runB, firstDivergenceIndex: null, commonPrefixLen: minLen }\n }\n const longer: Trajectory = a.steps.length > b.steps.length ? a : b\n return {\n runA,\n runB,\n firstDivergenceIndex: minLen,\n aStep: a.steps[minLen],\n bStep: b.steps[minLen],\n reason: `one trajectory has ${longer.steps.length - minLen} more step(s) after index ${minLen - 1}`,\n commonPrefixLen: minLen,\n }\n}\n\nfunction defaultStepEquals(a: TrajectoryStep, b: TrajectoryStep): boolean {\n if (a.span.kind !== b.span.kind) return false\n if (a.span.kind === 'tool' && b.span.kind === 'tool') return a.span.toolName === b.span.toolName\n if (a.span.kind === 'llm' && b.span.kind === 'llm') return a.span.model === b.span.model\n if (a.span.kind === 'judge' && b.span.kind === 'judge')\n return a.span.dimension === b.span.dimension\n return a.span.name === b.span.name\n}\n\nfunction describeDifference(a: TrajectoryStep, b: TrajectoryStep): string {\n if (a.span.kind !== b.span.kind) return `kind ${a.span.kind} vs ${b.span.kind}`\n if (a.span.kind === 'tool' && b.span.kind === 'tool' && a.span.toolName !== b.span.toolName) {\n return `tool ${a.span.toolName} vs ${b.span.toolName}`\n }\n if (a.span.kind === 'llm' && b.span.kind === 'llm' && a.span.model !== b.span.model) {\n return `model ${a.span.model} vs ${b.span.model}`\n }\n return `name \"${a.span.name}\" vs \"${b.span.name}\"`\n}\n","/\n JudgeAgreementView — pairwise agreement between judges across the\n * corpus, grouped by dimension.\n \n Output drives two workflows:\n * - Judge robustness audit: \"does Claude agree with GPT at κ ≥ 0.6?\"\n * - Calibration tracking: κ vs golden human labels over time (by\n * providing a `humanGoldenJudgeId`).\n /\n\nimport { interRaterReliability } from '../statistics'\nimport type { JudgeSpan } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface JudgePair {\n judgeA: string\n judgeB: string\n dimension: string\n /* Number of (targetSpanId, dimension) tuples both judges scored. /\n commonItems: number\n pearson: number\n krippendorff: number\n}\n\nexport interface JudgeAgreementReport {\n pairs: JudgePair[]\n dimensions: string[]\n judgeIds: string[]\n}\n\nexport async function judgeAgreementView(store: TraceStore): Promise<JudgeAgreementReport> {\n const all = (await store.spans({ kind: 'judge' })).filter(\n (s): s is JudgeSpan => s.kind === 'judge',\n )\n if (all.length === 0) return { pairs: [], dimensions: [], judgeIds: [] }\n\n const byDimension = new Map<string, JudgeSpan[]>()\n for (const s of all) {\n const arr = byDimension.get(s.dimension) ?? []\n arr.push(s)\n byDimension.set(s.dimension, arr)\n }\n\n const judgeIds = [...new Set(all.map((s) => s.judgeId))].sort()\n const pairs: JudgePair[] = []\n for (const [dim, spans] of byDimension) {\n const byJudge = new Map<string, Map<string, number>>()\n for (const s of spans) {\n const m = byJudge.get(s.judgeId) ?? new Map<string, number>()\n m.set(s.targetSpanId, s.score)\n byJudge.set(s.judgeId, m)\n }\n const judgesHere = [...byJudge.keys()]\n for (let i = 0; i < judgesHere.length; i++) {\n for (let j = i + 1; j < judgesHere.length; j++) {\n const judgeI = judgesHere[i]!\n const judgeJ = judgesHere[j]!\n const a = byJudge.get(judgeI)!\n const b = byJudge.get(judgeJ)!\n const common: Array<[number, number]> = []\n for (const [target, scoreA] of a) {\n const scoreB = b.get(target)\n if (scoreB !== undefined) common.push([scoreA, scoreB])\n }\n if (common.length < 2) continue\n const judgeScores = common.map(\n ([scoreA, scoreB]) =>\n [\n { judgeName: judgeI, dimension: dim, score: scoreA, reasoning: '' },\n { judgeName: judgeJ, dimension: dim, score: scoreB, reasoning: '' },\n ] as const,\n )\n const k = interRaterReliability(\n judgeScores[0]!.map((_, k2) => judgeScores.map((pair) => pair[k2]!)),\n )\n pairs.push({\n judgeA: judgeI,\n judgeB: judgeJ,\n dimension: dim,\n commonItems: common.length,\n pearson: pearson(\n common.map((c) => c[0]),\n common.map((c) => c[1]),\n ),\n krippendorff: k,\n })\n }\n }\n }\n\n return {\n pairs: pairs.sort((a, b) => b.commonItems - a.commonItems),\n dimensions: [...byDimension.keys()].sort(),\n judgeIds,\n }\n}\n\nfunction pearson(a: number[], b: number[]): number {\n if (a.length !== b.length \|\| a.length < 2) return NaN\n const mA = a.reduce((s, v) => s + v, 0) / a.length\n const mB = b.reduce((s, v) => s + v, 0) / b.length\n let num = 0,\n denA = 0,\n denB = 0\n for (let i = 0; i < a.length; i++) {\n const dA = a[i]! - mA\n const dB = b[i]! - mB\n num += dA dB\n denA += dA * dA\n denB += dB * dB\n }\n if (denA === 0 \|\| denB === 0) return denA === 0 && denB === 0 ? 1 : 0\n return num / Math.sqrt(denA * denB)\n}\n","/*\n RegressionView — compares a candidate slice to a baseline slice on a\n * named metric. Delegates the statistics (Welch's t-test, Cohen's d,\n * IQR stability) to `baseline.ts`.\n \n This is the entry point for CI regression gates: \"given runs tagged\n * release=A and release=B, did any metric regress?\"\n /\n\nimport { type BaselineOptions, type BaselineReport, compareToBaseline } from '../baseline'\nimport { aggregateLlm, llmSpans, runFailureClass } from '../trace/query'\nimport type { Run } from '../trace/schema'\nimport type { RunFilter, TraceStore } from '../trace/store'\n\nexport interface RegressionSpec {\n metric: string\n higherIsBetter: boolean\n /* Extract a scalar from a run. Default extractors handle common metrics. /\n extract?: (run: Run, store: TraceStore) => Promise<number \| null>\n}\n\nexport interface RegressionOptions extends BaselineOptions {\n baseline: RunFilter\n candidate: RunFilter\n}\n\nexport async function regressionView(\n store: TraceStore,\n metrics: RegressionSpec[],\n options: RegressionOptions,\n): Promise<BaselineReport> {\n const baselineRuns = await store.listRuns(options.baseline)\n const candidateRuns = await store.listRuns(options.candidate)\n const samples = await Promise.all(\n metrics.map(async (m) => {\n const extract = m.extract ?? defaultExtract(m.metric)\n const baseline = await extractAll(baselineRuns, extract, store)\n const candidate = await extractAll(candidateRuns, extract, store)\n return { metric: m.metric, higherIsBetter: m.higherIsBetter, baseline, candidate }\n }),\n )\n return compareToBaseline(samples, options)\n}\n\nasync function extractAll(\n runs: Run[],\n extract: (r: Run, s: TraceStore) => Promise<number \| null>,\n store: TraceStore,\n): Promise<number[]> {\n const out: number[] = []\n for (const r of runs) {\n const v = await extract(r, store)\n if (v !== null && Number.isFinite(v)) out.push(v)\n }\n return out\n}\n\nfunction defaultExtract(metric: string): (run: Run, store: TraceStore) => Promise<number \| null> {\n return async (run, store) => {\n switch (metric) {\n case 'score':\n case 'overallScore':\n return run.outcome?.score ?? null\n case 'pass':\n return run.outcome?.pass === true ? 1 : 0\n case 'durationMs':\n return run.endedAt && run.startedAt ? run.endedAt - run.startedAt : null\n case 'costUsd': {\n const llm = await llmSpans(store, run.runId)\n return aggregateLlm(llm).costUsd\n }\n case 'inputTokens': {\n const llm = await llmSpans(store, run.runId)\n return aggregateLlm(llm).inputTokens\n }\n case 'outputTokens': {\n const llm = await llmSpans(store, run.runId)\n return aggregateLlm(llm).outputTokens\n }\n case 'failureClass': {\n return runFailureClass(run) === 'success' ? 1 : 0\n }\n default:\n return null\n }\n }\n}\n","/\n StuckLoopView — detects when an agent calls the same tool with the\n * same (or structurally similar) arguments ≥ N times in a short window.\n \n Rationale: agents that loop are the number-one production failure\n * mode on long-horizon flows. The view returns (runId, toolName,\n * argHash, occurrences, windowMs) for each detected loop plus a\n * fraction of runs affected.\n /\n\nimport { argHash, toolSpans } from '../trace/query'\nimport type { TraceStore } from '../trace/store'\n\nexport interface StuckLoopFinding {\n runId: string\n toolName: string\n argHash: string\n occurrences: number\n spanIds: string[]\n /* Milliseconds between first and last call in the loop. /\n windowMs: number\n}\n\nexport interface StuckLoopReport {\n findings: StuckLoopFinding[]\n affectedRunRatio: number\n totalRuns: number\n}\n\nexport interface StuckLoopOptions {\n /* Minimum call count to flag a loop (default 3). /\n minOccurrences?: number\n /* Filter to a specific runId; omit to scan the entire corpus. /\n runId?: string\n}\n\nexport async function stuckLoopView(\n store: TraceStore,\n options: StuckLoopOptions = {},\n): Promise<StuckLoopReport> {\n const minOccurrences = options.minOccurrences ?? 3\n const runs = options.runId\n ? [{ runId: options.runId }]\n : (await store.listRuns()).map((r) => ({ runId: r.runId }))\n\n const findings: StuckLoopFinding[] = []\n for (const { runId } of runs) {\n const tools = await toolSpans(store, runId)\n const byKey = new Map<string, { spans: typeof tools; argHash: string }>()\n for (const t of tools) {\n const h = argHash(t.args)\n const key = `${t.toolName}\|${h}`\n const bucket = byKey.get(key) ?? { spans: [], argHash: h }\n bucket.spans.push(t)\n byKey.set(key, bucket)\n }\n for (const [key, { spans, argHash: h }] of byKey) {\n if (spans.length < minOccurrences) continue\n const sorted = [...spans].sort((a, b) => a.startedAt - b.startedAt)\n const first = sorted[0]!.startedAt\n const last = sorted[sorted.length - 1]!.startedAt\n findings.push({\n runId,\n toolName: key.split('\|')[0]!,\n argHash: h,\n occurrences: sorted.length,\n spanIds: sorted.map((s) => s.spanId),\n windowMs: last - first,\n })\n }\n }\n\n const affectedRuns = new Set(findings.map((f) => f.runId))\n return {\n findings,\n affectedRunRatio: runs.length > 0 ? affectedRuns.size / runs.length : 0,\n totalRuns: runs.length,\n }\n}\n","/\n ToolWasteView — fraction of tool calls whose results weren't used\n * downstream. Without a \"used\" signal we fall back to structural\n * proxies: error calls, duplicate calls, and tool calls followed by\n * zero subsequent LLM spans are all considered waste.\n \n Consumers can pass a `usageOracle` that inspects a tool span and\n * returns true iff the tool's result appears in a later LLM message,\n * artifact, or state mutation — that's the canonical definition; the\n * default heuristic is a reasonable fallback.\n */\n\nimport { computeToolUseMetrics } from '../tool-use-metrics'\nimport { llmSpans, toolSpans } from '../trace/query'\nimport type { ToolSpan } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface ToolWasteFinding {\n runId: string\n wastedCalls: number\n totalCalls: number\n wasteRate: number\n}\n\nexport interface ToolWasteReport {\n byRun: ToolWasteFinding[]\n overallWasteRate: number\n}\n\nexport interface ToolWasteOptions {\n runId?: string\n usageOracle?: (tool: ToolSpan, later: { llm: Awaited<ReturnType<typeof llmSpans>> }) => boolean\n}\n\nexport async function toolWasteView(\n store: TraceStore,\n options: ToolWasteOptions = {},\n): Promise<ToolWasteReport> {\n const runs = options.runId ? [options.runId] : (await store.listRuns()).map((r) => r.runId)\n\n const byRun: ToolWasteFinding[] = []\n let totalCalls = 0\n let totalWasted = 0\n for (const runId of runs) {\n const tools = await toolSpans(store, runId)\n if (tools.length === 0) {\n byRun.push({ runId, wastedCalls: 0, totalCalls: 0, wasteRate: 0 })\n continue\n }\n const llms = await llmSpans(store, runId)\n let wasted = 0\n for (const t of tools) {\n if (t.status === 'error') {\n wasted++\n continue\n }\n const laterLlm = llms.filter((l) => l.startedAt > t.startedAt)\n if (options.usageOracle) {\n if (!options.usageOracle(t, { llm: laterLlm })) wasted++\n } else {\n // Default heuristic: a tool whose result is NOT mentioned in any\n // later LLM input message is likely wasted.\n const resultStr = stringify(t.result)\n const used = laterLlm.some((l) =>\n l.messages.some(\n (m) =>\n typeof m.content === 'string' &&\n resultStr &&\n m.content.includes(resultStr.slice(0, 120)),\n ),\n )\n if (!used) wasted++\n }\n }\n const wasteRate = wasted / tools.length\n byRun.push({ runId, wastedCalls: wasted, totalCalls: tools.length, wasteRate })\n totalCalls += tools.length\n totalWasted += wasted\n }\n return { byRun, overallWasteRate: totalCalls > 0 ? totalWasted / totalCalls : 0 }\n}\n\nfunction stringify(v: unknown): string {\n if (v === null \|\| v === undefined) return ''\n if (typeof v === 'string') return v\n try {\n return JSON.stringify(v)\n } catch {\n return String(v)\n }\n}\n\n// Re-export for convenience in consumers that want both descriptive and usage metrics.\nexport { computeToolUseMetrics }\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA8BA,eAAsB,iBACpB,OACA,UAAuD,CAAC,GAC3B;AAC7B,QAAM,OAAO,MAAM,MAAM,SAAS;AAAA,IAChC,YAAY,QAAQ;AAAA,IACpB,WAAW,QAAQ;AAAA,EACrB,CAAC;AACD,QAAM,WAAkC,CAAC;AACzC,QAAM,cAAsC,CAAC;AAC7C,QAAM,aAAqC,CAAC;AAC5C,QAAM,YAAoC,CAAC;AAE3C,aAAW,OAAO,MAAM;AACtB,UAAM,UAAU,MAAM,MAAM,OAAO,IAAI,KAAK;AAC5C,eAAW,KAAK,SAAS;AACvB,UAAI,CAAC,EAAE,SAAU;AACjB,YAAM,cAAc,EAAE,QAAQ,IAAI,EAAE,WAAW,EAAE,QAAQ;AACzD,eAAS,KAAK;AAAA,QACZ,OAAO,IAAI;AAAA,QACX,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,WAAW,EAAE;AAAA,QACb,OAAO,EAAE;AAAA,QACT,UAAU,EAAE;AAAA,QACZ;AAAA,QACA,WAAW,EAAE;AAAA,MACf,CAAC;AACD,kBAAY,EAAE,SAAS,KAAK,YAAY,EAAE,SAAS,KAAK,KAAK;AAC7D,iBAAW,IAAI,UAAU,KAAK,WAAW,IAAI,UAAU,KAAK,KAAK;AACjE,UAAI,IAAI,UAAW,WAAU,IAAI,SAAS,KAAK,UAAU,IAAI,SAAS,KAAK,KAAK;AAAA,IAClF;AAAA,EACF;AAEA,QAAM,eAAe,IAAI,IAAI,SAAS,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC;AACzD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW,KAAK;AAAA,IAChB,kBAAkB,KAAK,SAAS,IAAI,aAAa,OAAO,KAAK,SAAS;AAAA,EACxE;AACF;;;ACnCA,eAAsB,mBACpB,OACA,UAA8D,CAAC,GAChC;AAC/B,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,UAAU,QAAQ,kBAAkB;AAC1C,QAAM,OAAO,MAAM,MAAM,SAAS;AAGlC,QAAM,WAAW,oBAAI,IAAyB;AAC9C,MAAI,gBAAgB;AAEpB,aAAW,OAAO,MAAM;AACtB,QAAI,IAAI,WAAW,eAAe,IAAI,SAAS,SAAS,MAAO;AAC/D;AACA,UAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,IAAI,MAAM,CAAC;AACpD,UAAM,SAAS,MAAM,MAAM,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC;AACtD,UAAM,MAAM,gBAAgB,EAAE,KAAK,OAAO,OAAO,GAAG,KAAK;AAEzD,QAAI;AACJ,QAAI;AACJ,QAAI;AACJ,QAAI,IAAI,eAAe;AACrB,YAAM,OAAO,MAAM,KAAK,CAAC,MAAM,EAAE,WAAW,IAAI,aAAa;AAC7D,UAAI,MAAM,SAAS,QAAQ;AACzB,mBAAW,KAAK;AAChB,oBAAY,QAAQ,KAAK,IAAI,EAAE,MAAM,GAAG,EAAE;AAAA,MAC5C,WAAW,MAAM,SAAS,SAAS;AACjC,oBAAY,KAAK;AAAA,MACnB;AAAA,IACF;AAEA,QAAI,CAAC,UAAU;AACb,YAAM,KAAK,MAAM,UAAU,OAAO,IAAI,KAAK;AAC3C,YAAM,UAAU,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,OAAO,EAAE,IAAI;AAC3D,UAAI,SAAS;AACX,mBAAW,QAAQ;AACnB,oBAAY,QAAQ,QAAQ,IAAI,EAAE,MAAM,GAAG,EAAE;AAAA,MAC/C;AAAA,IACF;AAIA,QAAI,CAAC,WAAW;AACd,YAAM,QAAQ,MAAM,KAAK,CAAC,MAAM,EAAE,SAAS,WAAW,OAAO,EAAE,cAAc,QAAQ;AACrF,UAAI,OAAO,SAAS,QAAS,aAAY,MAAM;AAAA,IACjD;AAEA,UAAM,MAAM,GAAG,IAAI,YAAY,IAAI,YAAY,EAAE,IAAI,aAAa,EAAE,IAAI,aAAa,EAAE;AACvF,QAAI,UAAU,SAAS,IAAI,GAAG;AAC9B,QAAI,CAAC,SAAS;AACZ,gBAAU;AAAA,QACR,cAAc,IAAI;AAAA,QAClB;AAAA,QACA;AAAA,QACA;AAAA,QACA,UAAU;AAAA,QACV,aAAa,CAAC;AAAA,QACd,cAAc,IAAI;AAAA,QAClB,cAAc,kBAAkB,KAAK,KAAK,IAAI;AAAA,MAChD;AACA,eAAS,IAAI,KAAK,OAAO;AAAA,IAC3B;AACA,YAAQ;AACR,QAAI,CAAC,QAAQ,YAAY,SAAS,IAAI,UAAU,EAAG,SAAQ,YAAY,KAAK,IAAI,UAAU;AAAA,EAC5F;AAEA,QAAM,MAAM,CAAC,GAAG,SAAS,OAAO,CAAC,EAC9B,OAAO,CAAC,MAAM,EAAE,YAAY,OAAO,EACnC,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAEzC,SAAO,EAAE,UAAU,KAAK,eAAe,WAAW,KAAK,OAAO;AAChE;AAEA,SAAS,kBAAkB,OAAmC;AAC5D,QAAM,UAAU,MAAM,KAAK,CAAC,MAAM,EAAE,WAAW,OAAO;AACtD,SAAO,SAAS;AAClB;;;ACvFA,eAAsB,oBACpB,OACA,MACA,MACA,UAA6B,CAAC,GACH;AAC3B,QAAM,CAAC,GAAG,CAAC,IAAI,MAAM,QAAQ,IAAI,CAAC,gBAAgB,OAAO,IAAI,GAAG,gBAAgB,OAAO,IAAI,CAAC,CAAC;AAC7F,QAAM,KAAK,QAAQ,cAAc;AACjC,QAAM,SAAS,KAAK,IAAI,EAAE,MAAM,QAAQ,EAAE,MAAM,MAAM;AACtD,WAAS,IAAI,GAAG,IAAI,QAAQ,KAAK;AAC/B,UAAM,QAAQ,EAAE,MAAM,CAAC;AACvB,UAAM,QAAQ,EAAE,MAAM,CAAC;AACvB,QAAI,CAAC,GAAG,OAAO,KAAK,GAAG;AACrB,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,sBAAsB;AAAA,QACtB;AAAA,QACA;AAAA,QACA,QAAQ,mBAAmB,OAAO,KAAK;AAAA,QACvC,iBAAiB;AAAA,MACnB;AAAA,IACF;AAAA,EACF;AACA,MAAI,EAAE,MAAM,WAAW,EAAE,MAAM,QAAQ;AACrC,WAAO,EAAE,MAAM,MAAM,sBAAsB,MAAM,iBAAiB,OAAO;AAAA,EAC3E;AACA,QAAM,SAAqB,EAAE,MAAM,SAAS,EAAE,MAAM,SAAS,IAAI;AACjE,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,sBAAsB;AAAA,IACtB,OAAO,EAAE,MAAM,MAAM;AAAA,IACrB,OAAO,EAAE,MAAM,MAAM;AAAA,IACrB,QAAQ,sBAAsB,OAAO,MAAM,SAAS,MAAM,6BAA6B,SAAS,CAAC;AAAA,IACjG,iBAAiB;AAAA,EACnB;AACF;AAEA,SAAS,kBAAkB,GAAmB,GAA4B;AACxE,MAAI,EAAE,KAAK,SAAS,EAAE,KAAK,KAAM,QAAO;AACxC,MAAI,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,SAAS,OAAQ,QAAO,EAAE,KAAK,aAAa,EAAE,KAAK;AACxF,MAAI,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,SAAS,MAAO,QAAO,EAAE,KAAK,UAAU,EAAE,KAAK;AACnF,MAAI,EAAE,KAAK,SAAS,WAAW,EAAE,KAAK,SAAS;AAC7C,WAAO,EAAE,KAAK,cAAc,EAAE,KAAK;AACrC,SAAO,EAAE,KAAK,SAAS,EAAE,KAAK;AAChC;AAEA,SAAS,mBAAmB,GAAmB,GAA2B;AACxE,MAAI,EAAE,KAAK,SAAS,EAAE,KAAK,KAAM,QAAO,QAAQ,EAAE,KAAK,IAAI,OAAO,EAAE,KAAK,IAAI;AAC7E,MAAI,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,aAAa,EAAE,KAAK,UAAU;AAC3F,WAAO,QAAQ,EAAE,KAAK,QAAQ,OAAO,EAAE,KAAK,QAAQ;AAAA,EACtD;AACA,MAAI,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,OAAO;AACnF,WAAO,SAAS,EAAE,KAAK,KAAK,OAAO,EAAE,KAAK,KAAK;AAAA,EACjD;AACA,SAAO,SAAS,EAAE,KAAK,IAAI,SAAS,EAAE,KAAK,IAAI;AACjD;;;ACvDA,eAAsB,mBAAmB,OAAkD;AACzF,QAAM,OAAO,MAAM,MAAM,MAAM,EAAE,MAAM,QAAQ,CAAC,GAAG;AAAA,IACjD,CAAC,MAAsB,EAAE,SAAS;AAAA,EACpC;AACA,MAAI,IAAI,WAAW,EAAG,QAAO,EAAE,OAAO,CAAC,GAAG,YAAY,CAAC,GAAG,UAAU,CAAC,EAAE;AAEvE,QAAM,cAAc,oBAAI,IAAyB;AACjD,aAAW,KAAK,KAAK;AACnB,UAAM,MAAM,YAAY,IAAI,EAAE,SAAS,KAAK,CAAC;AAC7C,QAAI,KAAK,CAAC;AACV,gBAAY,IAAI,EAAE,WAAW,GAAG;AAAA,EAClC;AAEA,QAAM,WAAW,CAAC,GAAG,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,KAAK;AAC9D,QAAM,QAAqB,CAAC;AAC5B,aAAW,CAAC,KAAK,KAAK,KAAK,aAAa;AACtC,UAAM,UAAU,oBAAI,IAAiC;AACrD,eAAW,KAAK,OAAO;AACrB,YAAM,IAAI,QAAQ,IAAI,EAAE,OAAO,KAAK,oBAAI,IAAoB;AAC5D,QAAE,IAAI,EAAE,cAAc,EAAE,KAAK;AAC7B,cAAQ,IAAI,EAAE,SAAS,CAAC;AAAA,IAC1B;AACA,UAAM,aAAa,CAAC,GAAG,QAAQ,KAAK,CAAC;AACrC,aAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,eAAS,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC9C,cAAM,SAAS,WAAW,CAAC;AAC3B,cAAM,SAAS,WAAW,CAAC;AAC3B,cAAM,IAAI,QAAQ,IAAI,MAAM;AAC5B,cAAM,IAAI,QAAQ,IAAI,MAAM;AAC5B,cAAM,SAAkC,CAAC;AACzC,mBAAW,CAAC,QAAQ,MAAM,KAAK,GAAG;AAChC,gBAAM,SAAS,EAAE,IAAI,MAAM;AAC3B,cAAI,WAAW,OAAW,QAAO,KAAK,CAAC,QAAQ,MAAM,CAAC;AAAA,QACxD;AACA,YAAI,OAAO,SAAS,EAAG;AACvB,cAAM,cAAc,OAAO;AAAA,UACzB,CAAC,CAAC,QAAQ,MAAM,MACd;AAAA,YACE,EAAE,WAAW,QAAQ,WAAW,KAAK,OAAO,QAAQ,WAAW,GAAG;AAAA,YAClE,EAAE,WAAW,QAAQ,WAAW,KAAK,OAAO,QAAQ,WAAW,GAAG;AAAA,UACpE;AAAA,QACJ;AACA,cAAM,IAAI;AAAA,UACR,YAAY,CAAC,EAAG,IAAI,CAAC,GAAG,OAAO,YAAY,IAAI,CAAC,SAAS,KAAK,EAAE,CAAE,CAAC;AAAA,QACrE;AACA,cAAM,KAAK;AAAA,UACT,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR,WAAW;AAAA,UACX,aAAa,OAAO;AAAA,UACpB,SAAS;AAAA,YACP,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAAA,YACtB,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAAA,UACxB;AAAA,UACA,cAAc;AAAA,QAChB,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,OAAO,MAAM,KAAK,CAAC,GAAG,MAAM,EAAE,cAAc,EAAE,WAAW;AAAA,IACzD,YAAY,CAAC,GAAG,YAAY,KAAK,CAAC,EAAE,KAAK;AAAA,IACzC;AAAA,EACF;AACF;AAEA,SAAS,QAAQ,GAAa,GAAqB;AACjD,MAAI,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAG,QAAO;AAClD,QAAM,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE;AAC5C,QAAM,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE;AAC5C,MAAI,MAAM,GACR,OAAO,GACP,OAAO;AACT,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,UAAM,KAAK,EAAE,CAAC,IAAK;AACnB,UAAM,KAAK,EAAE,CAAC,IAAK;AACnB,WAAO,KAAK;AACZ,YAAQ,KAAK;AACb,YAAQ,KAAK;AAAA,EACf;AACA,MAAI,SAAS,KAAK,SAAS,EAAG,QAAO,SAAS,KAAK,SAAS,IAAI,IAAI;AACpE,SAAO,MAAM,KAAK,KAAK,OAAO,IAAI;AACpC;;;ACvFA,eAAsB,eACpB,OACA,SACA,SACyB;AACzB,QAAM,eAAe,MAAM,MAAM,SAAS,QAAQ,QAAQ;AAC1D,QAAM,gBAAgB,MAAM,MAAM,SAAS,QAAQ,SAAS;AAC5D,QAAM,UAAU,MAAM,QAAQ;AAAA,IAC5B,QAAQ,IAAI,OAAO,MAAM;AACvB,YAAM,UAAU,EAAE,WAAW,eAAe,EAAE,MAAM;AACpD,YAAM,WAAW,MAAM,WAAW,cAAc,SAAS,KAAK;AAC9D,YAAM,YAAY,MAAM,WAAW,eAAe,SAAS,KAAK;AAChE,aAAO,EAAE,QAAQ,EAAE,QAAQ,gBAAgB,EAAE,gBAAgB,UAAU,UAAU;AAAA,IACnF,CAAC;AAAA,EACH;AACA,SAAO,kBAAkB,SAAS,OAAO;AAC3C;AAEA,eAAe,WACb,MACA,SACA,OACmB;AACnB,QAAM,MAAgB,CAAC;AACvB,aAAW,KAAK,MAAM;AACpB,UAAM,IAAI,MAAM,QAAQ,GAAG,KAAK;AAChC,QAAI,MAAM,QAAQ,OAAO,SAAS,CAAC,EAAG,KAAI,KAAK,CAAC;AAAA,EAClD;AACA,SAAO;AACT;AAEA,SAAS,eAAe,QAAyE;AAC/F,SAAO,OAAO,KAAK,UAAU;AAC3B,YAAQ,QAAQ;AAAA,MACd,KAAK;AAAA,MACL,KAAK;AACH,eAAO,IAAI,SAAS,SAAS;AAAA,MAC/B,KAAK;AACH,eAAO,IAAI,SAAS,SAAS,OAAO,IAAI;AAAA,MAC1C,KAAK;AACH,eAAO,IAAI,WAAW,IAAI,YAAY,IAAI,UAAU,IAAI,YAAY;AAAA,MACtE,KAAK,WAAW;AACd,cAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;AAC3C,eAAO,aAAa,GAAG,EAAE;AAAA,MAC3B;AAAA,MACA,KAAK,eAAe;AAClB,cAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;AAC3C,eAAO,aAAa,GAAG,EAAE;AAAA,MAC3B;AAAA,MACA,KAAK,gBAAgB;AACnB,cAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;AAC3C,eAAO,aAAa,GAAG,EAAE;AAAA,MAC3B;AAAA,MACA,KAAK,gBAAgB;AACnB,eAAO,gBAAgB,GAAG,MAAM,YAAY,IAAI;AAAA,MAClD;AAAA,MACA;AACE,eAAO;AAAA,IACX;AAAA,EACF;AACF;;;AClDA,eAAsB,cACpB,OACA,UAA4B,CAAC,GACH;AAC1B,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,OAAO,QAAQ,QACjB,CAAC,EAAE,OAAO,QAAQ,MAAM,CAAC,KACxB,MAAM,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE;AAE5D,QAAM,WAA+B,CAAC;AACtC,aAAW,EAAE,MAAM,KAAK,MAAM;AAC5B,UAAM,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC1C,UAAM,QAAQ,oBAAI,IAAsD;AACxE,eAAW,KAAK,OAAO;AACrB,YAAM,IAAI,QAAQ,EAAE,IAAI;AACxB,YAAM,MAAM,GAAG,EAAE,QAAQ,IAAI,CAAC;AAC9B,YAAM,SAAS,MAAM,IAAI,GAAG,KAAK,EAAE,OAAO,CAAC,GAAG,SAAS,EAAE;AACzD,aAAO,MAAM,KAAK,CAAC;AACnB,YAAM,IAAI,KAAK,MAAM;AAAA,IACvB;AACA,eAAW,CAAC,KAAK,EAAE,OAAO,SAAS,EAAE,CAAC,KAAK,OAAO;AAChD,UAAI,MAAM,SAAS,eAAgB;AACnC,YAAM,SAAS,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAClE,YAAM,QAAQ,OAAO,CAAC,EAAG;AACzB,YAAM,OAAO,OAAO,OAAO,SAAS,CAAC,EAAG;AACxC,eAAS,KAAK;AAAA,QACZ;AAAA,QACA,UAAU,IAAI,MAAM,GAAG,EAAE,CAAC;AAAA,QAC1B,SAAS;AAAA,QACT,aAAa,OAAO;AAAA,QACpB,SAAS,OAAO,IAAI,CAAC,MAAM,EAAE,MAAM;AAAA,QACnC,UAAU,OAAO;AAAA,MACnB,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,eAAe,IAAI,IAAI,SAAS,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC;AACzD,SAAO;AAAA,IACL;AAAA,IACA,kBAAkB,KAAK,SAAS,IAAI,aAAa,OAAO,KAAK,SAAS;AAAA,IACtE,WAAW,KAAK;AAAA,EAClB;AACF;;;AC5CA,eAAsB,cACpB,OACA,UAA4B,CAAC,GACH;AAC1B,QAAM,OAAO,QAAQ,QAAQ,CAAC,QAAQ,KAAK,KAAK,MAAM,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK;AAE1F,QAAM,QAA4B,CAAC;AACnC,MAAI,aAAa;AACjB,MAAI,cAAc;AAClB,aAAW,SAAS,MAAM;AACxB,UAAM,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC1C,QAAI,MAAM,WAAW,GAAG;AACtB,YAAM,KAAK,EAAE,OAAO,aAAa,GAAG,YAAY,GAAG,WAAW,EAAE,CAAC;AACjE;AAAA,IACF;AACA,UAAM,OAAO,MAAM,SAAS,OAAO,KAAK;AACxC,QAAI,SAAS;AACb,eAAW,KAAK,OAAO;AACrB,UAAI,EAAE,WAAW,SAAS;AACxB;AACA;AAAA,MACF;AACA,YAAM,WAAW,KAAK,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,SAAS;AAC7D,UAAI,QAAQ,aAAa;AACvB,YAAI,CAAC,QAAQ,YAAY,GAAG,EAAE,KAAK,SAAS,CAAC,EAAG;AAAA,MAClD,OAAO;AAGL,cAAM,YAAY,UAAU,EAAE,MAAM;AACpC,cAAM,OAAO,SAAS;AAAA,UAAK,CAAC,MAC1B,EAAE,SAAS;AAAA,YACT,CAAC,MACC,OAAO,EAAE,YAAY,YACrB,aACA,EAAE,QAAQ,SAAS,UAAU,MAAM,GAAG,GAAG,CAAC;AAAA,UAC9C;AAAA,QACF;AACA,YAAI,CAAC,KAAM;AAAA,MACb;AAAA,IACF;AACA,UAAM,YAAY,SAAS,MAAM;AACjC,UAAM,KAAK,EAAE,OAAO,aAAa,QAAQ,YAAY,MAAM,QAAQ,UAAU,CAAC;AAC9E,kBAAc,MAAM;AACpB,mBAAe;AAAA,EACjB;AACA,SAAO,EAAE,OAAO,kBAAkB,aAAa,IAAI,cAAc,aAAa,EAAE;AAClF;AAEA,SAAS,UAAU,GAAoB;AACrC,MAAI,MAAM,QAAQ,MAAM,OAAW,QAAO;AAC1C,MAAI,OAAO,MAAM,SAAU,QAAO;AAClC,MAAI;AACF,WAAO,KAAK,UAAU,CAAC;AAAA,EACzB,QAAQ;AACN,WAAO,OAAO,CAAC;AAAA,EACjB;AACF;","names":[]}
1	+ {"version":3,"sources":["../../src/pipelines/budget-breach.ts","../../src/pipelines/failure-cluster.ts","../../src/pipelines/first-divergence.ts","../../src/pipelines/judge-agreement.ts","../../src/pipelines/regression.ts","../../src/pipelines/stuck-loop.ts","../../src/pipelines/tool-waste.ts"],"sourcesContent":["/*\n BudgetBreachView — aggregates breach events across the corpus.\n \n Answers: which dimensions get hit most often? Which scenarios are\n * underbudgeted? Which variants trigger the most breaches?\n /\n\nimport type { BudgetSpec } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface BudgetBreachFinding {\n runId: string\n scenarioId: string\n variantId?: string\n dimension: keyof BudgetSpec\n limit: number\n consumed: number\n excessRatio: number\n timestamp: number\n}\n\nexport interface BudgetBreachReport {\n findings: BudgetBreachFinding[]\n byDimension: Record<string, number>\n byScenario: Record<string, number>\n byVariant: Record<string, number>\n totalRuns: number\n breachedRunRatio: number\n}\n\nexport async function budgetBreachView(\n store: TraceStore,\n options: { scenarioId?: string; variantId?: string } = {},\n): Promise<BudgetBreachReport> {\n const runs = await store.listRuns({\n scenarioId: options.scenarioId,\n variantId: options.variantId,\n })\n const findings: BudgetBreachFinding[] = []\n const byDimension: Record<string, number> = {}\n const byScenario: Record<string, number> = {}\n const byVariant: Record<string, number> = {}\n\n for (const run of runs) {\n const entries = await store.budget(run.runId)\n for (const e of entries) {\n if (!e.breached) continue\n const excessRatio = e.limit > 0 ? e.consumed / e.limit : Infinity\n findings.push({\n runId: run.runId,\n scenarioId: run.scenarioId,\n variantId: run.variantId,\n dimension: e.dimension,\n limit: e.limit,\n consumed: e.consumed,\n excessRatio,\n timestamp: e.timestamp,\n })\n byDimension[e.dimension] = (byDimension[e.dimension] ?? 0) + 1\n byScenario[run.scenarioId] = (byScenario[run.scenarioId] ?? 0) + 1\n if (run.variantId) byVariant[run.variantId] = (byVariant[run.variantId] ?? 0) + 1\n }\n }\n\n const breachedRuns = new Set(findings.map((f) => f.runId))\n return {\n findings,\n byDimension,\n byScenario,\n byVariant,\n totalRuns: runs.length,\n breachedRunRatio: runs.length > 0 ? breachedRuns.size / runs.length : 0,\n }\n}\n","/\n FailureClusterView — groups failed runs by (failureClass, triggerTool,\n * argHash-prefix) so weekly reviews can prioritize the top-N clusters.\n \n Each cluster includes: N runs, scenarios affected, representative\n * error message, a proposed mitigation hint (rule → action table).\n /\n\nimport { classifyFailure, DEFAULT_RULES, type FailureRule } from '../failure-taxonomy'\nimport { argHash, toolSpans } from '../trace/query'\nimport type { FailureClass, Span } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface FailureCluster {\n failureClass: FailureClass\n /* Tool name when the trigger was a tool span, else undefined. /\n toolName?: string\n /* First 16 chars of argHash — clusters similar args. /\n argPrefix?: string\n /\n Source dimension when the trigger was a judge span (e.g. `'format'`,\n * `'safety'`, `'correctness'`). Lets cross-template aggregators\n * group failures by the dimension that fired without overloading\n * `argPrefix`. Optional — clusters without this field deserialize cleanly.\n /\n dimension?: string\n runCount: number\n scenarioIds: string[]\n exampleError?: string\n exampleRunId: string\n}\n\nexport interface FailureClusterReport {\n clusters: FailureCluster[]\n totalFailures: number\n totalRuns: number\n}\n\nexport async function failureClusterView(\n store: TraceStore,\n options: { rules?: FailureRule[]; minClusterSize?: number } = {},\n): Promise<FailureClusterReport> {\n const rules = options.rules ?? DEFAULT_RULES\n const minSize = options.minClusterSize ?? 1\n const runs = await store.listRuns()\n\n type Key = string\n const clusters = new Map<Key, FailureCluster>()\n let totalFailures = 0\n\n for (const run of runs) {\n if (run.status === 'completed' && run.outcome?.pass !== false) continue\n totalFailures++\n const spans = await store.spans({ runId: run.runId })\n const events = await store.events({ runId: run.runId })\n const cls = classifyFailure({ run, spans, events }, rules)\n\n let toolName: string \| undefined\n let argPrefix: string \| undefined\n let dimension: string \| undefined\n if (cls.triggerSpanId) {\n const trig = spans.find((s) => s.spanId === cls.triggerSpanId)\n if (trig?.kind === 'tool') {\n toolName = trig.toolName\n argPrefix = argHash(trig.args).slice(0, 16)\n } else if (trig?.kind === 'judge') {\n dimension = trig.dimension\n }\n }\n // Fallback: look at the last errored tool span\n if (!toolName) {\n const ts = await toolSpans(store, run.runId)\n const errored = ts.filter((t) => t.status === 'error').pop()\n if (errored) {\n toolName = errored.toolName\n argPrefix = argHash(errored.args).slice(0, 16)\n }\n }\n // Secondary signal: any judge span on the failed run carries a\n // dimension. Useful when the rule classified by judge score but\n // didn't surface the trigger span (or surfaced a non-judge span).\n if (!dimension) {\n const judge = spans.find((s) => s.kind === 'judge' && typeof s.dimension === 'string')\n if (judge?.kind === 'judge') dimension = judge.dimension\n }\n\n const key = `${cls.failureClass}\|${toolName ?? ''}\|${argPrefix ?? ''}\|${dimension ?? ''}`\n let cluster = clusters.get(key)\n if (!cluster) {\n cluster = {\n failureClass: cls.failureClass,\n toolName,\n argPrefix,\n dimension,\n runCount: 0,\n scenarioIds: [],\n exampleRunId: run.runId,\n exampleError: firstErrorMessage(spans) ?? cls.reason,\n }\n clusters.set(key, cluster)\n }\n cluster.runCount++\n if (!cluster.scenarioIds.includes(run.scenarioId)) cluster.scenarioIds.push(run.scenarioId)\n }\n\n const arr = [...clusters.values()]\n .filter((c) => c.runCount >= minSize)\n .sort((a, b) => b.runCount - a.runCount)\n\n return { clusters: arr, totalFailures, totalRuns: runs.length }\n}\n\nfunction firstErrorMessage(spans: Span[]): string \| undefined {\n const errored = spans.find((s) => s.status === 'error')\n return errored?.error\n}\n","/\n FirstDivergenceView — aligns two trajectories by step index, reports\n * the first step where they differ.\n \n \"Differ\" is configurable — default is (kind, toolName if tool, model\n * if llm). Use this view to attribute \"why is variant B better?\" to a\n * specific step rather than an aggregate mean delta.\n /\n\nimport type { TraceStore } from '../trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from '../trajectory'\n\nexport interface DivergenceReport {\n runA: string\n runB: string\n firstDivergenceIndex: number \| null\n aStep?: TrajectoryStep\n bStep?: TrajectoryStep\n reason?: string\n /* Common prefix length (steps that matched). /\n commonPrefixLen: number\n}\n\nexport interface DivergenceOptions {\n /* Returns true if two steps are considered equal. Default: kind + tool/model match. /\n stepEquals?: (a: TrajectoryStep, b: TrajectoryStep) => boolean\n}\n\nexport async function firstDivergenceView(\n store: TraceStore,\n runA: string,\n runB: string,\n options: DivergenceOptions = {},\n): Promise<DivergenceReport> {\n const [a, b] = await Promise.all([buildTrajectory(store, runA), buildTrajectory(store, runB)])\n const eq = options.stepEquals ?? defaultStepEquals\n const minLen = Math.min(a.steps.length, b.steps.length)\n for (let i = 0; i < minLen; i++) {\n const aStep = a.steps[i]!\n const bStep = b.steps[i]!\n if (!eq(aStep, bStep)) {\n return {\n runA,\n runB,\n firstDivergenceIndex: i,\n aStep,\n bStep,\n reason: describeDifference(aStep, bStep),\n commonPrefixLen: i,\n }\n }\n }\n if (a.steps.length === b.steps.length) {\n return { runA, runB, firstDivergenceIndex: null, commonPrefixLen: minLen }\n }\n const longer: Trajectory = a.steps.length > b.steps.length ? a : b\n const extra = longer.steps.length - minLen\n // minLen === 0 means one trajectory is empty: divergence is the first step\n // itself (index 0), and the absent side has no step to surface.\n const reason =\n minLen === 0\n ? `one trajectory is empty; the other has ${extra} step(s) starting at index 0`\n : `one trajectory has ${extra} more step(s) after index ${minLen - 1}`\n return {\n runA,\n runB,\n firstDivergenceIndex: minLen,\n aStep: a.steps[minLen],\n bStep: b.steps[minLen],\n reason,\n commonPrefixLen: minLen,\n }\n}\n\nfunction defaultStepEquals(a: TrajectoryStep, b: TrajectoryStep): boolean {\n if (a.span.kind !== b.span.kind) return false\n if (a.span.kind === 'tool' && b.span.kind === 'tool') return a.span.toolName === b.span.toolName\n if (a.span.kind === 'llm' && b.span.kind === 'llm') return a.span.model === b.span.model\n if (a.span.kind === 'judge' && b.span.kind === 'judge')\n return a.span.dimension === b.span.dimension\n return a.span.name === b.span.name\n}\n\nfunction describeDifference(a: TrajectoryStep, b: TrajectoryStep): string {\n if (a.span.kind !== b.span.kind) return `kind ${a.span.kind} vs ${b.span.kind}`\n if (a.span.kind === 'tool' && b.span.kind === 'tool' && a.span.toolName !== b.span.toolName) {\n return `tool ${a.span.toolName} vs ${b.span.toolName}`\n }\n if (a.span.kind === 'llm' && b.span.kind === 'llm' && a.span.model !== b.span.model) {\n return `model ${a.span.model} vs ${b.span.model}`\n }\n return `name \"${a.span.name}\" vs \"${b.span.name}\"`\n}\n","/\n JudgeAgreementView — pairwise agreement between judges across the\n * corpus, grouped by dimension.\n \n Output drives two workflows:\n * - Judge robustness audit: \"does Claude agree with GPT at κ ≥ 0.6?\"\n * - Calibration tracking: κ vs golden human labels over time (by\n * providing a `humanGoldenJudgeId`).\n /\n\nimport { interRaterReliability } from '../statistics'\nimport type { JudgeSpan } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface JudgePair {\n judgeA: string\n judgeB: string\n dimension: string\n /* Number of (targetSpanId, dimension) tuples both judges scored. /\n commonItems: number\n pearson: number\n krippendorff: number\n}\n\nexport interface JudgeAgreementReport {\n pairs: JudgePair[]\n dimensions: string[]\n judgeIds: string[]\n}\n\nexport async function judgeAgreementView(store: TraceStore): Promise<JudgeAgreementReport> {\n const all = (await store.spans({ kind: 'judge' })).filter(\n (s): s is JudgeSpan => s.kind === 'judge',\n )\n if (all.length === 0) return { pairs: [], dimensions: [], judgeIds: [] }\n\n const byDimension = new Map<string, JudgeSpan[]>()\n for (const s of all) {\n const arr = byDimension.get(s.dimension) ?? []\n arr.push(s)\n byDimension.set(s.dimension, arr)\n }\n\n const judgeIds = [...new Set(all.map((s) => s.judgeId))].sort()\n const pairs: JudgePair[] = []\n for (const [dim, spans] of byDimension) {\n const byJudge = new Map<string, Map<string, number>>()\n for (const s of spans) {\n const m = byJudge.get(s.judgeId) ?? new Map<string, number>()\n m.set(s.targetSpanId, s.score)\n byJudge.set(s.judgeId, m)\n }\n const judgesHere = [...byJudge.keys()]\n for (let i = 0; i < judgesHere.length; i++) {\n for (let j = i + 1; j < judgesHere.length; j++) {\n const judgeI = judgesHere[i]!\n const judgeJ = judgesHere[j]!\n const a = byJudge.get(judgeI)!\n const b = byJudge.get(judgeJ)!\n const common: Array<[number, number]> = []\n for (const [target, scoreA] of a) {\n const scoreB = b.get(target)\n if (scoreB !== undefined) common.push([scoreA, scoreB])\n }\n if (common.length < 2) continue\n const judgeScores = common.map(\n ([scoreA, scoreB]) =>\n [\n { judgeName: judgeI, dimension: dim, score: scoreA, reasoning: '' },\n { judgeName: judgeJ, dimension: dim, score: scoreB, reasoning: '' },\n ] as const,\n )\n const k = interRaterReliability(\n judgeScores[0]!.map((_, k2) => judgeScores.map((pair) => pair[k2]!)),\n )\n pairs.push({\n judgeA: judgeI,\n judgeB: judgeJ,\n dimension: dim,\n commonItems: common.length,\n pearson: pearson(\n common.map((c) => c[0]),\n common.map((c) => c[1]),\n ),\n krippendorff: k,\n })\n }\n }\n }\n\n return {\n pairs: pairs.sort((a, b) => b.commonItems - a.commonItems),\n dimensions: [...byDimension.keys()].sort(),\n judgeIds,\n }\n}\n\nfunction pearson(a: number[], b: number[]): number {\n if (a.length !== b.length \|\| a.length < 2) return NaN\n const mA = a.reduce((s, v) => s + v, 0) / a.length\n const mB = b.reduce((s, v) => s + v, 0) / b.length\n let num = 0,\n denA = 0,\n denB = 0\n for (let i = 0; i < a.length; i++) {\n const dA = a[i]! - mA\n const dB = b[i]! - mB\n num += dA dB\n denA += dA * dA\n denB += dB * dB\n }\n if (denA === 0 \|\| denB === 0) return denA === 0 && denB === 0 ? 1 : 0\n return num / Math.sqrt(denA * denB)\n}\n","/*\n RegressionView — compares a candidate slice to a baseline slice on a\n * named metric. Delegates the statistics (Welch's t-test, Cohen's d,\n * IQR stability) to `baseline.ts`.\n \n This is the entry point for CI regression gates: \"given runs tagged\n * release=A and release=B, did any metric regress?\"\n /\n\nimport { type BaselineOptions, type BaselineReport, compareToBaseline } from '../baseline'\nimport { aggregateLlm, llmSpans, runFailureClass } from '../trace/query'\nimport type { Run } from '../trace/schema'\nimport type { RunFilter, TraceStore } from '../trace/store'\n\nexport interface RegressionSpec {\n metric: string\n higherIsBetter: boolean\n /* Extract a scalar from a run. Default extractors handle common metrics. /\n extract?: (run: Run, store: TraceStore) => Promise<number \| null>\n}\n\nexport interface RegressionOptions extends BaselineOptions {\n baseline: RunFilter\n candidate: RunFilter\n}\n\nexport async function regressionView(\n store: TraceStore,\n metrics: RegressionSpec[],\n options: RegressionOptions,\n): Promise<BaselineReport> {\n const baselineRuns = await store.listRuns(options.baseline)\n const candidateRuns = await store.listRuns(options.candidate)\n const samples = await Promise.all(\n metrics.map(async (m) => {\n const extract = m.extract ?? defaultExtract(m.metric)\n const baseline = await extractAll(baselineRuns, extract, store)\n const candidate = await extractAll(candidateRuns, extract, store)\n return { metric: m.metric, higherIsBetter: m.higherIsBetter, baseline, candidate }\n }),\n )\n return compareToBaseline(samples, options)\n}\n\nasync function extractAll(\n runs: Run[],\n extract: (r: Run, s: TraceStore) => Promise<number \| null>,\n store: TraceStore,\n): Promise<number[]> {\n const out: number[] = []\n for (const r of runs) {\n const v = await extract(r, store)\n if (v !== null && Number.isFinite(v)) out.push(v)\n }\n return out\n}\n\nfunction defaultExtract(metric: string): (run: Run, store: TraceStore) => Promise<number \| null> {\n return async (run, store) => {\n switch (metric) {\n case 'score':\n case 'overallScore':\n return run.outcome?.score ?? null\n case 'pass':\n return run.outcome?.pass === true ? 1 : 0\n case 'durationMs':\n return run.endedAt && run.startedAt ? run.endedAt - run.startedAt : null\n case 'costUsd': {\n const llm = await llmSpans(store, run.runId)\n return aggregateLlm(llm).costUsd\n }\n case 'inputTokens': {\n const llm = await llmSpans(store, run.runId)\n return aggregateLlm(llm).inputTokens\n }\n case 'outputTokens': {\n const llm = await llmSpans(store, run.runId)\n return aggregateLlm(llm).outputTokens\n }\n case 'failureClass': {\n return runFailureClass(run) === 'success' ? 1 : 0\n }\n default:\n return null\n }\n }\n}\n","/\n StuckLoopView — detects when an agent calls the same tool with the\n * same (or structurally similar) arguments ≥ N times in a short window.\n \n Rationale: agents that loop are the number-one production failure\n * mode on long-horizon flows. The view returns (runId, toolName,\n * argHash, occurrences, windowMs) for each detected loop plus a\n * fraction of runs affected.\n /\n\nimport { argHash, toolSpans } from '../trace/query'\nimport type { TraceStore } from '../trace/store'\n\nexport interface StuckLoopFinding {\n runId: string\n toolName: string\n argHash: string\n occurrences: number\n spanIds: string[]\n /* Milliseconds between first and last call in the loop. /\n windowMs: number\n}\n\nexport interface StuckLoopReport {\n findings: StuckLoopFinding[]\n affectedRunRatio: number\n totalRuns: number\n}\n\nexport interface StuckLoopOptions {\n /* Minimum call count to flag a loop (default 3). /\n minOccurrences?: number\n /* Filter to a specific runId; omit to scan the entire corpus. /\n runId?: string\n}\n\nexport async function stuckLoopView(\n store: TraceStore,\n options: StuckLoopOptions = {},\n): Promise<StuckLoopReport> {\n const minOccurrences = options.minOccurrences ?? 3\n const runs = options.runId\n ? [{ runId: options.runId }]\n : (await store.listRuns()).map((r) => ({ runId: r.runId }))\n\n const findings: StuckLoopFinding[] = []\n for (const { runId } of runs) {\n const tools = await toolSpans(store, runId)\n const byKey = new Map<string, { spans: typeof tools; argHash: string; toolName: string }>()\n for (const t of tools) {\n const h = argHash(t.args)\n // NUL delimiter never appears in a tool name or a JSON argHash, so the\n // key cannot collide; toolName is carried in the bucket rather than\n // re-derived by splitting the key (which mislabeled any tool whose name\n // contained the delimiter, e.g. \"shell\|grep\").\n const key = `${t.toolName}\\u0000${h}`\n const bucket = byKey.get(key) ?? { spans: [], argHash: h, toolName: t.toolName }\n bucket.spans.push(t)\n byKey.set(key, bucket)\n }\n for (const { spans, argHash: h, toolName } of byKey.values()) {\n if (spans.length < minOccurrences) continue\n const sorted = [...spans].sort((a, b) => a.startedAt - b.startedAt)\n const first = sorted[0]!.startedAt\n const last = sorted[sorted.length - 1]!.startedAt\n findings.push({\n runId,\n toolName,\n argHash: h,\n occurrences: sorted.length,\n spanIds: sorted.map((s) => s.spanId),\n windowMs: last - first,\n })\n }\n }\n\n const affectedRuns = new Set(findings.map((f) => f.runId))\n return {\n findings,\n affectedRunRatio: runs.length > 0 ? affectedRuns.size / runs.length : 0,\n totalRuns: runs.length,\n }\n}\n","/\n ToolWasteView — fraction of tool calls whose results weren't used\n * downstream. Without a \"used\" signal we fall back to structural\n * proxies: error calls, duplicate calls, and tool calls followed by\n * zero subsequent LLM spans are all considered waste.\n \n Consumers can pass a `usageOracle` that inspects a tool span and\n * returns true iff the tool's result appears in a later LLM message,\n * artifact, or state mutation — that's the canonical definition; the\n * default heuristic is a reasonable fallback.\n /\n\nimport { computeToolUseMetrics } from '../tool-use-metrics'\nimport { llmSpans, toolSpans } from '../trace/query'\nimport type { LlmSpan, ToolSpan } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface ToolWasteFinding {\n runId: string\n wastedCalls: number\n totalCalls: number\n wasteRate: number\n}\n\nexport interface ToolWasteReport {\n byRun: ToolWasteFinding[]\n overallWasteRate: number\n}\n\nexport interface ToolWasteOptions {\n runId?: string\n usageOracle?: (tool: ToolSpan, later: { llm: Awaited<ReturnType<typeof llmSpans>> }) => boolean\n}\n\nexport async function toolWasteView(\n store: TraceStore,\n options: ToolWasteOptions = {},\n): Promise<ToolWasteReport> {\n const runs = options.runId ? [options.runId] : (await store.listRuns()).map((r) => r.runId)\n\n const byRun: ToolWasteFinding[] = []\n let totalCalls = 0\n let totalWasted = 0\n for (const runId of runs) {\n const tools = await toolSpans(store, runId)\n if (tools.length === 0) {\n byRun.push({ runId, wastedCalls: 0, totalCalls: 0, wasteRate: 0 })\n continue\n }\n const llms = await llmSpans(store, runId)\n // Sort LLM spans once by start time, then build a suffix index of the\n // concatenated message text. `suffixText[i]` is the haystack of every\n // string message content in spans[i..]. Per tool we binary-search the\n // first span started strictly after the tool, then test that one suffix\n // — turning the per-tool O(llms × messages × content) scan into a single\n // O(log llms) lookup over precomputed text.\n const sortedLlm = [...llms].sort((a, b) => a.startedAt - b.startedAt)\n const startTimes = sortedLlm.map((l) => l.startedAt)\n const suffixText = buildSuffixText(sortedLlm)\n let wasted = 0\n for (const t of tools) {\n if (t.status === 'error') {\n wasted++\n continue\n }\n // First LLM span started strictly after this tool (upper-bound search).\n const cutoff = upperBound(startTimes, t.startedAt)\n if (options.usageOracle) {\n if (!options.usageOracle(t, { llm: sortedLlm.slice(cutoff) })) wasted++\n } else {\n // Default heuristic: a tool whose result is NOT mentioned in any\n // later LLM input message is likely wasted. An empty/null result has\n // no payload to propagate downstream — there is nothing to find in a\n // later message, so it is not evidence of waste; skip it.\n const resultStr = stringify(t.result)\n if (resultStr === '') continue\n const haystack = suffixText[cutoff] ?? ''\n const used = haystack.includes(resultStr.slice(0, 120))\n if (!used) wasted++\n }\n }\n const wasteRate = wasted / tools.length\n byRun.push({ runId, wastedCalls: wasted, totalCalls: tools.length, wasteRate })\n totalCalls += tools.length\n totalWasted += wasted\n }\n return { byRun, overallWasteRate: totalCalls > 0 ? totalWasted / totalCalls : 0 }\n}\n\n/\n Build per-position suffix haystacks: result[i] is the concatenation of every\n * string message content in spans[i..end]. Built back-to-front so each entry\n * reuses the next one — O(total message text) rather than O(spans²).\n /\nfunction buildSuffixText(spans: LlmSpan[]): string[] {\n const result = new Array<string>(spans.length + 1)\n result[spans.length] = ''\n for (let i = spans.length - 1; i >= 0; i--) {\n const own = spans[i]!.messages.map((m) =>\n typeof m.content === 'string' ? m.content : '',\n ).join('\\n')\n result[i] = `${own}\\n${result[i + 1]}`\n }\n return result\n}\n\n/* Index of the first element strictly greater than `target` in a sorted array. */\nfunction upperBound(sorted: number[], target: number): number {\n let lo = 0\n let hi = sorted.length\n while (lo < hi) {\n const mid = (lo + hi) >>> 1\n if (sorted[mid]! <= target) lo = mid + 1\n else hi = mid\n }\n return lo\n}\n\nfunction stringify(v: unknown): string {\n if (v === null \|\| v === undefined) return ''\n if (typeof v === 'string') return v\n try {\n return JSON.stringify(v)\n } catch {\n return String(v)\n }\n}\n\n// Re-export for convenience in consumers that want both descriptive and usage metrics.\nexport { computeToolUseMetrics }\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA8BA,eAAsB,iBACpB,OACA,UAAuD,CAAC,GAC3B;AAC7B,QAAM,OAAO,MAAM,MAAM,SAAS;AAAA,IAChC,YAAY,QAAQ;AAAA,IACpB,WAAW,QAAQ;AAAA,EACrB,CAAC;AACD,QAAM,WAAkC,CAAC;AACzC,QAAM,cAAsC,CAAC;AAC7C,QAAM,aAAqC,CAAC;AAC5C,QAAM,YAAoC,CAAC;AAE3C,aAAW,OAAO,MAAM;AACtB,UAAM,UAAU,MAAM,MAAM,OAAO,IAAI,KAAK;AAC5C,eAAW,KAAK,SAAS;AACvB,UAAI,CAAC,EAAE,SAAU;AACjB,YAAM,cAAc,EAAE,QAAQ,IAAI,EAAE,WAAW,EAAE,QAAQ;AACzD,eAAS,KAAK;AAAA,QACZ,OAAO,IAAI;AAAA,QACX,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,WAAW,EAAE;AAAA,QACb,OAAO,EAAE;AAAA,QACT,UAAU,EAAE;AAAA,QACZ;AAAA,QACA,WAAW,EAAE;AAAA,MACf,CAAC;AACD,kBAAY,EAAE,SAAS,KAAK,YAAY,EAAE,SAAS,KAAK,KAAK;AAC7D,iBAAW,IAAI,UAAU,KAAK,WAAW,IAAI,UAAU,KAAK,KAAK;AACjE,UAAI,IAAI,UAAW,WAAU,IAAI,SAAS,KAAK,UAAU,IAAI,SAAS,KAAK,KAAK;AAAA,IAClF;AAAA,EACF;AAEA,QAAM,eAAe,IAAI,IAAI,SAAS,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC;AACzD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW,KAAK;AAAA,IAChB,kBAAkB,KAAK,SAAS,IAAI,aAAa,OAAO,KAAK,SAAS;AAAA,EACxE;AACF;;;ACnCA,eAAsB,mBACpB,OACA,UAA8D,CAAC,GAChC;AAC/B,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,UAAU,QAAQ,kBAAkB;AAC1C,QAAM,OAAO,MAAM,MAAM,SAAS;AAGlC,QAAM,WAAW,oBAAI,IAAyB;AAC9C,MAAI,gBAAgB;AAEpB,aAAW,OAAO,MAAM;AACtB,QAAI,IAAI,WAAW,eAAe,IAAI,SAAS,SAAS,MAAO;AAC/D;AACA,UAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,IAAI,MAAM,CAAC;AACpD,UAAM,SAAS,MAAM,MAAM,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC;AACtD,UAAM,MAAM,gBAAgB,EAAE,KAAK,OAAO,OAAO,GAAG,KAAK;AAEzD,QAAI;AACJ,QAAI;AACJ,QAAI;AACJ,QAAI,IAAI,eAAe;AACrB,YAAM,OAAO,MAAM,KAAK,CAAC,MAAM,EAAE,WAAW,IAAI,aAAa;AAC7D,UAAI,MAAM,SAAS,QAAQ;AACzB,mBAAW,KAAK;AAChB,oBAAY,QAAQ,KAAK,IAAI,EAAE,MAAM,GAAG,EAAE;AAAA,MAC5C,WAAW,MAAM,SAAS,SAAS;AACjC,oBAAY,KAAK;AAAA,MACnB;AAAA,IACF;AAEA,QAAI,CAAC,UAAU;AACb,YAAM,KAAK,MAAM,UAAU,OAAO,IAAI,KAAK;AAC3C,YAAM,UAAU,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,OAAO,EAAE,IAAI;AAC3D,UAAI,SAAS;AACX,mBAAW,QAAQ;AACnB,oBAAY,QAAQ,QAAQ,IAAI,EAAE,MAAM,GAAG,EAAE;AAAA,MAC/C;AAAA,IACF;AAIA,QAAI,CAAC,WAAW;AACd,YAAM,QAAQ,MAAM,KAAK,CAAC,MAAM,EAAE,SAAS,WAAW,OAAO,EAAE,cAAc,QAAQ;AACrF,UAAI,OAAO,SAAS,QAAS,aAAY,MAAM;AAAA,IACjD;AAEA,UAAM,MAAM,GAAG,IAAI,YAAY,IAAI,YAAY,EAAE,IAAI,aAAa,EAAE,IAAI,aAAa,EAAE;AACvF,QAAI,UAAU,SAAS,IAAI,GAAG;AAC9B,QAAI,CAAC,SAAS;AACZ,gBAAU;AAAA,QACR,cAAc,IAAI;AAAA,QAClB;AAAA,QACA;AAAA,QACA;AAAA,QACA,UAAU;AAAA,QACV,aAAa,CAAC;AAAA,QACd,cAAc,IAAI;AAAA,QAClB,cAAc,kBAAkB,KAAK,KAAK,IAAI;AAAA,MAChD;AACA,eAAS,IAAI,KAAK,OAAO;AAAA,IAC3B;AACA,YAAQ;AACR,QAAI,CAAC,QAAQ,YAAY,SAAS,IAAI,UAAU,EAAG,SAAQ,YAAY,KAAK,IAAI,UAAU;AAAA,EAC5F;AAEA,QAAM,MAAM,CAAC,GAAG,SAAS,OAAO,CAAC,EAC9B,OAAO,CAAC,MAAM,EAAE,YAAY,OAAO,EACnC,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAEzC,SAAO,EAAE,UAAU,KAAK,eAAe,WAAW,KAAK,OAAO;AAChE;AAEA,SAAS,kBAAkB,OAAmC;AAC5D,QAAM,UAAU,MAAM,KAAK,CAAC,MAAM,EAAE,WAAW,OAAO;AACtD,SAAO,SAAS;AAClB;;;ACvFA,eAAsB,oBACpB,OACA,MACA,MACA,UAA6B,CAAC,GACH;AAC3B,QAAM,CAAC,GAAG,CAAC,IAAI,MAAM,QAAQ,IAAI,CAAC,gBAAgB,OAAO,IAAI,GAAG,gBAAgB,OAAO,IAAI,CAAC,CAAC;AAC7F,QAAM,KAAK,QAAQ,cAAc;AACjC,QAAM,SAAS,KAAK,IAAI,EAAE,MAAM,QAAQ,EAAE,MAAM,MAAM;AACtD,WAAS,IAAI,GAAG,IAAI,QAAQ,KAAK;AAC/B,UAAM,QAAQ,EAAE,MAAM,CAAC;AACvB,UAAM,QAAQ,EAAE,MAAM,CAAC;AACvB,QAAI,CAAC,GAAG,OAAO,KAAK,GAAG;AACrB,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,sBAAsB;AAAA,QACtB;AAAA,QACA;AAAA,QACA,QAAQ,mBAAmB,OAAO,KAAK;AAAA,QACvC,iBAAiB;AAAA,MACnB;AAAA,IACF;AAAA,EACF;AACA,MAAI,EAAE,MAAM,WAAW,EAAE,MAAM,QAAQ;AACrC,WAAO,EAAE,MAAM,MAAM,sBAAsB,MAAM,iBAAiB,OAAO;AAAA,EAC3E;AACA,QAAM,SAAqB,EAAE,MAAM,SAAS,EAAE,MAAM,SAAS,IAAI;AACjE,QAAM,QAAQ,OAAO,MAAM,SAAS;AAGpC,QAAM,SACJ,WAAW,IACP,0CAA0C,KAAK,iCAC/C,sBAAsB,KAAK,6BAA6B,SAAS,CAAC;AACxE,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,sBAAsB;AAAA,IACtB,OAAO,EAAE,MAAM,MAAM;AAAA,IACrB,OAAO,EAAE,MAAM,MAAM;AAAA,IACrB;AAAA,IACA,iBAAiB;AAAA,EACnB;AACF;AAEA,SAAS,kBAAkB,GAAmB,GAA4B;AACxE,MAAI,EAAE,KAAK,SAAS,EAAE,KAAK,KAAM,QAAO;AACxC,MAAI,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,SAAS,OAAQ,QAAO,EAAE,KAAK,aAAa,EAAE,KAAK;AACxF,MAAI,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,SAAS,MAAO,QAAO,EAAE,KAAK,UAAU,EAAE,KAAK;AACnF,MAAI,EAAE,KAAK,SAAS,WAAW,EAAE,KAAK,SAAS;AAC7C,WAAO,EAAE,KAAK,cAAc,EAAE,KAAK;AACrC,SAAO,EAAE,KAAK,SAAS,EAAE,KAAK;AAChC;AAEA,SAAS,mBAAmB,GAAmB,GAA2B;AACxE,MAAI,EAAE,KAAK,SAAS,EAAE,KAAK,KAAM,QAAO,QAAQ,EAAE,KAAK,IAAI,OAAO,EAAE,KAAK,IAAI;AAC7E,MAAI,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,SAAS,UAAU,EAAE,KAAK,aAAa,EAAE,KAAK,UAAU;AAC3F,WAAO,QAAQ,EAAE,KAAK,QAAQ,OAAO,EAAE,KAAK,QAAQ;AAAA,EACtD;AACA,MAAI,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,SAAS,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,OAAO;AACnF,WAAO,SAAS,EAAE,KAAK,KAAK,OAAO,EAAE,KAAK,KAAK;AAAA,EACjD;AACA,SAAO,SAAS,EAAE,KAAK,IAAI,SAAS,EAAE,KAAK,IAAI;AACjD;;;AC9DA,eAAsB,mBAAmB,OAAkD;AACzF,QAAM,OAAO,MAAM,MAAM,MAAM,EAAE,MAAM,QAAQ,CAAC,GAAG;AAAA,IACjD,CAAC,MAAsB,EAAE,SAAS;AAAA,EACpC;AACA,MAAI,IAAI,WAAW,EAAG,QAAO,EAAE,OAAO,CAAC,GAAG,YAAY,CAAC,GAAG,UAAU,CAAC,EAAE;AAEvE,QAAM,cAAc,oBAAI,IAAyB;AACjD,aAAW,KAAK,KAAK;AACnB,UAAM,MAAM,YAAY,IAAI,EAAE,SAAS,KAAK,CAAC;AAC7C,QAAI,KAAK,CAAC;AACV,gBAAY,IAAI,EAAE,WAAW,GAAG;AAAA,EAClC;AAEA,QAAM,WAAW,CAAC,GAAG,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,KAAK;AAC9D,QAAM,QAAqB,CAAC;AAC5B,aAAW,CAAC,KAAK,KAAK,KAAK,aAAa;AACtC,UAAM,UAAU,oBAAI,IAAiC;AACrD,eAAW,KAAK,OAAO;AACrB,YAAM,IAAI,QAAQ,IAAI,EAAE,OAAO,KAAK,oBAAI,IAAoB;AAC5D,QAAE,IAAI,EAAE,cAAc,EAAE,KAAK;AAC7B,cAAQ,IAAI,EAAE,SAAS,CAAC;AAAA,IAC1B;AACA,UAAM,aAAa,CAAC,GAAG,QAAQ,KAAK,CAAC;AACrC,aAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,eAAS,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC9C,cAAM,SAAS,WAAW,CAAC;AAC3B,cAAM,SAAS,WAAW,CAAC;AAC3B,cAAM,IAAI,QAAQ,IAAI,MAAM;AAC5B,cAAM,IAAI,QAAQ,IAAI,MAAM;AAC5B,cAAM,SAAkC,CAAC;AACzC,mBAAW,CAAC,QAAQ,MAAM,KAAK,GAAG;AAChC,gBAAM,SAAS,EAAE,IAAI,MAAM;AAC3B,cAAI,WAAW,OAAW,QAAO,KAAK,CAAC,QAAQ,MAAM,CAAC;AAAA,QACxD;AACA,YAAI,OAAO,SAAS,EAAG;AACvB,cAAM,cAAc,OAAO;AAAA,UACzB,CAAC,CAAC,QAAQ,MAAM,MACd;AAAA,YACE,EAAE,WAAW,QAAQ,WAAW,KAAK,OAAO,QAAQ,WAAW,GAAG;AAAA,YAClE,EAAE,WAAW,QAAQ,WAAW,KAAK,OAAO,QAAQ,WAAW,GAAG;AAAA,UACpE;AAAA,QACJ;AACA,cAAM,IAAI;AAAA,UACR,YAAY,CAAC,EAAG,IAAI,CAAC,GAAG,OAAO,YAAY,IAAI,CAAC,SAAS,KAAK,EAAE,CAAE,CAAC;AAAA,QACrE;AACA,cAAM,KAAK;AAAA,UACT,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR,WAAW;AAAA,UACX,aAAa,OAAO;AAAA,UACpB,SAAS;AAAA,YACP,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAAA,YACtB,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAAA,UACxB;AAAA,UACA,cAAc;AAAA,QAChB,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,OAAO,MAAM,KAAK,CAAC,GAAG,MAAM,EAAE,cAAc,EAAE,WAAW;AAAA,IACzD,YAAY,CAAC,GAAG,YAAY,KAAK,CAAC,EAAE,KAAK;AAAA,IACzC;AAAA,EACF;AACF;AAEA,SAAS,QAAQ,GAAa,GAAqB;AACjD,MAAI,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAG,QAAO;AAClD,QAAM,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE;AAC5C,QAAM,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE;AAC5C,MAAI,MAAM,GACR,OAAO,GACP,OAAO;AACT,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,UAAM,KAAK,EAAE,CAAC,IAAK;AACnB,UAAM,KAAK,EAAE,CAAC,IAAK;AACnB,WAAO,KAAK;AACZ,YAAQ,KAAK;AACb,YAAQ,KAAK;AAAA,EACf;AACA,MAAI,SAAS,KAAK,SAAS,EAAG,QAAO,SAAS,KAAK,SAAS,IAAI,IAAI;AACpE,SAAO,MAAM,KAAK,KAAK,OAAO,IAAI;AACpC;;;ACvFA,eAAsB,eACpB,OACA,SACA,SACyB;AACzB,QAAM,eAAe,MAAM,MAAM,SAAS,QAAQ,QAAQ;AAC1D,QAAM,gBAAgB,MAAM,MAAM,SAAS,QAAQ,SAAS;AAC5D,QAAM,UAAU,MAAM,QAAQ;AAAA,IAC5B,QAAQ,IAAI,OAAO,MAAM;AACvB,YAAM,UAAU,EAAE,WAAW,eAAe,EAAE,MAAM;AACpD,YAAM,WAAW,MAAM,WAAW,cAAc,SAAS,KAAK;AAC9D,YAAM,YAAY,MAAM,WAAW,eAAe,SAAS,KAAK;AAChE,aAAO,EAAE,QAAQ,EAAE,QAAQ,gBAAgB,EAAE,gBAAgB,UAAU,UAAU;AAAA,IACnF,CAAC;AAAA,EACH;AACA,SAAO,kBAAkB,SAAS,OAAO;AAC3C;AAEA,eAAe,WACb,MACA,SACA,OACmB;AACnB,QAAM,MAAgB,CAAC;AACvB,aAAW,KAAK,MAAM;AACpB,UAAM,IAAI,MAAM,QAAQ,GAAG,KAAK;AAChC,QAAI,MAAM,QAAQ,OAAO,SAAS,CAAC,EAAG,KAAI,KAAK,CAAC;AAAA,EAClD;AACA,SAAO;AACT;AAEA,SAAS,eAAe,QAAyE;AAC/F,SAAO,OAAO,KAAK,UAAU;AAC3B,YAAQ,QAAQ;AAAA,MACd,KAAK;AAAA,MACL,KAAK;AACH,eAAO,IAAI,SAAS,SAAS;AAAA,MAC/B,KAAK;AACH,eAAO,IAAI,SAAS,SAAS,OAAO,IAAI;AAAA,MAC1C,KAAK;AACH,eAAO,IAAI,WAAW,IAAI,YAAY,IAAI,UAAU,IAAI,YAAY;AAAA,MACtE,KAAK,WAAW;AACd,cAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;AAC3C,eAAO,aAAa,GAAG,EAAE;AAAA,MAC3B;AAAA,MACA,KAAK,eAAe;AAClB,cAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;AAC3C,eAAO,aAAa,GAAG,EAAE;AAAA,MAC3B;AAAA,MACA,KAAK,gBAAgB;AACnB,cAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;AAC3C,eAAO,aAAa,GAAG,EAAE;AAAA,MAC3B;AAAA,MACA,KAAK,gBAAgB;AACnB,eAAO,gBAAgB,GAAG,MAAM,YAAY,IAAI;AAAA,MAClD;AAAA,MACA;AACE,eAAO;AAAA,IACX;AAAA,EACF;AACF;;;AClDA,eAAsB,cACpB,OACA,UAA4B,CAAC,GACH;AAC1B,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,OAAO,QAAQ,QACjB,CAAC,EAAE,OAAO,QAAQ,MAAM,CAAC,KACxB,MAAM,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE;AAE5D,QAAM,WAA+B,CAAC;AACtC,aAAW,EAAE,MAAM,KAAK,MAAM;AAC5B,UAAM,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC1C,UAAM,QAAQ,oBAAI,IAAwE;AAC1F,eAAW,KAAK,OAAO;AACrB,YAAM,IAAI,QAAQ,EAAE,IAAI;AAKxB,YAAM,MAAM,GAAG,EAAE,QAAQ,KAAS,CAAC;AACnC,YAAM,SAAS,MAAM,IAAI,GAAG,KAAK,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,UAAU,EAAE,SAAS;AAC/E,aAAO,MAAM,KAAK,CAAC;AACnB,YAAM,IAAI,KAAK,MAAM;AAAA,IACvB;AACA,eAAW,EAAE,OAAO,SAAS,GAAG,SAAS,KAAK,MAAM,OAAO,GAAG;AAC5D,UAAI,MAAM,SAAS,eAAgB;AACnC,YAAM,SAAS,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAClE,YAAM,QAAQ,OAAO,CAAC,EAAG;AACzB,YAAM,OAAO,OAAO,OAAO,SAAS,CAAC,EAAG;AACxC,eAAS,KAAK;AAAA,QACZ;AAAA,QACA;AAAA,QACA,SAAS;AAAA,QACT,aAAa,OAAO;AAAA,QACpB,SAAS,OAAO,IAAI,CAAC,MAAM,EAAE,MAAM;AAAA,QACnC,UAAU,OAAO;AAAA,MACnB,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,eAAe,IAAI,IAAI,SAAS,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC;AACzD,SAAO;AAAA,IACL;AAAA,IACA,kBAAkB,KAAK,SAAS,IAAI,aAAa,OAAO,KAAK,SAAS;AAAA,IACtE,WAAW,KAAK;AAAA,EAClB;AACF;;;AChDA,eAAsB,cACpB,OACA,UAA4B,CAAC,GACH;AAC1B,QAAM,OAAO,QAAQ,QAAQ,CAAC,QAAQ,KAAK,KAAK,MAAM,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK;AAE1F,QAAM,QAA4B,CAAC;AACnC,MAAI,aAAa;AACjB,MAAI,cAAc;AAClB,aAAW,SAAS,MAAM;AACxB,UAAM,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC1C,QAAI,MAAM,WAAW,GAAG;AACtB,YAAM,KAAK,EAAE,OAAO,aAAa,GAAG,YAAY,GAAG,WAAW,EAAE,CAAC;AACjE;AAAA,IACF;AACA,UAAM,OAAO,MAAM,SAAS,OAAO,KAAK;AAOxC,UAAM,YAAY,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AACpE,UAAM,aAAa,UAAU,IAAI,CAAC,MAAM,EAAE,SAAS;AACnD,UAAM,aAAa,gBAAgB,SAAS;AAC5C,QAAI,SAAS;AACb,eAAW,KAAK,OAAO;AACrB,UAAI,EAAE,WAAW,SAAS;AACxB;AACA;AAAA,MACF;AAEA,YAAM,SAAS,WAAW,YAAY,EAAE,SAAS;AACjD,UAAI,QAAQ,aAAa;AACvB,YAAI,CAAC,QAAQ,YAAY,GAAG,EAAE,KAAK,UAAU,MAAM,MAAM,EAAE,CAAC,EAAG;AAAA,MACjE,OAAO;AAKL,cAAM,YAAY,UAAU,EAAE,MAAM;AACpC,YAAI,cAAc,GAAI;AACtB,cAAM,WAAW,WAAW,MAAM,KAAK;AACvC,cAAM,OAAO,SAAS,SAAS,UAAU,MAAM,GAAG,GAAG,CAAC;AACtD,YAAI,CAAC,KAAM;AAAA,MACb;AAAA,IACF;AACA,UAAM,YAAY,SAAS,MAAM;AACjC,UAAM,KAAK,EAAE,OAAO,aAAa,QAAQ,YAAY,MAAM,QAAQ,UAAU,CAAC;AAC9E,kBAAc,MAAM;AACpB,mBAAe;AAAA,EACjB;AACA,SAAO,EAAE,OAAO,kBAAkB,aAAa,IAAI,cAAc,aAAa,EAAE;AAClF;AAOA,SAAS,gBAAgB,OAA4B;AACnD,QAAM,SAAS,IAAI,MAAc,MAAM,SAAS,CAAC;AACjD,SAAO,MAAM,MAAM,IAAI;AACvB,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK;AAC1C,UAAM,MAAM,MAAM,CAAC,EAAG,SAAS;AAAA,MAAI,CAAC,MAClC,OAAO,EAAE,YAAY,WAAW,EAAE,UAAU;AAAA,IAC9C,EAAE,KAAK,IAAI;AACX,WAAO,CAAC,IAAI,GAAG,GAAG;AAAA,EAAK,OAAO,IAAI,CAAC,CAAC;AAAA,EACtC;AACA,SAAO;AACT;AAGA,SAAS,WAAW,QAAkB,QAAwB;AAC5D,MAAI,KAAK;AACT,MAAI,KAAK,OAAO;AAChB,SAAO,KAAK,IAAI;AACd,UAAM,MAAO,KAAK,OAAQ;AAC1B,QAAI,OAAO,GAAG,KAAM,OAAQ,MAAK,MAAM;AAAA,QAClC,MAAK;AAAA,EACZ;AACA,SAAO;AACT;AAEA,SAAS,UAAU,GAAoB;AACrC,MAAI,MAAM,QAAQ,MAAM,OAAW,QAAO;AAC1C,MAAI,OAAO,MAAM,SAAU,QAAO;AAClC,MAAI;AACF,WAAO,KAAK,UAAU,CAAC;AAAA,EACzB,QAAQ;AACN,WAAO,OAAO,CAAC;AAAA,EACjB;AACF;","names":[]}

package/dist/prm/index.d.ts CHANGED Viewed

@@ -1,8 +1,8 @@
-import { P as PrmGradedTrace, S as StepRubric, a as PrmGrader } from '../rubric-BOfxn4ja.js';
-export { G as GradedStep, b as StepContext, i as isPrmVerdict } from '../rubric-BOfxn4ja.js';
-import { T as TraceStore } from '../store-CKUAgsJz.js';
+import { P as PrmGradedTrace, S as StepRubric, a as PrmGrader } from '../rubric-Cc6UHvUb.js';
+export { G as GradedStep, b as StepContext, i as isPrmVerdict } from '../rubric-Cc6UHvUb.js';
+import { T as TraceStore } from '../store-BcFXE6LG.js';
 import { S as Span } from '../schema-m0gsnbt3.js';
-import '../trajectory-GEdXJCL5.js';
+import '../trajectory-2TkpSEVh.js';
 /**
  * Export PRM-graded traces as training data for downstream reward-model
@@ -88,13 +88,17 @@ interface BestOfNResult {
     /** Standard deviation of aggregate scores — small = candidates were homogenous. */
     stdDev: number;
 }
-declare function prmBestOfN(store: TraceStore, grader: PrmGrader, runIds: string[]): Promise<BestOfNResult>;
+interface PrmBestOfNOptions {
+    /** Max concurrent `grader.grade` calls. Default 4. */
+    concurrency?: number;
+}
+declare function prmBestOfN(store: TraceStore, grader: PrmGrader, runIds: string[], options?: PrmBestOfNOptions): Promise<BestOfNResult>;
 /**
  * Weighted vote across multiple graders — use when you want a PRM ensemble
  * (e.g. rule-based + LLM-based + trained model). Each grader produces its
  * own ranking; we aggregate via rank-sum (Borda count) so no single grader
  * dominates via a different score scale.
  */
-declare function prmEnsembleBestOfN(store: TraceStore, graders: PrmGrader[], runIds: string[]): Promise<BestOfNResult>;
+declare function prmEnsembleBestOfN(store: TraceStore, graders: PrmGrader[], runIds: string[], options?: PrmBestOfNOptions): Promise<BestOfNResult>;
-export { type BestOfNResult, PrmGradedTrace, PrmGrader, type PrmTrainingSample, StepRubric, exportTrainingData, nonRefusalRubric, outputLengthRubric, prmBestOfN, prmEnsembleBestOfN, toNdjson, toolIntentAlignmentRubric, toolNonRedundantRubric, toolSuccessRubric };
+export { type BestOfNResult, type PrmBestOfNOptions, PrmGradedTrace, PrmGrader, type PrmTrainingSample, StepRubric, exportTrainingData, nonRefusalRubric, outputLengthRubric, prmBestOfN, prmEnsembleBestOfN, toNdjson, toolIntentAlignmentRubric, toolNonRedundantRubric, toolSuccessRubric };

package/dist/prm/index.js CHANGED Viewed

@@ -114,30 +114,73 @@ function stableStringify(value) {
 }
 // src/prm/inference.ts
-async function prmBestOfN(store, grader, runIds) {
+var DEFAULT_PRM_CONCURRENCY = 4;
+async function prmBestOfN(store, grader, runIds, options = {}) {
   if (runIds.length === 0) throw new Error("prmBestOfN: at least 1 candidate required");
-  const graded = await Promise.all(runIds.map((id) => grader.grade(store, id)));
+  const concurrency = Math.max(1, options.concurrency ?? DEFAULT_PRM_CONCURRENCY);
+  const graded = new Array(runIds.length);
+  let cursor = 0;
+  async function worker() {
+    while (true) {
+      const i = cursor++;
+      if (i >= runIds.length) return;
+      graded[i] = await grader.grade(store, runIds[i]);
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(concurrency, runIds.length) }, () => worker()));
   const ranked = [...graded].sort((a, b) => b.aggregateScore - a.aggregateScore);
   const mean = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length;
   const variance = graded.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / graded.length;
   return { winner: ranked[0], ranked, stdDev: Math.sqrt(variance) };
 }
-async function prmEnsembleBestOfN(store, graders, runIds) {
+async function prmEnsembleBestOfN(store, graders, runIds, options = {}) {
   if (graders.length === 0) throw new Error("prmEnsembleBestOfN: at least 1 grader");
-  const perGrader = await Promise.all(
-    graders.map(async (g) => {
-      const graded = await Promise.all(runIds.map((id) => g.grade(store, id)));
-      return graded.sort((a, b) => b.aggregateScore - a.aggregateScore);
-    })
-  );
+  if (runIds.length === 0) throw new Error("prmEnsembleBestOfN: at least 1 candidate required");
+  const concurrency = Math.max(1, options.concurrency ?? DEFAULT_PRM_CONCURRENCY);
+  const jobs = [];
+  for (let gi = 0; gi < graders.length; gi++) {
+    for (const runId of runIds) jobs.push({ graderIdx: gi, runId });
+  }
+  const settled = new Array(jobs.length);
+  let cursor = 0;
+  async function worker() {
+    while (true) {
+      const i = cursor++;
+      if (i >= jobs.length) return;
+      const job = jobs[i];
+      settled[i] = await graders[job.graderIdx].grade(store, job.runId).then((value) => ({ status: "fulfilled", value })).catch((reason) => ({ status: "rejected", reason }));
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(concurrency, jobs.length) }, () => worker()));
+  const perGrader = graders.map(() => []);
+  const failures = [];
+  for (let i = 0; i < jobs.length; i++) {
+    const job = jobs[i];
+    const result = settled[i];
+    if (result.status === "fulfilled") perGrader[job.graderIdx].push(result.value);
+    else {
+      const error = result.reason instanceof Error ? result.reason.message : String(result.reason);
+      failures.push({ graderIdx: job.graderIdx, runId: job.runId, error });
+    }
+  }
+  const survivingGraders = perGrader.filter((ranking) => ranking.length > 0);
+  if (survivingGraders.length === 0) {
+    throw new Error(
+      `prmEnsembleBestOfN: every grader failed on every candidate (${failures.length} call(s)). First error: ${failures[0]?.error ?? "unknown"}`
+    );
+  }
+  for (const ranking of survivingGraders)
+    ranking.sort((a, b) => b.aggregateScore - a.aggregateScore);
   const bordaScores = /* @__PURE__ */ new Map();
-  for (const ranking of perGrader) {
+  for (const ranking of survivingGraders) {
     ranking.forEach((g, rank) => {
       bordaScores.set(g.runId, (bordaScores.get(g.runId) ?? 0) + (ranking.length - rank));
     });
   }
-  const canonical = perGrader[0];
-  const byRun = new Map(canonical.map((g) => [g.runId, g]));
+  const byRun = /* @__PURE__ */ new Map();
+  for (const ranking of survivingGraders) {
+    for (const g of ranking) if (!byRun.has(g.runId)) byRun.set(g.runId, g);
+  }
   const ranked = [...byRun.values()].sort(
     (a, b) => (bordaScores.get(b.runId) ?? 0) - (bordaScores.get(a.runId) ?? 0)
   );