@tangle-network/agent-eval 0.144.11 → 0.144.13

package/dist/tool-waste-BDdBZG1F.js

@@ -0,0 +1,803 @@
+import { s as ValidationError } from "./errors-Dngq5h35.js";
+import { r as pearsonR } from "./descriptive-B5MwKfbf.js";
+import { r as continuousAgreement } from "./judge-calibration-DZkWrm5H.js";
+import { i as hasCapturedToolArgs, l as toolSpans, n as argHash, o as llmSpans, r as groupBy } from "./query-Di7eEQ79.js";
+//#region src/statistics/agreement-irr.ts
+/**
+* Inter-rater reliability — Krippendorff's α under the squared-difference
+* metric, pooled across dimensions.
+*
+* Each inner array is one judge's scores. Items are matched by position
+* WITHIN a dimension: the k-th score a judge supplies carrying dimension
+* `d` is item k of `d`, and the ratings compared against each other are
+* the ones different judges gave to the same item. Every judge that scores
+* a dimension at all must supply the same number of scores for it —
+* ragged input cannot be aligned into items and throws rather than
+* comparing mismatched items.
+*
+* α = 1 − D_observed / D_expected: D_observed averages the squared
+* difference over within-item judge pairs, D_expected over every pair of
+* ratings irrespective of item. α = 1 is perfect agreement, 0 is chance,
+* negative is systematic disagreement.
+*/
+function interRaterReliability(judgeScores) {
+	if (judgeScores.length < 2) return 1;
+	const perDimension = /* @__PURE__ */ new Map();
+	for (let judgeIndex = 0; judgeIndex < judgeScores.length; judgeIndex++) for (const s of judgeScores[judgeIndex]) {
+		let byJudge = perDimension.get(s.dimension);
+		if (byJudge === void 0) {
+			byJudge = Array.from({ length: judgeScores.length }, () => []);
+			perDimension.set(s.dimension, byJudge);
+		}
+		byJudge[judgeIndex].push(s.score);
+	}
+	const allValues = [];
+	const pairDiffs = [];
+	for (const [dimension, byJudge] of perDimension) {
+		const scoring = byJudge.filter((scores) => scores.length > 0);
+		if (scoring.length < 2) continue;
+		const itemCount = scoring[0].length;
+		if (scoring.some((scores) => scores.length !== itemCount)) throw new ValidationError(`interRaterReliability: dimension '${dimension}' has judges supplying ${scoring.map((scores) => scores.length).join("/")} scores — items cannot be aligned`);
+		for (let item = 0; item < itemCount; item++) {
+			const ratings = scoring.map((scores) => scores[item]);
+			for (const v of ratings) allValues.push(v);
+			for (let i = 0; i < ratings.length; i++) for (let j = i + 1; j < ratings.length; j++) pairDiffs.push((ratings[i] - ratings[j]) ** 2);
+		}
+	}
+	if (pairDiffs.length === 0 || allValues.length < 2) return 1;
+	const observedDisagreement = pairDiffs.reduce((a, b) => a + b, 0) / pairDiffs.length;
+	let expectedDisagreement = 0;
+	let expectedCount = 0;
+	for (let i = 0; i < allValues.length; i++) for (let j = i + 1; j < allValues.length; j++) {
+		expectedDisagreement += (allValues[i] - allValues[j]) ** 2;
+		expectedCount++;
+	}
+	expectedDisagreement = expectedCount > 0 ? expectedDisagreement / expectedCount : 0;
+	if (expectedDisagreement === 0) return 1;
+	return 1 - observedDisagreement / expectedDisagreement;
+}
+/**
+* Corpus-wide inter-rater agreement across N items × M judges × D dimensions.
+*
+* For each dimension, builds the [n_items][n_judges] matrix of scores
+* (keeping only items every judge rated on that dimension), then runs
+* `continuousAgreement` to get ICC(2,1), κ_w, Pearson, Spearman, and
+* bootstrap CIs. Reports a pooled mean across dimensions as a single
+* "is this judge panel reliable on this corpus?" number.
+*
+* Fail-loud contract:
+*   - Empty input throws.
+*   - Fewer than 2 judges or fewer than 2 items per dimension throws.
+*   - A judge present in some dimensions but with zero scored items on
+*     another dimension throws (would silently shrink the matrix).
+*   - Duplicate (itemId, judgeName, dimension) records throw.
+*/
+function corpusInterRaterAgreement(records, opts = {}) {
+	if (records.length === 0) throw new ValidationError("corpusInterRaterAgreement: no score records supplied");
+	const judgesSeen = /* @__PURE__ */ new Set();
+	const dimsSeen = /* @__PURE__ */ new Set();
+	const grid = /* @__PURE__ */ new Map();
+	for (const r of records) {
+		if (!Number.isFinite(r.score)) throw new ValidationError(`corpusInterRaterAgreement: non-finite score for (item=${r.itemId}, judge=${r.judgeName}, dim=${r.dimension})`);
+		judgesSeen.add(r.judgeName);
+		dimsSeen.add(r.dimension);
+		const byJudge = grid.get(r.dimension) ?? /* @__PURE__ */ new Map();
+		const byItem = byJudge.get(r.judgeName) ?? /* @__PURE__ */ new Map();
+		if (byItem.has(r.itemId)) throw new ValidationError(`corpusInterRaterAgreement: duplicate record for (item=${r.itemId}, judge=${r.judgeName}, dim=${r.dimension})`);
+		byItem.set(r.itemId, r.score);
+		byJudge.set(r.judgeName, byItem);
+		grid.set(r.dimension, byJudge);
+	}
+	const targetDims = opts.dimensions ?? [...dimsSeen].sort();
+	for (const d of targetDims) if (!dimsSeen.has(d)) throw new ValidationError(`corpusInterRaterAgreement: dimension '${d}' was requested but no records carry it`);
+	const targetJudges = opts.judges ? [...opts.judges] : [...judgesSeen].sort();
+	for (const j of targetJudges) if (!judgesSeen.has(j)) throw new ValidationError(`corpusInterRaterAgreement: judge '${j}' was requested but produced no records`);
+	if (targetJudges.length < 2) throw new ValidationError(`corpusInterRaterAgreement: need ≥2 judges, got ${targetJudges.length}`);
+	const perDimension = [];
+	const iccs = [];
+	const kappas = [];
+	for (const dim of targetDims) {
+		const byJudge = grid.get(dim);
+		const judgeItemCounts = {};
+		for (const j of targetJudges) judgeItemCounts[j] = byJudge.get(j)?.size ?? 0;
+		const emptyJudges = targetJudges.filter((j) => judgeItemCounts[j] === 0);
+		if (emptyJudges.length > 0) throw new ValidationError(`corpusInterRaterAgreement: dimension '${dim}' has no scores from judge(s) ${emptyJudges.join(", ")} (counts: ${JSON.stringify(judgeItemCounts)})`);
+		let commonItems = null;
+		for (const j of targetJudges) {
+			const ids = new Set(byJudge.get(j).keys());
+			if (commonItems === null) commonItems = ids;
+			else commonItems = new Set([...commonItems].filter((x) => ids.has(x)));
+		}
+		const sortedItems = [...commonItems ?? /* @__PURE__ */ new Set()].sort();
+		if (sortedItems.length < 2) throw new ValidationError(`corpusInterRaterAgreement: dimension '${dim}' has ${sortedItems.length} item(s) rated by all ${targetJudges.length} judges (need ≥2)`);
+		const agreement = continuousAgreement(sortedItems.map((itemId) => targetJudges.map((j) => byJudge.get(j).get(itemId))), opts);
+		perDimension.push({
+			...agreement,
+			dimension: dim,
+			itemIds: sortedItems,
+			judgeIds: [...targetJudges]
+		});
+		if (Number.isFinite(agreement.icc)) iccs.push(agreement.icc);
+		if (Number.isFinite(agreement.weightedKappa)) kappas.push(agreement.weightedKappa);
+	}
+	const mean = (xs) => xs.length === 0 ? NaN : xs.reduce((a, b) => a + b, 0) / xs.length;
+	return {
+		perDimension,
+		overallIcc: mean(iccs),
+		overallWeightedKappa: mean(kappas),
+		dimensions: targetDims,
+		judgeIds: targetJudges
+	};
+}
+/**
+* Convenience adapter for `JudgeScore[]` data keyed externally by item.
+*
+* Use when you have per-item arrays of `JudgeScore[]` (e.g. one
+* `ScenarioResult.judgeScores` per scenario) and want corpus-wide
+* agreement without manually flattening. `itemId` must be unique per
+* row of `itemsScores`.
+*/
+function corpusInterRaterAgreementFromJudgeScores(itemsScores, opts = {}) {
+	const records = [];
+	const seen = /* @__PURE__ */ new Set();
+	for (const { itemId, scores } of itemsScores) {
+		if (seen.has(itemId)) throw new ValidationError(`corpusInterRaterAgreementFromJudgeScores: duplicate itemId '${itemId}'`);
+		seen.add(itemId);
+		for (const s of scores) records.push({
+			itemId,
+			judgeName: s.judgeName,
+			dimension: s.dimension,
+			score: s.score
+		});
+	}
+	return corpusInterRaterAgreement(records, opts);
+}
+//#endregion
+//#region src/failure-taxonomy.ts
+const DEFAULT_RULES = [
+	{
+		id: "explicit-outcome",
+		match: ({ run }) => {
+			const fc = run.outcome?.failureClass;
+			if (fc && fc !== "unknown") return {
+				failureClass: fc,
+				reason: "outcome.failureClass set explicitly"
+			};
+			return null;
+		}
+	},
+	{
+		id: "knowledge-readiness-blocked",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "readiness_scored" && e.payload.passed === false);
+			return event ? {
+				failureClass: "knowledge_readiness_blocked",
+				reason: "knowledge readiness report blocked execution",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "bad-integration-manifest",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && (e.payload.kind === "integration_manifest_validated" && e.payload.valid === false || e.payload.kind === "integration_invoke_failed" && e.payload.code === "manifest_invalid"));
+			return event ? {
+				failureClass: "bad_integration_manifest",
+				reason: "integration manifest validation failed before launch",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "missing-integration-connection",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "integration_manifest_resolved" && hasResolutionStatus(e.payload, "missing_connection"));
+			return event ? {
+				failureClass: "missing_integration_connection",
+				reason: "required integration connection was missing",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "missing-integration-scope",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && (e.payload.kind === "integration_manifest_resolved" && hasMissingScopes(e.payload) || e.payload.kind === "integration_invoke_failed" && e.payload.code === "scope_denied"));
+			return event ? {
+				failureClass: "missing_integration_scope",
+				reason: "integration grant or connection lacks required scopes",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "integration-approval-required",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && (e.payload.kind === "integration_invoke" && e.payload.status === "approval_required" || e.payload.kind === "integration_invoke_failed" && e.payload.code === "approval_required" || e.payload.kind === "integration_approval_required"));
+			return event ? {
+				failureClass: "integration_approval_required",
+				reason: "integration write paused for user approval",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "integration-auth-expired",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && (e.payload.code === "auth_expired" || e.payload.code === "connection_not_active" || e.payload.code === "capability_expired" || e.payload.status === "expired"));
+			return event ? {
+				failureClass: "integration_auth_expired",
+				reason: "integration connection or capability expired",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "unsafe-integration-write-denied",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && (e.payload.code === "unsafe_write_denied" || e.payload.code === "policy_denied" || e.payload.code === "action_denied"));
+			return event ? {
+				failureClass: "unsafe_integration_write_denied",
+				reason: "integration write was denied by policy or capability scope",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "integration-provider-failure",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && ![
+				"scope_denied",
+				"approval_required",
+				"auth_expired",
+				"connection_not_active",
+				"capability_expired",
+				"unsafe_write_denied",
+				"policy_denied",
+				"action_denied",
+				"manifest_invalid"
+			].includes(String(e.payload.code)));
+			return event ? {
+				failureClass: "integration_provider_failure",
+				reason: "integration provider invocation failed",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "missing-credentials",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.category === "credential_or_secret");
+			return event ? {
+				failureClass: "missing_credentials",
+				reason: "required credential or secret was missing",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "bad-retrieval",
+		match: ({ run, spans }) => {
+			if (run.outcome?.pass !== false) return null;
+			const retrieval = spans.find((s) => s.kind === "retrieval" && (s.hits.length === 0 || s.hits.every((hit) => hit.score <= 0)));
+			return retrieval ? {
+				failureClass: "bad_retrieval",
+				reason: "retrieval returned no useful hits for a failed run",
+				triggerSpanId: retrieval.spanId
+			} : null;
+		}
+	},
+	{
+		id: "insufficient-evidence",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.reason === "insufficient_evidence");
+			return event ? {
+				failureClass: "insufficient_evidence",
+				reason: "task proceeded with insufficient supporting evidence",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "contradictory-evidence",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.reason === "contradictory_evidence");
+			return event ? {
+				failureClass: "contradictory_evidence",
+				reason: "supporting evidence contradicted itself",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "budget-breach",
+		match: ({ events }) => {
+			const breach = events.find((e) => e.kind === "budget_breach");
+			return breach ? {
+				failureClass: "budget_exceeded",
+				reason: `budget breached on ${breach.payload.dimension ?? "unknown dimension"}`,
+				triggerEventId: breach.eventId
+			} : null;
+		}
+	},
+	{
+		id: "policy-violation",
+		match: ({ events }) => {
+			const e = events.find((x) => x.kind === "policy_violation");
+			return e ? {
+				failureClass: "policy_violation",
+				reason: "policy_violation event emitted",
+				triggerEventId: e.eventId
+			} : null;
+		}
+	},
+	{
+		id: "sandbox-failure",
+		match: ({ spans }) => {
+			const s = spans.find((x) => x.kind === "sandbox" && typeof x.exitCode === "number" && x.exitCode !== 0);
+			if (!s) return null;
+			return {
+				failureClass: "sandbox_failure",
+				reason: `sandbox exited ${s.exitCode}`,
+				triggerSpanId: s.spanId
+			};
+		}
+	},
+	{
+		id: "timeout",
+		match: ({ run, events }) => {
+			if (run.status !== "aborted") return null;
+			const hasTimeout = events.some((e) => e.kind === "error" && String(e.payload.reason ?? "").toLowerCase().includes("timeout"));
+			const note = (run.outcome?.notes ?? "").toLowerCase();
+			if (hasTimeout || note.includes("timeout") || note.includes("deadline")) return {
+				failureClass: "timeout",
+				reason: "timeout signal observed"
+			};
+			return null;
+		}
+	},
+	{
+		id: "tool-recovery-failure",
+		match: ({ spans }) => {
+			const tools = spans.filter((s) => s.kind === "tool");
+			const byTool = /* @__PURE__ */ new Map();
+			for (const t of tools) {
+				const name = t.toolName;
+				const arr = byTool.get(name) ?? [];
+				arr.push(t);
+				byTool.set(name, arr);
+			}
+			for (const [name, arr] of byTool) {
+				const errs = arr.filter((s) => s.status === "error");
+				if (errs.length >= 3 && errs.length === arr.length) return {
+					failureClass: "tool_recovery_failure",
+					reason: `${errs.length} consecutive errors on tool "${name}"`,
+					triggerSpanId: errs[errs.length - 1].spanId
+				};
+			}
+			return null;
+		}
+	},
+	{
+		id: "tool-selection-error",
+		match: ({ run, spans }) => {
+			if (run.outcome?.pass !== false) return null;
+			const hasToolsAvailable = spans.some((s) => s.kind === "agent" && s.attributes?.toolsAvailable !== void 0 && s.attributes?.toolsAvailable > 0);
+			const tools = spans.filter((s) => s.kind === "tool");
+			if (hasToolsAvailable && tools.length === 0) return {
+				failureClass: "tool_selection_error",
+				reason: "tools were available but none were called"
+			};
+			return null;
+		}
+	},
+	{
+		id: "format-drift",
+		match: ({ spans }) => {
+			const judge = spans.find((s) => s.kind === "judge" && s.dimension === "format" && s.score < .5);
+			return judge ? {
+				failureClass: "format_drift",
+				reason: "format judge scored below 0.5",
+				triggerSpanId: judge.spanId
+			} : null;
+		}
+	}
+];
+function hasResolutionStatus(payload, status) {
+	if (status === "missing_connection" && stringArray(payload.missingConnections).length > 0) return true;
+	return resolutionItems(payload).some((item) => item.status === status);
+}
+function hasMissingScopes(payload) {
+	if (stringArray(payload.missingScopes).length > 0) return true;
+	return resolutionItems(payload).some((item) => Array.isArray(item.missingScopes) && item.missingScopes.length > 0);
+}
+function resolutionItems(payload) {
+	return [
+		...records(payload.missing),
+		...records(payload.optionalMissing),
+		...records(payload.ready)
+	];
+}
+function records(value) {
+	if (!Array.isArray(value)) return [];
+	return value.filter((item) => Boolean(item) && typeof item === "object" && !Array.isArray(item));
+}
+function stringArray(value) {
+	return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
+}
+/** Classify the failure mode of a run using an ordered rule list. */
+function classifyFailure(ctx, rules = DEFAULT_RULES) {
+	if (ctx.run.outcome?.pass !== false && ctx.run.status === "completed") return {
+		failureClass: "success",
+		reason: "run completed with pass=true (or no explicit fail)"
+	};
+	for (const rule of rules) {
+		const hit = rule.match(ctx);
+		if (hit) return hit;
+	}
+	return {
+		failureClass: "unknown",
+		reason: "no rule matched; run failed for unclassified reason"
+	};
+}
+//#endregion
+//#region src/pipelines/budget-breach.ts
+async function budgetBreachView(store, options = {}) {
+	const runs = await store.listRuns({
+		scenarioId: options.scenarioId,
+		variantId: options.variantId
+	});
+	const findings = [];
+	const byDimension = {};
+	const byScenario = {};
+	const byVariant = {};
+	for (const run of runs) {
+		const entries = await store.budget(run.runId);
+		for (const e of entries) {
+			if (!e.breached) continue;
+			const excessRatio = e.limit > 0 ? e.consumed / e.limit : Infinity;
+			findings.push({
+				runId: run.runId,
+				scenarioId: run.scenarioId,
+				variantId: run.variantId,
+				dimension: e.dimension,
+				limit: e.limit,
+				consumed: e.consumed,
+				excessRatio,
+				timestamp: e.timestamp
+			});
+			byDimension[e.dimension] = (byDimension[e.dimension] ?? 0) + 1;
+			byScenario[run.scenarioId] = (byScenario[run.scenarioId] ?? 0) + 1;
+			if (run.variantId) byVariant[run.variantId] = (byVariant[run.variantId] ?? 0) + 1;
+		}
+	}
+	const breachedRuns = new Set(findings.map((f) => f.runId));
+	return {
+		findings,
+		byDimension,
+		byScenario,
+		byVariant,
+		totalRuns: runs.length,
+		breachedRunRatio: runs.length > 0 ? breachedRuns.size / runs.length : 0
+	};
+}
+//#endregion
+//#region src/pipelines/failure-cluster.ts
+/**
+* FailureClusterView — groups failed runs by (failureClass, triggerTool,
+* argHash-prefix) so weekly reviews can prioritize the top-N clusters.
+*
+* Each cluster includes: N runs, scenarios affected, representative
+* error message, a proposed mitigation hint (rule → action table).
+*/
+async function failureClusterView(store, options = {}) {
+	const rules = options.rules ?? DEFAULT_RULES;
+	const minSize = options.minClusterSize ?? 1;
+	const runs = await store.listRuns();
+	const clusters = /* @__PURE__ */ new Map();
+	let totalFailures = 0;
+	for (const run of runs) {
+		if (run.status === "completed" && run.outcome?.pass !== false) continue;
+		totalFailures++;
+		const spans = await store.spans({ runId: run.runId });
+		const cls = classifyFailure({
+			run,
+			spans,
+			events: await store.events({ runId: run.runId })
+		}, rules);
+		let toolName;
+		let argPrefix;
+		let dimension;
+		if (cls.triggerSpanId) {
+			const trig = spans.find((s) => s.spanId === cls.triggerSpanId);
+			if (trig?.kind === "tool") {
+				toolName = trig.toolName;
+				if (hasCapturedToolArgs(trig)) argPrefix = argHash(trig.args).slice(0, 16);
+			} else if (trig?.kind === "judge") dimension = trig.dimension;
+		}
+		if (!toolName) {
+			const errored = (await toolSpans(store, run.runId)).filter((t) => t.status === "error").pop();
+			if (errored) {
+				toolName = errored.toolName;
+				if (hasCapturedToolArgs(errored)) argPrefix = argHash(errored.args).slice(0, 16);
+			}
+		}
+		if (!dimension) {
+			const judge = spans.find((s) => s.kind === "judge" && typeof s.dimension === "string");
+			if (judge?.kind === "judge") dimension = judge.dimension;
+		}
+		const key = `${cls.failureClass}|${toolName ?? ""}|${argPrefix ?? ""}|${dimension ?? ""}`;
+		let cluster = clusters.get(key);
+		if (!cluster) {
+			cluster = {
+				failureClass: cls.failureClass,
+				toolName,
+				argPrefix,
+				dimension,
+				runCount: 0,
+				scenarioIds: [],
+				exampleRunId: run.runId,
+				exampleError: firstErrorMessage(spans) ?? cls.reason
+			};
+			clusters.set(key, cluster);
+		}
+		cluster.runCount++;
+		if (!cluster.scenarioIds.includes(run.scenarioId)) cluster.scenarioIds.push(run.scenarioId);
+	}
+	return {
+		clusters: [...clusters.values()].filter((c) => c.runCount >= minSize).sort((a, b) => b.runCount - a.runCount),
+		totalFailures,
+		totalRuns: runs.length
+	};
+}
+function firstErrorMessage(spans) {
+	return spans.find((s) => s.status === "error")?.error;
+}
+//#endregion
+//#region src/pipelines/judge-agreement.ts
+/**
+* JudgeAgreementView — pairwise agreement between judges across the
+* corpus, grouped by dimension.
+*
+* Output drives two workflows:
+*   - Judge robustness audit: "does Claude agree with GPT at κ ≥ 0.6?"
+*   - Calibration tracking: κ vs golden human labels over time (by
+*     providing a `humanGoldenJudgeId`).
+*/
+async function judgeAgreementView(store) {
+	const all = (await store.spans({ kind: "judge" })).filter((s) => s.kind === "judge");
+	if (all.length === 0) return {
+		pairs: [],
+		dimensions: [],
+		judgeIds: []
+	};
+	const byDimension = /* @__PURE__ */ new Map();
+	for (const s of all) {
+		const arr = byDimension.get(s.dimension) ?? [];
+		arr.push(s);
+		byDimension.set(s.dimension, arr);
+	}
+	const judgeIds = [...new Set(all.map((s) => s.judgeId))].sort();
+	const pairs = [];
+	for (const [dim, spans] of byDimension) {
+		const byJudge = /* @__PURE__ */ new Map();
+		for (const s of spans) {
+			const m = byJudge.get(s.judgeId) ?? /* @__PURE__ */ new Map();
+			m.set(s.targetSpanId, s.score);
+			byJudge.set(s.judgeId, m);
+		}
+		const judgesHere = [...byJudge.keys()];
+		for (let i = 0; i < judgesHere.length; i++) for (let j = i + 1; j < judgesHere.length; j++) {
+			const judgeI = judgesHere[i];
+			const judgeJ = judgesHere[j];
+			const a = byJudge.get(judgeI);
+			const b = byJudge.get(judgeJ);
+			const common = [];
+			for (const [target, scoreA] of a) {
+				const scoreB = b.get(target);
+				if (scoreB !== void 0) common.push([scoreA, scoreB]);
+			}
+			if (common.length < 2) continue;
+			const judgeScores = common.map(([scoreA, scoreB]) => [{
+				judgeName: judgeI,
+				dimension: dim,
+				score: scoreA,
+				reasoning: ""
+			}, {
+				judgeName: judgeJ,
+				dimension: dim,
+				score: scoreB,
+				reasoning: ""
+			}]);
+			const k = interRaterReliability(judgeScores[0].map((_, k2) => judgeScores.map((pair) => pair[k2])));
+			pairs.push({
+				judgeA: judgeI,
+				judgeB: judgeJ,
+				dimension: dim,
+				commonItems: common.length,
+				pearson: pearsonR(common.map((c) => c[0]), common.map((c) => c[1])),
+				krippendorff: k
+			});
+		}
+	}
+	return {
+		pairs: pairs.sort((a, b) => b.commonItems - a.commonItems),
+		dimensions: [...byDimension.keys()].sort(),
+		judgeIds
+	};
+}
+//#endregion
+//#region src/tool-use-metrics.ts
+/**
+* Tool-use metrics — derived purely from trace data.
+*
+* No scoring assumptions: consumers supply optional ground-truth tool
+* selections per turn + optional "information used downstream" signals.
+* Without those, we still compute descriptive metrics (error rate,
+* retry rate, duplicate-call rate) that are useful on their own.
+*/
+async function computeToolUseMetrics(store, runId, options = {}) {
+	const tools = await toolSpans(store, runId);
+	if (tools.length === 0) return {
+		runId,
+		totalCalls: 0,
+		callsWithCapturedArgs: 0,
+		byTool: {},
+		errorRate: 0,
+		duplicateRate: 0,
+		retryRate: 0
+	};
+	const byTool = {};
+	let totalErrors = 0;
+	let totalDuplicates = 0;
+	let callsWithCapturedArgs = 0;
+	const sortedTools = [...tools].sort((a, b) => a.startedAt - b.startedAt);
+	const seenSignatures = /* @__PURE__ */ new Set();
+	for (const t of sortedTools) {
+		byTool[t.toolName] ??= {
+			calls: 0,
+			callsWithCapturedArgs: 0,
+			errors: 0,
+			avgLatencyMs: 0,
+			duplicates: 0
+		};
+		const stat = byTool[t.toolName];
+		stat.calls += 1;
+		if (t.status === "error") {
+			stat.errors += 1;
+			totalErrors += 1;
+		}
+		if (typeof t.latencyMs === "number") stat.avgLatencyMs += t.latencyMs;
+		if (hasCapturedToolArgs(t)) {
+			callsWithCapturedArgs += 1;
+			stat.callsWithCapturedArgs += 1;
+			const sig = `${t.toolName}|${argHash(t.args)}`;
+			if (seenSignatures.has(sig)) {
+				stat.duplicates += 1;
+				totalDuplicates += 1;
+			}
+			seenSignatures.add(sig);
+		}
+	}
+	for (const stat of Object.values(byTool)) stat.avgLatencyMs = stat.calls > 0 ? stat.avgLatencyMs / stat.calls : 0;
+	let retryOpportunities = 0;
+	let retriesFollowed = 0;
+	for (const [, arr] of groupBy(sortedTools, (t) => t.toolName)) for (let i = 0; i < arr.length; i++) {
+		if (arr[i].status !== "error") continue;
+		retryOpportunities += 1;
+		if (arr[i + 1]) retriesFollowed += 1;
+	}
+	const retryRate = retryOpportunities > 0 ? retriesFollowed / retryOpportunities : 0;
+	let selectionAccuracy;
+	if (options.selectionLabels) {
+		const labeled = sortedTools.filter((t) => t.spanId in options.selectionLabels);
+		if (labeled.length > 0) selectionAccuracy = labeled.filter((t) => options.selectionLabels[t.spanId]).length / labeled.length;
+	}
+	return {
+		runId,
+		totalCalls: sortedTools.length,
+		callsWithCapturedArgs,
+		byTool,
+		errorRate: totalErrors / sortedTools.length,
+		duplicateRate: callsWithCapturedArgs > 0 ? totalDuplicates / callsWithCapturedArgs : 0,
+		retryRate,
+		selectionAccuracy
+	};
+}
+//#endregion
+//#region src/pipelines/tool-waste.ts
+/**
+* ToolWasteView — fraction of tool calls whose results weren't used
+* downstream. Without a "used" signal we fall back to structural
+* proxies: error calls, duplicate calls, and tool calls followed by
+* zero subsequent LLM spans are all considered waste.
+*
+* Consumers can pass a `usageOracle` that inspects a tool span and
+* returns true iff the tool's result appears in a later LLM message,
+* artifact, or state mutation — that's the canonical definition; the
+* default heuristic is a reasonable fallback.
+*/
+async function toolWasteView(store, options = {}) {
+	const runs = options.runId ? [options.runId] : (await store.listRuns()).map((r) => r.runId);
+	const byRun = [];
+	let totalCalls = 0;
+	let totalWasted = 0;
+	for (const runId of runs) {
+		const tools = await toolSpans(store, runId);
+		if (tools.length === 0) {
+			byRun.push({
+				runId,
+				wastedCalls: 0,
+				totalCalls: 0,
+				wasteRate: 0
+			});
+			continue;
+		}
+		const sortedLlm = [...await llmSpans(store, runId)].sort((a, b) => a.startedAt - b.startedAt);
+		const startTimes = sortedLlm.map((l) => l.startedAt);
+		const suffixText = buildSuffixText(sortedLlm);
+		let wasted = 0;
+		for (const t of tools) {
+			if (t.status === "error") {
+				wasted++;
+				continue;
+			}
+			const cutoff = upperBound(startTimes, t.startedAt);
+			if (options.usageOracle) {
+				if (!options.usageOracle(t, { llm: sortedLlm.slice(cutoff) })) wasted++;
+			} else {
+				const resultStr = stringify(t.result);
+				if (resultStr === "") continue;
+				if (!(suffixText[cutoff] ?? "").includes(resultStr.slice(0, 120))) wasted++;
+			}
+		}
+		const wasteRate = wasted / tools.length;
+		byRun.push({
+			runId,
+			wastedCalls: wasted,
+			totalCalls: tools.length,
+			wasteRate
+		});
+		totalCalls += tools.length;
+		totalWasted += wasted;
+	}
+	return {
+		byRun,
+		overallWasteRate: totalCalls > 0 ? totalWasted / totalCalls : 0
+	};
+}
+/**
+* Build per-position suffix haystacks: result[i] is the concatenation of every
+* string message content in spans[i..end]. Built back-to-front so each entry
+* reuses the next one — O(total message text) rather than O(spans²).
+*/
+function buildSuffixText(spans) {
+	const result = new Array(spans.length + 1);
+	result[spans.length] = "";
+	for (let i = spans.length - 1; i >= 0; i--) result[i] = `${spans[i].messages.map((m) => typeof m.content === "string" ? m.content : "").join("\n")}\n${result[i + 1]}`;
+	return result;
+}
+/** Index of the first element strictly greater than `target` in a sorted array. */
+function upperBound(sorted, target) {
+	let lo = 0;
+	let hi = sorted.length;
+	while (lo < hi) {
+		const mid = lo + hi >>> 1;
+		if (sorted[mid] <= target) lo = mid + 1;
+		else hi = mid;
+	}
+	return lo;
+}
+function stringify(v) {
+	if (v === null || v === void 0) return "";
+	if (typeof v === "string") return v;
+	try {
+		return JSON.stringify(v);
+	} catch {
+		return String(v);
+	}
+}
+//#endregion
+export { budgetBreachView as a, corpusInterRaterAgreementFromJudgeScores as c, failureClusterView as i, interRaterReliability as l, computeToolUseMetrics as n, classifyFailure as o, judgeAgreementView as r, corpusInterRaterAgreement as s, toolWasteView as t };
+
+//# sourceMappingURL=tool-waste-BDdBZG1F.js.map