@tangle-network/agent-eval 0.144.11 → 0.144.13

package/dist/tool-use-metrics-DEGMKycK.js

@@ -1,370 +0,0 @@
-import { i as hasCapturedToolArgs, l as toolSpans, n as argHash, r as groupBy } from "./query-Di7eEQ79.js";
-//#region src/failure-taxonomy.ts
-const DEFAULT_RULES = [
-	{
-		id: "explicit-outcome",
-		match: ({ run }) => {
-			const fc = run.outcome?.failureClass;
-			if (fc && fc !== "unknown") return {
-				failureClass: fc,
-				reason: "outcome.failureClass set explicitly"
-			};
-			return null;
-		}
-	},
-	{
-		id: "knowledge-readiness-blocked",
-		match: ({ events }) => {
-			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "readiness_scored" && e.payload.passed === false);
-			return event ? {
-				failureClass: "knowledge_readiness_blocked",
-				reason: "knowledge readiness report blocked execution",
-				triggerEventId: event.eventId
-			} : null;
-		}
-	},
-	{
-		id: "bad-integration-manifest",
-		match: ({ events }) => {
-			const event = events.find((e) => e.kind === "custom" && (e.payload.kind === "integration_manifest_validated" && e.payload.valid === false || e.payload.kind === "integration_invoke_failed" && e.payload.code === "manifest_invalid"));
-			return event ? {
-				failureClass: "bad_integration_manifest",
-				reason: "integration manifest validation failed before launch",
-				triggerEventId: event.eventId
-			} : null;
-		}
-	},
-	{
-		id: "missing-integration-connection",
-		match: ({ events }) => {
-			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "integration_manifest_resolved" && hasResolutionStatus(e.payload, "missing_connection"));
-			return event ? {
-				failureClass: "missing_integration_connection",
-				reason: "required integration connection was missing",
-				triggerEventId: event.eventId
-			} : null;
-		}
-	},
-	{
-		id: "missing-integration-scope",
-		match: ({ events }) => {
-			const event = events.find((e) => e.kind === "custom" && (e.payload.kind === "integration_manifest_resolved" && hasMissingScopes(e.payload) || e.payload.kind === "integration_invoke_failed" && e.payload.code === "scope_denied"));
-			return event ? {
-				failureClass: "missing_integration_scope",
-				reason: "integration grant or connection lacks required scopes",
-				triggerEventId: event.eventId
-			} : null;
-		}
-	},
-	{
-		id: "integration-approval-required",
-		match: ({ events }) => {
-			const event = events.find((e) => e.kind === "custom" && (e.payload.kind === "integration_invoke" && e.payload.status === "approval_required" || e.payload.kind === "integration_invoke_failed" && e.payload.code === "approval_required" || e.payload.kind === "integration_approval_required"));
-			return event ? {
-				failureClass: "integration_approval_required",
-				reason: "integration write paused for user approval",
-				triggerEventId: event.eventId
-			} : null;
-		}
-	},
-	{
-		id: "integration-auth-expired",
-		match: ({ events }) => {
-			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && (e.payload.code === "auth_expired" || e.payload.code === "connection_not_active" || e.payload.code === "capability_expired" || e.payload.status === "expired"));
-			return event ? {
-				failureClass: "integration_auth_expired",
-				reason: "integration connection or capability expired",
-				triggerEventId: event.eventId
-			} : null;
-		}
-	},
-	{
-		id: "unsafe-integration-write-denied",
-		match: ({ events }) => {
-			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && (e.payload.code === "unsafe_write_denied" || e.payload.code === "policy_denied" || e.payload.code === "action_denied"));
-			return event ? {
-				failureClass: "unsafe_integration_write_denied",
-				reason: "integration write was denied by policy or capability scope",
-				triggerEventId: event.eventId
-			} : null;
-		}
-	},
-	{
-		id: "integration-provider-failure",
-		match: ({ events }) => {
-			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && ![
-				"scope_denied",
-				"approval_required",
-				"auth_expired",
-				"connection_not_active",
-				"capability_expired",
-				"unsafe_write_denied",
-				"policy_denied",
-				"action_denied",
-				"manifest_invalid"
-			].includes(String(e.payload.code)));
-			return event ? {
-				failureClass: "integration_provider_failure",
-				reason: "integration provider invocation failed",
-				triggerEventId: event.eventId
-			} : null;
-		}
-	},
-	{
-		id: "missing-credentials",
-		match: ({ events }) => {
-			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.category === "credential_or_secret");
-			return event ? {
-				failureClass: "missing_credentials",
-				reason: "required credential or secret was missing",
-				triggerEventId: event.eventId
-			} : null;
-		}
-	},
-	{
-		id: "bad-retrieval",
-		match: ({ run, spans }) => {
-			if (run.outcome?.pass !== false) return null;
-			const retrieval = spans.find((s) => s.kind === "retrieval" && (s.hits.length === 0 || s.hits.every((hit) => hit.score <= 0)));
-			return retrieval ? {
-				failureClass: "bad_retrieval",
-				reason: "retrieval returned no useful hits for a failed run",
-				triggerSpanId: retrieval.spanId
-			} : null;
-		}
-	},
-	{
-		id: "insufficient-evidence",
-		match: ({ events }) => {
-			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.reason === "insufficient_evidence");
-			return event ? {
-				failureClass: "insufficient_evidence",
-				reason: "task proceeded with insufficient supporting evidence",
-				triggerEventId: event.eventId
-			} : null;
-		}
-	},
-	{
-		id: "contradictory-evidence",
-		match: ({ events }) => {
-			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.reason === "contradictory_evidence");
-			return event ? {
-				failureClass: "contradictory_evidence",
-				reason: "supporting evidence contradicted itself",
-				triggerEventId: event.eventId
-			} : null;
-		}
-	},
-	{
-		id: "budget-breach",
-		match: ({ events }) => {
-			const breach = events.find((e) => e.kind === "budget_breach");
-			return breach ? {
-				failureClass: "budget_exceeded",
-				reason: `budget breached on ${breach.payload.dimension ?? "unknown dimension"}`,
-				triggerEventId: breach.eventId
-			} : null;
-		}
-	},
-	{
-		id: "policy-violation",
-		match: ({ events }) => {
-			const e = events.find((x) => x.kind === "policy_violation");
-			return e ? {
-				failureClass: "policy_violation",
-				reason: "policy_violation event emitted",
-				triggerEventId: e.eventId
-			} : null;
-		}
-	},
-	{
-		id: "sandbox-failure",
-		match: ({ spans }) => {
-			const s = spans.find((x) => x.kind === "sandbox" && typeof x.exitCode === "number" && x.exitCode !== 0);
-			if (!s) return null;
-			return {
-				failureClass: "sandbox_failure",
-				reason: `sandbox exited ${s.exitCode}`,
-				triggerSpanId: s.spanId
-			};
-		}
-	},
-	{
-		id: "timeout",
-		match: ({ run, events }) => {
-			if (run.status !== "aborted") return null;
-			const hasTimeout = events.some((e) => e.kind === "error" && String(e.payload.reason ?? "").toLowerCase().includes("timeout"));
-			const note = (run.outcome?.notes ?? "").toLowerCase();
-			if (hasTimeout || note.includes("timeout") || note.includes("deadline")) return {
-				failureClass: "timeout",
-				reason: "timeout signal observed"
-			};
-			return null;
-		}
-	},
-	{
-		id: "tool-recovery-failure",
-		match: ({ spans }) => {
-			const tools = spans.filter((s) => s.kind === "tool");
-			const byTool = /* @__PURE__ */ new Map();
-			for (const t of tools) {
-				const name = t.toolName;
-				const arr = byTool.get(name) ?? [];
-				arr.push(t);
-				byTool.set(name, arr);
-			}
-			for (const [name, arr] of byTool) {
-				const errs = arr.filter((s) => s.status === "error");
-				if (errs.length >= 3 && errs.length === arr.length) return {
-					failureClass: "tool_recovery_failure",
-					reason: `${errs.length} consecutive errors on tool "${name}"`,
-					triggerSpanId: errs[errs.length - 1].spanId
-				};
-			}
-			return null;
-		}
-	},
-	{
-		id: "tool-selection-error",
-		match: ({ run, spans }) => {
-			if (run.outcome?.pass !== false) return null;
-			const hasToolsAvailable = spans.some((s) => s.kind === "agent" && s.attributes?.toolsAvailable !== void 0 && s.attributes?.toolsAvailable > 0);
-			const tools = spans.filter((s) => s.kind === "tool");
-			if (hasToolsAvailable && tools.length === 0) return {
-				failureClass: "tool_selection_error",
-				reason: "tools were available but none were called"
-			};
-			return null;
-		}
-	},
-	{
-		id: "format-drift",
-		match: ({ spans }) => {
-			const judge = spans.find((s) => s.kind === "judge" && s.dimension === "format" && s.score < .5);
-			return judge ? {
-				failureClass: "format_drift",
-				reason: "format judge scored below 0.5",
-				triggerSpanId: judge.spanId
-			} : null;
-		}
-	}
-];
-function hasResolutionStatus(payload, status) {
-	if (status === "missing_connection" && stringArray(payload.missingConnections).length > 0) return true;
-	return resolutionItems(payload).some((item) => item.status === status);
-}
-function hasMissingScopes(payload) {
-	if (stringArray(payload.missingScopes).length > 0) return true;
-	return resolutionItems(payload).some((item) => Array.isArray(item.missingScopes) && item.missingScopes.length > 0);
-}
-function resolutionItems(payload) {
-	return [
-		...records(payload.missing),
-		...records(payload.optionalMissing),
-		...records(payload.ready)
-	];
-}
-function records(value) {
-	if (!Array.isArray(value)) return [];
-	return value.filter((item) => Boolean(item) && typeof item === "object" && !Array.isArray(item));
-}
-function stringArray(value) {
-	return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
-}
-/** Classify the failure mode of a run using an ordered rule list. */
-function classifyFailure(ctx, rules = DEFAULT_RULES) {
-	if (ctx.run.outcome?.pass !== false && ctx.run.status === "completed") return {
-		failureClass: "success",
-		reason: "run completed with pass=true (or no explicit fail)"
-	};
-	for (const rule of rules) {
-		const hit = rule.match(ctx);
-		if (hit) return hit;
-	}
-	return {
-		failureClass: "unknown",
-		reason: "no rule matched; run failed for unclassified reason"
-	};
-}
-//#endregion
-//#region src/tool-use-metrics.ts
-/**
-* Tool-use metrics — derived purely from trace data.
-*
-* No scoring assumptions: consumers supply optional ground-truth tool
-* selections per turn + optional "information used downstream" signals.
-* Without those, we still compute descriptive metrics (error rate,
-* retry rate, duplicate-call rate) that are useful on their own.
-*/
-async function computeToolUseMetrics(store, runId, options = {}) {
-	const tools = await toolSpans(store, runId);
-	if (tools.length === 0) return {
-		runId,
-		totalCalls: 0,
-		callsWithCapturedArgs: 0,
-		byTool: {},
-		errorRate: 0,
-		duplicateRate: 0,
-		retryRate: 0
-	};
-	const byTool = {};
-	let totalErrors = 0;
-	let totalDuplicates = 0;
-	let callsWithCapturedArgs = 0;
-	const sortedTools = [...tools].sort((a, b) => a.startedAt - b.startedAt);
-	const seenSignatures = /* @__PURE__ */ new Set();
-	for (const t of sortedTools) {
-		byTool[t.toolName] ??= {
-			calls: 0,
-			callsWithCapturedArgs: 0,
-			errors: 0,
-			avgLatencyMs: 0,
-			duplicates: 0
-		};
-		const stat = byTool[t.toolName];
-		stat.calls += 1;
-		if (t.status === "error") {
-			stat.errors += 1;
-			totalErrors += 1;
-		}
-		if (typeof t.latencyMs === "number") stat.avgLatencyMs += t.latencyMs;
-		if (hasCapturedToolArgs(t)) {
-			callsWithCapturedArgs += 1;
-			stat.callsWithCapturedArgs += 1;
-			const sig = `${t.toolName}|${argHash(t.args)}`;
-			if (seenSignatures.has(sig)) {
-				stat.duplicates += 1;
-				totalDuplicates += 1;
-			}
-			seenSignatures.add(sig);
-		}
-	}
-	for (const stat of Object.values(byTool)) stat.avgLatencyMs = stat.calls > 0 ? stat.avgLatencyMs / stat.calls : 0;
-	let retryOpportunities = 0;
-	let retriesFollowed = 0;
-	for (const [, arr] of groupBy(sortedTools, (t) => t.toolName)) for (let i = 0; i < arr.length; i++) {
-		if (arr[i].status !== "error") continue;
-		retryOpportunities += 1;
-		if (arr[i + 1]) retriesFollowed += 1;
-	}
-	const retryRate = retryOpportunities > 0 ? retriesFollowed / retryOpportunities : 0;
-	let selectionAccuracy;
-	if (options.selectionLabels) {
-		const labeled = sortedTools.filter((t) => t.spanId in options.selectionLabels);
-		if (labeled.length > 0) selectionAccuracy = labeled.filter((t) => options.selectionLabels[t.spanId]).length / labeled.length;
-	}
-	return {
-		runId,
-		totalCalls: sortedTools.length,
-		callsWithCapturedArgs,
-		byTool,
-		errorRate: totalErrors / sortedTools.length,
-		duplicateRate: callsWithCapturedArgs > 0 ? totalDuplicates / callsWithCapturedArgs : 0,
-		retryRate,
-		selectionAccuracy
-	};
-}
-//#endregion
-export { DEFAULT_RULES as n, classifyFailure as r, computeToolUseMetrics as t };
-
-//# sourceMappingURL=tool-use-metrics-DEGMKycK.js.map

package/dist/tool-use-metrics-DEGMKycK.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"tool-use-metrics-DEGMKycK.js","names":[],"sources":["../src/failure-taxonomy.ts","../src/tool-use-metrics.ts"],"sourcesContent":["/**\n * Failure taxonomy — canonical classes + a default classifier.\n *\n * Every failed run should end up in a named class. The classifier here\n * is rule-based (fast, deterministic); an LLM fallback can be added by\n * the consumer for novel cases and trained into the rule base over time.\n *\n * Consumers call `classifyFailure(run, spans, events)` and persist the\n * returned class as `Run.outcome.failureClass`.\n */\n\nimport type { FailureClass, Run, Span, TraceEvent } from './trace/schema'\nimport { FAILURE_CLASSES } from './trace/schema'\n\nexport { FAILURE_CLASSES, type FailureClass }\n\nexport interface FailureContext {\n  run: Run\n  spans: Span[]\n  events: TraceEvent[]\n}\n\nexport interface FailureClassification {\n  failureClass: FailureClass\n  reason: string\n  triggerSpanId?: string\n  triggerEventId?: string\n}\n\n/** Ordered rules — first match wins. */\nexport interface FailureRule {\n  id: string\n  match: (ctx: FailureContext) => {\n    failureClass: FailureClass\n    reason: string\n    triggerSpanId?: string\n    triggerEventId?: string\n  } | null\n}\n\nexport const DEFAULT_RULES: FailureRule[] = [\n  // Outcome already named? Respect it.\n  {\n    id: 'explicit-outcome',\n    match: ({ run }) => {\n      const fc = run.outcome?.failureClass\n      if (fc && fc !== 'unknown')\n        return { failureClass: fc, reason: 'outcome.failureClass set explicitly' }\n      return null\n    },\n  },\n  {\n    id: 'knowledge-readiness-blocked',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'readiness_scored' &&\n          e.payload.passed === false,\n      )\n      return event\n        ? {\n            failureClass: 'knowledge_readiness_blocked',\n            reason: 'knowledge readiness report blocked execution',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'bad-integration-manifest',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          ((e.payload.kind === 'integration_manifest_validated' && e.payload.valid === false) ||\n            (e.payload.kind === 'integration_invoke_failed' &&\n              e.payload.code === 'manifest_invalid')),\n      )\n      return event\n        ? {\n            failureClass: 'bad_integration_manifest',\n            reason: 'integration manifest validation failed before launch',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'missing-integration-connection',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'integration_manifest_resolved' &&\n          hasResolutionStatus(e.payload, 'missing_connection'),\n      )\n      return event\n        ? {\n            failureClass: 'missing_integration_connection',\n            reason: 'required integration connection was missing',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'missing-integration-scope',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          ((e.payload.kind === 'integration_manifest_resolved' && hasMissingScopes(e.payload)) ||\n            (e.payload.kind === 'integration_invoke_failed' && e.payload.code === 'scope_denied')),\n      )\n      return event\n        ? {\n            failureClass: 'missing_integration_scope',\n            reason: 'integration grant or connection lacks required scopes',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'integration-approval-required',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          ((e.payload.kind === 'integration_invoke' && e.payload.status === 'approval_required') ||\n            (e.payload.kind === 'integration_invoke_failed' &&\n              e.payload.code === 'approval_required') ||\n            e.payload.kind === 'integration_approval_required'),\n      )\n      return event\n        ? {\n            failureClass: 'integration_approval_required',\n            reason: 'integration write paused for user approval',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'integration-auth-expired',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'integration_invoke_failed' &&\n          (e.payload.code === 'auth_expired' ||\n            e.payload.code === 'connection_not_active' ||\n            e.payload.code === 'capability_expired' ||\n            e.payload.status === 'expired'),\n      )\n      return event\n        ? {\n            failureClass: 'integration_auth_expired',\n            reason: 'integration connection or capability expired',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'unsafe-integration-write-denied',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'integration_invoke_failed' &&\n          (e.payload.code === 'unsafe_write_denied' ||\n            e.payload.code === 'policy_denied' ||\n            e.payload.code === 'action_denied'),\n      )\n      return event\n        ? {\n            failureClass: 'unsafe_integration_write_denied',\n            reason: 'integration write was denied by policy or capability scope',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'integration-provider-failure',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'integration_invoke_failed' &&\n          ![\n            'scope_denied',\n            'approval_required',\n            'auth_expired',\n            'connection_not_active',\n            'capability_expired',\n            'unsafe_write_denied',\n            'policy_denied',\n            'action_denied',\n            'manifest_invalid',\n          ].includes(String(e.payload.code)),\n      )\n      return event\n        ? {\n            failureClass: 'integration_provider_failure',\n            reason: 'integration provider invocation failed',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'missing-credentials',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'knowledge_gap' &&\n          e.payload.category === 'credential_or_secret',\n      )\n      return event\n        ? {\n            failureClass: 'missing_credentials',\n            reason: 'required credential or secret was missing',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'bad-retrieval',\n    match: ({ run, spans }) => {\n      if (run.outcome?.pass !== false) return null\n      const retrieval = spans.find(\n        (s) =>\n          s.kind === 'retrieval' && (s.hits.length === 0 || s.hits.every((hit) => hit.score <= 0)),\n      )\n      return retrieval\n        ? {\n            failureClass: 'bad_retrieval',\n            reason: 'retrieval returned no useful hits for a failed run',\n            triggerSpanId: retrieval.spanId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'insufficient-evidence',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'knowledge_gap' &&\n          e.payload.reason === 'insufficient_evidence',\n      )\n      return event\n        ? {\n            failureClass: 'insufficient_evidence',\n            reason: 'task proceeded with insufficient supporting evidence',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'contradictory-evidence',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'knowledge_gap' &&\n          e.payload.reason === 'contradictory_evidence',\n      )\n      return event\n        ? {\n            failureClass: 'contradictory_evidence',\n            reason: 'supporting evidence contradicted itself',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  // Budget breach events\n  {\n    id: 'budget-breach',\n    match: ({ events }) => {\n      const breach = events.find((e) => e.kind === 'budget_breach')\n      return breach\n        ? {\n            failureClass: 'budget_exceeded',\n            reason: `budget breached on ${breach.payload.dimension ?? 'unknown dimension'}`,\n            triggerEventId: breach.eventId,\n          }\n        : null\n    },\n  },\n  // Policy violations\n  {\n    id: 'policy-violation',\n    match: ({ events }) => {\n      const e = events.find((x) => x.kind === 'policy_violation')\n      return e\n        ? {\n            failureClass: 'policy_violation',\n            reason: 'policy_violation event emitted',\n            triggerEventId: e.eventId,\n          }\n        : null\n    },\n  },\n  // Sandbox non-zero exit code\n  {\n    id: 'sandbox-failure',\n    match: ({ spans }) => {\n      const s = spans.find(\n        (x) => x.kind === 'sandbox' && typeof x.exitCode === 'number' && x.exitCode !== 0,\n      )\n      if (!s) return null\n      return {\n        failureClass: 'sandbox_failure',\n        reason: `sandbox exited ${(s as Extract<Span, { kind: 'sandbox' }>).exitCode}`,\n        triggerSpanId: s.spanId,\n      }\n    },\n  },\n  // Timeout: run aborted by external signal\n  {\n    id: 'timeout',\n    match: ({ run, events }) => {\n      if (run.status !== 'aborted') return null\n      const hasTimeout = events.some(\n        (e) =>\n          e.kind === 'error' &&\n          String(e.payload.reason ?? '')\n            .toLowerCase()\n            .includes('timeout'),\n      )\n      const note = (run.outcome?.notes ?? '').toLowerCase()\n      if (hasTimeout || note.includes('timeout') || note.includes('deadline')) {\n        return { failureClass: 'timeout', reason: 'timeout signal observed' }\n      }\n      return null\n    },\n  },\n  // Tool recovery failure: many consecutive tool errors on the same tool\n  {\n    id: 'tool-recovery-failure',\n    match: ({ spans }) => {\n      const tools = spans.filter((s) => s.kind === 'tool')\n      const byTool = new Map<string, Span[]>()\n      for (const t of tools) {\n        const name = (t as Extract<Span, { kind: 'tool' }>).toolName\n        const arr = byTool.get(name) ?? []\n        arr.push(t)\n        byTool.set(name, arr)\n      }\n      for (const [name, arr] of byTool) {\n        const errs = arr.filter((s) => s.status === 'error')\n        if (errs.length >= 3 && errs.length === arr.length) {\n          return {\n            failureClass: 'tool_recovery_failure',\n            reason: `${errs.length} consecutive errors on tool \"${name}\"`,\n            triggerSpanId: errs[errs.length - 1]!.spanId,\n          }\n        }\n      }\n      return null\n    },\n  },\n  // Tool selection error: the run failed and agent called zero tools despite having them\n  {\n    id: 'tool-selection-error',\n    match: ({ run, spans }) => {\n      if (run.outcome?.pass !== false) return null\n      const hasToolsAvailable = spans.some(\n        (s) =>\n          s.kind === 'agent' &&\n          (s.attributes?.toolsAvailable as number | undefined) !== undefined &&\n          (s.attributes?.toolsAvailable as number) > 0,\n      )\n      const tools = spans.filter((s) => s.kind === 'tool')\n      if (hasToolsAvailable && tools.length === 0) {\n        return {\n          failureClass: 'tool_selection_error',\n          reason: 'tools were available but none were called',\n        }\n      }\n      return null\n    },\n  },\n  // Format drift: scored by a judge with dimension='format' below threshold\n  {\n    id: 'format-drift',\n    match: ({ spans }) => {\n      const judge = spans.find(\n        (s) =>\n          s.kind === 'judge' &&\n          (s as Extract<Span, { kind: 'judge' }>).dimension === 'format' &&\n          (s as Extract<Span, { kind: 'judge' }>).score < 0.5,\n      )\n      return judge\n        ? {\n            failureClass: 'format_drift',\n            reason: 'format judge scored below 0.5',\n            triggerSpanId: judge.spanId,\n          }\n        : null\n    },\n  },\n]\n\nfunction hasResolutionStatus(payload: Record<string, unknown>, status: string): boolean {\n  if (status === 'missing_connection' && stringArray(payload.missingConnections).length > 0)\n    return true\n  return resolutionItems(payload).some((item) => item.status === status)\n}\n\nfunction hasMissingScopes(payload: Record<string, unknown>): boolean {\n  if (stringArray(payload.missingScopes).length > 0) return true\n  return resolutionItems(payload).some(\n    (item) => Array.isArray(item.missingScopes) && item.missingScopes.length > 0,\n  )\n}\n\nfunction resolutionItems(payload: Record<string, unknown>): Array<Record<string, unknown>> {\n  return [\n    ...records(payload.missing),\n    ...records(payload.optionalMissing),\n    ...records(payload.ready),\n  ]\n}\n\nfunction records(value: unknown): Array<Record<string, unknown>> {\n  if (!Array.isArray(value)) return []\n  return value.filter(\n    (item): item is Record<string, unknown> =>\n      Boolean(item) && typeof item === 'object' && !Array.isArray(item),\n  )\n}\n\nfunction stringArray(value: unknown): string[] {\n  return Array.isArray(value)\n    ? value.filter((item): item is string => typeof item === 'string')\n    : []\n}\n\n/** Classify the failure mode of a run using an ordered rule list. */\nexport function classifyFailure(\n  ctx: FailureContext,\n  rules: FailureRule[] = DEFAULT_RULES,\n): FailureClassification {\n  if (ctx.run.outcome?.pass !== false && ctx.run.status === 'completed') {\n    return { failureClass: 'success', reason: 'run completed with pass=true (or no explicit fail)' }\n  }\n  for (const rule of rules) {\n    const hit = rule.match(ctx)\n    if (hit) return hit\n  }\n  return { failureClass: 'unknown', reason: 'no rule matched; run failed for unclassified reason' }\n}\n","/**\n * Tool-use metrics — derived purely from trace data.\n *\n * No scoring assumptions: consumers supply optional ground-truth tool\n * selections per turn + optional \"information used downstream\" signals.\n * Without those, we still compute descriptive metrics (error rate,\n * retry rate, duplicate-call rate) that are useful on their own.\n */\n\nimport { argHash, groupBy, hasCapturedToolArgs, toolSpans } from './trace/query'\nimport type { Span } from './trace/schema'\nimport type { TraceStore } from './trace/store'\n\nexport interface ToolUseMetrics {\n  runId: string\n  totalCalls: number\n  /** Calls whose arguments were captured and can be compared for duplication. */\n  callsWithCapturedArgs: number\n  byTool: Record<string, ToolStats>\n  errorRate: number\n  /** Ratio of captured-argument calls already seen with the same tool name and arguments. */\n  duplicateRate: number\n  /** Ratio of error calls followed by ≥1 retry on same tool. */\n  retryRate: number\n  /** Optional: of the calls agent made, fraction the evaluator marked as \"correct selection\". */\n  selectionAccuracy?: number\n}\n\nexport interface ToolStats {\n  calls: number\n  callsWithCapturedArgs: number\n  errors: number\n  avgLatencyMs: number\n  duplicates: number\n}\n\nexport interface ToolUseOptions {\n  /** Map of spanId → whether the evaluator judged the tool selection correct. Optional. */\n  selectionLabels?: Record<string, boolean>\n}\n\nexport async function computeToolUseMetrics(\n  store: TraceStore,\n  runId: string,\n  options: ToolUseOptions = {},\n): Promise<ToolUseMetrics> {\n  const tools = await toolSpans(store, runId)\n  if (tools.length === 0) {\n    return {\n      runId,\n      totalCalls: 0,\n      callsWithCapturedArgs: 0,\n      byTool: {},\n      errorRate: 0,\n      duplicateRate: 0,\n      retryRate: 0,\n    }\n  }\n\n  const byTool: Record<string, ToolStats> = {}\n  let totalErrors = 0\n  let totalDuplicates = 0\n  let callsWithCapturedArgs = 0\n  const sortedTools = [...tools].sort((a, b) => a.startedAt - b.startedAt)\n  const seenSignatures = new Set<string>()\n\n  // duplicate detection + per-tool aggregation\n  for (const t of sortedTools) {\n    byTool[t.toolName] ??= {\n      calls: 0,\n      callsWithCapturedArgs: 0,\n      errors: 0,\n      avgLatencyMs: 0,\n      duplicates: 0,\n    }\n    const stat = byTool[t.toolName]!\n    stat.calls += 1\n    if (t.status === 'error') {\n      stat.errors += 1\n      totalErrors += 1\n    }\n    if (typeof t.latencyMs === 'number') stat.avgLatencyMs += t.latencyMs\n    if (hasCapturedToolArgs(t)) {\n      callsWithCapturedArgs += 1\n      stat.callsWithCapturedArgs += 1\n      const sig = `${t.toolName}|${argHash(t.args)}`\n      if (seenSignatures.has(sig)) {\n        stat.duplicates += 1\n        totalDuplicates += 1\n      }\n      seenSignatures.add(sig)\n    }\n  }\n\n  for (const stat of Object.values(byTool)) {\n    stat.avgLatencyMs = stat.calls > 0 ? stat.avgLatencyMs / stat.calls : 0\n  }\n\n  // retry detection: per-tool chronological adjacency where error → next same-tool call\n  let retryOpportunities = 0\n  let retriesFollowed = 0\n  for (const [, arr] of groupBy(sortedTools, (t) => t.toolName)) {\n    for (let i = 0; i < arr.length; i++) {\n      if (arr[i]!.status !== 'error') continue\n      retryOpportunities += 1\n      if (arr[i + 1]) retriesFollowed += 1\n    }\n  }\n  const retryRate = retryOpportunities > 0 ? retriesFollowed / retryOpportunities : 0\n\n  let selectionAccuracy: number | undefined\n  if (options.selectionLabels) {\n    const labeled = sortedTools.filter((t) => t.spanId in options.selectionLabels!)\n    if (labeled.length > 0) {\n      selectionAccuracy =\n        labeled.filter((t) => options.selectionLabels![t.spanId]).length / labeled.length\n    }\n  }\n\n  return {\n    runId,\n    totalCalls: sortedTools.length,\n    callsWithCapturedArgs,\n    byTool,\n    errorRate: totalErrors / sortedTools.length,\n    duplicateRate: callsWithCapturedArgs > 0 ? totalDuplicates / callsWithCapturedArgs : 0,\n    retryRate,\n    selectionAccuracy,\n  }\n}\n\nexport type { Span }\n"],"mappings":";;AAwCA,MAAa,gBAA+B;CAE1C;EACE,IAAI;EACJ,QAAQ,EAAE,UAAU;GAClB,MAAM,KAAK,IAAI,SAAS;GACxB,IAAI,MAAM,OAAO,WACf,OAAO;IAAE,cAAc;IAAI,QAAQ;GAAsC;GAC3E,OAAO;EACT;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,sBACnB,EAAE,QAAQ,WAAW,KACzB;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,aACT,EAAE,QAAQ,SAAS,oCAAoC,EAAE,QAAQ,UAAU,SAC1E,EAAE,QAAQ,SAAS,+BAClB,EAAE,QAAQ,SAAS,mBAC3B;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,mCACnB,oBAAoB,EAAE,SAAS,oBAAoB,CACvD;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,aACT,EAAE,QAAQ,SAAS,mCAAmC,iBAAiB,EAAE,OAAO,KAC/E,EAAE,QAAQ,SAAS,+BAA+B,EAAE,QAAQ,SAAS,eAC5E;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,aACT,EAAE,QAAQ,SAAS,wBAAwB,EAAE,QAAQ,WAAW,uBAC/D,EAAE,QAAQ,SAAS,+BAClB,EAAE,QAAQ,SAAS,uBACrB,EAAE,QAAQ,SAAS,gCACzB;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,gCAClB,EAAE,QAAQ,SAAS,kBAClB,EAAE,QAAQ,SAAS,2BACnB,EAAE,QAAQ,SAAS,wBACnB,EAAE,QAAQ,WAAW,UAC3B;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,gCAClB,EAAE,QAAQ,SAAS,yBAClB,EAAE,QAAQ,SAAS,mBACnB,EAAE,QAAQ,SAAS,gBACzB;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,+BACnB,CAAC;IACC;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;GACF,CAAC,CAAC,SAAS,OAAO,EAAE,QAAQ,IAAI,CAAC,CACrC;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,mBACnB,EAAE,QAAQ,aAAa,sBAC3B;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,KAAK,YAAY;GACzB,IAAI,IAAI,SAAS,SAAS,OAAO,OAAO;GACxC,MAAM,YAAY,MAAM,MACrB,MACC,EAAE,SAAS,gBAAgB,EAAE,KAAK,WAAW,KAAK,EAAE,KAAK,OAAO,QAAQ,IAAI,SAAS,CAAC,EAC1F;GACA,OAAO,YACH;IACE,cAAc;IACd,QAAQ;IACR,eAAe,UAAU;GAC3B,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,mBACnB,EAAE,QAAQ,WAAW,uBACzB;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,mBACnB,EAAE,QAAQ,WAAW,wBACzB;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,SAAS,OAAO,MAAM,MAAM,EAAE,SAAS,eAAe;GAC5D,OAAO,SACH;IACE,cAAc;IACd,QAAQ,sBAAsB,OAAO,QAAQ,aAAa;IAC1D,gBAAgB,OAAO;GACzB,IACA;EACN;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,IAAI,OAAO,MAAM,MAAM,EAAE,SAAS,kBAAkB;GAC1D,OAAO,IACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,EAAE;GACpB,IACA;EACN;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,YAAY;GACpB,MAAM,IAAI,MAAM,MACb,MAAM,EAAE,SAAS,aAAa,OAAO,EAAE,aAAa,YAAY,EAAE,aAAa,CAClF;GACA,IAAI,CAAC,GAAG,OAAO;GACf,OAAO;IACL,cAAc;IACd,QAAQ,kBAAmB,EAAyC;IACpE,eAAe,EAAE;GACnB;EACF;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,KAAK,aAAa;GAC1B,IAAI,IAAI,WAAW,WAAW,OAAO;GACrC,MAAM,aAAa,OAAO,MACvB,MACC,EAAE,SAAS,WACX,OAAO,EAAE,QAAQ,UAAU,EAAE,CAAC,CAC3B,YAAY,CAAC,CACb,SAAS,SAAS,CACzB;GACA,MAAM,QAAQ,IAAI,SAAS,SAAS,GAAA,CAAI,YAAY;GACpD,IAAI,cAAc,KAAK,SAAS,SAAS,KAAK,KAAK,SAAS,UAAU,GACpE,OAAO;IAAE,cAAc;IAAW,QAAQ;GAA0B;GAEtE,OAAO;EACT;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,YAAY;GACpB,MAAM,QAAQ,MAAM,QAAQ,MAAM,EAAE,SAAS,MAAM;GACnD,MAAM,yBAAS,IAAI,IAAoB;GACvC,KAAK,MAAM,KAAK,OAAO;IACrB,MAAM,OAAQ,EAAsC;IACpD,MAAM,MAAM,OAAO,IAAI,IAAI,KAAK,CAAC;IACjC,IAAI,KAAK,CAAC;IACV,OAAO,IAAI,MAAM,GAAG;GACtB;GACA,KAAK,MAAM,CAAC,MAAM,QAAQ,QAAQ;IAChC,MAAM,OAAO,IAAI,QAAQ,MAAM,EAAE,WAAW,OAAO;IACnD,IAAI,KAAK,UAAU,KAAK,KAAK,WAAW,IAAI,QAC1C,OAAO;KACL,cAAc;KACd,QAAQ,GAAG,KAAK,OAAO,+BAA+B,KAAK;KAC3D,eAAe,KAAK,KAAK,SAAS,EAAE,CAAE;IACxC;GAEJ;GACA,OAAO;EACT;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,KAAK,YAAY;GACzB,IAAI,IAAI,SAAS,SAAS,OAAO,OAAO;GACxC,MAAM,oBAAoB,MAAM,MAC7B,MACC,EAAE,SAAS,WACV,EAAE,YAAY,mBAA0C,KAAA,KACxD,EAAE,YAAY,iBAA4B,CAC/C;GACA,MAAM,QAAQ,MAAM,QAAQ,MAAM,EAAE,SAAS,MAAM;GACnD,IAAI,qBAAqB,MAAM,WAAW,GACxC,OAAO;IACL,cAAc;IACd,QAAQ;GACV;GAEF,OAAO;EACT;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,YAAY;GACpB,MAAM,QAAQ,MAAM,MACjB,MACC,EAAE,SAAS,WACV,EAAuC,cAAc,YACrD,EAAuC,QAAQ,EACpD;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,eAAe,MAAM;GACvB,IACA;EACN;CACF;AACF;AAEA,SAAS,oBAAoB,SAAkC,QAAyB;CACtF,IAAI,WAAW,wBAAwB,YAAY,QAAQ,kBAAkB,CAAC,CAAC,SAAS,GACtF,OAAO;CACT,OAAO,gBAAgB,OAAO,CAAC,CAAC,MAAM,SAAS,KAAK,WAAW,MAAM;AACvE;AAEA,SAAS,iBAAiB,SAA2C;CACnE,IAAI,YAAY,QAAQ,aAAa,CAAC,CAAC,SAAS,GAAG,OAAO;CAC1D,OAAO,gBAAgB,OAAO,CAAC,CAAC,MAC7B,SAAS,MAAM,QAAQ,KAAK,aAAa,KAAK,KAAK,cAAc,SAAS,CAC7E;AACF;AAEA,SAAS,gBAAgB,SAAkE;CACzF,OAAO;EACL,GAAG,QAAQ,QAAQ,OAAO;EAC1B,GAAG,QAAQ,QAAQ,eAAe;EAClC,GAAG,QAAQ,QAAQ,KAAK;CAC1B;AACF;AAEA,SAAS,QAAQ,OAAgD;CAC/D,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,OAAO,MAAM,QACV,SACC,QAAQ,IAAI,KAAK,OAAO,SAAS,YAAY,CAAC,MAAM,QAAQ,IAAI,CACpE;AACF;AAEA,SAAS,YAAY,OAA0B;CAC7C,OAAO,MAAM,QAAQ,KAAK,IACtB,MAAM,QAAQ,SAAyB,OAAO,SAAS,QAAQ,IAC/D,CAAC;AACP;;AAGA,SAAgB,gBACd,KACA,QAAuB,eACA;CACvB,IAAI,IAAI,IAAI,SAAS,SAAS,SAAS,IAAI,IAAI,WAAW,aACxD,OAAO;EAAE,cAAc;EAAW,QAAQ;CAAqD;CAEjG,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,MAAM,KAAK,MAAM,GAAG;EAC1B,IAAI,KAAK,OAAO;CAClB;CACA,OAAO;EAAE,cAAc;EAAW,QAAQ;CAAsD;AAClG;;;;;;;;;;;ACpaA,eAAsB,sBACpB,OACA,OACA,UAA0B,CAAC,GACF;CACzB,MAAM,QAAQ,MAAM,UAAU,OAAO,KAAK;CAC1C,IAAI,MAAM,WAAW,GACnB,OAAO;EACL;EACA,YAAY;EACZ,uBAAuB;EACvB,QAAQ,CAAC;EACT,WAAW;EACX,eAAe;EACf,WAAW;CACb;CAGF,MAAM,SAAoC,CAAC;CAC3C,IAAI,cAAc;CAClB,IAAI,kBAAkB;CACtB,IAAI,wBAAwB;CAC5B,MAAM,cAAc,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;CACvE,MAAM,iCAAiB,IAAI,IAAY;CAGvC,KAAK,MAAM,KAAK,aAAa;EAC3B,OAAO,EAAE,cAAc;GACrB,OAAO;GACP,uBAAuB;GACvB,QAAQ;GACR,cAAc;GACd,YAAY;EACd;EACA,MAAM,OAAO,OAAO,EAAE;EACtB,KAAK,SAAS;EACd,IAAI,EAAE,WAAW,SAAS;GACxB,KAAK,UAAU;GACf,eAAe;EACjB;EACA,IAAI,OAAO,EAAE,cAAc,UAAU,KAAK,gBAAgB,EAAE;EAC5D,IAAI,oBAAoB,CAAC,GAAG;GAC1B,yBAAyB;GACzB,KAAK,yBAAyB;GAC9B,MAAM,MAAM,GAAG,EAAE,SAAS,GAAG,QAAQ,EAAE,IAAI;GAC3C,IAAI,eAAe,IAAI,GAAG,GAAG;IAC3B,KAAK,cAAc;IACnB,mBAAmB;GACrB;GACA,eAAe,IAAI,GAAG;EACxB;CACF;CAEA,KAAK,MAAM,QAAQ,OAAO,OAAO,MAAM,GACrC,KAAK,eAAe,KAAK,QAAQ,IAAI,KAAK,eAAe,KAAK,QAAQ;CAIxE,IAAI,qBAAqB;CACzB,IAAI,kBAAkB;CACtB,KAAK,MAAM,GAAG,QAAQ,QAAQ,cAAc,MAAM,EAAE,QAAQ,GAC1D,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;EACnC,IAAI,IAAI,EAAE,CAAE,WAAW,SAAS;EAChC,sBAAsB;EACtB,IAAI,IAAI,IAAI,IAAI,mBAAmB;CACrC;CAEF,MAAM,YAAY,qBAAqB,IAAI,kBAAkB,qBAAqB;CAElF,IAAI;CACJ,IAAI,QAAQ,iBAAiB;EAC3B,MAAM,UAAU,YAAY,QAAQ,MAAM,EAAE,UAAU,QAAQ,eAAgB;EAC9E,IAAI,QAAQ,SAAS,GACnB,oBACE,QAAQ,QAAQ,MAAM,QAAQ,gBAAiB,EAAE,OAAO,CAAC,CAAC,SAAS,QAAQ;CAEjF;CAEA,OAAO;EACL;EACA,YAAY,YAAY;EACxB;EACA;EACA,WAAW,cAAc,YAAY;EACrC,eAAe,wBAAwB,IAAI,kBAAkB,wBAAwB;EACrF;EACA;CACF;AACF"}

package/dist/types-D216SgwM.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types-D216SgwM.d.ts","names":[],"sources":["../src/trace/raw-provider-sink.ts","../src/judge-families.ts","../src/integrity/served-model.ts","../src/llm-client.ts","../src/analyst/chat-client.ts","../src/types.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA8BY;UAEK;;EAEf;;EAEA;EACA;;;;;;EAMA;EACA;;EAEA;;EAEA;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA,iBAAiB;EACjB;EACA,kBAAkB;EAClB;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA,YAAY;EACZ;;UAGe;EACf,OAAO,OAAO,mBAAmB;;EAEjC,MAAM,SAAS,wBAAwB,QAAQ;;EAE/C,UAAU;;KAGA,oBAAoB,OAAO,qBAAqB;;;;;;iBAmB5C,wBAAwB,OAAO,mBAAmB;UA8CjD;EACf,WAAW;;cAGA,mCAAmC;UACtC;UACA;EAER,YAAY,OAAM;EAIZ,OAAO,OAAO,mBAAmB;EAIjC,KAAK,SAAQ,wBAA6B,QAAQ;EAUxD;;cAKW,+BAA+B;EACpC,UAAU;;;;;;;EASV,QAAQ,QAAQ;;UAOP;;EAEf;;EAEA;;EAEA;EACA,WAAW;;cAGA,qCAAqC;UACxC;UACA;UACA;UACA;UACA;UACA;UACA;EAER,YAAY,MAAM;UAOJ;UAON;EAKF,OAAO,OAAO,mBAAmB;EAYjC,KAAK,SAAQ,wBAA6B,QAAQ;;;;;;iBAkC1C,oBAAoB;;;;;;;;;;;;;KC5QxB;;;;;;;iBAkEI,YAAY,kBAAkB;UAc7B;;EAEf;;;;EAIA;;cAGW,yBAAyB;WAGlB,UAAU;WACV;EAHlB,YACE,iBACgB,UAAU,eACV;;;;;;;;;;;;;;;iBAoBJ,kBACd,kBACA,OAAM,2BACL;;;;;;;;;;;;;;;;;;;cCtFU;;KAGD;;;;;;;;;;;UAYK;;EAEf;;EAEA;EACA,iBAAiB;;EAEjB,cAAc;EACd,SAAS;;EAET;;;;;;;;;;;;;;;;;iBAkBc,iBAAiB;;;;;;;;;;;iBA0BjB,iBACd,mBACA,oCACC;cA4CU,+BAA+B;WAGxB,QAAQ,cAAc;EAFxC,YACE,iBACgB,QAAQ,cAAc;;UAOzB;;;;;;EAMf;;;;;EAKA;;EAEA;;;;;;;iBAQc,sBACd,OAAO,kBACP,OAAM;;;;;;iBA6BQ,kBACd,mBACA,mCACA,OAAM,2BACL;;;;;iBAea,mBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,2BACL;UAec,uCAAuC;;EAEtD;;EAEA;;cAGW,+BAA+B;WAGxB,UAAU;WACV,QAAQ,cAAc;EAHxC,YACE,iBACgB,UAAU,eACV,QAAQ,cAAc;;;;;;;;;;;iBAgB1B,wBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,iCACL;;;UCjPc;EACf;;;;;EAKA,kBAEI;IACM;IAAc;;IACd;IAAmB;MAAa;MAAa;;;;KAI7C;UAEK;EACf;EACA,UAAU;;EAEV;;EAEA;IAAe;IAAc,QAAQ;;EACrC;EACA;;EAEA,WAAW;;EAEX;;;;;;iBAOc,2BACd,SAAS,KAAK,iFACd,UAAS,mBACR;UAgCc;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;EACA,OAAO;;;;;EAKP;;;;;;;;;EASA;;;;;;;;;;;;EAYA;;EAEA;;;;;;;;;EASA;;;;;;;EAOA;;EAEA,KAAK;;KAGK,kBAAkB,KAAK;;iBAGnB,mBACd,QAAQ,eACR,qBAAqB,qBACpB;;iBA2Ba,wBACd,OAAO,OACP,qBAAqB,qBACpB;cAMU,qBAAqB;WAGd;WACA;WACA;EAJlB,YACE,iBACgB,gBACA,cACA;;;;;cASP,yBAAyB;WAGlB,QAAQ;EAF1B,YACE,iBACgB,QAAQ,eACxB;IAAY;;;UAMC;;EAEf;;EAEA;EACA;;EAEA;IAAe;IAAc;;;EAE7B;;EAEA;;;;;;;EAOA,SAAS;;;;;;;;EAQT;;EAEA;;EAEA,qBAAqB;;;;;;;EAOrB;;;;;;EAMA;;EAEA,WAAW;;EAEX,eAAe;;;;;;;;EAQf,UAAU;;;;;EAKV;;EAEA;IAAiB;IAAgB;;;EAEjC,WAAW;;;;;;;;;;EAUX,8BAA8B;;;;;;;;;;;;;iBAsEhB,oBAAoB;;iBAgCpB,UAAU;;;;;;iBA6GV,gBAAgB;;;;;;iBA6EV,QACpB,KAAK,gBACL,OAAM,mBACL,QAAQ;;;;;;;iBAoVW,YAAY,aAChC,KAAK,gBACL,OAAM,mBACL;EAAU,OAAO;EAAG,QAAQ;;KA4DnB;cAOC,+BAA+B;WAGxB,QAAQ;WACR;EAHlB,YACE,iBACgB,QAAQ,yBACR;;UAMH;;;;;;;EAOf;;;;;EAKA,kBAAkB,eAAe;;EAEjC,kBAAkB,eAAe;;EAEjC;;;;;EAKA;;;;;;;;;;;iBAYc,eAAe,MAAM,kBAAkB,MAAK;;;;;;;;;;;;;;;;;;iBA6EtC,SACpB,eACA,OAAM;EAAqB;IAC1B;EACD;EACA;EACA;;EAEA;;EAEA;;;;;;;cAoCW;WACF;mBACQ;EAEjB,YAAY,OAAM;EAKlB,KAAK,KAAK,gBAAgB,MAAM,mBAAmB,QAAQ;EAK3D,SAAS,aACP,KAAK,gBACL,MAAM,mBACL;IAAU,OAAO;IAAG,QAAQ;;;;;;;;UCjqChB;;WAEN,WAAW;;WAEX;;WAEA;;EAGT,KAAK,KAAK,aAAa,OAAO,eAAe,QAAQ;;KAG3C;UAQK,oBAAoB,KAAK;;EAExC;;KAGU,eAAe;UAEV;;EAEf,SAAS;;EAET;;EAEA;;EAEA;;KAKU,uBACR,sBACA,yBACA,8BACA,0BACA,sBACA;UAEM;EACR;;EAEA;;UAGe,4BAA4B;EAC3C;EACA;EACA;;UAGe,+BAA+B;EAC9C;EACA;EACA;;UAGe,oCAAoC;EACnD;EACA;EACA;;;;;;UAOe,gCAAgC;EAC/C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;UAI1C,4BAA4B;EAC3C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;UAO1C,0BAA0B;EACzC;EACA,UAAU,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;iBAO9C,iBAAiB,MAAM,uBAAuB;;;UCjH7C;EACf;EACA;EACA;EACA;EACA;EACA,OAAO;EACP,gBAAgB;EAChB;;UAGe;EACf;EACA;EACA;EACA;;UAKe;EACf;EAQA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,QAAQ;;UAGO;EACf;EACA;EACA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,OAAO;EACP,iBAAiB;EACjB,aAAa;EACb;EACA;EACA;EACA,WAAW;;EAEX,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;IAAmB;IAAc;;EACjC;EACA;;UAGe;EACf,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;IAAc;IAAc;;EAC5B;IAAmB;IAAc,QAAQ;;EACzC;IAAc;IAAkB;;EAChC;;UAKe;EACf;EACA;EACA;EACA;EACA,SAAS;EACT,OAAO;EACP;IACE;IACA,WAAW;MAAiB;MAAa;MAAgB;;IACzD,aAAa;MAAiB;MAAa;;IAC3C;MAAW;MAAkB;MAAe;;IAC5C;MAAa;MAAkB;MAAe;;;;UAMjC;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;GACC;;UAGc;EACf;EACA,QAAQ;;EAER;;UAKe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;IACE;MACE;MACA;MACA;;;EAGJ,OAAO;EACP,gBAAgB;;UAKD;EACf;EACA,QAAQ,OAAO;EACf,YAAY,OAAO;;UAGJ;EACf;EACA;;;;;;;;;;KAWU;UAEK;EACf;EACA;EACA;EACA,oBAAoB;EACpB,mBAAmB;EACnB;EACA;;EAEA,QAAQ;;;;;;;EAOR;;;;;;;EAOA;;;;;;EAMA;;UAGe;EACf;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;;EAEA;;EAEA;;;;;;EAMA;EACA;EACA,SAAS;EACT,YAAY;EACZ;EACA;EACA;;UAKe;EACf,WAAW;EACX,QAAQ;EACR;EACA;EACA;EACA;EACA;EACA;;EAEA,aAAa;;UAGE;EACf,UAAU;EACV,OAAO;EACP,WAAW;;EAEX,aAAa;EACb;EACA,WAAW;EACX,SAAS;;KAGC,WAAW,MAAM,YAAY,OAAO,eAAe,QAAQ;UAItD;EACf;EACA;EACA;EACA;EACA,QAAQ;;UAGO;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA"}

package/dist/usage-receipt-t7vAzCRQ.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"usage-receipt-t7vAzCRQ.js","names":[],"sources":["../src/analyst/types.ts","../src/ledger-core/deep-freeze.ts","../src/analyst/usage-receipt.ts"],"sourcesContent":["/**\n * Analyst contract — the missing orchestration layer over agent-eval's\n * existing analyzers (analyzeTraces, MultiLayerVerifier, RunCritic,\n * SemanticConceptJudge, JudgeFn, ...).\n *\n * Each existing primitive returns its own output shape. The Analyst\n * contract is the single envelope every primitive lifts into, so a\n * registry can run N analysts against a run and a single renderer can\n * compose findings without knowing which analyzer produced them.\n *\n * The contract is intentionally domain-agnostic: nothing here knows\n * about code, voice, RAG, or any particular agent stack. Analysts\n * declare what INPUT KIND they need (a trace store, an artifact dir,\n * a RunRecord, a JudgeInput, or `custom`), and the registry routes\n * the matching input from `AnalystRunInputs`.\n */\n\nimport { createHash } from 'node:crypto'\nimport type { CostLedgerHandle } from '../cost-ledger'\nimport type { RunCostProvenance, RunRecord, RunTokenUsage } from '../run-record'\nimport type { TraceAnalysisStore } from '../trace-analyst/store'\nimport type { JudgeInput } from '../types'\nimport type { ChatClient } from './chat-client'\n\n/**\n * Unified envelope every analyst emits. Schema-versioned so renderers\n * and time-series diffs survive future field additions.\n */\nexport interface AnalystFinding {\n  schema_version: '1.0.0'\n  /**\n   * Stable hash over identity-defining fields (analyst_id + canonical\n   * claim + area + optional subject). Two findings from two runs that\n   * \"are the same finding\" share this id — that's what `diffFindings`\n   * uses to compute appeared/disappeared sets across runs.\n   */\n  finding_id: string\n  analyst_id: string\n  produced_at: string\n  severity: AnalystSeverity\n  /**\n   * Coarse classification. Renderers group by this. Free-form so\n   * domain-specific analysts can introduce categories without a\n   * schema change ('agent-reasoning', 'verification', 'cost',\n   * 'tool-use', 'safety', 'latency', 'data-quality', ...).\n   */\n  area: string\n  claim: string\n  rationale?: string\n  evidence_refs: EvidenceRef[]\n  recommended_action?: string\n  validation_plan?: string\n  /** 0..1 — the analyst's own confidence. Not calibrated across analysts. */\n  confidence: number\n  /**\n   * Optional subject the finding is about — leaf id, agent id, request\n   * id. Included in finding_id when present so per-subject findings\n   * diff cleanly across runs.\n   */\n  subject?: string\n  /** True when this finding was lifted from a judge result rather than observed\n   *  directly in a trace or artifact. Descriptive only: proposal access is\n   *  controlled by `ProposalFinding.proposal_origin`. */\n  derived_from_judge?: boolean\n  /** Analyst-private extras; renderers ignore unless they know the analyst. */\n  metadata?: Record<string, unknown>\n}\n\nexport type AnalystSeverity = 'critical' | 'high' | 'medium' | 'low' | 'info'\n\n/** Data sources that candidate generation may intentionally learn from. */\nexport type ProposalFindingOrigin = 'search' | 'production'\n\n/** A finding explicitly admitted as candidate-generation input. */\nexport type ProposalFinding = AnalystFinding & {\n  readonly proposal_origin: ProposalFindingOrigin\n}\n\nexport interface EvidenceRef {\n  /**\n   * Where the evidence lives. `span` and `event` refer to OTLP trace\n   * elements; `artifact` to a file inside the run's artifact tree;\n   * `finding` to another AnalystFinding (cross-analyst chaining);\n   * `metric` to a named scalar reading the renderer knows how to read.\n   */\n  kind: 'span' | 'event' | 'artifact' | 'finding' | 'metric'\n  uri: string\n  excerpt?: string\n}\n\n// ── Analyst contract ─────────────────────────────────────────────────\n\n/**\n * The discriminator the registry uses to pass the right input.\n * `custom` is the escape hatch — analysts that need something else\n * (e.g. an embedding cache, a partner SDK handle) read it from\n * `AnalystRunInputs.custom[<analyst id>]`.\n */\nexport type AnalystInputKind =\n  | 'trace-store'\n  | 'artifact-dir'\n  | 'run-record'\n  | 'judge-input'\n  | 'custom'\n\nexport interface AnalystCost {\n  /** `deterministic` analysts MUST NOT call the LLM. */\n  kind: 'deterministic' | 'llm'\n  /** Optional declared upper bound; the registry can enforce a budget. */\n  est_usd_per_run?: number\n  /** Models the analyst expects to use (informational). */\n  models?: string[]\n  /** Maximum post-cancellation wait for provider usage. Model analysts default to 5 seconds. */\n  settlement_timeout_ms?: number\n}\n\nexport interface AnalystRequirements {\n  /** Min number of shots / samples the analyst needs to produce signal. */\n  min_shots?: number\n  /** Capabilities the runtime must supply (e.g. ['network', 'gpu']). */\n  capabilities?: string[]\n}\n\n/**\n * What's passed to every analyst call. The registry resolves which\n * field the analyst's `inputKind` selects and asserts it's present.\n */\nexport interface AnalystRunInputs {\n  traceStore?: TraceAnalysisStore\n  artifactDir?: string\n  runRecord?: RunRecord\n  judgeInput?: JudgeInput\n  /** Keyed by analyst id; populated by callers that registered custom analysts. */\n  custom?: Record<string, unknown>\n}\n\nexport interface AnalystContext {\n  runId: string\n  /** Stable correlation id so logs from a single registry.run() share a tag. */\n  correlationId: string\n  /** Enforced wall-clock deadline (epoch ms). */\n  deadlineMs?: number\n  /** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */\n  budgetUsd?: number\n  /** Shared paid-call account when the analyst runs inside a larger campaign. */\n  costLedger?: CostLedgerHandle\n  /** Attribution phase used when writing to the shared paid-call account. */\n  costPhase?: string\n  /**\n   * Shared chat client. Analysts that call an LLM go through this so\n   * the operator picks transport (sandbox-sdk | router | cli-bridge |\n   * direct-provider | mock) at the registry boundary without touching\n   * analyst code.\n   */\n  chat?: ChatClient\n  /**\n   * Findings from a prior run the operator wants the analyst to see as\n   * retrieval context. Kinds that take advantage of cross-run memory\n   * (failure-mode \"I saw this cluster last run\", knowledge-gap \"the wiki\n   * page I asked for is still missing\") render these into the actor's\n   * working set. Filtering is the operator's job: pass the slice that\n   * matches the analyst's id, or pass everything and let the kind\n   * filter. Empty / absent means no cross-run context.\n   */\n  priorFindings?: ReadonlyArray<AnalystFinding>\n  /**\n   * Findings emitted by analysts that completed earlier in this registry run.\n   * This is separate from `priorFindings`: upstream findings are dependency\n   * context for the current pass, while prior findings are cross-run memory.\n   * The registry populates this only when `RegistryRunOpts.chainFindings` is on.\n   */\n  upstreamFindings?: ReadonlyArray<AnalystFinding>\n  /**\n   * Report metered work independently of findings. This keeps an empty finding\n   * set from erasing token/cost telemetry. Multiple receipts are accumulated.\n   */\n  recordUsage?: (receipt: AnalystUsageReceipt) => void\n  /** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */\n  tags?: Record<string, string>\n  /** Logger callback — analysts SHOULD prefer this over console.* for testability. */\n  log?: (msg: string, fields?: Record<string, unknown>) => void\n  /** Optional abort signal. Analysts SHOULD pass it through to LLM calls. */\n  signal?: AbortSignal\n  /**\n   * Optional live-execution port. A runtime that owns a sandbox or checkout\n   * fills it so an analyst can execute a bounded probe against the run's\n   * produced state instead of reasoning about it from the trace alone. This\n   * package defines only the port: no field here reaches for an agent loop,\n   * and an absent probe means the analyst works from recorded evidence.\n   */\n  probe?: ExecutionProbe\n}\n\n// ── Live-execution port ─────────────────────────────────────────────\n\n/** One bounded command an analyst asks the probe to run. */\nexport interface ExecutionProbeRequest {\n  command: string\n  /** Working directory inside the probed environment. */\n  cwd?: string\n  /** Hard wall-clock deadline for this one execution. */\n  timeoutMs: number\n  /** Bytes of combined output retained; the prober truncates beyond it. */\n  maxOutputBytes?: number\n  signal?: AbortSignal\n}\n\n/**\n * Typed outcome of one probe execution. `succeeded: false` is a PROBE failure\n * (the environment could not run the command); a command that ran and exited\n * non-zero is a successful observation with a non-zero `exitCode`.\n */\nexport type ExecutionProbeOutcome =\n  | {\n      succeeded: true\n      exitCode: number\n      stdout: string\n      stderr: string\n      durationMs: number\n      /** True when output was cut at `maxOutputBytes`. */\n      truncated: boolean\n    }\n  | { succeeded: false; error: { class: string; message: string } }\n\n/**\n * The seam a runtime fills to let analysts observe produced state live.\n * Implementations own sandboxing, credentials, and cleanup; analysts only\n * submit bounded requests and read typed outcomes.\n */\nexport interface ExecutionProbe {\n  /** One plain sentence naming what is being probed (e.g. a sandbox id). */\n  readonly description: string\n  execute(request: ExecutionProbeRequest): Promise<ExecutionProbeOutcome>\n}\n\n/**\n * The minimal contract. Concrete analysts can refine `TInput` so\n * implementations stay type-safe (e.g. a trace analyst's `TInput` is\n * `TraceAnalysisStore`); the registry passes the right field from\n * `AnalystRunInputs` based on `inputKind`.\n */\nexport interface Analyst<TInput = unknown> {\n  /** Stable identifier — appears in finding_id, telemetry, and registry exclusion lists. */\n  readonly id: string\n  /** Human-readable. One sentence. */\n  readonly description: string\n  readonly inputKind: AnalystInputKind\n  readonly cost: AnalystCost\n  readonly requires?: AnalystRequirements\n  /** Bump on breaking changes to claim wording or area so old finding_ids don't collide. */\n  readonly version: string\n  analyze(input: TInput, ctx: AnalystContext): Promise<AnalystFinding[]>\n}\n\n/** Metered work performed by one analyst call. */\nexport interface AnalystUsageReceipt {\n  /** Number of model-usage records observed at the provider boundary. */\n  calls: number | null\n  /** Null when the provider did not return token accounting. */\n  tokens: RunTokenUsage | null\n  /** Observed, estimated, or explicitly uncaptured dollar cost. */\n  cost: RunCostProvenance\n  /** Known lower bound when one or more calls have uncaptured cost. */\n  knownCostUsd?: number\n  /**\n   * Token counts the provider reported on only one side. Present exactly when\n   * `tokens` is null and at least one side WAS reported: `RunTokenUsage` has no\n   * nullable side, so a one-sided count cannot live in `tokens` without writing\n   * a zero nobody measured. Read it as a lower bound, never as a total — the\n   * field exists so a null `tokens` cannot hide a real count.\n   */\n  partialTokens?: { input: number | null; output: number | null }\n  /**\n   * True when the token counts were DERIVED by the transport (from character\n   * lengths, say) rather than measured by the model provider. `cost.kind` is\n   * `estimated` both for a rate estimate over exact tokens and for one over\n   * derived tokens; this is the field that separates them.\n   */\n  tokensEstimated?: boolean\n}\n\n// ── finding_id stability ─────────────────────────────────────────────\n\n/**\n * Compute the stable finding_id from the identity-defining fields.\n * Default implementation hashes {analyst_id, area, subject, normalized claim}.\n * Analysts that emit findings whose claim text varies per run (timestamps,\n * counts) SHOULD either: (a) pass an explicit `id_basis` to fix the hash,\n * or (b) move the variable part into `rationale`/`metadata` and keep the\n * `claim` static.\n */\nexport function computeFindingId(input: {\n  analyst_id: string\n  area: string\n  subject?: string\n  claim: string\n  /** Override the claim for hashing — use when the displayed claim has run-specific bits. */\n  id_basis?: string\n}): string {\n  const basis = JSON.stringify({\n    a: input.analyst_id,\n    r: input.area,\n    s: input.subject ?? '',\n    c: normalizeClaim(input.id_basis ?? input.claim),\n  })\n  return `f_${createHash('sha256').update(basis).digest('hex').slice(0, 20)}`\n}\n\nfunction normalizeClaim(c: string): string {\n  // Lowercase, collapse whitespace, strip trailing punctuation. Goal:\n  // \"Leaf X failed install\" and \"Leaf X failed install.\" hash the same.\n  return c\n    .toLowerCase()\n    .replace(/\\s+/g, ' ')\n    .replace(/[.!?;:,]+$/g, '')\n    .trim()\n}\n\n/**\n * Convenience factory: produce a fully-formed AnalystFinding with the\n * id computed automatically. Analyst code stays terse.\n */\nexport function makeFinding(\n  init: Omit<AnalystFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {\n    id_basis?: string\n    produced_at?: string\n  },\n): AnalystFinding {\n  const { id_basis, produced_at, ...rest } = init\n  return {\n    schema_version: '1.0.0',\n    finding_id: computeFindingId({\n      analyst_id: rest.analyst_id,\n      area: rest.area,\n      subject: rest.subject,\n      claim: rest.claim,\n      id_basis,\n    }),\n    produced_at: produced_at ?? new Date().toISOString(),\n    ...rest,\n  }\n}\n\n/** Build a finding whose source is explicitly allowed during candidate generation. */\nexport function makeProposalFinding(\n  init: Omit<ProposalFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {\n    id_basis?: string\n    produced_at?: string\n  },\n): ProposalFinding {\n  const { proposal_origin, ...finding } = init\n  return { ...makeFinding(finding), proposal_origin }\n}\n\n// ── Registry result envelope ────────────────────────────────────────\n\nexport interface AnalystRunSummary {\n  analyst_id: string\n  status: 'ok' | 'skipped' | 'failed'\n  /** Why skipped — missing input, budget exceeded, capability unmet. */\n  reason?: string\n  findings_count: number\n  latency_ms: number\n  /** Additive model usage and cost provenance for this analyst. */\n  usage: AnalystUsageReceipt\n  /** When `status='failed'`: the error class + message, never the full stack. */\n  error?: { class: string; message: string }\n}\n\nexport interface AnalystRunResult {\n  run_id: string\n  correlation_id: string\n  started_at: string\n  ended_at: string\n  findings: AnalystFinding[]\n  per_analyst: AnalystRunSummary[]\n  /** Total LLM cost in USD across all analysts in this registry.run(). */\n  total_cost_usd: number\n  /**\n   * Provenance for `total_cost_usd`. When uncaptured, the numeric field is only\n   * the known subtotal and must not be treated as the run's total spend.\n   */\n  total_cost_provenance?: RunCostProvenance\n}\n\n// ── Streaming event envelope ────────────────────────────────────────\n\n/**\n * Events emitted by `AnalystRegistry.runStream(...)` in real time as\n * the registry executes. UIs subscribe via `for await (const ev of\n * registry.runStream(...))`; `registry.run(...)` is a thin collector\n * over the same stream, so the two surfaces share their invariants.\n *\n * Per-finding events are intentionally omitted — analyzers are batch\n * operations (a recursive engine returns the full `findings:json[]` at the\n * end of the responder), so streaming inside one analyst would only\n * emit partial JSON consumers can't render. The kind-completion event\n * is the right granularity; subscribers wanting per-finding rendering\n * iterate `event.findings` themselves.\n */\nexport type AnalystRunEvent =\n  | {\n      type: 'run-started'\n      run_id: string\n      correlation_id: string\n      started_at: string\n      /** The ordered list of analyst ids the registry will run. */\n      analyst_ids: ReadonlyArray<string>\n    }\n  | {\n      type: 'analyst-skipped'\n      summary: AnalystRunSummary\n    }\n  | {\n      type: 'analyst-started'\n      analyst_id: string\n      started_at: string\n    }\n  | {\n      type: 'analyst-completed'\n      /** `summary.status` is `'ok'` for clean completion or `'failed'` for thrown analysts. */\n      summary: AnalystRunSummary\n      findings: ReadonlyArray<AnalystFinding>\n    }\n  | {\n      type: 'run-completed'\n      result: AnalystRunResult\n    }\n","/** Freeze a detached canonical-JSON graph. Canonicalization has already ruled out cycles.\n *\n * Lives outside canonical.ts so the analyst-benchmark implementation digest,\n * which covers canonical.ts, stays bound to the published benchmark evidence. */\nexport function deepFreezeCanonicalJson<T>(value: T): T {\n  if (value && typeof value === 'object' && !Object.isFrozen(value)) {\n    Object.freeze(value)\n    for (const nested of Object.values(value)) deepFreezeCanonicalJson(nested)\n  }\n  return value\n}\n","import type { CostChannel, CostLedgerFilter, CostLedgerHandle } from '../cost-ledger'\nimport type { AnalystUsageReceipt } from './types'\n\nexport const DEFAULT_USAGE_SETTLEMENT_TIMEOUT_MS = 5_000\n\n/** Convert one ledger channel's complete call set into one analyst receipt. */\nexport function usageReceiptFromCostLedger(\n  ledger: CostLedgerHandle,\n  filter: CostChannel | CostLedgerFilter = 'analyst',\n): AnalystUsageReceipt {\n  const resolvedFilter = typeof filter === 'string' ? { channel: filter } : filter\n  const summary = ledger.summary(resolvedFilter)\n  const receipts = ledger.list(resolvedFilter)\n  const hasReasoningUsage = receipts.some((receipt) => receipt.reasoningTokens !== undefined)\n  const hasCacheWriteUsage = receipts.some((receipt) => receipt.cacheWriteTokens !== undefined)\n  const cost = summary.costProvenance\n  return {\n    calls: summary.totalCalls + summary.pendingCalls,\n    tokens: summary.usageComplete\n      ? {\n          input: summary.inputTokens,\n          output: summary.outputTokens,\n          ...(hasReasoningUsage ? { reasoning: summary.reasoningTokens ?? 0 } : {}),\n          ...(summary.cachedTokens > 0 ? { cached: summary.cachedTokens } : {}),\n          ...(hasCacheWriteUsage ? { cacheWrite: summary.cacheWriteTokens ?? 0 } : {}),\n        }\n      : null,\n    cost,\n    ...(cost.kind === 'uncaptured' ? { knownCostUsd: summary.totalCostUsd } : {}),\n  }\n}\n\nexport interface SettledUsageReceipt {\n  settled: boolean\n  pendingCalls: number\n  receipt: AnalystUsageReceipt\n}\n\n/** Wait a bounded time for late provider receipts, then take one immutable snapshot. */\nexport async function settleUsageReceiptFromCostLedger(\n  ledger: CostLedgerHandle,\n  options: CostLedgerFilter & { timeoutMs?: number } = {},\n): Promise<SettledUsageReceipt> {\n  const { timeoutMs: requestedTimeoutMs, ...requestedFilter } = options\n  const filter: CostLedgerFilter = {\n    channel: requestedFilter.channel ?? 'analyst',\n    ...(requestedFilter.phase === undefined ? {} : { phase: requestedFilter.phase }),\n    ...(requestedFilter.tags === undefined ? {} : { tags: requestedFilter.tags }),\n  }\n  const timeoutMs = validateUsageSettlementTimeout(requestedTimeoutMs)\n  const initial = ledger.summary(filter)\n  const waitResult =\n    initial.pendingCalls === 0\n      ? true\n      : ledger.waitForIdle\n        ? await ledger.waitForIdle({ timeoutMs })\n        : false\n  const pendingCalls = ledger.summary(filter).pendingCalls\n  return {\n    settled: waitResult && pendingCalls === 0,\n    pendingCalls,\n    receipt: usageReceiptFromCostLedger(ledger, filter),\n  }\n}\n\nexport function validateUsageSettlementTimeout(timeoutMs?: number): number {\n  const resolved = timeoutMs ?? DEFAULT_USAGE_SETTLEMENT_TIMEOUT_MS\n  if (!Number.isSafeInteger(resolved) || resolved < 0 || resolved > 2_147_483_647) {\n    throw new TypeError(\n      'settlementTimeoutMs must be a non-negative safe integer no greater than 2147483647',\n    )\n  }\n  return resolved\n}\n\nexport function assertValidAnalystUsageReceipt(\n  receipt: AnalystUsageReceipt,\n  context = 'AnalystContext.recordUsage',\n): void {\n  if (receipt.calls !== null && (!Number.isSafeInteger(receipt.calls) || receipt.calls < 0)) {\n    throw new Error(`${context}: calls must be a non-negative safe integer or null`)\n  }\n  if (receipt.tokens) {\n    assertNonNegativeSafeInteger(receipt.tokens.input, 'tokens.input', context)\n    assertNonNegativeSafeInteger(receipt.tokens.output, 'tokens.output', context)\n    if (receipt.tokens.reasoning !== undefined) {\n      assertNonNegativeSafeInteger(receipt.tokens.reasoning, 'tokens.reasoning', context)\n      if (receipt.tokens.reasoning > receipt.tokens.output) {\n        throw new Error(`${context}: tokens.reasoning must not exceed tokens.output`)\n      }\n    }\n    if (receipt.tokens.cached !== undefined) {\n      assertNonNegativeSafeInteger(receipt.tokens.cached, 'tokens.cached', context)\n    }\n    if (receipt.tokens.cacheWrite !== undefined) {\n      assertNonNegativeSafeInteger(receipt.tokens.cacheWrite, 'tokens.cacheWrite', context)\n    }\n  }\n  if (receipt.cost.kind !== 'uncaptured') {\n    assertNonNegativeFinite(receipt.cost.usd, 'cost.usd', context)\n  } else if (receipt.cost.usd !== null) {\n    throw new Error(`${context}: uncaptured cost.usd must be null`)\n  }\n  if (receipt.knownCostUsd !== undefined) {\n    assertNonNegativeFinite(receipt.knownCostUsd, 'knownCostUsd', context)\n  }\n  if (receipt.partialTokens) {\n    const { input, output } = receipt.partialTokens\n    if (receipt.tokens) {\n      throw new Error(`${context}: partialTokens must be absent when tokens is complete`)\n    }\n    if (input === null && output === null) {\n      throw new Error(`${context}: partialTokens must carry at least one reported side`)\n    }\n    if (input !== null) assertNonNegativeSafeInteger(input, 'partialTokens.input', context)\n    if (output !== null) assertNonNegativeSafeInteger(output, 'partialTokens.output', context)\n  }\n}\n\nfunction assertNonNegativeSafeInteger(value: number, field: string, context: string): void {\n  if (!Number.isSafeInteger(value) || value < 0) {\n    throw new Error(`${context}: ${field} must be a non-negative safe integer`)\n  }\n}\n\nfunction assertNonNegativeFinite(value: number, field: string, context: string): void {\n  if (!Number.isFinite(value) || value < 0) {\n    throw new Error(`${context}: ${field} must be a non-negative finite number`)\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAmSA,SAAgB,iBAAiB,OAOtB;CACT,MAAM,QAAQ,KAAK,UAAU;EAC3B,GAAG,MAAM;EACT,GAAG,MAAM;EACT,GAAG,MAAM,WAAW;EACpB,GAAG,eAAe,MAAM,YAAY,MAAM,KAAK;CACjD,CAAC;CACD,OAAO,KAAK,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,EAAE;AAC1E;AAEA,SAAS,eAAe,GAAmB;CAGzC,OAAO,EACJ,YAAY,CAAC,CACb,QAAQ,QAAQ,GAAG,CAAC,CACpB,QAAQ,eAAe,EAAE,CAAC,CAC1B,KAAK;AACV;;;;;AAMA,SAAgB,YACd,MAIgB;CAChB,MAAM,EAAE,UAAU,aAAa,GAAG,SAAS;CAC3C,OAAO;EACL,gBAAgB;EAChB,YAAY,iBAAiB;GAC3B,YAAY,KAAK;GACjB,MAAM,KAAK;GACX,SAAS,KAAK;GACd,OAAO,KAAK;GACZ;EACF,CAAC;EACD,aAAa,gCAAe,IAAI,KAAK,EAAA,CAAE,YAAY;EACnD,GAAG;CACL;AACF;;AAGA,SAAgB,oBACd,MAIiB;CACjB,MAAM,EAAE,iBAAiB,GAAG,YAAY;CACxC,OAAO;EAAE,GAAG,YAAY,OAAO;EAAG;CAAgB;AACpD;;;;;;;AC5VA,SAAgB,wBAA2B,OAAa;CACtD,IAAI,SAAS,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG;EACjE,OAAO,OAAO,KAAK;EACnB,KAAK,MAAM,UAAU,OAAO,OAAO,KAAK,GAAG,wBAAwB,MAAM;CAC3E;CACA,OAAO;AACT;;ACJA,SAAgB,2BACd,QACA,SAAyC,WACpB;CACrB,MAAM,iBAAiB,OAAO,WAAW,WAAW,EAAE,SAAS,OAAO,IAAI;CAC1E,MAAM,UAAU,OAAO,QAAQ,cAAc;CAC7C,MAAM,WAAW,OAAO,KAAK,cAAc;CAC3C,MAAM,oBAAoB,SAAS,MAAM,YAAY,QAAQ,oBAAoB,KAAA,CAAS;CAC1F,MAAM,qBAAqB,SAAS,MAAM,YAAY,QAAQ,qBAAqB,KAAA,CAAS;CAC5F,MAAM,OAAO,QAAQ;CACrB,OAAO;EACL,OAAO,QAAQ,aAAa,QAAQ;EACpC,QAAQ,QAAQ,gBACZ;GACE,OAAO,QAAQ;GACf,QAAQ,QAAQ;GAChB,GAAI,oBAAoB,EAAE,WAAW,QAAQ,mBAAmB,EAAE,IAAI,CAAC;GACvE,GAAI,QAAQ,eAAe,IAAI,EAAE,QAAQ,QAAQ,aAAa,IAAI,CAAC;GACnE,GAAI,qBAAqB,EAAE,YAAY,QAAQ,oBAAoB,EAAE,IAAI,CAAC;EAC5E,IACA;EACJ;EACA,GAAI,KAAK,SAAS,eAAe,EAAE,cAAc,QAAQ,aAAa,IAAI,CAAC;CAC7E;AACF;;AASA,eAAsB,iCACpB,QACA,UAAqD,CAAC,GACxB;CAC9B,MAAM,EAAE,WAAW,oBAAoB,GAAG,oBAAoB;CAC9D,MAAM,SAA2B;EAC/B,SAAS,gBAAgB,WAAW;EACpC,GAAI,gBAAgB,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,gBAAgB,MAAM;EAC9E,GAAI,gBAAgB,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,MAAM,gBAAgB,KAAK;CAC7E;CACA,MAAM,YAAY,+BAA+B,kBAAkB;CAEnE,MAAM,aADU,OAAO,QAAQ,MAEvB,CAAC,CAAC,iBAAiB,IACrB,OACA,OAAO,cACL,MAAM,OAAO,YAAY,EAAE,UAAU,CAAC,IACtC;CACR,MAAM,eAAe,OAAO,QAAQ,MAAM,CAAC,CAAC;CAC5C,OAAO;EACL,SAAS,cAAc,iBAAiB;EACxC;EACA,SAAS,2BAA2B,QAAQ,MAAM;CACpD;AACF;AAEA,SAAgB,+BAA+B,WAA4B;CACzE,MAAM,WAAW,aAAA;CACjB,IAAI,CAAC,OAAO,cAAc,QAAQ,KAAK,WAAW,KAAK,WAAW,YAChE,MAAM,IAAI,UACR,oFACF;CAEF,OAAO;AACT;AAEA,SAAgB,+BACd,SACA,UAAU,8BACJ;CACN,IAAI,QAAQ,UAAU,SAAS,CAAC,OAAO,cAAc,QAAQ,KAAK,KAAK,QAAQ,QAAQ,IACrF,MAAM,IAAI,MAAM,GAAG,QAAQ,oDAAoD;CAEjF,IAAI,QAAQ,QAAQ;EAClB,6BAA6B,QAAQ,OAAO,OAAO,gBAAgB,OAAO;EAC1E,6BAA6B,QAAQ,OAAO,QAAQ,iBAAiB,OAAO;EAC5E,IAAI,QAAQ,OAAO,cAAc,KAAA,GAAW;GAC1C,6BAA6B,QAAQ,OAAO,WAAW,oBAAoB,OAAO;GAClF,IAAI,QAAQ,OAAO,YAAY,QAAQ,OAAO,QAC5C,MAAM,IAAI,MAAM,GAAG,QAAQ,iDAAiD;EAEhF;EACA,IAAI,QAAQ,OAAO,WAAW,KAAA,GAC5B,6BAA6B,QAAQ,OAAO,QAAQ,iBAAiB,OAAO;EAE9E,IAAI,QAAQ,OAAO,eAAe,KAAA,GAChC,6BAA6B,QAAQ,OAAO,YAAY,qBAAqB,OAAO;CAExF;CACA,IAAI,QAAQ,KAAK,SAAS,cACxB,wBAAwB,QAAQ,KAAK,KAAK,YAAY,OAAO;MACxD,IAAI,QAAQ,KAAK,QAAQ,MAC9B,MAAM,IAAI,MAAM,GAAG,QAAQ,mCAAmC;CAEhE,IAAI,QAAQ,iBAAiB,KAAA,GAC3B,wBAAwB,QAAQ,cAAc,gBAAgB,OAAO;CAEvE,IAAI,QAAQ,eAAe;EACzB,MAAM,EAAE,OAAO,WAAW,QAAQ;EAClC,IAAI,QAAQ,QACV,MAAM,IAAI,MAAM,GAAG,QAAQ,uDAAuD;EAEpF,IAAI,UAAU,QAAQ,WAAW,MAC/B,MAAM,IAAI,MAAM,GAAG,QAAQ,sDAAsD;EAEnF,IAAI,UAAU,MAAM,6BAA6B,OAAO,uBAAuB,OAAO;EACtF,IAAI,WAAW,MAAM,6BAA6B,QAAQ,wBAAwB,OAAO;CAC3F;AACF;AAEA,SAAS,6BAA6B,OAAe,OAAe,SAAuB;CACzF,IAAI,CAAC,OAAO,cAAc,KAAK,KAAK,QAAQ,GAC1C,MAAM,IAAI,MAAM,GAAG,QAAQ,IAAI,MAAM,qCAAqC;AAE9E;AAEA,SAAS,wBAAwB,OAAe,OAAe,SAAuB;CACpF,IAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GACrC,MAAM,IAAI,MAAM,GAAG,QAAQ,IAAI,MAAM,sCAAsC;AAE/E"}