@tangle-network/agent-eval 0.187.2 → 0.189.0

package/dist/{failure-cluster-OldNRoAt.d.ts → failure-cluster-CbgEDcFJ.d.ts}

@@ -1,5 +1,5 @@
-import { C as TraceEvent, _ as Span, f as Run, o as FailureClass } from "./schema-CR5cpjQ3.js";
-import { s as TraceStore } from "./store-BErPvYBr.js";
+import { f as Run, o as FailureClass, v as Span, w as TraceEvent } from "./schema-2g7trUwH.js";
+import { s as TraceStore } from "./store-C4avFjIi.js";
 //#region src/failure-taxonomy.d.ts
 /**
  * Who a failure is charged to. This is the axis a comparison reads before it
@@ -151,4 +151,4 @@ declare function failureClusterView(store: TraceStore, options?: {
 }): Promise<FailureClusterReport>;
 //#endregion
 export { FAILURE_BLAME as a, FailureContext as c, INFRA_FAILURE_BLAMES as d, classifyFailure as f, DEFAULT_FAILURE_REASON_RULES as i, FailureReasonRule as l, failureBlame as m, FailureClusterReport as n, FailureBlame as o, classifyFailureReason as p, failureClusterView as r, FailureClassification as s, FailureCluster as t, FailureRule as u };
-//# sourceMappingURL=failure-cluster-OldNRoAt.d.ts.map
+//# sourceMappingURL=failure-cluster-CbgEDcFJ.d.ts.map

package/dist/{failure-cluster-OldNRoAt.d.ts.map → failure-cluster-CbgEDcFJ.d.ts.map}

@@ -1 +1 @@
-{"version":3,"file":"failure-cluster-OldNRoAt.d.ts","names":[],"sources":["../src/failure-taxonomy.ts","../src/pipelines/failure-cluster.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;KA6CY;;;;;;;;;;;;;cAcC,eAAe,SAAS,OAAO,cAAc;;;;;;;;;;;;;;;;;cAmG7C,sBAAsB,YAAY;;iBAM/B,aAAa,cAAc,eAAe;UAIzC;EACf,KAAK;EACL,OAAO;EACP,QAAQ;;UAGO;EACf,cAAc;;EAEd,OAAO;EACP;EACA;EACA;;;UAIe;EACf;EACA,QAAQ,KAAK;IACX,cAAc;IACd;;IAEA,QAAQ;IACR;IACA;;;;;;;;;;UAWa;;EAEf,cAAc;EACd,SAAS;;EAET,QAAQ;;;;;;;;;;cAWG,uCAAuC;;;;;;;;;;;iBA6DpC,sBACd,mCACA,iBAAgB,sBACf;;;;;;;;;;iBAkba,gBACd,KAAK,gBACL,QAAO,eACP,uBAAsB,sBACrB;;;UCpsBc;EACf,cAAc;;EAEd;;EAEA;;;;;;;EAOA;EACA;EACA;EACA;EACA;;UAGe;EACf,UAAU;EACV;EACA;;iBAGoB,mBACpB,OAAO,YACP;EAAW,QAAQ;EAAe;IACjC,QAAQ"}
+{"version":3,"file":"failure-cluster-CbgEDcFJ.d.ts","names":[],"sources":["../src/failure-taxonomy.ts","../src/pipelines/failure-cluster.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;KA6CY;;;;;;;;;;;;;cAcC,eAAe,SAAS,OAAO,cAAc;;;;;;;;;;;;;;;;;cAmG7C,sBAAsB,YAAY;;iBAM/B,aAAa,cAAc,eAAe;UAIzC;EACf,KAAK;EACL,OAAO;EACP,QAAQ;;UAGO;EACf,cAAc;;EAEd,OAAO;EACP;EACA;EACA;;;UAIe;EACf;EACA,QAAQ,KAAK;IACX,cAAc;IACd;;IAEA,QAAQ;IACR;IACA;;;;;;;;;;UAWa;;EAEf,cAAc;EACd,SAAS;;EAET,QAAQ;;;;;;;;;;cAWG,uCAAuC;;;;;;;;;;;iBA6DpC,sBACd,mCACA,iBAAgB,sBACf;;;;;;;;;;iBAkba,gBACd,KAAK,gBACL,QAAO,eACP,uBAAsB,sBACrB;;;UCpsBc;EACf,cAAc;;EAEd;;EAEA;;;;;;;EAOA;EACA;EACA;EACA;EACA;;UAGe;EACf,UAAU;EACV;EACA;;iBAGoB,mBACpB,OAAO,YACP;EAAW,QAAQ;EAAe;IACjC,QAAQ"}

package/dist/failure-taxonomy-CjabTATj.js

@@ -0,0 +1,538 @@
+//#region src/failure-taxonomy.ts
+/**
+* Blame for every canonical class. The `Record` is exhaustive on purpose: a
+* class added to `FAILURE_CLASSES` without a blame here fails to compile, so
+* the axis cannot silently acquire a hole that reads as `undefined` — and a
+* missing blame would void or charge a run by accident.
+*
+* The assignment rule, applied member by member: a class is `machine` only
+* when the run demonstrably never reached a model. An environment that refuses
+* one specific action the agent chose (a denied write, an out-of-scope call,
+* an approval hold) stays `agent`, because the agent selected the action and
+* every arm faces the same policy.
+*/
+const FAILURE_BLAME = {
+	success: "none",
+	reasoning_error: "agent",
+	tool_selection_error: "agent",
+	tool_argument_error: "agent",
+	tool_recovery_failure: "agent",
+	hallucination: "agent",
+	instruction_following: "agent",
+	safety_refusal_miss: "agent",
+	policy_violation: "agent",
+	budget_exceeded: "agent",
+	format_drift: "agent",
+	permission_escalation: "agent",
+	pii_leak: "agent",
+	cost_overrun: "agent",
+	timeout: "agent",
+	sandbox_failure: "agent",
+	missing_user_data: "agent",
+	missing_domain_data: "agent",
+	missing_codebase_context: "agent",
+	missing_runtime_context: "agent",
+	stale_external_data: "agent",
+	bad_retrieval: "agent",
+	insufficient_evidence: "agent",
+	contradictory_evidence: "agent",
+	ambiguous_user_intent: "agent",
+	missing_credentials: "machine",
+	missing_integration_connection: "machine",
+	integration_auth_expired: "machine",
+	bad_integration_manifest: "machine",
+	knowledge_readiness_blocked: "machine",
+	missing_integration_scope: "agent",
+	integration_approval_required: "agent",
+	unsafe_integration_write_denied: "agent",
+	integration_provider_failure: "provider",
+	bridge_unreachable: "machine",
+	bridge_lane_saturated: "machine",
+	router_usage_mismatch: "machine",
+	session_profile_conflict: "machine",
+	runtime_restart: "machine",
+	root_driver_failed: "machine",
+	box_provision_failure: "machine",
+	box_teardown_failure: "machine",
+	profile_refused: "agent",
+	empty_completion: "provider",
+	stream_protocol_error: "provider",
+	stream_incomplete: "provider",
+	message_sealed: "provider",
+	process_killed: "provider",
+	terminated: "provider",
+	cancelled: "unknown",
+	traversal_cap_exhausted: "agent",
+	unreported: "unknown",
+	unknown: "unknown"
+};
+/**
+* The blames that mean the run got no usable turn out of the machine or the
+* provider.
+*
+* This answers a different question from `machine` alone, and the two must not
+* be confused. "Is this run evidence about the agent?" is answered by `machine`
+* and by nothing else — a `provider` death stays charged, because every arm
+* faces the same provider on the same day. "Is this seat producing usable turns
+* at all?" is answered by this set, because a seat that only ever returns empty
+* completions and torn streams is as dead as one whose bridge refuses, however
+* the individual runs are charged.
+*
+* So a reader counting an outage streak counts these; a reader deciding whether
+* to void a run reads `machine`. Nothing an operator did lands here: `cancelled`
+* is blamed `unknown` for exactly that reason.
+*/
+const INFRA_FAILURE_BLAMES = /* @__PURE__ */ new Set(["machine", "provider"]);
+/** Who the named class is charged to. */
+function failureBlame(failureClass) {
+	return FAILURE_BLAME[failureClass];
+}
+/**
+* The default ordered reason rules, measured against the transport death
+* strings this stack records — runtime settle reasons, bridge errors, sandbox
+* provisioning errors, and provider stream errors.
+*
+* A consumer with a different transport passes its own set to
+* `classifyFailureReason`; nothing here is a gate.
+*/
+const DEFAULT_FAILURE_REASON_RULES = [
+	{
+		failureClass: "bridge_unreachable",
+		pattern: /ECONNREFUSED|disconnected before terminal acknowledgement|bridge .* unreachable/i
+	},
+	{
+		failureClass: "bridge_lane_saturated",
+		pattern: /acquire timeout after \d+ms|in_flight=\d+\/\d+/i
+	},
+	{
+		failureClass: "router_usage_mismatch",
+		pattern: /did not match its recorded usage/i
+	},
+	{
+		failureClass: "session_profile_conflict",
+		pattern: /bound to a different AgentProfile|profile materialization changed across session turns/i
+	},
+	{
+		failureClass: "runtime_restart",
+		pattern: /agent runtime restarted/i
+	},
+	{
+		failureClass: "root_driver_failed",
+		pattern: /root driver failed/i
+	},
+	{
+		failureClass: "box_provision_failure",
+		pattern: /Sandbox create did not complete within|failed to (create|provision) (a )?box|box .*: (create|provision) failed/i
+	},
+	{
+		failureClass: "box_teardown_failure",
+		pattern: /teardown reported destroyed=false/i
+	},
+	{
+		failureClass: "profile_refused",
+		pattern: /workspace materialization failed|cannot replace its harness's system prompt|has no explicit entry in .*models\.json|conflicts with agent_profile\.model/i
+	},
+	{
+		failureClass: "empty_completion",
+		pattern: /without emitting any visible output/i
+	},
+	{
+		failureClass: "stream_protocol_error",
+		pattern: /JSON error injected into SSE stream/i
+	},
+	{
+		failureClass: "stream_incomplete",
+		pattern: /Prompt stream ended before a terminal event/i
+	},
+	{
+		failureClass: "message_sealed",
+		pattern: /already rolled up \(sealed\)/i
+	},
+	{
+		failureClass: "process_killed",
+		pattern: /killed by SIGKILL|exited with code null/i
+	},
+	{
+		failureClass: "cancelled",
+		pattern: /cancelled by user/i
+	},
+	{
+		failureClass: "terminated",
+		pattern: /(^|: )terminated\b/i
+	},
+	{
+		failureClass: "budget_exceeded",
+		pattern: /budget pool: ticket \d+ spent/i
+	},
+	{
+		failureClass: "timeout",
+		pattern: /deadline|timed out|timeout/i
+	},
+	{
+		failureClass: "traversal_cap_exhausted",
+		pattern: /traversal|iteration cap|exhausted/i
+	}
+];
+/**
+* Name the failure one reason string describes.
+*
+* This is the entry point for a run that died before it produced a span: a
+* terminal record, a settle row, a spawn journal line. An unmatched reason is
+* never dropped — it returns `unknown` and keeps the text, so a new failure
+* mode shows up as a named gap instead of a silent bucket. An absent reason
+* returns `unreported`, which is a different fact: the record itself is
+* incomplete.
+*/
+function classifyFailureReason(reason, rules = DEFAULT_FAILURE_REASON_RULES) {
+	const text = String(reason ?? "").trim();
+	if (text === "") return {
+		failureClass: "unreported",
+		blame: FAILURE_BLAME.unreported,
+		reason: ""
+	};
+	for (const rule of rules) if (rule.pattern.test(text)) return {
+		failureClass: rule.failureClass,
+		blame: rule.blame ?? FAILURE_BLAME[rule.failureClass],
+		reason: text
+	};
+	return {
+		failureClass: "unknown",
+		blame: FAILURE_BLAME.unknown,
+		reason: text
+	};
+}
+const DEFAULT_RULES = [
+	{
+		id: "explicit-outcome",
+		match: ({ run }) => {
+			const fc = run.outcome?.failureClass;
+			if (fc && fc !== "unknown") return {
+				failureClass: fc,
+				reason: "outcome.failureClass set explicitly"
+			};
+			return null;
+		}
+	},
+	{
+		id: "knowledge-readiness-blocked",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "readiness_scored" && e.payload.passed === false);
+			return event ? {
+				failureClass: "knowledge_readiness_blocked",
+				reason: "knowledge readiness report blocked execution",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "bad-integration-manifest",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && (e.payload.kind === "integration_manifest_validated" && e.payload.valid === false || e.payload.kind === "integration_invoke_failed" && e.payload.code === "manifest_invalid"));
+			return event ? {
+				failureClass: "bad_integration_manifest",
+				reason: "integration manifest validation failed before launch",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "missing-integration-connection",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "integration_manifest_resolved" && hasResolutionStatus(e.payload, "missing_connection"));
+			return event ? {
+				failureClass: "missing_integration_connection",
+				reason: "required integration connection was missing",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "missing-integration-scope",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && (e.payload.kind === "integration_manifest_resolved" && hasMissingScopes(e.payload) || e.payload.kind === "integration_invoke_failed" && e.payload.code === "scope_denied"));
+			return event ? {
+				failureClass: "missing_integration_scope",
+				reason: "integration grant or connection lacks required scopes",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "integration-approval-required",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && (e.payload.kind === "integration_invoke" && e.payload.status === "approval_required" || e.payload.kind === "integration_invoke_failed" && e.payload.code === "approval_required" || e.payload.kind === "integration_approval_required"));
+			return event ? {
+				failureClass: "integration_approval_required",
+				reason: "integration write paused for user approval",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "integration-auth-expired",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && (e.payload.code === "auth_expired" || e.payload.code === "connection_not_active" || e.payload.code === "capability_expired" || e.payload.status === "expired"));
+			return event ? {
+				failureClass: "integration_auth_expired",
+				reason: "integration connection or capability expired",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "unsafe-integration-write-denied",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && (e.payload.code === "unsafe_write_denied" || e.payload.code === "policy_denied" || e.payload.code === "action_denied"));
+			return event ? {
+				failureClass: "unsafe_integration_write_denied",
+				reason: "integration write was denied by policy or capability scope",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "integration-provider-failure",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && ![
+				"scope_denied",
+				"approval_required",
+				"auth_expired",
+				"connection_not_active",
+				"capability_expired",
+				"unsafe_write_denied",
+				"policy_denied",
+				"action_denied",
+				"manifest_invalid"
+			].includes(String(e.payload.code)));
+			return event ? {
+				failureClass: "integration_provider_failure",
+				reason: "integration provider invocation failed",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "missing-credentials",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.category === "credential_or_secret");
+			return event ? {
+				failureClass: "missing_credentials",
+				reason: "required credential or secret was missing",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "bad-retrieval",
+		match: ({ run, spans }) => {
+			if (run.outcome?.pass !== false) return null;
+			const retrieval = spans.find((s) => s.kind === "retrieval" && (s.hits.length === 0 || s.hits.every((hit) => hit.score <= 0)));
+			return retrieval ? {
+				failureClass: "bad_retrieval",
+				reason: "retrieval returned no useful hits for a failed run",
+				triggerSpanId: retrieval.spanId
+			} : null;
+		}
+	},
+	{
+		id: "insufficient-evidence",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.reason === "insufficient_evidence");
+			return event ? {
+				failureClass: "insufficient_evidence",
+				reason: "task proceeded with insufficient supporting evidence",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "contradictory-evidence",
+		match: ({ events }) => {
+			const event = events.find((e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.reason === "contradictory_evidence");
+			return event ? {
+				failureClass: "contradictory_evidence",
+				reason: "supporting evidence contradicted itself",
+				triggerEventId: event.eventId
+			} : null;
+		}
+	},
+	{
+		id: "budget-breach",
+		match: ({ events }) => {
+			const breach = events.find((e) => e.kind === "budget_breach");
+			return breach ? {
+				failureClass: "budget_exceeded",
+				reason: `budget breached on ${breach.payload.dimension ?? "unknown dimension"}`,
+				triggerEventId: breach.eventId
+			} : null;
+		}
+	},
+	{
+		id: "policy-violation",
+		match: ({ events }) => {
+			const e = events.find((x) => x.kind === "policy_violation");
+			return e ? {
+				failureClass: "policy_violation",
+				reason: "policy_violation event emitted",
+				triggerEventId: e.eventId
+			} : null;
+		}
+	},
+	{
+		id: "sandbox-failure",
+		match: ({ spans }) => {
+			const s = spans.find((x) => x.kind === "sandbox" && typeof x.exitCode === "number" && x.exitCode !== 0);
+			if (!s) return null;
+			return {
+				failureClass: "sandbox_failure",
+				reason: `sandbox exited ${s.exitCode}`,
+				triggerSpanId: s.spanId
+			};
+		}
+	},
+	{
+		id: "timeout",
+		match: ({ run, events }) => {
+			if (run.status !== "aborted") return null;
+			const hasTimeout = events.some((e) => e.kind === "error" && String(e.payload.reason ?? "").toLowerCase().includes("timeout"));
+			const note = (run.outcome?.notes ?? "").toLowerCase();
+			if (hasTimeout || note.includes("timeout") || note.includes("deadline")) return {
+				failureClass: "timeout",
+				reason: "timeout signal observed"
+			};
+			return null;
+		}
+	},
+	{
+		id: "tool-recovery-failure",
+		match: ({ spans }) => {
+			const tools = spans.filter((s) => s.kind === "tool");
+			const byTool = /* @__PURE__ */ new Map();
+			for (const t of tools) {
+				const name = t.toolName;
+				const arr = byTool.get(name) ?? [];
+				arr.push(t);
+				byTool.set(name, arr);
+			}
+			for (const [name, arr] of byTool) {
+				const errs = arr.filter((s) => s.status === "error");
+				if (errs.length >= 3 && errs.length === arr.length) return {
+					failureClass: "tool_recovery_failure",
+					reason: `${errs.length} consecutive errors on tool "${name}"`,
+					triggerSpanId: errs[errs.length - 1].spanId
+				};
+			}
+			return null;
+		}
+	},
+	{
+		id: "tool-selection-error",
+		match: ({ run, spans }) => {
+			if (run.outcome?.pass !== false) return null;
+			const hasToolsAvailable = spans.some((s) => s.kind === "agent" && s.attributes?.toolsAvailable !== void 0 && s.attributes?.toolsAvailable > 0);
+			const tools = spans.filter((s) => s.kind === "tool");
+			if (hasToolsAvailable && tools.length === 0) return {
+				failureClass: "tool_selection_error",
+				reason: "tools were available but none were called"
+			};
+			return null;
+		}
+	},
+	{
+		id: "format-drift",
+		match: ({ spans }) => {
+			const judge = spans.find((s) => s.kind === "judge" && s.dimension === "format" && s.score < .5);
+			return judge ? {
+				failureClass: "format_drift",
+				reason: "format judge scored below 0.5",
+				triggerSpanId: judge.spanId
+			} : null;
+		}
+	}
+];
+function hasResolutionStatus(payload, status) {
+	if (status === "missing_connection" && stringArray(payload.missingConnections).length > 0) return true;
+	return resolutionItems(payload).some((item) => item.status === status);
+}
+function hasMissingScopes(payload) {
+	if (stringArray(payload.missingScopes).length > 0) return true;
+	return resolutionItems(payload).some((item) => Array.isArray(item.missingScopes) && item.missingScopes.length > 0);
+}
+function resolutionItems(payload) {
+	return [
+		...records(payload.missing),
+		...records(payload.optionalMissing),
+		...records(payload.ready)
+	];
+}
+function records(value) {
+	if (!Array.isArray(value)) return [];
+	return value.filter((item) => Boolean(item) && typeof item === "object" && !Array.isArray(item));
+}
+function stringArray(value) {
+	return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
+}
+/**
+* Classify the failure mode of a run using an ordered rule list.
+*
+* Structural rules run first, in their declared order. When none matches, the
+* reason strings the run did record are classified with
+* `classifyFailureReason`, so a run whose only evidence is a transport error
+* on an error event is named instead of landing in `unknown`. Only a run that
+* would already have returned `unknown` can change class this way.
+*/
+function classifyFailure(ctx, rules = DEFAULT_RULES, reasonRules = DEFAULT_FAILURE_REASON_RULES) {
+	if (ctx.run.outcome?.pass !== false && ctx.run.status === "completed") return {
+		failureClass: "success",
+		blame: FAILURE_BLAME.success,
+		reason: "run completed with pass=true (or no explicit fail)"
+	};
+	for (const rule of rules) {
+		const hit = rule.match(ctx);
+		if (hit) return {
+			...hit,
+			blame: hit.blame ?? FAILURE_BLAME[hit.failureClass]
+		};
+	}
+	for (const candidate of recordedReasons(ctx)) {
+		const classified = classifyFailureReason(candidate.text, reasonRules);
+		if (classified.failureClass === "unknown" || classified.failureClass === "unreported") continue;
+		return {
+			...classified,
+			...candidate.eventId === void 0 ? {} : { triggerEventId: candidate.eventId }
+		};
+	}
+	return {
+		failureClass: "unknown",
+		blame: FAILURE_BLAME.unknown,
+		reason: "no rule matched; run failed for unclassified reason"
+	};
+}
+/**
+* The reason strings a failed run recorded, error events first because they
+* sit closest to the failure, then the outcome notes.
+*/
+function recordedReasons(ctx) {
+	const found = [];
+	for (const event of ctx.events) {
+		if (event.kind !== "error") continue;
+		for (const field of [
+			"reason",
+			"message",
+			"error"
+		]) {
+			const value = event.payload[field];
+			if (typeof value === "string" && value.trim()) found.push({
+				text: value,
+				eventId: event.eventId
+			});
+		}
+	}
+	const notes = ctx.run.outcome?.notes;
+	if (typeof notes === "string" && notes.trim()) found.push({ text: notes });
+	return found;
+}
+//#endregion
+export { classifyFailure as a, INFRA_FAILURE_BLAMES as i, DEFAULT_RULES as n, classifyFailureReason as o, FAILURE_BLAME as r, failureBlame as s, DEFAULT_FAILURE_REASON_RULES as t };
+
+//# sourceMappingURL=failure-taxonomy-CjabTATj.js.map

package/dist/failure-taxonomy-CjabTATj.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"failure-taxonomy-CjabTATj.js","names":[],"sources":["../src/failure-taxonomy.ts"],"sourcesContent":["/**\n * Failure taxonomy — canonical classes, who each one is charged to, and two\n * default classifiers.\n *\n * Every failed run should end up in a named class. The classifiers here are\n * rule-based (fast, deterministic); an LLM fallback can be added by the\n * consumer for novel cases and trained into the rule base over time.\n *\n * Two entry points, because a dead run does not always leave a trace:\n *   - `classifyFailure(ctx)` reads a run with its spans and events, and\n *     persists as `Run.outcome.failureClass`;\n *   - `classifyFailureReason(text)` reads the one thing a run that died before\n *     any span always leaves — a reason string on its terminal record.\n * `classifyFailure` calls the reason classifier itself when no structural rule\n * matches, so a run whose only evidence is `ECONNREFUSED` no longer lands in\n * `unknown`.\n *\n * Every classification carries a `blame`. Without it a comparison charges\n * infrastructure outages to the agent under test: a run that never reached a\n * model is not evidence about the agent and must be voided, not counted as a\n * loss.\n */\n\nimport type { FailureClass, Run, Span, TraceEvent } from './trace/schema'\nimport { FAILURE_CLASSES } from './trace/schema'\n\nexport { FAILURE_CLASSES, type FailureClass }\n\n/**\n * Who a failure is charged to. This is the axis a comparison reads before it\n * counts a loss.\n *\n * `machine`  — the run never reached a model. The bridge refused, its lane was\n *              full, the box never provisioned, the credential was absent. An\n *              agent cannot route around this, so a run that recorded nothing\n *              behind only machine failures is not evidence about the agent.\n * `provider` — the model was reached and the turn did not complete: an empty\n *              completion, a torn stream, a killed process. Every arm faces\n *              the same provider on the same day, so this stays charged.\n * `agent`    — the agent's own reasoning, tool use, budgeting, or scheduling\n *              produced the failure. Charged.\n * `unknown`  — the record does not say. Never voids a run: an unread reason is\n *              not a proven outage.\n * `none`     — no failure to charge.\n */\nexport type FailureBlame = 'none' | 'machine' | 'provider' | 'agent' | 'unknown'\n\n/**\n * Blame for every canonical class. The `Record` is exhaustive on purpose: a\n * class added to `FAILURE_CLASSES` without a blame here fails to compile, so\n * the axis cannot silently acquire a hole that reads as `undefined` — and a\n * missing blame would void or charge a run by accident.\n *\n * The assignment rule, applied member by member: a class is `machine` only\n * when the run demonstrably never reached a model. An environment that refuses\n * one specific action the agent chose (a denied write, an out-of-scope call,\n * an approval hold) stays `agent`, because the agent selected the action and\n * every arm faces the same policy.\n */\nexport const FAILURE_BLAME: Readonly<Record<FailureClass, FailureBlame>> = {\n  success: 'none',\n\n  // Behaviour: the agent ran and its own work is what failed.\n  reasoning_error: 'agent',\n  tool_selection_error: 'agent',\n  tool_argument_error: 'agent',\n  tool_recovery_failure: 'agent',\n  hallucination: 'agent',\n  instruction_following: 'agent',\n  safety_refusal_miss: 'agent',\n  policy_violation: 'agent',\n  budget_exceeded: 'agent',\n  format_drift: 'agent',\n  permission_escalation: 'agent',\n  pii_leak: 'agent',\n  cost_overrun: 'agent',\n  timeout: 'agent',\n  // A command the agent ran exited non-zero inside a live box. The box worked.\n  sandbox_failure: 'agent',\n\n  // Knowledge and evidence: a model answered, and how the agent handled the\n  // gap is exactly what is under test. Every arm faces the same gap.\n  missing_user_data: 'agent',\n  missing_domain_data: 'agent',\n  missing_codebase_context: 'agent',\n  missing_runtime_context: 'agent',\n  stale_external_data: 'agent',\n  bad_retrieval: 'agent',\n  insufficient_evidence: 'agent',\n  contradictory_evidence: 'agent',\n  ambiguous_user_intent: 'agent',\n\n  // Environment provisioning: the run could not reach the model or was stopped\n  // before execution, through no act of the agent.\n  missing_credentials: 'machine',\n  missing_integration_connection: 'machine',\n  integration_auth_expired: 'machine',\n  bad_integration_manifest: 'machine',\n  knowledge_readiness_blocked: 'machine',\n\n  // Environment refusing one action the agent chose.\n  missing_integration_scope: 'agent',\n  integration_approval_required: 'agent',\n  unsafe_integration_write_denied: 'agent',\n\n  // A third-party invocation the agent reached and that did not complete.\n  integration_provider_failure: 'provider',\n\n  // Transport: never reached a model.\n  bridge_unreachable: 'machine',\n  bridge_lane_saturated: 'machine',\n  router_usage_mismatch: 'machine',\n  session_profile_conflict: 'machine',\n  runtime_restart: 'machine',\n  root_driver_failed: 'machine',\n  box_provision_failure: 'machine',\n  box_teardown_failure: 'machine',\n\n  // The agent wrote the broken instrument, so the run is charged: the machine\n  // answered, and a run dying on its own profile is not an outage.\n  profile_refused: 'agent',\n\n  // Transport: reached the model, the turn did not complete.\n  empty_completion: 'provider',\n  stream_protocol_error: 'provider',\n  stream_incomplete: 'provider',\n  message_sealed: 'provider',\n  process_killed: 'provider',\n  terminated: 'provider',\n\n  // An operator stopping a run is not the machine, the provider or the agent.\n  // Charging it to any of the three would make a deliberate stop read as a\n  // failure of the thing that was stopped, and counting it as infrastructure\n  // would turn a cancelled batch into an outage streak.\n  cancelled: 'unknown',\n\n  traversal_cap_exhausted: 'agent',\n\n  unreported: 'unknown',\n  unknown: 'unknown',\n}\n\n/**\n * The blames that mean the run got no usable turn out of the machine or the\n * provider.\n *\n * This answers a different question from `machine` alone, and the two must not\n * be confused. \"Is this run evidence about the agent?\" is answered by `machine`\n * and by nothing else — a `provider` death stays charged, because every arm\n * faces the same provider on the same day. \"Is this seat producing usable turns\n * at all?\" is answered by this set, because a seat that only ever returns empty\n * completions and torn streams is as dead as one whose bridge refuses, however\n * the individual runs are charged.\n *\n * So a reader counting an outage streak counts these; a reader deciding whether\n * to void a run reads `machine`. Nothing an operator did lands here: `cancelled`\n * is blamed `unknown` for exactly that reason.\n */\nexport const INFRA_FAILURE_BLAMES: ReadonlySet<FailureBlame> = new Set<FailureBlame>([\n  'machine',\n  'provider',\n])\n\n/** Who the named class is charged to. */\nexport function failureBlame(failureClass: FailureClass): FailureBlame {\n  return FAILURE_BLAME[failureClass]\n}\n\nexport interface FailureContext {\n  run: Run\n  spans: Span[]\n  events: TraceEvent[]\n}\n\nexport interface FailureClassification {\n  failureClass: FailureClass\n  /** Who the failure is charged to. Derived from the class unless a rule overrode it. */\n  blame: FailureBlame\n  reason: string\n  triggerSpanId?: string\n  triggerEventId?: string\n}\n\n/** Ordered rules — first match wins. */\nexport interface FailureRule {\n  id: string\n  match: (ctx: FailureContext) => {\n    failureClass: FailureClass\n    reason: string\n    /** Overrides the class's canonical blame. State why at the call site. */\n    blame?: FailureBlame\n    triggerSpanId?: string\n    triggerEventId?: string\n  } | null\n}\n\n/**\n * One ordered rule over a failure reason string.\n *\n * Order is load-bearing, because one reason string carries two signals: a seat\n * killed by SIGKILL while the bridge refuses connections is a bridge failure,\n * not a process failure, so the bridge patterns are tested first.\n */\nexport interface FailureReasonRule {\n  /** The class the first matching pattern names. */\n  failureClass: FailureClass\n  pattern: RegExp\n  /** Overrides the class's canonical blame. State why at the call site. */\n  blame?: FailureBlame\n}\n\n/**\n * The default ordered reason rules, measured against the transport death\n * strings this stack records — runtime settle reasons, bridge errors, sandbox\n * provisioning errors, and provider stream errors.\n *\n * A consumer with a different transport passes its own set to\n * `classifyFailureReason`; nothing here is a gate.\n */\nexport const DEFAULT_FAILURE_REASON_RULES: readonly FailureReasonRule[] = [\n  {\n    failureClass: 'bridge_unreachable',\n    pattern: /ECONNREFUSED|disconnected before terminal acknowledgement|bridge .* unreachable/i,\n  },\n  {\n    failureClass: 'bridge_lane_saturated',\n    pattern: /acquire timeout after \\d+ms|in_flight=\\d+\\/\\d+/i,\n  },\n  { failureClass: 'router_usage_mismatch', pattern: /did not match its recorded usage/i },\n  {\n    failureClass: 'session_profile_conflict',\n    pattern:\n      /bound to a different AgentProfile|profile materialization changed across session turns/i,\n  },\n  { failureClass: 'runtime_restart', pattern: /agent runtime restarted/i },\n  { failureClass: 'root_driver_failed', pattern: /root driver failed/i },\n  {\n    failureClass: 'box_provision_failure',\n    pattern:\n      /Sandbox create did not complete within|failed to (create|provision) (a )?box|box .*: (create|provision) failed/i,\n  },\n  { failureClass: 'box_teardown_failure', pattern: /teardown reported destroyed=false/i },\n  {\n    failureClass: 'profile_refused',\n    pattern:\n      /workspace materialization failed|cannot replace its harness's system prompt|has no explicit entry in .*models\\.json|conflicts with agent_profile\\.model/i,\n  },\n  { failureClass: 'empty_completion', pattern: /without emitting any visible output/i },\n  { failureClass: 'stream_protocol_error', pattern: /JSON error injected into SSE stream/i },\n  {\n    failureClass: 'stream_incomplete',\n    pattern: /Prompt stream ended before a terminal event/i,\n  },\n  { failureClass: 'message_sealed', pattern: /already rolled up \\(sealed\\)/i },\n  { failureClass: 'process_killed', pattern: /killed by SIGKILL|exited with code null/i },\n  { failureClass: 'cancelled', pattern: /cancelled by user/i },\n  { failureClass: 'terminated', pattern: /(^|: )terminated\\b/i },\n  { failureClass: 'budget_exceeded', pattern: /budget pool: ticket \\d+ spent/i },\n  // Deliberately last and deliberately broad: by the time a reason reaches this\n  // rule every transport pattern above has declined it, so what remains is the\n  // agent's own clock. That boundary is the rule set's, not a fact about the\n  // word: a transport whose deadline wording is not matched above would be\n  // charged to the agent here, which is the mis-charge this taxonomy exists to\n  // stop. A consumer whose machine has its own deadline wording adds a rule for\n  // it AHEAD of this one rather than widening this one — inventing patterns for\n  // strings nobody has recorded is how a rule set stops describing anything.\n  { failureClass: 'timeout', pattern: /deadline|timed out|timeout/i },\n  { failureClass: 'traversal_cap_exhausted', pattern: /traversal|iteration cap|exhausted/i },\n]\n\n/**\n * Name the failure one reason string describes.\n *\n * This is the entry point for a run that died before it produced a span: a\n * terminal record, a settle row, a spawn journal line. An unmatched reason is\n * never dropped — it returns `unknown` and keeps the text, so a new failure\n * mode shows up as a named gap instead of a silent bucket. An absent reason\n * returns `unreported`, which is a different fact: the record itself is\n * incomplete.\n */\nexport function classifyFailureReason(\n  reason: string | null | undefined,\n  rules: readonly FailureReasonRule[] = DEFAULT_FAILURE_REASON_RULES,\n): FailureClassification {\n  const text = String(reason ?? '').trim()\n  if (text === '') {\n    return { failureClass: 'unreported', blame: FAILURE_BLAME.unreported, reason: '' }\n  }\n  for (const rule of rules) {\n    if (rule.pattern.test(text)) {\n      return {\n        failureClass: rule.failureClass,\n        blame: rule.blame ?? FAILURE_BLAME[rule.failureClass],\n        reason: text,\n      }\n    }\n  }\n  return { failureClass: 'unknown', blame: FAILURE_BLAME.unknown, reason: text }\n}\n\nexport const DEFAULT_RULES: FailureRule[] = [\n  // Outcome already named? Respect it.\n  {\n    id: 'explicit-outcome',\n    match: ({ run }) => {\n      const fc = run.outcome?.failureClass\n      if (fc && fc !== 'unknown')\n        return { failureClass: fc, reason: 'outcome.failureClass set explicitly' }\n      return null\n    },\n  },\n  {\n    id: 'knowledge-readiness-blocked',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'readiness_scored' &&\n          e.payload.passed === false,\n      )\n      return event\n        ? {\n            failureClass: 'knowledge_readiness_blocked',\n            reason: 'knowledge readiness report blocked execution',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'bad-integration-manifest',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          ((e.payload.kind === 'integration_manifest_validated' && e.payload.valid === false) ||\n            (e.payload.kind === 'integration_invoke_failed' &&\n              e.payload.code === 'manifest_invalid')),\n      )\n      return event\n        ? {\n            failureClass: 'bad_integration_manifest',\n            reason: 'integration manifest validation failed before launch',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'missing-integration-connection',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'integration_manifest_resolved' &&\n          hasResolutionStatus(e.payload, 'missing_connection'),\n      )\n      return event\n        ? {\n            failureClass: 'missing_integration_connection',\n            reason: 'required integration connection was missing',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'missing-integration-scope',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          ((e.payload.kind === 'integration_manifest_resolved' && hasMissingScopes(e.payload)) ||\n            (e.payload.kind === 'integration_invoke_failed' && e.payload.code === 'scope_denied')),\n      )\n      return event\n        ? {\n            failureClass: 'missing_integration_scope',\n            reason: 'integration grant or connection lacks required scopes',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'integration-approval-required',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          ((e.payload.kind === 'integration_invoke' && e.payload.status === 'approval_required') ||\n            (e.payload.kind === 'integration_invoke_failed' &&\n              e.payload.code === 'approval_required') ||\n            e.payload.kind === 'integration_approval_required'),\n      )\n      return event\n        ? {\n            failureClass: 'integration_approval_required',\n            reason: 'integration write paused for user approval',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'integration-auth-expired',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'integration_invoke_failed' &&\n          (e.payload.code === 'auth_expired' ||\n            e.payload.code === 'connection_not_active' ||\n            e.payload.code === 'capability_expired' ||\n            e.payload.status === 'expired'),\n      )\n      return event\n        ? {\n            failureClass: 'integration_auth_expired',\n            reason: 'integration connection or capability expired',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'unsafe-integration-write-denied',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'integration_invoke_failed' &&\n          (e.payload.code === 'unsafe_write_denied' ||\n            e.payload.code === 'policy_denied' ||\n            e.payload.code === 'action_denied'),\n      )\n      return event\n        ? {\n            failureClass: 'unsafe_integration_write_denied',\n            reason: 'integration write was denied by policy or capability scope',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'integration-provider-failure',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'integration_invoke_failed' &&\n          ![\n            'scope_denied',\n            'approval_required',\n            'auth_expired',\n            'connection_not_active',\n            'capability_expired',\n            'unsafe_write_denied',\n            'policy_denied',\n            'action_denied',\n            'manifest_invalid',\n          ].includes(String(e.payload.code)),\n      )\n      return event\n        ? {\n            failureClass: 'integration_provider_failure',\n            reason: 'integration provider invocation failed',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'missing-credentials',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'knowledge_gap' &&\n          e.payload.category === 'credential_or_secret',\n      )\n      return event\n        ? {\n            failureClass: 'missing_credentials',\n            reason: 'required credential or secret was missing',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'bad-retrieval',\n    match: ({ run, spans }) => {\n      if (run.outcome?.pass !== false) return null\n      const retrieval = spans.find(\n        (s) =>\n          s.kind === 'retrieval' && (s.hits.length === 0 || s.hits.every((hit) => hit.score <= 0)),\n      )\n      return retrieval\n        ? {\n            failureClass: 'bad_retrieval',\n            reason: 'retrieval returned no useful hits for a failed run',\n            triggerSpanId: retrieval.spanId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'insufficient-evidence',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'knowledge_gap' &&\n          e.payload.reason === 'insufficient_evidence',\n      )\n      return event\n        ? {\n            failureClass: 'insufficient_evidence',\n            reason: 'task proceeded with insufficient supporting evidence',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  {\n    id: 'contradictory-evidence',\n    match: ({ events }) => {\n      const event = events.find(\n        (e) =>\n          e.kind === 'custom' &&\n          e.payload.kind === 'knowledge_gap' &&\n          e.payload.reason === 'contradictory_evidence',\n      )\n      return event\n        ? {\n            failureClass: 'contradictory_evidence',\n            reason: 'supporting evidence contradicted itself',\n            triggerEventId: event.eventId,\n          }\n        : null\n    },\n  },\n  // Budget breach events\n  {\n    id: 'budget-breach',\n    match: ({ events }) => {\n      const breach = events.find((e) => e.kind === 'budget_breach')\n      return breach\n        ? {\n            failureClass: 'budget_exceeded',\n            reason: `budget breached on ${breach.payload.dimension ?? 'unknown dimension'}`,\n            triggerEventId: breach.eventId,\n          }\n        : null\n    },\n  },\n  // Policy violations\n  {\n    id: 'policy-violation',\n    match: ({ events }) => {\n      const e = events.find((x) => x.kind === 'policy_violation')\n      return e\n        ? {\n            failureClass: 'policy_violation',\n            reason: 'policy_violation event emitted',\n            triggerEventId: e.eventId,\n          }\n        : null\n    },\n  },\n  // Sandbox non-zero exit code\n  {\n    id: 'sandbox-failure',\n    match: ({ spans }) => {\n      const s = spans.find(\n        (x) => x.kind === 'sandbox' && typeof x.exitCode === 'number' && x.exitCode !== 0,\n      )\n      if (!s) return null\n      return {\n        failureClass: 'sandbox_failure',\n        reason: `sandbox exited ${(s as Extract<Span, { kind: 'sandbox' }>).exitCode}`,\n        triggerSpanId: s.spanId,\n      }\n    },\n  },\n  // Timeout: run aborted by external signal\n  {\n    id: 'timeout',\n    match: ({ run, events }) => {\n      if (run.status !== 'aborted') return null\n      const hasTimeout = events.some(\n        (e) =>\n          e.kind === 'error' &&\n          String(e.payload.reason ?? '')\n            .toLowerCase()\n            .includes('timeout'),\n      )\n      const note = (run.outcome?.notes ?? '').toLowerCase()\n      if (hasTimeout || note.includes('timeout') || note.includes('deadline')) {\n        return { failureClass: 'timeout', reason: 'timeout signal observed' }\n      }\n      return null\n    },\n  },\n  // Tool recovery failure: many consecutive tool errors on the same tool\n  {\n    id: 'tool-recovery-failure',\n    match: ({ spans }) => {\n      const tools = spans.filter((s) => s.kind === 'tool')\n      const byTool = new Map<string, Span[]>()\n      for (const t of tools) {\n        const name = (t as Extract<Span, { kind: 'tool' }>).toolName\n        const arr = byTool.get(name) ?? []\n        arr.push(t)\n        byTool.set(name, arr)\n      }\n      for (const [name, arr] of byTool) {\n        const errs = arr.filter((s) => s.status === 'error')\n        if (errs.length >= 3 && errs.length === arr.length) {\n          return {\n            failureClass: 'tool_recovery_failure',\n            reason: `${errs.length} consecutive errors on tool \"${name}\"`,\n            triggerSpanId: errs[errs.length - 1]!.spanId,\n          }\n        }\n      }\n      return null\n    },\n  },\n  // Tool selection error: the run failed and agent called zero tools despite having them\n  {\n    id: 'tool-selection-error',\n    match: ({ run, spans }) => {\n      if (run.outcome?.pass !== false) return null\n      const hasToolsAvailable = spans.some(\n        (s) =>\n          s.kind === 'agent' &&\n          (s.attributes?.toolsAvailable as number | undefined) !== undefined &&\n          (s.attributes?.toolsAvailable as number) > 0,\n      )\n      const tools = spans.filter((s) => s.kind === 'tool')\n      if (hasToolsAvailable && tools.length === 0) {\n        return {\n          failureClass: 'tool_selection_error',\n          reason: 'tools were available but none were called',\n        }\n      }\n      return null\n    },\n  },\n  // Format drift: scored by a judge with dimension='format' below threshold\n  {\n    id: 'format-drift',\n    match: ({ spans }) => {\n      const judge = spans.find(\n        (s) =>\n          s.kind === 'judge' &&\n          (s as Extract<Span, { kind: 'judge' }>).dimension === 'format' &&\n          (s as Extract<Span, { kind: 'judge' }>).score < 0.5,\n      )\n      return judge\n        ? {\n            failureClass: 'format_drift',\n            reason: 'format judge scored below 0.5',\n            triggerSpanId: judge.spanId,\n          }\n        : null\n    },\n  },\n]\n\nfunction hasResolutionStatus(payload: Record<string, unknown>, status: string): boolean {\n  if (status === 'missing_connection' && stringArray(payload.missingConnections).length > 0)\n    return true\n  return resolutionItems(payload).some((item) => item.status === status)\n}\n\nfunction hasMissingScopes(payload: Record<string, unknown>): boolean {\n  if (stringArray(payload.missingScopes).length > 0) return true\n  return resolutionItems(payload).some(\n    (item) => Array.isArray(item.missingScopes) && item.missingScopes.length > 0,\n  )\n}\n\nfunction resolutionItems(payload: Record<string, unknown>): Array<Record<string, unknown>> {\n  return [\n    ...records(payload.missing),\n    ...records(payload.optionalMissing),\n    ...records(payload.ready),\n  ]\n}\n\nfunction records(value: unknown): Array<Record<string, unknown>> {\n  if (!Array.isArray(value)) return []\n  return value.filter(\n    (item): item is Record<string, unknown> =>\n      Boolean(item) && typeof item === 'object' && !Array.isArray(item),\n  )\n}\n\nfunction stringArray(value: unknown): string[] {\n  return Array.isArray(value)\n    ? value.filter((item): item is string => typeof item === 'string')\n    : []\n}\n\n/**\n * Classify the failure mode of a run using an ordered rule list.\n *\n * Structural rules run first, in their declared order. When none matches, the\n * reason strings the run did record are classified with\n * `classifyFailureReason`, so a run whose only evidence is a transport error\n * on an error event is named instead of landing in `unknown`. Only a run that\n * would already have returned `unknown` can change class this way.\n */\nexport function classifyFailure(\n  ctx: FailureContext,\n  rules: FailureRule[] = DEFAULT_RULES,\n  reasonRules: readonly FailureReasonRule[] = DEFAULT_FAILURE_REASON_RULES,\n): FailureClassification {\n  if (ctx.run.outcome?.pass !== false && ctx.run.status === 'completed') {\n    return {\n      failureClass: 'success',\n      blame: FAILURE_BLAME.success,\n      reason: 'run completed with pass=true (or no explicit fail)',\n    }\n  }\n  for (const rule of rules) {\n    const hit = rule.match(ctx)\n    if (hit) return { ...hit, blame: hit.blame ?? FAILURE_BLAME[hit.failureClass] }\n  }\n  for (const candidate of recordedReasons(ctx)) {\n    const classified = classifyFailureReason(candidate.text, reasonRules)\n    if (classified.failureClass === 'unknown' || classified.failureClass === 'unreported') continue\n    return {\n      ...classified,\n      ...(candidate.eventId === undefined ? {} : { triggerEventId: candidate.eventId }),\n    }\n  }\n  return {\n    failureClass: 'unknown',\n    blame: FAILURE_BLAME.unknown,\n    reason: 'no rule matched; run failed for unclassified reason',\n  }\n}\n\n/**\n * The reason strings a failed run recorded, error events first because they\n * sit closest to the failure, then the outcome notes.\n */\nfunction recordedReasons(ctx: FailureContext): Array<{ text: string; eventId?: string }> {\n  const found: Array<{ text: string; eventId?: string }> = []\n  for (const event of ctx.events) {\n    if (event.kind !== 'error') continue\n    for (const field of ['reason', 'message', 'error'] as const) {\n      const value = event.payload[field]\n      if (typeof value === 'string' && value.trim()) {\n        found.push({ text: value, eventId: event.eventId })\n      }\n    }\n  }\n  const notes = ctx.run.outcome?.notes\n  if (typeof notes === 'string' && notes.trim()) found.push({ text: notes })\n  return found\n}\n"],"mappings":";;;;;;;;;;;;;AA2DA,MAAa,gBAA8D;CACzE,SAAS;CAGT,iBAAiB;CACjB,sBAAsB;CACtB,qBAAqB;CACrB,uBAAuB;CACvB,eAAe;CACf,uBAAuB;CACvB,qBAAqB;CACrB,kBAAkB;CAClB,iBAAiB;CACjB,cAAc;CACd,uBAAuB;CACvB,UAAU;CACV,cAAc;CACd,SAAS;CAET,iBAAiB;CAIjB,mBAAmB;CACnB,qBAAqB;CACrB,0BAA0B;CAC1B,yBAAyB;CACzB,qBAAqB;CACrB,eAAe;CACf,uBAAuB;CACvB,wBAAwB;CACxB,uBAAuB;CAIvB,qBAAqB;CACrB,gCAAgC;CAChC,0BAA0B;CAC1B,0BAA0B;CAC1B,6BAA6B;CAG7B,2BAA2B;CAC3B,+BAA+B;CAC/B,iCAAiC;CAGjC,8BAA8B;CAG9B,oBAAoB;CACpB,uBAAuB;CACvB,uBAAuB;CACvB,0BAA0B;CAC1B,iBAAiB;CACjB,oBAAoB;CACpB,uBAAuB;CACvB,sBAAsB;CAItB,iBAAiB;CAGjB,kBAAkB;CAClB,uBAAuB;CACvB,mBAAmB;CACnB,gBAAgB;CAChB,gBAAgB;CAChB,YAAY;CAMZ,WAAW;CAEX,yBAAyB;CAEzB,YAAY;CACZ,SAAS;AACX;;;;;;;;;;;;;;;;;AAkBA,MAAa,uCAAkD,IAAI,IAAkB,CACnF,WACA,UACF,CAAC;;AAGD,SAAgB,aAAa,cAA0C;CACrE,OAAO,cAAc;AACvB;;;;;;;;;AAqDA,MAAa,+BAA6D;CACxE;EACE,cAAc;EACd,SAAS;CACX;CACA;EACE,cAAc;EACd,SAAS;CACX;CACA;EAAE,cAAc;EAAyB,SAAS;CAAoC;CACtF;EACE,cAAc;EACd,SACE;CACJ;CACA;EAAE,cAAc;EAAmB,SAAS;CAA2B;CACvE;EAAE,cAAc;EAAsB,SAAS;CAAsB;CACrE;EACE,cAAc;EACd,SACE;CACJ;CACA;EAAE,cAAc;EAAwB,SAAS;CAAqC;CACtF;EACE,cAAc;EACd,SACE;CACJ;CACA;EAAE,cAAc;EAAoB,SAAS;CAAuC;CACpF;EAAE,cAAc;EAAyB,SAAS;CAAuC;CACzF;EACE,cAAc;EACd,SAAS;CACX;CACA;EAAE,cAAc;EAAkB,SAAS;CAAgC;CAC3E;EAAE,cAAc;EAAkB,SAAS;CAA2C;CACtF;EAAE,cAAc;EAAa,SAAS;CAAqB;CAC3D;EAAE,cAAc;EAAc,SAAS;CAAsB;CAC7D;EAAE,cAAc;EAAmB,SAAS;CAAiC;CAS7E;EAAE,cAAc;EAAW,SAAS;CAA8B;CAClE;EAAE,cAAc;EAA2B,SAAS;CAAqC;AAC3F;;;;;;;;;;;AAYA,SAAgB,sBACd,QACA,QAAsC,8BACf;CACvB,MAAM,OAAO,OAAO,UAAU,EAAE,CAAC,CAAC,KAAK;CACvC,IAAI,SAAS,IACX,OAAO;EAAE,cAAc;EAAc,OAAO,cAAc;EAAY,QAAQ;CAAG;CAEnF,KAAK,MAAM,QAAQ,OACjB,IAAI,KAAK,QAAQ,KAAK,IAAI,GACxB,OAAO;EACL,cAAc,KAAK;EACnB,OAAO,KAAK,SAAS,cAAc,KAAK;EACxC,QAAQ;CACV;CAGJ,OAAO;EAAE,cAAc;EAAW,OAAO,cAAc;EAAS,QAAQ;CAAK;AAC/E;AAEA,MAAa,gBAA+B;CAE1C;EACE,IAAI;EACJ,QAAQ,EAAE,UAAU;GAClB,MAAM,KAAK,IAAI,SAAS;GACxB,IAAI,MAAM,OAAO,WACf,OAAO;IAAE,cAAc;IAAI,QAAQ;GAAsC;GAC3E,OAAO;EACT;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,sBACnB,EAAE,QAAQ,WAAW,KACzB;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,aACT,EAAE,QAAQ,SAAS,oCAAoC,EAAE,QAAQ,UAAU,SAC1E,EAAE,QAAQ,SAAS,+BAClB,EAAE,QAAQ,SAAS,mBAC3B;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,mCACnB,oBAAoB,EAAE,SAAS,oBAAoB,CACvD;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,aACT,EAAE,QAAQ,SAAS,mCAAmC,iBAAiB,EAAE,OAAO,KAC/E,EAAE,QAAQ,SAAS,+BAA+B,EAAE,QAAQ,SAAS,eAC5E;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,aACT,EAAE,QAAQ,SAAS,wBAAwB,EAAE,QAAQ,WAAW,uBAC/D,EAAE,QAAQ,SAAS,+BAClB,EAAE,QAAQ,SAAS,uBACrB,EAAE,QAAQ,SAAS,gCACzB;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,gCAClB,EAAE,QAAQ,SAAS,kBAClB,EAAE,QAAQ,SAAS,2BACnB,EAAE,QAAQ,SAAS,wBACnB,EAAE,QAAQ,WAAW,UAC3B;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,gCAClB,EAAE,QAAQ,SAAS,yBAClB,EAAE,QAAQ,SAAS,mBACnB,EAAE,QAAQ,SAAS,gBACzB;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,+BACnB,CAAC;IACC;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;GACF,CAAC,CAAC,SAAS,OAAO,EAAE,QAAQ,IAAI,CAAC,CACrC;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,mBACnB,EAAE,QAAQ,aAAa,sBAC3B;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,KAAK,YAAY;GACzB,IAAI,IAAI,SAAS,SAAS,OAAO,OAAO;GACxC,MAAM,YAAY,MAAM,MACrB,MACC,EAAE,SAAS,gBAAgB,EAAE,KAAK,WAAW,KAAK,EAAE,KAAK,OAAO,QAAQ,IAAI,SAAS,CAAC,EAC1F;GACA,OAAO,YACH;IACE,cAAc;IACd,QAAQ;IACR,eAAe,UAAU;GAC3B,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,mBACnB,EAAE,QAAQ,WAAW,uBACzB;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CACA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,QAAQ,OAAO,MAClB,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,mBACnB,EAAE,QAAQ,WAAW,wBACzB;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,MAAM;GACxB,IACA;EACN;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,SAAS,OAAO,MAAM,MAAM,EAAE,SAAS,eAAe;GAC5D,OAAO,SACH;IACE,cAAc;IACd,QAAQ,sBAAsB,OAAO,QAAQ,aAAa;IAC1D,gBAAgB,OAAO;GACzB,IACA;EACN;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,aAAa;GACrB,MAAM,IAAI,OAAO,MAAM,MAAM,EAAE,SAAS,kBAAkB;GAC1D,OAAO,IACH;IACE,cAAc;IACd,QAAQ;IACR,gBAAgB,EAAE;GACpB,IACA;EACN;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,YAAY;GACpB,MAAM,IAAI,MAAM,MACb,MAAM,EAAE,SAAS,aAAa,OAAO,EAAE,aAAa,YAAY,EAAE,aAAa,CAClF;GACA,IAAI,CAAC,GAAG,OAAO;GACf,OAAO;IACL,cAAc;IACd,QAAQ,kBAAmB,EAAyC;IACpE,eAAe,EAAE;GACnB;EACF;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,KAAK,aAAa;GAC1B,IAAI,IAAI,WAAW,WAAW,OAAO;GACrC,MAAM,aAAa,OAAO,MACvB,MACC,EAAE,SAAS,WACX,OAAO,EAAE,QAAQ,UAAU,EAAE,CAAC,CAC3B,YAAY,CAAC,CACb,SAAS,SAAS,CACzB;GACA,MAAM,QAAQ,IAAI,SAAS,SAAS,GAAA,CAAI,YAAY;GACpD,IAAI,cAAc,KAAK,SAAS,SAAS,KAAK,KAAK,SAAS,UAAU,GACpE,OAAO;IAAE,cAAc;IAAW,QAAQ;GAA0B;GAEtE,OAAO;EACT;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,YAAY;GACpB,MAAM,QAAQ,MAAM,QAAQ,MAAM,EAAE,SAAS,MAAM;GACnD,MAAM,yBAAS,IAAI,IAAoB;GACvC,KAAK,MAAM,KAAK,OAAO;IACrB,MAAM,OAAQ,EAAsC;IACpD,MAAM,MAAM,OAAO,IAAI,IAAI,KAAK,CAAC;IACjC,IAAI,KAAK,CAAC;IACV,OAAO,IAAI,MAAM,GAAG;GACtB;GACA,KAAK,MAAM,CAAC,MAAM,QAAQ,QAAQ;IAChC,MAAM,OAAO,IAAI,QAAQ,MAAM,EAAE,WAAW,OAAO;IACnD,IAAI,KAAK,UAAU,KAAK,KAAK,WAAW,IAAI,QAC1C,OAAO;KACL,cAAc;KACd,QAAQ,GAAG,KAAK,OAAO,+BAA+B,KAAK;KAC3D,eAAe,KAAK,KAAK,SAAS,EAAE,CAAE;IACxC;GAEJ;GACA,OAAO;EACT;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,KAAK,YAAY;GACzB,IAAI,IAAI,SAAS,SAAS,OAAO,OAAO;GACxC,MAAM,oBAAoB,MAAM,MAC7B,MACC,EAAE,SAAS,WACV,EAAE,YAAY,mBAA0C,KAAA,KACxD,EAAE,YAAY,iBAA4B,CAC/C;GACA,MAAM,QAAQ,MAAM,QAAQ,MAAM,EAAE,SAAS,MAAM;GACnD,IAAI,qBAAqB,MAAM,WAAW,GACxC,OAAO;IACL,cAAc;IACd,QAAQ;GACV;GAEF,OAAO;EACT;CACF;CAEA;EACE,IAAI;EACJ,QAAQ,EAAE,YAAY;GACpB,MAAM,QAAQ,MAAM,MACjB,MACC,EAAE,SAAS,WACV,EAAuC,cAAc,YACrD,EAAuC,QAAQ,EACpD;GACA,OAAO,QACH;IACE,cAAc;IACd,QAAQ;IACR,eAAe,MAAM;GACvB,IACA;EACN;CACF;AACF;AAEA,SAAS,oBAAoB,SAAkC,QAAyB;CACtF,IAAI,WAAW,wBAAwB,YAAY,QAAQ,kBAAkB,CAAC,CAAC,SAAS,GACtF,OAAO;CACT,OAAO,gBAAgB,OAAO,CAAC,CAAC,MAAM,SAAS,KAAK,WAAW,MAAM;AACvE;AAEA,SAAS,iBAAiB,SAA2C;CACnE,IAAI,YAAY,QAAQ,aAAa,CAAC,CAAC,SAAS,GAAG,OAAO;CAC1D,OAAO,gBAAgB,OAAO,CAAC,CAAC,MAC7B,SAAS,MAAM,QAAQ,KAAK,aAAa,KAAK,KAAK,cAAc,SAAS,CAC7E;AACF;AAEA,SAAS,gBAAgB,SAAkE;CACzF,OAAO;EACL,GAAG,QAAQ,QAAQ,OAAO;EAC1B,GAAG,QAAQ,QAAQ,eAAe;EAClC,GAAG,QAAQ,QAAQ,KAAK;CAC1B;AACF;AAEA,SAAS,QAAQ,OAAgD;CAC/D,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,OAAO,MAAM,QACV,SACC,QAAQ,IAAI,KAAK,OAAO,SAAS,YAAY,CAAC,MAAM,QAAQ,IAAI,CACpE;AACF;AAEA,SAAS,YAAY,OAA0B;CAC7C,OAAO,MAAM,QAAQ,KAAK,IACtB,MAAM,QAAQ,SAAyB,OAAO,SAAS,QAAQ,IAC/D,CAAC;AACP;;;;;;;;;;AAWA,SAAgB,gBACd,KACA,QAAuB,eACvB,cAA4C,8BACrB;CACvB,IAAI,IAAI,IAAI,SAAS,SAAS,SAAS,IAAI,IAAI,WAAW,aACxD,OAAO;EACL,cAAc;EACd,OAAO,cAAc;EACrB,QAAQ;CACV;CAEF,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,MAAM,KAAK,MAAM,GAAG;EAC1B,IAAI,KAAK,OAAO;GAAE,GAAG;GAAK,OAAO,IAAI,SAAS,cAAc,IAAI;EAAc;CAChF;CACA,KAAK,MAAM,aAAa,gBAAgB,GAAG,GAAG;EAC5C,MAAM,aAAa,sBAAsB,UAAU,MAAM,WAAW;EACpE,IAAI,WAAW,iBAAiB,aAAa,WAAW,iBAAiB,cAAc;EACvF,OAAO;GACL,GAAG;GACH,GAAI,UAAU,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,gBAAgB,UAAU,QAAQ;EACjF;CACF;CACA,OAAO;EACL,cAAc;EACd,OAAO,cAAc;EACrB,QAAQ;CACV;AACF;;;;;AAMA,SAAS,gBAAgB,KAAgE;CACvF,MAAM,QAAmD,CAAC;CAC1D,KAAK,MAAM,SAAS,IAAI,QAAQ;EAC9B,IAAI,MAAM,SAAS,SAAS;EAC5B,KAAK,MAAM,SAAS;GAAC;GAAU;GAAW;EAAO,GAAY;GAC3D,MAAM,QAAQ,MAAM,QAAQ;GAC5B,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,GAC1C,MAAM,KAAK;IAAE,MAAM;IAAO,SAAS,MAAM;GAAQ,CAAC;EAEtD;CACF;CACA,MAAM,QAAQ,IAAI,IAAI,SAAS;CAC/B,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,GAAG,MAAM,KAAK,EAAE,MAAM,MAAM,CAAC;CACzE,OAAO;AACT"}

package/dist/{feedback-trajectory-CytTR7u4.d.ts → feedback-trajectory-B3ZSkfAq.d.ts}

@@ -1,9 +1,9 @@
-import { o as FailureClass } from "./schema-CR5cpjQ3.js";
-import { i as AnalystFinding, l as AnalystRunResult, p as EvidenceRef } from "./types-BjB2wZM9.js";
+import { o as FailureClass } from "./schema-2g7trUwH.js";
+import { i as AnalystFinding, l as AnalystRunResult, p as EvidenceRef } from "./types-Bay1dh2o.js";
 import { i as DatasetSplit, r as DatasetScenario } from "./dataset-DQqhOCPt.js";
-import { s as TraceStore } from "./store-BErPvYBr.js";
-import "./exact-types-BEHN7tnq.js";
-import { i as TraceEmitter } from "./emitter-Cs0egaFd.js";
+import { s as TraceStore } from "./store-C4avFjIi.js";
+import "./exact-types-CfpP2A4Z.js";
+import { o as TraceEmitter } from "./emitter-HcCmFQl_.js";
 //#region src/control-runtime.d.ts
 type ControlSeverity = 'info' | 'warning' | 'error' | 'critical';
 type ControlActionFailureMode = 'continue' | 'stop';
@@ -394,4 +394,4 @@ declare function controlRunToFeedbackTrajectory<TState, TAction, TActionResult>(
 }): FeedbackTrajectory;
 //#endregion
 export { AnalystFindingDigest as A, ControlRunResult as B, createFeedbackTrajectory as C, renderPreferenceMemoryMarkdown as D, feedbackTrajectoryToOptimizerRow as E, ControlActionOutcome as F, StopDecision as G, ControlRuntimeError as H, ControlBudget as I, subjectiveEval as J, objectiveEval as K, ControlContext as L, AnalystRunDigest as M, analystFindingDigest as N, summarizePreferenceMemory as O, analystRunDigest as P, ControlDecision as R, controlRunToFeedbackTrajectory as S, feedbackTrajectoriesToOptimizerRows as T, ControlSeverity as U, ControlRuntimeConfig as V, ControlStep as W, PreferenceMemoryEntry as _, FeedbackLabel as a, analystRunToReviewRequests as b, FeedbackOptimizerRow as c, FeedbackTask as d, FeedbackTrajectory as f, InMemoryFeedbackTrajectoryStore as g, FileSystemFeedbackTrajectoryStore as h, FeedbackAttempt as i, AnalystReviewDecision as j, withAssignedFeedbackSplit as k, FeedbackOutcome as l, FeedbackTrajectoryStore as m, AnalystReviewRequest as n, FeedbackLabelKind as o, FeedbackTrajectoryFilter as p, runAgentControlLoop as q, FeedbackArtifactType as r, FeedbackLabelSource as s, AnalystFeedbackTrajectoryOptions as t, FeedbackSplitPolicy as u, ProposedSideEffect as v, feedbackTrajectoriesToDatasetScenarios as w, assignFeedbackSplit as x, analystRunToFeedbackTrajectory as y, ControlEvalResult as z };
-//# sourceMappingURL=feedback-trajectory-CytTR7u4.d.ts.map
+//# sourceMappingURL=feedback-trajectory-B3ZSkfAq.d.ts.map