@obsunified/collector 2.1.1 → 2.2.1

package/src/plugins/evidence-retrieval-routes.ts

@@ -12,6 +12,23 @@ import type {
 	TelemetryTraceDetailResponse,
 } from "@obsunified/types";
 import type { CollectorPlugin } from "../framework/collector";
+import {
+	computeChangeBlastRadius,
+	getChangeEvent,
+	listChangeEvents,
+} from "../lib/autosre-change-correlation";
+import {
+	getInvestigationFinding,
+	searchInvestigationFindings,
+} from "../lib/autosre-findings";
+import {
+	getOntologyGraph,
+	loadTraceFaultLocusFromDb,
+} from "../lib/autosre-ontology";
+import {
+	getIncidentInvestigation,
+	listRemediationProposals,
+} from "../lib/autosre-remediation-gate";
 import type {
 	ActionRef,
 	EntityManifestExtended,
@@ -75,7 +92,7 @@ interface ReplayEvidenceRow {
 
 interface AICallEvidenceRow {
 	call_id: string;
-	trace_id: string | null;
+	trace_id: string;
 	span_id?: string | null;
 	service_name?: string | null;
 	model_name: string;
@@ -105,6 +122,7 @@ const DEFAULT_PROFILE_FRAME_LIMIT = 50;
 const MAX_PROFILE_FRAME_LIMIT = 200;
 const EVIDENCE_INTENTS = new Set<EvidenceBundleIntent>([
 	"debug_failure",
+	"triage_incident",
 	"explain_latency",
 	"explain_cost",
 	"inspect_agent_run",
@@ -1460,6 +1478,280 @@ const buildActionGraphBundle = (
 	return bundle;
 };
 
+/**
+ * Resolves the trace to enrich an anchor with. Prefers the anchor's own
+ * symptom entity (a trace id, or the trace owning a span); only when that
+ * cannot be resolved does it fall back to the most recent error span in the
+ * environment (a heuristic: it may belong to an unrelated request).
+ */
+const resolveAnchorTraceId = async (
+	db: SqlDb,
+	projectId: string,
+	environment: string,
+	symptom?: { kind: string; id: string } | null,
+): Promise<string | null> => {
+	if (symptom?.id) {
+		if (symptom.kind === "trace") {
+			const row = await db
+				.prepare(
+					`SELECT trace_id FROM telemetry_spans
+					 WHERE project_id = ? AND trace_id = ? LIMIT 1`,
+				)
+				.bind(projectId, symptom.id)
+				.first<{ trace_id: string }>();
+			if (row?.trace_id) return row.trace_id;
+		} else if (symptom.kind === "span") {
+			const row = await db
+				.prepare(
+					`SELECT trace_id FROM telemetry_spans
+					 WHERE project_id = ? AND span_id = ? LIMIT 1`,
+				)
+				.bind(projectId, symptom.id)
+				.first<{ trace_id: string }>();
+			if (row?.trace_id) return row.trace_id;
+		}
+	}
+	const recent = await db
+		.prepare(
+			`SELECT trace_id FROM telemetry_spans
+			 WHERE project_id = ? AND environment = ? AND status_code = 2
+			 ORDER BY start_time DESC LIMIT 1`,
+		)
+		.bind(projectId, environment)
+		.first<{ trace_id: string }>();
+	return recent?.trace_id ?? null;
+};
+
+const attachAutoSreTriageContext = async (
+	db: SqlDb,
+	projectId: string,
+	traceId: string | null,
+	bundle: EvidenceBundle,
+	logger?: { warn: (message: string, meta?: Record<string, unknown>) => void },
+): Promise<EvidenceBundle> => {
+	try {
+		const faultLocus = traceId
+			? await loadTraceFaultLocusFromDb(db, projectId, traceId)
+			: null;
+		const environment = (
+			faultLocus?.faultLocusSpan?.environment ??
+			bundle.environment ??
+			"default"
+		).toLowerCase();
+		const locusService = faultLocus?.faultLocusSpan?.service_name ?? null;
+		const allRecentChanges = await listChangeEvents(db, {
+			projectId,
+			environment,
+			limit: locusService ? 25 : 5,
+		});
+		// When the fault locus service is known, only changes that target it
+		// (or its node) are candidates; unrelated deploys are noise.
+		const recentChanges = locusService
+			? allRecentChanges
+					.filter(
+						(ch) =>
+							ch.serviceName === locusService ||
+							(faultLocus?.faultLocusNodeId &&
+								ch.targetNodeId === faultLocus.faultLocusNodeId),
+					)
+					.slice(0, 5)
+			: allRecentChanges;
+		const blastRadii = (
+			await Promise.all(
+				recentChanges
+					.slice(0, 3)
+					.map((ch) =>
+						computeChangeBlastRadius(db, projectId, ch.id, { environment }),
+					),
+			)
+		).filter((br): br is NonNullable<typeof br> => Boolean(br));
+
+		const candidateChanges = blastRadii.map((br) => ({
+			changeId: br.change.id,
+			kind: br.change.kind,
+			version: br.change.version,
+			previousVersion: br.change.previousVersion,
+			occurredAt: br.change.occurredAt,
+			cohortVerdict: br.cohortVerdict,
+			causalProofTier: br.causalProofTier,
+			rateNew: br.newCohort.errorRate,
+			rateOld: br.oldCohort.errorRate,
+			sampleNew: br.newCohort.requestCount,
+			sampleOld: br.oldCohort.requestCount,
+			overlappingCodePaths: br.overlappingCodePaths,
+		}));
+		const contentionCandidates = blastRadii
+			.flatMap((br) => br.sharedResourceAggressors)
+			.map((ag) => ({
+				aggressorTraceId: ag.aggressorTraceId,
+				aggressorSpanId: ag.aggressorSpanId,
+				aggressorServiceName: ag.aggressorServiceName,
+				sharedTargetNodeId: ag.sharedTargetNodeId,
+				relationshipCertainty: ag.relationshipCertainty,
+				totalResourceTimeMs: ag.totalResourceTimeMs,
+			}));
+		let priorFindingsList = await searchInvestigationFindings(db, {
+			projectId,
+			environment,
+			rootCauseEntityId:
+				faultLocus?.faultLocusNodeId ??
+				faultLocus?.faultLocusSpan?.span_id ??
+				undefined,
+			limit: 5,
+		});
+		if (priorFindingsList.length === 0 && faultLocus?.faultLocusSpan?.span_id) {
+			priorFindingsList = await searchInvestigationFindings(db, {
+				projectId,
+				environment,
+				rootCauseEntityId: faultLocus.faultLocusSpan.span_id,
+				limit: 5,
+			});
+		}
+		// No unfiltered fallback: unrelated findings would read as relevant
+		// prior art. An empty list is the honest answer.
+		const priorFindings = priorFindingsList.map((f) => ({
+			findingId: f.id,
+			title: f.title,
+			causalProofTier: f.causalProofTier,
+			causalVerificationStatus: f.causalVerificationStatus,
+			recoveryStatus: f.recoveryStatus,
+			contributingChangeId: f.contributingChangeId,
+			occurredAt: f.createdAt,
+		}));
+
+		for (const br of blastRadii) {
+			const changeId = br.change.id;
+			const changeRoute = `#/changes/${encodeURIComponent(changeId)}`;
+			bundle.evidenceReferences.push({
+				evidenceId: `change:${changeId}`,
+				entityKind: "change",
+				entityId: changeId,
+				route: changeRoute,
+				source: br.overlappingCodePaths.length
+					? "stack_frame_overlap"
+					: "cohort_comparison",
+				confidence: br.cohortVerdict === "isolated_to_new_version" ? 0.9 : 0.65,
+				relationshipCertainty: br.relationshipCertainty,
+				causalProofTier: br.causalProofTier,
+				reason: br.summary,
+				citations: [
+					{
+						label: `change ${changeId} (${br.change.version ?? br.change.kind})`,
+						entityKind: "change",
+						entityId: changeId,
+						route: changeRoute,
+					},
+				],
+				suggestedNextPivots: [
+					{
+						label: "Inspect change blast radius",
+						entityKind: "change",
+						entityId: changeId,
+						route: changeRoute,
+						reason: "Inspect traffic-normalized cohort lift and contention.",
+					},
+				],
+			});
+		}
+
+		for (const finding of priorFindingsList) {
+			const findingRoute = `#/findings/${encodeURIComponent(finding.id)}`;
+			bundle.evidenceReferences.push({
+				evidenceId: `finding:${finding.id}`,
+				entityKind: "finding",
+				entityId: finding.id,
+				route: findingRoute,
+				source: "incident_memory_fingerprint",
+				confidence: finding.confidence,
+				relationshipCertainty: "exact",
+				causalProofTier: finding.causalProofTier,
+				reason: `Prior finding (${finding.causalProofTier}): ${finding.title}`,
+				citations: [
+					{
+						label: finding.title,
+						entityKind: "finding",
+						entityId: finding.id,
+						route: findingRoute,
+					},
+				],
+				suggestedNextPivots: [
+					{
+						label: "Open prior finding",
+						entityKind: "finding",
+						entityId: finding.id,
+						route: findingRoute,
+						reason: "Search incident memory for similar recurring findings.",
+					},
+				],
+			});
+		}
+
+		let isBlackoutSuspected = false;
+		if (faultLocus?.faultLocusNodeId) {
+			try {
+				const graph = await getOntologyGraph(db, { projectId, environment });
+				isBlackoutSuspected = Boolean(
+					graph.nodes.find((n) => n.id === faultLocus.faultLocusNodeId)
+						?.isBlackoutSuspected,
+				);
+			} catch (err) {
+				logger?.warn("[evidence-retrieval] blackout check failed", {
+					error: err instanceof Error ? err.message : String(err),
+				});
+			}
+		}
+
+		bundle.autosre = {
+			faultLocus: faultLocus?.faultLocusNodeId
+				? {
+						nodeId: faultLocus.faultLocusNodeId,
+						spanId: faultLocus.faultLocusSpan?.span_id,
+						relationshipCertainty: faultLocus.relationshipCertainty,
+						isBlackoutSuspected,
+					}
+				: undefined,
+			candidateChanges,
+			contentionCandidates,
+			priorFindings,
+		};
+		if (bundle.budget) {
+			// Keep the autosre section inside the caller's token budget by
+			// dropping the least-relevant tail entries (and their evidence refs).
+			const autosre = bundle.autosre;
+			const target = bundle.budget.targetTokens ?? Number.POSITIVE_INFINITY;
+			const dropRef = (evidenceId: string) => {
+				bundle.evidenceReferences = bundle.evidenceReferences.filter(
+					(ref) => ref.evidenceId !== evidenceId,
+				);
+			};
+			while (estimateTokens(bundle) > target) {
+				if (autosre.priorFindings.length > 0) {
+					const dropped = autosre.priorFindings.pop();
+					if (dropped) dropRef(`finding:${dropped.findingId}`);
+				} else if (autosre.contentionCandidates.length > 0) {
+					autosre.contentionCandidates.pop();
+				} else if (autosre.candidateChanges.length > 0) {
+					const dropped = autosre.candidateChanges.pop();
+					if (dropped) dropRef(`change:${dropped.changeId}`);
+				} else {
+					break;
+				}
+			}
+			bundle.budget = {
+				...bundle.budget,
+				estimatedTokens: estimateTokens(bundle),
+			};
+		}
+	} catch (err) {
+		// Gracefully degrade if AutoSRE tables are absent in minimal test setups.
+		logger?.warn("[evidence-retrieval] autosre triage enrichment skipped", {
+			trace_id: traceId,
+			error: err instanceof Error ? err.message : String(err),
+		});
+	}
+	return bundle;
+};
+
 const parseBundleRequest = async (request: Request) => {
 	try {
 		const body = await request.json();
@@ -1490,11 +1782,17 @@ export const evidenceRetrievalRoutesPlugin: CollectorPlugin = {
 				);
 			}
 
+			const isAutoSreAnchor =
+				anchor.entityKind === "incident" ||
+				anchor.entityKind === "change" ||
+				anchor.entityKind === "finding";
 			const intent =
 				typeof body.intent === "string" &&
 				EVIDENCE_INTENTS.has(body.intent as EvidenceBundleIntent)
 					? (body.intent as EvidenceBundleIntent)
-					: "debug_failure";
+					: isAutoSreAnchor
+						? "triage_incident"
+						: "debug_failure";
 			const budget = body.budget as Record<string, unknown> | undefined;
 			const targetTokens = clampInt(
 				budget?.targetTokens,
@@ -1550,7 +1848,7 @@ export const evidenceRetrievalRoutesPlugin: CollectorPlugin = {
 							all.findIndex((candidate) => candidate.id === profile.id) ===
 							index,
 					);
-				const bundle = buildActionGraphBundle(
+				let bundle = buildActionGraphBundle(
 					projectId,
 					anchor.entityKind,
 					anchor.entityId,
@@ -1560,15 +1858,346 @@ export const evidenceRetrievalRoutesPlugin: CollectorPlugin = {
 					targetTokens,
 					detailLevel,
 				);
+				if (intent === "triage_incident") {
+					bundle = await attachAutoSreTriageContext(
+						db,
+						projectId,
+						traceIds[0] ?? null,
+						bundle,
+						runtime.logger,
+					);
+				}
 				return c.json(await materializeBundle(db, projectId, bundle));
 			}
 
+			if (anchor.entityKind === "incident") {
+				const incident = await getIncidentInvestigation(
+					db,
+					projectId,
+					anchor.entityId,
+				);
+				if (!incident) {
+					return c.json(
+						{ error: "Not Found", message: "Incident not found" },
+						404,
+					);
+				}
+				const anchorTraceId = await resolveAnchorTraceId(
+					db,
+					projectId,
+					incident.environment,
+					{ kind: incident.symptomAnchorKind, id: incident.symptomAnchorId },
+				);
+
+				const incRoute = `#/incidents/${encodeURIComponent(incident.id)}`;
+				const baseBundle: EvidenceBundle = {
+					schemaVersion: BUNDLE_SCHEMA_VERSION,
+					intent,
+					environment: incident.environment,
+					incidentId: incident.id,
+					symptomFingerprint: incident.symptomFingerprint ?? undefined,
+					anchor: { entityKind: "incident", entityId: incident.id },
+					budget: { targetTokens, detailLevel },
+					summary: `Incident ${incident.id} (${incident.status}) on ${incident.targetNodeId} in environment ${incident.environment}.`,
+					derivedSummaries: [
+						{
+							title: `Incident ${incident.id}`,
+							reason: `Trigger ${incident.symptomAnchorKind}:${incident.symptomAnchorId} on ${incident.targetNodeId} (status=${incident.status}).`,
+							confidence: 1.0,
+							evidenceIds: [`incident:${incident.id}`],
+							retrievalRefIds: [],
+						},
+					],
+					findings: [],
+					compactions: [],
+					evidenceReferences: [
+						{
+							evidenceId: `incident:${incident.id}`,
+							entityKind: "incident",
+							entityId: incident.id,
+							route: incRoute,
+							source: "incident_investigations",
+							confidence: 1.0,
+							relationshipCertainty: "exact",
+							reason: `Incident triggered by ${incident.symptomAnchorKind}:${incident.symptomAnchorId}.`,
+							citations: [
+								{
+									label: incident.targetNodeId,
+									entityKind: "incident",
+									entityId: incident.id,
+									route: incRoute,
+								},
+							],
+							suggestedNextPivots: [
+								{
+									label: "Open incident",
+									entityKind: "incident",
+									entityId: incident.id,
+									route: incRoute,
+									reason:
+										"Inspect incident investigation and candidate changes.",
+								},
+							],
+						},
+					],
+					retrievalRefs: [],
+					suggestedNextPivots: [
+						{
+							label: "Open incident",
+							entityKind: "incident",
+							entityId: incident.id,
+							route: incRoute,
+							reason: "Inspect incident investigation and candidate changes.",
+						},
+					],
+				};
+				const enriched = await attachAutoSreTriageContext(
+					db,
+					projectId,
+					anchorTraceId,
+					baseBundle,
+					runtime.logger,
+				);
+				return c.json(await materializeBundle(db, projectId, enriched));
+			}
+
+			if (anchor.entityKind === "change") {
+				const change = await getChangeEvent(db, projectId, anchor.entityId);
+				if (!change) {
+					return c.json(
+						{ error: "Not Found", message: "Change event not found" },
+						404,
+					);
+				}
+				const blastRadius = await computeChangeBlastRadius(
+					db,
+					projectId,
+					change.id,
+					{ environment: change.environment },
+				);
+				const anchorTraceId = await resolveAnchorTraceId(
+					db,
+					projectId,
+					change.environment,
+					null,
+				);
+
+				const changeRoute = `#/changes/${encodeURIComponent(change.id)}`;
+				const summary =
+					blastRadius?.summary ??
+					`Change ${change.id} (${change.kind} ${change.version ?? ""}) in ${change.environment}: ${change.summary}`;
+				const baseBundle: EvidenceBundle = {
+					schemaVersion: BUNDLE_SCHEMA_VERSION,
+					intent,
+					environment: change.environment,
+					anchor: { entityKind: "change", entityId: change.id },
+					budget: { targetTokens, detailLevel },
+					summary,
+					derivedSummaries: [
+						{
+							title: `Change ${change.id} (${change.kind})`,
+							reason: summary,
+							confidence:
+								blastRadius?.cohortVerdict === "isolated_to_new_version"
+									? 0.9
+									: 0.7,
+							evidenceIds: [`change:${change.id}`],
+							retrievalRefIds: [],
+						},
+					],
+					findings: blastRadius
+						? [
+								{
+									title: `Blast radius (${blastRadius.cohortVerdict})`,
+									reason: blastRadius.summary,
+									confidence:
+										blastRadius.cohortVerdict === "isolated_to_new_version"
+											? 0.9
+											: 0.65,
+									evidenceIds: [`change:${change.id}`],
+									retrievalRefIds: [],
+								},
+							]
+						: [],
+					compactions: [],
+					evidenceReferences: [
+						{
+							evidenceId: `change:${change.id}`,
+							entityKind: "change",
+							entityId: change.id,
+							route: changeRoute,
+							source: "change_events",
+							confidence: 1.0,
+							relationshipCertainty:
+								blastRadius?.relationshipCertainty ??
+								change.relationshipCertainty,
+							causalProofTier: blastRadius?.causalProofTier,
+							reason: summary,
+							citations: [
+								{
+									label: `${change.kind} ${change.version ?? change.id}`,
+									entityKind: "change",
+									entityId: change.id,
+									route: changeRoute,
+								},
+							],
+							suggestedNextPivots: [
+								{
+									label: "Inspect change blast radius",
+									entityKind: "change",
+									entityId: change.id,
+									route: changeRoute,
+									reason:
+										"Inspect traffic-normalized cohort lift and contention.",
+								},
+							],
+						},
+					],
+					retrievalRefs: [],
+					suggestedNextPivots: [
+						{
+							label: "Inspect change blast radius",
+							entityKind: "change",
+							entityId: change.id,
+							route: changeRoute,
+							reason: "Inspect traffic-normalized cohort lift and contention.",
+						},
+					],
+				};
+				const enriched = await attachAutoSreTriageContext(
+					db,
+					projectId,
+					anchorTraceId,
+					baseBundle,
+					runtime.logger,
+				);
+				return c.json(await materializeBundle(db, projectId, enriched));
+			}
+
+			if (anchor.entityKind === "finding") {
+				const finding = await getInvestigationFinding(
+					db,
+					projectId,
+					anchor.entityId,
+				);
+				if (!finding) {
+					return c.json(
+						{ error: "Not Found", message: "Finding not found" },
+						404,
+					);
+				}
+				const proposals = await listRemediationProposals(db, {
+					projectId,
+					environment: finding.environment,
+					findingId: finding.id,
+					limit: 10,
+				});
+				const anchorTraceId = await resolveAnchorTraceId(
+					db,
+					projectId,
+					finding.environment,
+					{ kind: finding.symptomEntityKind, id: finding.symptomEntityId },
+				);
+
+				const findingRoute = `#/findings/${encodeURIComponent(finding.id)}`;
+				const summary = `${finding.title} (${finding.causalProofTier}, verification=${finding.causalVerificationStatus}, recovery=${finding.recoveryStatus}): ${finding.summary}`;
+				const baseBundle: EvidenceBundle = {
+					schemaVersion: BUNDLE_SCHEMA_VERSION,
+					intent,
+					environment: finding.environment,
+					incidentId: finding.incidentId ?? undefined,
+					symptomFingerprint: finding.fingerprint,
+					anchor: { entityKind: "finding", entityId: finding.id },
+					budget: { targetTokens, detailLevel },
+					summary,
+					derivedSummaries: [
+						{
+							title: finding.title,
+							reason: summary,
+							confidence: finding.confidence,
+							evidenceIds: [`finding:${finding.id}`],
+							retrievalRefIds: [],
+						},
+					],
+					findings: [
+						{
+							title: finding.title,
+							reason: finding.summary,
+							confidence: finding.confidence,
+							evidenceIds: [
+								`finding:${finding.id}`,
+								...finding.citedEvidenceIds,
+							],
+							retrievalRefIds: [],
+						},
+						...proposals.map((p) => ({
+							title: `Remediation proposal ${p.id} (${p.actionKind})`,
+							reason: `status=${p.status}, autonomy=${p.effectiveAutonomy} — ${p.policyDecisionReason}`,
+							confidence: finding.confidence,
+							evidenceIds: [`finding:${finding.id}`],
+							retrievalRefIds: [],
+						})),
+					],
+					compactions: [],
+					evidenceReferences: [
+						{
+							evidenceId: `finding:${finding.id}`,
+							entityKind: "finding",
+							entityId: finding.id,
+							route: findingRoute,
+							source: "investigation_findings",
+							confidence: finding.confidence,
+							relationshipCertainty: "exact",
+							causalProofTier: finding.causalProofTier,
+							reason: summary,
+							citations: [
+								{
+									label: finding.title,
+									entityKind: "finding",
+									entityId: finding.id,
+									route: findingRoute,
+								},
+							],
+							suggestedNextPivots: [
+								{
+									label: "Open finding",
+									entityKind: "finding",
+									entityId: finding.id,
+									route: findingRoute,
+									reason:
+										"Inspect causal chain, counterfactual verification, and remediation proposals.",
+								},
+							],
+						},
+					],
+					retrievalRefs: [],
+					suggestedNextPivots: [
+						{
+							label: "Open finding",
+							entityKind: "finding",
+							entityId: finding.id,
+							route: findingRoute,
+							reason:
+								"Inspect causal chain, counterfactual verification, and remediation proposals.",
+						},
+					],
+				};
+				const enriched = await attachAutoSreTriageContext(
+					db,
+					projectId,
+					anchorTraceId,
+					baseBundle,
+					runtime.logger,
+				);
+				return c.json(await materializeBundle(db, projectId, enriched));
+			}
+
 			if (anchor.entityKind !== "trace") {
 				return c.json(
 					{
 						error: "Bad Request",
 						message:
-							"Supported evidence bundle anchors are trace, action, agent_run, and tool_call.",
+							"Supported evidence bundle anchors are trace, action, agent_run, tool_call, change, finding, and incident.",
 					},
 					400,
 				);
@@ -1588,7 +2217,7 @@ export const evidenceRetrievalRoutesPlugin: CollectorPlugin = {
 				limit: MAX_LOGS_FOR_BUNDLE,
 			});
 			const profiles = await profilesForTrace(db, projectId, traceId);
-			const bundle = await buildTraceBundle(
+			let bundle = await buildTraceBundle(
 				projectId,
 				detail,
 				logs.logs,
@@ -1598,6 +2227,15 @@ export const evidenceRetrievalRoutesPlugin: CollectorPlugin = {
 				detailLevel,
 				hours,
 			);
+			if (intent === "triage_incident") {
+				bundle = await attachAutoSreTriageContext(
+					db,
+					projectId,
+					traceId,
+					bundle,
+					runtime.logger,
+				);
+			}
 			return c.json(await materializeBundle(db, projectId, bundle));
 		});