@obsunified/collector 2.1.1 → 2.2.1

package/src/lib/eval-cases-store.ts

@@ -76,6 +76,10 @@ export interface EvalRunCandidate {
 	modelProvider: string | null;
 	model: string | null;
 	modelVersion: string | null;
+	environment?: string | null;
+	serviceVersion?: string | null;
+	commitSha?: string | null;
+	configDigest?: string | null;
 }
 
 export interface EvalRunSourceCase {
@@ -99,6 +103,12 @@ export interface EvalRun {
 	evalCaseId: string | null;
 	status: EvalRunStatus;
 	candidate: EvalRunCandidate;
+	environment?: string;
+	serviceVersion?: string | null;
+	commitSha?: string | null;
+	configDigest?: string | null;
+	inputPayloadHash?: string | null;
+	failureFingerprint?: string | null;
 	startedAt: string | null;
 	endedAt: string | null;
 	totalCount: number;
@@ -117,6 +127,12 @@ export interface EvalRunInput {
 	evalCaseId?: string | null;
 	status?: EvalRunStatus;
 	candidate?: Partial<EvalRunCandidate>;
+	environment?: string | null;
+	serviceVersion?: string | null;
+	commitSha?: string | null;
+	configDigest?: string | null;
+	inputPayloadHash?: string | null;
+	failureFingerprint?: string | null;
 	startedAt?: string | null;
 	endedAt?: string | null;
 	totalCount?: number;
@@ -144,6 +160,12 @@ interface EvalRunRow {
 	candidate_model_provider: string | null;
 	candidate_model: string | null;
 	candidate_model_version: string | null;
+	environment?: string | null;
+	service_version?: string | null;
+	commit_sha?: string | null;
+	config_digest?: string | null;
+	input_payload_hash?: string | null;
+	failure_fingerprint?: string | null;
 	status: string;
 	started_at: string | null;
 	ended_at: string | null;
@@ -412,7 +434,17 @@ const rowToEvalRun = (row: EvalRunRow): EvalRun => {
 			modelProvider: row.candidate_model_provider ?? null,
 			model: row.candidate_model ?? null,
 			modelVersion: row.candidate_model_version ?? null,
+			environment: row.environment ?? "default",
+			serviceVersion: row.service_version ?? null,
+			commitSha: row.commit_sha ?? null,
+			configDigest: row.config_digest ?? null,
 		},
+		environment: row.environment ?? "default",
+		serviceVersion: row.service_version ?? null,
+		commitSha: row.commit_sha ?? null,
+		configDigest: row.config_digest ?? null,
+		inputPayloadHash: row.input_payload_hash ?? null,
+		failureFingerprint: row.failure_fingerprint ?? null,
 		startedAt: row.started_at ?? null,
 		endedAt: row.ended_at ?? null,
 		totalCount: row.total_count ?? 0,
@@ -560,6 +592,16 @@ export class EvalCasesStore {
 		}
 
 		const candidate = input.candidate ?? {};
+		const environment =
+			input.environment?.trim() || candidate.environment?.trim() || "default";
+		const serviceVersion =
+			input.serviceVersion?.trim() || candidate.serviceVersion?.trim() || null;
+		const commitSha =
+			input.commitSha?.trim() || candidate.commitSha?.trim() || null;
+		const configDigest =
+			input.configDigest?.trim() || candidate.configDigest?.trim() || null;
+		const inputPayloadHash = input.inputPayloadHash?.trim() || null;
+		const failureFingerprint = input.failureFingerprint?.trim() || null;
 		const totalCount = nonNegativeInteger(input.totalCount);
 		const passCount = nonNegativeInteger(input.passCount);
 		const failCount = nonNegativeInteger(input.failCount);
@@ -580,10 +622,12 @@ export class EvalCasesStore {
 					candidate_agent_id, candidate_agent_version,
 					candidate_prompt_id, candidate_prompt_version,
 					candidate_model_provider, candidate_model, candidate_model_version,
+					environment, service_version, commit_sha, config_digest,
+					input_payload_hash, failure_fingerprint,
 					status, started_at, ended_at,
 					total_count, pass_count, fail_count, average_score,
 					metadata_json, created_at
-				) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+				) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 			)
 			.bind(
 				id,
@@ -596,6 +640,12 @@ export class EvalCasesStore {
 				candidate.modelProvider ?? null,
 				candidate.model ?? null,
 				candidate.modelVersion ?? null,
+				environment,
+				serviceVersion,
+				commitSha,
+				configDigest,
+				inputPayloadHash,
+				failureFingerprint,
 				status,
 				startedAt,
 				endedAt,
@@ -679,6 +729,14 @@ export class EvalCasesStore {
 					average_score = (
 						SELECT AVG(score) FROM eval_case_results
 						WHERE project_id = ? AND run_id = ? AND score IS NOT NULL
+					),
+					input_payload_hash = COALESCE(
+						input_payload_hash,
+						(SELECT input_payload_hash FROM eval_case_results WHERE project_id = ? AND run_id = ? AND input_payload_hash IS NOT NULL LIMIT 1)
+					),
+					failure_fingerprint = COALESCE(
+						failure_fingerprint,
+						(SELECT failure_fingerprint FROM eval_case_results WHERE project_id = ? AND run_id = ? AND failure_fingerprint IS NOT NULL LIMIT 1)
 					)
 				WHERE project_id = ? AND id = ?`,
 			)
@@ -693,6 +751,10 @@ export class EvalCasesStore {
 				id,
 				projectId,
 				id,
+				projectId,
+				id,
+				projectId,
+				id,
 			)
 			.run();
 	}

package/src/lib/evidence-references.ts

@@ -146,6 +146,12 @@ export const evidenceRouteFor = (
 			return traceRoute(entityId);
 		case "docs":
 			return entityId.startsWith("#/") ? entityId : `#/docs/${entityId}`;
+		case "change":
+			return `#/changes/${encodeURIComponent(entityId)}`;
+		case "finding":
+			return `#/findings/${encodeURIComponent(entityId)}`;
+		case "incident":
+			return `#/incidents/${encodeURIComponent(entityId)}`;
 	}
 	const exhaustive: never = entityKind;
 	return exhaustive;

package/src/lib/housekeeping.test.ts

@@ -58,6 +58,13 @@ describe("purgeSecondaryRows", () => {
 			metric_series: 1,
 			evidence_refs: 1,
 			evidence_ref_expansions: 1,
+			ontology_edges: 0,
+			ontology_nodes: 0,
+			change_events: 0,
+			incident_investigations: 0,
+			incident_leases_reaped: 0,
+			remediation_proposals: 0,
+			investigation_findings: 0,
 		});
 		const ids = (sql: string) =>
 			(raw.prepare(sql).all() as Array<{ id: string }>).map((r) => r.id).sort();
@@ -70,4 +77,147 @@ describe("purgeSecondaryRows", () => {
 		expect(ids("SELECT id FROM evidence_ref_expansions")).toEqual(["e2"]);
 		db.close();
 	});
+
+	it("enforces RFC 0013 AutoSRE retention windows while preserving counterfactual_verified and operationally_corroborated findings", async () => {
+		const db = createMigratedSqliteDb();
+		const raw = db.raw;
+		raw.exec(`
+			INSERT INTO projects (id, name, slug, created_at) VALUES ('alpha', 'A', 'a', '${iso(3000)}');
+		`);
+		// Ontology edges/nodes (30d = 720h)
+		raw
+			.prepare(
+				`INSERT INTO ontology_nodes (id, project_id, environment, kind, name, first_seen_at, last_seen_at) VALUES (?, 'alpha', 'prod', 'service', ?, ?, ?)`,
+			)
+			.run("n-old", "old-svc", iso(800), iso(750));
+		raw
+			.prepare(
+				`INSERT INTO ontology_nodes (id, project_id, environment, kind, name, first_seen_at, last_seen_at) VALUES (?, 'alpha', 'prod', 'service', ?, ?, ?)`,
+			)
+			.run("n-fresh", "fresh-svc", iso(100), iso(10));
+		raw
+			.prepare(
+				`INSERT INTO ontology_edges (id, project_id, environment, source_node_id, predicate, target_node_id, relationship_certainty, first_seen_at, last_seen_at) VALUES (?, 'alpha', 'prod', 'n-old', 'calls', 'n-fresh', 'exact', ?, ?)`,
+			)
+			.run("edge-old", iso(800), iso(750));
+
+		// Change events (90d = 2160h)
+		raw
+			.prepare(
+				`INSERT INTO change_events (id, project_id, environment, kind, source, target_node_id, summary, occurred_at, received_at) VALUES (?, 'alpha', 'prod', 'deploy', 'ci_webhook', 'n-fresh', 'old deploy', ?, ?)`,
+			)
+			.run("chg-old", iso(2200), iso(2200));
+		raw
+			.prepare(
+				`INSERT INTO change_events (id, project_id, environment, kind, source, target_node_id, summary, occurred_at, received_at) VALUES (?, 'alpha', 'prod', 'deploy', 'ci_webhook', 'n-fresh', 'recent deploy', ?, ?)`,
+			)
+			.run("chg-recent", iso(100), iso(100));
+
+		// Findings (90d = 2160h, unless counterfactual_verified or operationally_corroborated)
+		const insFinding = raw.prepare(
+			`INSERT INTO investigation_findings (id, project_id, environment, fingerprint, fingerprint_input_json, title, summary, causal_proof_tier, causal_verification_status, confidence, confidence_breakdown_json, symptom_entity_kind, symptom_entity_id, fault_locus_entity_kind, fault_locus_entity_id, causal_chain_json, cited_evidence_ids_json, actor_type, actor_id, created_at, updated_at) VALUES (?, 'alpha', 'prod', 'fp_1', '{}', 't', 's', 'corroborated_candidate', ?, 0.8, '{}', 'trace', 't1', 'span', 's1', '[]', '["t1"]', 'agent', 'autosre', ?, ?)`,
+		);
+		insFinding.run("fnd-old-unverified", "unverified", iso(2200), iso(2200));
+		insFinding.run(
+			"fnd-old-verified",
+			"counterfactual_verified",
+			iso(2200),
+			iso(2200),
+		);
+		insFinding.run(
+			"fnd-old-corroborated",
+			"operationally_corroborated",
+			iso(2200),
+			iso(2200),
+		);
+
+		const summary = await purgeSecondaryRows(db, {
+			retentionHours: 72,
+			now: () => NOW,
+		});
+		expect(summary.ontology_edges).toBe(1);
+		expect(summary.ontology_nodes).toBe(1);
+		expect(summary.change_events).toBe(1);
+		expect(summary.investigation_findings).toBe(1);
+
+		const remainingFindings = (
+			raw
+				.prepare(`SELECT id FROM investigation_findings ORDER BY id ASC`)
+				.all() as Array<{ id: string }>
+		).map((r) => r.id);
+		expect(remainingFindings).toEqual([
+			"fnd-old-corroborated",
+			"fnd-old-verified",
+		]);
+		db.close();
+	});
+
+	it("reaps expired leases back to open without touching live leases or non-leased incidents (RFC 0013 § 11)", async () => {
+		const db = createMigratedSqliteDb();
+		const ins = db.raw.prepare(
+			`INSERT INTO incident_investigations (project_id, environment, id, dedup_key, status, target_node_id, symptom_anchor_kind, symptom_anchor_id, lease_owner_id, lease_expires_at, opened_at, updated_at, expires_at)
+			 VALUES ('alpha', 'prod', ?, ?, ?, 'n', 'trace', 't', ?, ?, ?, ?, ?)`,
+		);
+		const farFuture = iso(-24 * 365);
+		ins.run(
+			"lease-expired",
+			"k1",
+			"leased",
+			"runner-a",
+			iso(1),
+			iso(5),
+			iso(5),
+			farFuture,
+		);
+		ins.run(
+			"lease-live",
+			"k2",
+			"leased",
+			"runner-b",
+			iso(-1),
+			iso(5),
+			iso(5),
+			farFuture,
+		);
+		ins.run(
+			"remediating",
+			"k3",
+			"remediating",
+			"runner-c",
+			iso(1),
+			iso(5),
+			iso(5),
+			farFuture,
+		);
+
+		const summary = await purgeSecondaryRows(db, {
+			retentionHours: 72,
+			now: () => NOW,
+		});
+		expect(summary.incident_leases_reaped).toBe(1);
+		const rows = db.raw
+			.prepare(
+				`SELECT id, status, lease_owner_id FROM incident_investigations ORDER BY id`,
+			)
+			.all() as Array<{
+			id: string;
+			status: string;
+			lease_owner_id: string | null;
+		}>;
+		expect(rows).toEqual([
+			{ id: "lease-expired", status: "open", lease_owner_id: null },
+			{ id: "lease-live", status: "leased", lease_owner_id: "runner-b" },
+			{ id: "remediating", status: "remediating", lease_owner_id: "runner-c" },
+		]);
+		db.close();
+	});
+
+	it("propagates storage errors instead of silently swallowing them", async () => {
+		const db = createMigratedSqliteDb();
+		db.raw.exec("DROP TABLE remediation_proposals");
+		await expect(
+			purgeSecondaryRows(db, { retentionHours: 72, now: () => NOW }),
+		).rejects.toThrow(/remediation_proposals/);
+		db.close();
+	});
 });

package/src/lib/housekeeping.ts

@@ -24,6 +24,14 @@ export interface HousekeepingSummary {
 	metric_series: number;
 	evidence_refs: number;
 	evidence_ref_expansions: number;
+	ontology_edges?: number;
+	ontology_nodes?: number;
+	change_events?: number;
+	incident_investigations?: number;
+	/** `leased` incidents whose lease expired and were returned to `open` (RFC 0013 § 11). */
+	incident_leases_reaped?: number;
+	investigation_findings?: number;
+	remediation_proposals?: number;
 }
 
 export interface HousekeepingOptions {
@@ -42,6 +50,7 @@ export async function purgeSecondaryRows(
 	const nowMs = options.now?.() ?? Date.now();
 	const isoBefore = (hours: number) =>
 		new Date(nowMs - hours * 3_600_000).toISOString();
+	const nowIso = isoBefore(0);
 
 	const alertEvaluations = await db
 		.prepare(`DELETE FROM alert_evaluations WHERE evaluated_at < ?`)
@@ -71,7 +80,7 @@ export async function purgeSecondaryRows(
 			`DELETE FROM evidence_retrieval_refs
 			 WHERE expires_at IS NOT NULL AND expires_at < ?`,
 		)
-		.bind(isoBefore(0))
+		.bind(nowIso)
 		.run();
 	const expansions = await db
 		.prepare(
@@ -83,11 +92,110 @@ export async function purgeSecondaryRows(
 		)
 		.run();
 
+	// RFC 0013 — AutoSRE ontology, change events, incident investigations,
+	// investigation findings, and remediation proposals retention, plus lease
+	// reaping. Errors propagate (the retention caller logs them); the
+	// post-remediation watchdog runs on the 1-minute analyses tick instead.
+	const thirtyDaysAgoIso = isoBefore(30 * 24);
+	const ninetyDaysAgoIso = isoBefore(90 * 24);
+	const edgesRes = await db
+		.prepare(`DELETE FROM ontology_edges WHERE last_seen_at < ?`)
+		.bind(thirtyDaysAgoIso)
+		.run();
+	const ontologyEdgesPurged = changes(edgesRes);
+
+	const nodesRes = await db
+		.prepare(
+			`DELETE FROM ontology_nodes
+			 WHERE last_seen_at < ?
+			   AND NOT EXISTS (
+			     SELECT 1 FROM ontology_edges e
+			     WHERE e.project_id = ontology_nodes.project_id
+			       AND e.environment = ontology_nodes.environment
+			       AND (e.source_node_id = ontology_nodes.id OR e.target_node_id = ontology_nodes.id)
+			   )`,
+		)
+		.bind(thirtyDaysAgoIso)
+		.run();
+	const ontologyNodesPurged = changes(nodesRes);
+
+	const proposalsRes = await db
+		.prepare(
+			`DELETE FROM remediation_proposals
+			 WHERE (expires_at IS NOT NULL AND expires_at < ?)
+			    OR (expires_at IS NULL AND created_at < ?)`,
+		)
+		.bind(nowIso, ninetyDaysAgoIso)
+		.run();
+	const remediationProposalsPurged = changes(proposalsRes);
+
+	const findingsRes = await db
+		.prepare(
+			`DELETE FROM investigation_findings
+			 WHERE causal_verification_status NOT IN ('counterfactual_verified', 'operationally_corroborated')
+			   AND ((expires_at IS NOT NULL AND expires_at < ?)
+			     OR (expires_at IS NULL AND created_at < ?))`,
+		)
+		.bind(nowIso, ninetyDaysAgoIso)
+		.run();
+	const investigationFindingsPurged = changes(findingsRes);
+
+	// Preserve change_events referenced by active investigation_findings
+	const changesRes = await db
+		.prepare(
+			`DELETE FROM change_events
+			 WHERE ((expires_at IS NOT NULL AND expires_at < ?)
+			     OR (expires_at IS NULL AND occurred_at < ?))
+			   AND NOT EXISTS (
+			     SELECT 1 FROM investigation_findings f
+			     WHERE f.project_id = change_events.project_id
+			       AND f.environment = change_events.environment
+			       AND f.contributing_change_id = change_events.id
+			   )`,
+		)
+		.bind(nowIso, ninetyDaysAgoIso)
+		.run();
+	const changeEventsPurged = changes(changesRes);
+
+	const incidentsRes = await db
+		.prepare(
+			`DELETE FROM incident_investigations
+			 WHERE status IN ('resolved', 'escalated')
+			   AND ((expires_at IS NOT NULL AND expires_at < ?)
+			     OR (expires_at IS NULL AND updated_at < ?))`,
+		)
+		.bind(nowIso, ninetyDaysAgoIso)
+		.run();
+	const incidentInvestigationsPurged = changes(incidentsRes);
+
+	// Lease reaping (RFC 0013 § 11): a runner that died holding a lease must not
+	// strand the incident. Expired `leased` incidents go back to `open` so the
+	// next runner can pick them up. `remediating`/`verifying` are owned by the
+	// proposal claim/watchdog timeouts, not by this reaper.
+	const leasesRes = await db
+		.prepare(
+			`UPDATE incident_investigations
+			 SET status = 'open', lease_owner_id = NULL, lease_expires_at = NULL, updated_at = ?
+			 WHERE status = 'leased'
+			   AND lease_expires_at IS NOT NULL
+			   AND lease_expires_at < ?`,
+		)
+		.bind(nowIso, nowIso)
+		.run();
+	const incidentLeasesReaped = changes(leasesRes);
+
 	return {
 		alert_evaluations: changes(alertEvaluations),
 		ask_evidence_events: changes(askEvents),
 		metric_series: changes(series),
 		evidence_refs: changes(refs),
 		evidence_ref_expansions: changes(expansions),
+		ontology_edges: ontologyEdgesPurged,
+		ontology_nodes: ontologyNodesPurged,
+		change_events: changeEventsPurged,
+		incident_investigations: incidentInvestigationsPurged,
+		incident_leases_reaped: incidentLeasesReaped,
+		investigation_findings: investigationFindingsPurged,
+		remediation_proposals: remediationProposalsPurged,
 	};
 }

package/src/lib/store/ingest.ts

@@ -1,16 +1,40 @@
 import type { StoredSpan } from "@obsunified/types";
+import {
+	extractSpanLineageAndTopology,
+	upsertOntologyFromSpans,
+} from "../autosre-ontology";
 import type { SqlDb } from "../sql-db";
 
 export async function ingestTelemetrySpans(
 	db: SqlDb,
 	spans: StoredSpan[],
+	logger: { warn: (message: string) => void } = console,
 ): Promise<{ inserted: number; traceCount: number }> {
 	if (spans.length === 0) return { inserted: 0, traceCount: 0 };
 
-	const statements = spans.map((span) => {
+	const enrichedSpans = spans.map((span) => {
 		if (!span.projectId)
 			throw new Error("TelemetryStore.ingest: span.projectId is required");
-		return db
+		if (
+			span.environment !== undefined &&
+			span.serviceVersion !== undefined &&
+			span.targetNodeId !== undefined
+		) {
+			return span;
+		}
+		const lineage = extractSpanLineageAndTopology(span);
+		return {
+			...span,
+			environment: span.environment ?? lineage.environment,
+			serviceVersion: span.serviceVersion ?? lineage.serviceVersion,
+			commitSha: span.commitSha ?? lineage.commitSha,
+			targetNodeId: span.targetNodeId ?? lineage.targetNode?.id ?? null,
+			errorSignature: span.errorSignature ?? lineage.errorSignature,
+		};
+	});
+
+	const statements = enrichedSpans.map((span) =>
+		db
 			.prepare(`
         INSERT OR IGNORE INTO telemetry_spans (
           project_id, trace_id, span_id, parent_span_id, trace_state,
@@ -19,8 +43,9 @@ export async function ingestTelemetrySpans(
           attributes_json, dropped_attributes_count,
           resource_attributes_json, events_json, dropped_events_count,
           links_json, dropped_links_count, received_at, expires_at,
-          session_id, interaction_id, telemetry_sdk_name
-        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+          session_id, interaction_id, telemetry_sdk_name,
+          environment, service_version, commit_sha, target_node_id, error_signature
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
       `)
 			.bind(
 				span.projectId,
@@ -50,11 +75,26 @@ export async function ingestTelemetrySpans(
 				span.sessionId ?? null,
 				span.interactionId ?? null,
 				span.telemetrySdkName ?? null,
-			);
-	});
+				span.environment ?? "default",
+				span.serviceVersion ?? null,
+				span.commitSha ?? null,
+				span.targetNodeId ?? null,
+				span.errorSignature ?? null,
+			),
+	);
 
 	await db.batch(statements);
 
+	try {
+		await upsertOntologyFromSpans(db, enrichedSpans);
+	} catch (err) {
+		// Ontology enrichment must never block span ingest, but a failure here
+		// means the causal graph is silently going stale, so surface it.
+		logger.warn(
+			`upsertOntologyFromSpans failed: ${err instanceof Error ? err.message : String(err)}`,
+		);
+	}
+
 	// Trace instrumentation gaps are computed lazily at read time (see
 	// `getTelemetryTraceGaps`), not materialized on ingest. This keeps the
 	// ingest hot path to a single batched span write and avoids recomputing