@obsunified/collector 2.2.0 → 2.2.1

package/src/lib/housekeeping.test.ts

@@ -62,6 +62,7 @@ describe("purgeSecondaryRows", () => {
 			ontology_nodes: 0,
 			change_events: 0,
 			incident_investigations: 0,
+			incident_leases_reaped: 0,
 			remediation_proposals: 0,
 			investigation_findings: 0,
 		});
@@ -150,4 +151,73 @@ describe("purgeSecondaryRows", () => {
 		]);
 		db.close();
 	});
+
+	it("reaps expired leases back to open without touching live leases or non-leased incidents (RFC 0013 § 11)", async () => {
+		const db = createMigratedSqliteDb();
+		const ins = db.raw.prepare(
+			`INSERT INTO incident_investigations (project_id, environment, id, dedup_key, status, target_node_id, symptom_anchor_kind, symptom_anchor_id, lease_owner_id, lease_expires_at, opened_at, updated_at, expires_at)
+			 VALUES ('alpha', 'prod', ?, ?, ?, 'n', 'trace', 't', ?, ?, ?, ?, ?)`,
+		);
+		const farFuture = iso(-24 * 365);
+		ins.run(
+			"lease-expired",
+			"k1",
+			"leased",
+			"runner-a",
+			iso(1),
+			iso(5),
+			iso(5),
+			farFuture,
+		);
+		ins.run(
+			"lease-live",
+			"k2",
+			"leased",
+			"runner-b",
+			iso(-1),
+			iso(5),
+			iso(5),
+			farFuture,
+		);
+		ins.run(
+			"remediating",
+			"k3",
+			"remediating",
+			"runner-c",
+			iso(1),
+			iso(5),
+			iso(5),
+			farFuture,
+		);
+
+		const summary = await purgeSecondaryRows(db, {
+			retentionHours: 72,
+			now: () => NOW,
+		});
+		expect(summary.incident_leases_reaped).toBe(1);
+		const rows = db.raw
+			.prepare(
+				`SELECT id, status, lease_owner_id FROM incident_investigations ORDER BY id`,
+			)
+			.all() as Array<{
+			id: string;
+			status: string;
+			lease_owner_id: string | null;
+		}>;
+		expect(rows).toEqual([
+			{ id: "lease-expired", status: "open", lease_owner_id: null },
+			{ id: "lease-live", status: "leased", lease_owner_id: "runner-b" },
+			{ id: "remediating", status: "remediating", lease_owner_id: "runner-c" },
+		]);
+		db.close();
+	});
+
+	it("propagates storage errors instead of silently swallowing them", async () => {
+		const db = createMigratedSqliteDb();
+		db.raw.exec("DROP TABLE remediation_proposals");
+		await expect(
+			purgeSecondaryRows(db, { retentionHours: 72, now: () => NOW }),
+		).rejects.toThrow(/remediation_proposals/);
+		db.close();
+	});
 });

package/src/lib/housekeeping.ts

@@ -13,7 +13,6 @@
  *    the signal they reference); expansions follow their ref.
  */
 
-import { evaluateRemediationWatchdogs } from "./autosre-remediation-gate";
 import type { SqlDb } from "./sql-db";
 
 export const ALERT_EVALUATIONS_MAX_AGE_HOURS = 720;
@@ -29,6 +28,8 @@ export interface HousekeepingSummary {
 	ontology_nodes?: number;
 	change_events?: number;
 	incident_investigations?: number;
+	/** `leased` incidents whose lease expired and were returned to `open` (RFC 0013 § 11). */
+	incident_leases_reaped?: number;
 	investigation_findings?: number;
 	remediation_proposals?: number;
 }
@@ -92,90 +93,96 @@ export async function purgeSecondaryRows(
 		.run();
 
 	// RFC 0013 — AutoSRE ontology, change events, incident investigations,
-	// investigation findings, and remediation proposals retention + watchdog.
+	// investigation findings, and remediation proposals retention, plus lease
+	// reaping. Errors propagate (the retention caller logs them); the
+	// post-remediation watchdog runs on the 1-minute analyses tick instead.
 	const thirtyDaysAgoIso = isoBefore(30 * 24);
 	const ninetyDaysAgoIso = isoBefore(90 * 24);
-	let ontologyEdgesPurged = 0;
-	let ontologyNodesPurged = 0;
-	let changeEventsPurged = 0;
-	let incidentInvestigationsPurged = 0;
-	let investigationFindingsPurged = 0;
-	let remediationProposalsPurged = 0;
-	try {
-		const edgesRes = await db
-			.prepare(`DELETE FROM ontology_edges WHERE last_seen_at < ?`)
-			.bind(thirtyDaysAgoIso)
-			.run();
-		ontologyEdgesPurged = changes(edgesRes);
-
-		const nodesRes = await db
-			.prepare(
-				`DELETE FROM ontology_nodes
-				 WHERE last_seen_at < ?
-				   AND NOT EXISTS (
-				     SELECT 1 FROM ontology_edges e
-				     WHERE e.project_id = ontology_nodes.project_id
-				       AND e.environment = ontology_nodes.environment
-				       AND (e.source_node_id = ontology_nodes.id OR e.target_node_id = ontology_nodes.id)
-				   )`,
-			)
-			.bind(thirtyDaysAgoIso)
-			.run();
-		ontologyNodesPurged = changes(nodesRes);
-
-		const proposalsRes = await db
-			.prepare(
-				`DELETE FROM remediation_proposals
-				 WHERE (expires_at IS NOT NULL AND expires_at < ?)
-				    OR (expires_at IS NULL AND created_at < ?)`,
-			)
-			.bind(nowIso, ninetyDaysAgoIso)
-			.run();
-		remediationProposalsPurged = changes(proposalsRes);
-
-		const findingsRes = await db
-			.prepare(
-				`DELETE FROM investigation_findings
-				 WHERE causal_verification_status NOT IN ('counterfactual_verified', 'operationally_corroborated')
-				   AND ((expires_at IS NOT NULL AND expires_at < ?)
-				     OR (expires_at IS NULL AND created_at < ?))`,
-			)
-			.bind(nowIso, ninetyDaysAgoIso)
-			.run();
-		investigationFindingsPurged = changes(findingsRes);
-
-		// Preserve change_events referenced by active investigation_findings
-		const changesRes = await db
-			.prepare(
-				`DELETE FROM change_events
-				 WHERE ((expires_at IS NOT NULL AND expires_at < ?)
-				     OR (expires_at IS NULL AND occurred_at < ?))
-				   AND NOT EXISTS (
-				     SELECT 1 FROM investigation_findings f
-				     WHERE f.project_id = change_events.project_id
-				       AND f.environment = change_events.environment
-				       AND f.contributing_change_id = change_events.id
-				   )`,
-			)
-			.bind(nowIso, ninetyDaysAgoIso)
-			.run();
-		changeEventsPurged = changes(changesRes);
-
-		const incidentsRes = await db
-			.prepare(
-				`DELETE FROM incident_investigations
-				 WHERE status IN ('resolved', 'escalated')
-				   AND ((expires_at IS NOT NULL AND expires_at < ?)
-				     OR (expires_at IS NULL AND updated_at < ?))`,
-			)
-			.bind(nowIso, ninetyDaysAgoIso)
-			.run();
-		incidentInvestigationsPurged = changes(incidentsRes);
-
-		await evaluateRemediationWatchdogs(db, { nowIso });
-	} catch {
-		// Gracefully ignore when running against partial test schemas without 050
-	}
+	const edgesRes = await db
+		.prepare(`DELETE FROM ontology_edges WHERE last_seen_at < ?`)
+		.bind(thirtyDaysAgoIso)
+		.run();
+	const ontologyEdgesPurged = changes(edgesRes);
+
+	const nodesRes = await db
+		.prepare(
+			`DELETE FROM ontology_nodes
+			 WHERE last_seen_at < ?
+			   AND NOT EXISTS (
+			     SELECT 1 FROM ontology_edges e
+			     WHERE e.project_id = ontology_nodes.project_id
+			       AND e.environment = ontology_nodes.environment
+			       AND (e.source_node_id = ontology_nodes.id OR e.target_node_id = ontology_nodes.id)
+			   )`,
+		)
+		.bind(thirtyDaysAgoIso)
+		.run();
+	const ontologyNodesPurged = changes(nodesRes);
+
+	const proposalsRes = await db
+		.prepare(
+			`DELETE FROM remediation_proposals
+			 WHERE (expires_at IS NOT NULL AND expires_at < ?)
+			    OR (expires_at IS NULL AND created_at < ?)`,
+		)
+		.bind(nowIso, ninetyDaysAgoIso)
+		.run();
+	const remediationProposalsPurged = changes(proposalsRes);
+
+	const findingsRes = await db
+		.prepare(
+			`DELETE FROM investigation_findings
+			 WHERE causal_verification_status NOT IN ('counterfactual_verified', 'operationally_corroborated')
+			   AND ((expires_at IS NOT NULL AND expires_at < ?)
+			     OR (expires_at IS NULL AND created_at < ?))`,
+		)
+		.bind(nowIso, ninetyDaysAgoIso)
+		.run();
+	const investigationFindingsPurged = changes(findingsRes);
+
+	// Preserve change_events referenced by active investigation_findings
+	const changesRes = await db
+		.prepare(
+			`DELETE FROM change_events
+			 WHERE ((expires_at IS NOT NULL AND expires_at < ?)
+			     OR (expires_at IS NULL AND occurred_at < ?))
+			   AND NOT EXISTS (
+			     SELECT 1 FROM investigation_findings f
+			     WHERE f.project_id = change_events.project_id
+			       AND f.environment = change_events.environment
+			       AND f.contributing_change_id = change_events.id
+			   )`,
+		)
+		.bind(nowIso, ninetyDaysAgoIso)
+		.run();
+	const changeEventsPurged = changes(changesRes);
+
+	const incidentsRes = await db
+		.prepare(
+			`DELETE FROM incident_investigations
+			 WHERE status IN ('resolved', 'escalated')
+			   AND ((expires_at IS NOT NULL AND expires_at < ?)
+			     OR (expires_at IS NULL AND updated_at < ?))`,
+		)
+		.bind(nowIso, ninetyDaysAgoIso)
+		.run();
+	const incidentInvestigationsPurged = changes(incidentsRes);
+
+	// Lease reaping (RFC 0013 § 11): a runner that died holding a lease must not
+	// strand the incident. Expired `leased` incidents go back to `open` so the
+	// next runner can pick them up. `remediating`/`verifying` are owned by the
+	// proposal claim/watchdog timeouts, not by this reaper.
+	const leasesRes = await db
+		.prepare(
+			`UPDATE incident_investigations
+			 SET status = 'open', lease_owner_id = NULL, lease_expires_at = NULL, updated_at = ?
+			 WHERE status = 'leased'
+			   AND lease_expires_at IS NOT NULL
+			   AND lease_expires_at < ?`,
+		)
+		.bind(nowIso, nowIso)
+		.run();
+	const incidentLeasesReaped = changes(leasesRes);
 
 	return {
 		alert_evaluations: changes(alertEvaluations),
@@ -187,6 +194,7 @@ export async function purgeSecondaryRows(
 		ontology_nodes: ontologyNodesPurged,
 		change_events: changeEventsPurged,
 		incident_investigations: incidentInvestigationsPurged,
+		incident_leases_reaped: incidentLeasesReaped,
 		investigation_findings: investigationFindingsPurged,
 		remediation_proposals: remediationProposalsPurged,
 	};

package/src/lib/store/ingest.ts

@@ -8,6 +8,7 @@ import type { SqlDb } from "../sql-db";
 export async function ingestTelemetrySpans(
 	db: SqlDb,
 	spans: StoredSpan[],
+	logger: { warn: (message: string) => void } = console,
 ): Promise<{ inserted: number; traceCount: number }> {
 	if (spans.length === 0) return { inserted: 0, traceCount: 0 };
 
@@ -82,65 +83,16 @@ export async function ingestTelemetrySpans(
 			),
 	);
 
-	try {
-		await db.batch(statements);
-	} catch (err) {
-		const msg = err instanceof Error ? err.message : String(err);
-		if (msg.includes("no such column: environment")) {
-			// Fallback for unit tests running against pre-050 partial schemas
-			const legacyStatements = enrichedSpans.map((span) =>
-				db
-					.prepare(`
-            INSERT OR IGNORE INTO telemetry_spans (
-              project_id, trace_id, span_id, parent_span_id, trace_state,
-              service_name, scope_name, scope_version, span_name, span_kind,
-              status_code, status_message, start_time, end_time, duration_ms,
-              attributes_json, dropped_attributes_count,
-              resource_attributes_json, events_json, dropped_events_count,
-              links_json, dropped_links_count, received_at, expires_at,
-              session_id, interaction_id, telemetry_sdk_name
-            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-          `)
-					.bind(
-						span.projectId,
-						span.traceId,
-						span.spanId,
-						span.parentSpanId,
-						span.traceState,
-						span.serviceName,
-						span.scopeName,
-						span.scopeVersion,
-						span.spanName,
-						span.spanKind,
-						span.statusCode,
-						span.statusMessage,
-						span.startTime,
-						span.endTime,
-						span.durationMs,
-						span.attributesJson,
-						span.droppedAttributesCount,
-						span.resourceAttributesJson,
-						span.eventsJson,
-						span.droppedEventsCount,
-						span.linksJson,
-						span.droppedLinksCount,
-						span.receivedAt,
-						span.expiresAt,
-						span.sessionId ?? null,
-						span.interactionId ?? null,
-						span.telemetrySdkName ?? null,
-					),
-			);
-			await db.batch(legacyStatements);
-		} else {
-			throw err;
-		}
-	}
+	await db.batch(statements);
 
 	try {
 		await upsertOntologyFromSpans(db, enrichedSpans);
-	} catch {
-		// Non-fatal on partial-schema unit test DBs
+	} catch (err) {
+		// Ontology enrichment must never block span ingest, but a failure here
+		// means the causal graph is silently going stale, so surface it.
+		logger.warn(
+			`upsertOntologyFromSpans failed: ${err instanceof Error ? err.message : String(err)}`,
+		);
 	}
 
 	// Trace instrumentation gaps are computed lazily at read time (see

package/src/migrations/051_incident_active_dedup_unique.sql

@@ -0,0 +1,31 @@
+-- Make active-incident deduplication race-free (RFC 0013 § 6.A, governance review B4).
+--
+-- `openOrDeduplicateIncident` used to SELECT-then-INSERT, so two concurrent
+-- alerts for the same symptom could each open an incident. A partial UNIQUE
+-- index lets the database arbitrate; the application inserts with
+-- ON CONFLICT DO NOTHING and re-reads the winner.
+
+-- Collapse any pre-existing duplicate active rows (keep the newest per key) so
+-- the index can be built. Older duplicates are closed as escalated, not deleted.
+UPDATE incident_investigations
+SET status = 'escalated'
+WHERE status IN ('open', 'dispatched', 'leased', 'remediating', 'verifying')
+  AND EXISTS (
+    SELECT 1 FROM incident_investigations o
+    WHERE o.project_id = incident_investigations.project_id
+      AND o.environment = incident_investigations.environment
+      AND o.dedup_key = incident_investigations.dedup_key
+      AND o.status IN ('open', 'dispatched', 'leased', 'remediating', 'verifying')
+      AND (
+        o.opened_at > incident_investigations.opened_at
+        OR (o.opened_at = incident_investigations.opened_at AND o.id > incident_investigations.id)
+      )
+  );
+
+CREATE UNIQUE INDEX IF NOT EXISTS uq_incidents_active_dedup
+  ON incident_investigations (project_id, environment, dedup_key)
+  WHERE status IN ('open', 'dispatched', 'leased', 'remediating', 'verifying');
+
+-- Watchdog (governance review B11): a finding is only `recovered` after two
+-- consecutive checkpoints with sufficient traffic and a healthy error rate.
+ALTER TABLE investigation_findings ADD COLUMN consecutive_ok_checks INTEGER NOT NULL DEFAULT 0;

package/src/migrations/alert-state-notified.test.ts

@@ -44,9 +44,9 @@ describe("migration 049 (alert_state.notified)", () => {
 			.map((c) => (c as { name: string }).name);
 		expect(columnsBefore).not.toContain("notified");
 
-		expect(
-			applySqliteMigrations(db.raw, { from: "049", upTo: "049" }),
-		).toEqual(["049_alert_state_notified.sql"]);
+		expect(applySqliteMigrations(db.raw, { from: "049", upTo: "049" })).toEqual(
+			["049_alert_state_notified.sql"],
+		);
 
 		const notified = db.raw
 			.prepare("PRAGMA table_info(alert_state)")

package/src/migrations-postgres/051_incident_active_dedup_unique.sql

@@ -0,0 +1,31 @@
+-- (PostgreSQL mirror) Make active-incident deduplication race-free (RFC 0013 § 6.A, governance review B4).
+--
+-- `openOrDeduplicateIncident` used to SELECT-then-INSERT, so two concurrent
+-- alerts for the same symptom could each open an incident. A partial UNIQUE
+-- index lets the database arbitrate; the application inserts with
+-- ON CONFLICT DO NOTHING and re-reads the winner.
+
+-- Collapse any pre-existing duplicate active rows (keep the newest per key) so
+-- the index can be built. Older duplicates are closed as escalated, not deleted.
+UPDATE incident_investigations
+SET status = 'escalated'
+WHERE status IN ('open', 'dispatched', 'leased', 'remediating', 'verifying')
+  AND EXISTS (
+    SELECT 1 FROM incident_investigations o
+    WHERE o.project_id = incident_investigations.project_id
+      AND o.environment = incident_investigations.environment
+      AND o.dedup_key = incident_investigations.dedup_key
+      AND o.status IN ('open', 'dispatched', 'leased', 'remediating', 'verifying')
+      AND (
+        o.opened_at > incident_investigations.opened_at
+        OR (o.opened_at = incident_investigations.opened_at AND o.id > incident_investigations.id)
+      )
+  );
+
+CREATE UNIQUE INDEX IF NOT EXISTS uq_incidents_active_dedup
+  ON incident_investigations (project_id, environment, dedup_key)
+  WHERE status IN ('open', 'dispatched', 'leased', 'remediating', 'verifying');
+
+-- Watchdog (governance review B11): a finding is only `recovered` after two
+-- consecutive checkpoints with sufficient traffic and a healthy error rate.
+ALTER TABLE investigation_findings ADD COLUMN IF NOT EXISTS consecutive_ok_checks BIGINT NOT NULL DEFAULT 0;

package/src/plugins/alerts-evaluator.ts

@@ -11,7 +11,10 @@ import {
 	type Logger,
 } from "../framework/logger";
 import { AlertsStore, compareValue } from "../lib/alerts-store";
-import { openOrDeduplicateIncident } from "../lib/autosre-remediation-gate";
+import {
+	openOrDeduplicateIncident,
+	resolveServiceIncidentTarget,
+} from "../lib/autosre-remediation-gate";
 import { sqlDbFor } from "../lib/sql-db";
 import {
 	assertSafeWebhookUrl,
@@ -439,9 +442,18 @@ export async function evaluateAllRules(
 		allowPrivateWebhookTargets,
 		hostLookup: options?.hostLookup,
 		onAlertFiring: async (rule) => {
+			// Target the rule's service when it names one (environment taken from
+			// that service's latest spans); otherwise fall back to the rule itself.
+			const target = await resolveServiceIncidentTarget(
+				db,
+				rule.projectId,
+				(rule.query as { serviceName?: string }).serviceName,
+				`alert:${rule.id}`,
+				logger,
+			);
 			await openOrDeduplicateIncident(db, rule.projectId, {
-				environment: "default",
-				targetNodeId: `alert:${rule.id}`,
+				environment: target.environment,
+				targetNodeId: target.targetNodeId,
 				symptomAnchorKind: "alert",
 				symptomAnchorId: rule.id,
 			});