@obsunified/collector 2.1.1 → 2.2.0

package/src/migrations/050_autosre_causal_ontology.sql

@@ -0,0 +1,216 @@
+-- ============================================================================
+-- Migration 050: AutoSRE Causal Ontology, Change Correlation & Incident Memory
+-- (RFC 0013)
+-- ============================================================================
+
+-- 0. Denormalize environment, version, and target_node_id onto existing tables
+--    for O(log N) cohort rate lift and cross-trace contention queries.
+ALTER TABLE telemetry_spans ADD COLUMN environment TEXT NOT NULL DEFAULT 'default';
+ALTER TABLE telemetry_spans ADD COLUMN service_version TEXT;
+ALTER TABLE telemetry_spans ADD COLUMN commit_sha TEXT;
+ALTER TABLE telemetry_spans ADD COLUMN target_node_id TEXT;
+ALTER TABLE telemetry_spans ADD COLUMN error_signature TEXT;
+
+CREATE INDEX IF NOT EXISTS idx_spans_cohort_attribution
+  ON telemetry_spans (project_id, environment, service_name, service_version, received_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_spans_contention_lookup
+  ON telemetry_spans (project_id, environment, target_node_id, start_time, end_time)
+  WHERE target_node_id IS NOT NULL;
+
+ALTER TABLE actions ADD COLUMN environment TEXT NOT NULL DEFAULT 'default';
+ALTER TABLE actions ADD COLUMN service_version TEXT;
+ALTER TABLE actions ADD COLUMN target_node_id TEXT;
+
+-- Extend eval_runs and add eval_case_results for verifiable PairedCounterfactualProof
+ALTER TABLE eval_runs ADD COLUMN environment TEXT NOT NULL DEFAULT 'default';
+ALTER TABLE eval_runs ADD COLUMN service_version TEXT;
+ALTER TABLE eval_runs ADD COLUMN commit_sha TEXT;
+ALTER TABLE eval_runs ADD COLUMN config_digest TEXT;
+ALTER TABLE eval_runs ADD COLUMN input_payload_hash TEXT;
+ALTER TABLE eval_runs ADD COLUMN failure_fingerprint TEXT;
+
+ALTER TABLE eval_case_results ADD COLUMN input_payload_hash TEXT;
+ALTER TABLE eval_case_results ADD COLUMN failure_fingerprint TEXT;
+
+CREATE INDEX IF NOT EXISTS idx_eval_case_results_counterfactual
+  ON eval_case_results (project_id, eval_case_id, input_payload_hash, passed);
+
+-- 1. Environment-Scoped Persistent Operational Ontology Nodes
+CREATE TABLE IF NOT EXISTS ontology_nodes (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  environment TEXT NOT NULL DEFAULT 'default',
+  id TEXT NOT NULL,
+  kind TEXT NOT NULL,
+  name TEXT NOT NULL,
+  namespace_or_endpoint TEXT NOT NULL DEFAULT 'default',
+  is_scoped_identity INTEGER NOT NULL DEFAULT 1,
+  current_version TEXT,
+  current_commit_sha TEXT,
+  current_config_digest TEXT,
+  attributes_json TEXT,
+  first_seen_at TEXT NOT NULL,
+  last_seen_at TEXT NOT NULL,
+  PRIMARY KEY (project_id, environment, id)
+);
+
+-- 2. Directed Operational Ontology Edges (Cycles Permitted; Handled via Tarjan SCC)
+CREATE TABLE IF NOT EXISTS ontology_edges (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  environment TEXT NOT NULL DEFAULT 'default',
+  id TEXT NOT NULL,
+  source_node_id TEXT NOT NULL,
+  predicate TEXT NOT NULL,
+  target_node_id TEXT NOT NULL,
+  relationship_certainty TEXT NOT NULL DEFAULT 'exact',
+  call_count INTEGER NOT NULL DEFAULT 0,
+  error_count INTEGER NOT NULL DEFAULT 0,
+  last_duration_ms REAL,
+  first_seen_at TEXT NOT NULL,
+  last_seen_at TEXT NOT NULL,
+  PRIMARY KEY (project_id, environment, id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_ontology_edges_source
+  ON ontology_edges (project_id, environment, source_node_id);
+
+CREATE INDEX IF NOT EXISTS idx_ontology_edges_target
+  ON ontology_edges (project_id, environment, target_node_id);
+
+-- 3. Explicit & Inferred Change Events
+CREATE TABLE IF NOT EXISTS change_events (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  environment TEXT NOT NULL DEFAULT 'default',
+  id TEXT NOT NULL,
+  target_node_id TEXT NOT NULL,
+  kind TEXT NOT NULL,
+  source TEXT NOT NULL,
+  relationship_certainty TEXT NOT NULL DEFAULT 'exact',
+  summary TEXT NOT NULL,
+  service_name TEXT,
+  version TEXT,
+  previous_version TEXT,
+  commit_sha TEXT,
+  previous_commit_sha TEXT,
+  config_digest TEXT,
+  repository_url TEXT,
+  pull_request_url TEXT,
+  actor_id TEXT,
+  caused_by_action_id TEXT,
+  agent_run_id TEXT,
+  touched_paths_json TEXT,
+  attributes_json TEXT,
+  occurred_at TEXT NOT NULL,
+  received_at TEXT NOT NULL,
+  expires_at TEXT,
+  PRIMARY KEY (project_id, environment, id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_change_events_node_time
+  ON change_events (project_id, environment, target_node_id, occurred_at DESC);
+
+-- 4. Durable Incident Investigations & Runner Lease Coordination
+CREATE TABLE IF NOT EXISTS incident_investigations (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  environment TEXT NOT NULL DEFAULT 'default',
+  id TEXT NOT NULL,
+  dedup_key TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'open',
+  target_node_id TEXT NOT NULL,
+  symptom_anchor_kind TEXT NOT NULL,
+  symptom_anchor_id TEXT NOT NULL,
+  symptom_fingerprint TEXT,
+  alert_count INTEGER NOT NULL DEFAULT 1,
+  latest_bundle_json TEXT,
+  lease_owner_id TEXT,
+  lease_expires_at TEXT,
+  dispatch_attempts INTEGER NOT NULL DEFAULT 0,
+  last_dispatched_at TEXT,
+  opened_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  resolved_at TEXT,
+  expires_at TEXT,
+  PRIMARY KEY (project_id, environment, id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_incidents_active_dedup
+  ON incident_investigations (project_id, environment, dedup_key, status);
+
+-- 5. Closed-Loop Investigation Findings (Separating Recovery vs Causal Proof)
+CREATE TABLE IF NOT EXISTS investigation_findings (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  environment TEXT NOT NULL DEFAULT 'default',
+  id TEXT NOT NULL,
+  incident_id TEXT,
+  fingerprint TEXT NOT NULL,
+  fingerprint_input_json TEXT NOT NULL,
+  title TEXT NOT NULL,
+  summary TEXT NOT NULL,
+  causal_proof_tier TEXT NOT NULL DEFAULT 'unverified_locus',
+  confidence REAL NOT NULL,
+  confidence_breakdown_json TEXT NOT NULL,
+  symptom_entity_kind TEXT NOT NULL,
+  symptom_entity_id TEXT NOT NULL,
+  fault_locus_entity_kind TEXT NOT NULL,
+  fault_locus_entity_id TEXT NOT NULL,
+  root_cause_entity_kind TEXT,
+  root_cause_entity_id TEXT,
+  contributing_change_id TEXT,
+  cohort_verdict TEXT,
+  contending_trace_id TEXT,
+  causal_chain_json TEXT NOT NULL,
+  cited_evidence_ids_json TEXT NOT NULL,
+  code_references_json TEXT,
+  counterfactual_proof_json TEXT,
+  recovery_status TEXT NOT NULL DEFAULT 'not_applicable',
+  causal_verification_status TEXT NOT NULL DEFAULT 'unverified',
+  watchdog_step INTEGER NOT NULL DEFAULT 0,
+  baseline_request_rate_per_min REAL,
+  next_watchdog_check_at TEXT,
+  actor_type TEXT NOT NULL,
+  actor_id TEXT NOT NULL,
+  agent_run_id TEXT,
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  expires_at TEXT,
+  PRIMARY KEY (project_id, environment, id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_findings_fingerprint
+  ON investigation_findings (project_id, environment, fingerprint, created_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_findings_watchdog
+  ON investigation_findings (project_id, recovery_status, next_watchdog_check_at);
+
+-- 6. Pre-Execution Remediation Proposals & Single-Use Grants
+CREATE TABLE IF NOT EXISTS remediation_proposals (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  environment TEXT NOT NULL DEFAULT 'default',
+  id TEXT NOT NULL,
+  incident_id TEXT NOT NULL,
+  finding_id TEXT NOT NULL,
+  idempotency_key TEXT NOT NULL,
+  action_kind TEXT NOT NULL,
+  target_node_id TEXT NOT NULL,
+  expected_current_state_json TEXT NOT NULL,
+  desired_target_state_json TEXT NOT NULL,
+  requested_autonomy TEXT NOT NULL,
+  effective_autonomy TEXT NOT NULL,
+  policy_decision_reason TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'proposed',
+  grant_token_hash TEXT,
+  grant_expires_at TEXT,
+  approved_by_user_id TEXT,
+  approved_at TEXT,
+  claimed_by_runner_id TEXT,
+  claimed_at TEXT,
+  claim_expires_at TEXT,
+  remediation_agent_run_id TEXT,
+  resulting_change_id TEXT,
+  failure_reason TEXT,
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  expires_at TEXT,
+  PRIMARY KEY (project_id, environment, id),
+  UNIQUE (project_id, environment, idempotency_key)
+);

package/src/migrations/alert-state-notified.test.ts

@@ -44,9 +44,9 @@ describe("migration 049 (alert_state.notified)", () => {
 			.map((c) => (c as { name: string }).name);
 		expect(columnsBefore).not.toContain("notified");
 
-		expect(applySqliteMigrations(db.raw, { from: "049" })).toEqual([
-			"049_alert_state_notified.sql",
-		]);
+		expect(
+			applySqliteMigrations(db.raw, { from: "049", upTo: "049" }),
+		).toEqual(["049_alert_state_notified.sql"]);
 
 		const notified = db.raw
 			.prepare("PRAGMA table_info(alert_state)")

package/src/migrations/parity.test.ts

@@ -28,6 +28,10 @@ const RETENTION_TABLES = [
 	"ai_calls",
 	"metric_point",
 	"metric_exemplars",
+	"change_events",
+	"incident_investigations",
+	"investigation_findings",
+	"remediation_proposals",
 ];
 
 describe("migration parity", () => {

package/src/migrations/project-scoped-keys.test.ts

@@ -259,7 +259,7 @@ describe("migrations 045–048 (project-scoped composite keys)", () => {
 		).toEqual({ input_hash: "hash", action_id: "01ACTION" });
 
 		// Re-applying the rebuild files is a no-op in effect.
-		applySqliteMigrations(raw, { from: "045" });
+		applySqliteMigrations(raw, { from: "045", upTo: "048" });
 		for (const [table, key] of Object.entries(EXPECTED_KEYS)) {
 			expect(primaryKeyColumns(raw, table), table).toEqual(key);
 			expect(

package/src/migrations-postgres/050_autosre_causal_ontology.sql

@@ -0,0 +1,216 @@
+-- ============================================================================
+-- Migration 050: AutoSRE Causal Ontology, Change Correlation & Incident Memory
+-- (RFC 0013 — PostgreSQL mirror)
+-- ============================================================================
+
+-- 0. Denormalize environment, version, and target_node_id onto existing tables
+--    for O(log N) cohort rate lift and cross-trace contention queries.
+ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS environment TEXT NOT NULL DEFAULT 'default';
+ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS service_version TEXT;
+ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS commit_sha TEXT;
+ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS target_node_id TEXT;
+ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS error_signature TEXT;
+
+CREATE INDEX IF NOT EXISTS idx_spans_cohort_attribution
+  ON telemetry_spans (project_id, environment, service_name, service_version, received_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_spans_contention_lookup
+  ON telemetry_spans (project_id, environment, target_node_id, start_time, end_time)
+  WHERE target_node_id IS NOT NULL;
+
+ALTER TABLE actions ADD COLUMN IF NOT EXISTS environment TEXT NOT NULL DEFAULT 'default';
+ALTER TABLE actions ADD COLUMN IF NOT EXISTS service_version TEXT;
+ALTER TABLE actions ADD COLUMN IF NOT EXISTS target_node_id TEXT;
+
+-- Extend eval_runs and add eval_case_results for verifiable PairedCounterfactualProof
+ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS environment TEXT NOT NULL DEFAULT 'default';
+ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS service_version TEXT;
+ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS commit_sha TEXT;
+ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS config_digest TEXT;
+ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS input_payload_hash TEXT;
+ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS failure_fingerprint TEXT;
+
+ALTER TABLE eval_case_results ADD COLUMN IF NOT EXISTS input_payload_hash TEXT;
+ALTER TABLE eval_case_results ADD COLUMN IF NOT EXISTS failure_fingerprint TEXT;
+
+CREATE INDEX IF NOT EXISTS idx_eval_case_results_counterfactual
+  ON eval_case_results (project_id, eval_case_id, input_payload_hash, passed);
+
+-- 1. Environment-Scoped Persistent Operational Ontology Nodes
+CREATE TABLE IF NOT EXISTS ontology_nodes (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  environment TEXT NOT NULL DEFAULT 'default',
+  id TEXT NOT NULL,
+  kind TEXT NOT NULL,
+  name TEXT NOT NULL,
+  namespace_or_endpoint TEXT NOT NULL DEFAULT 'default',
+  is_scoped_identity INTEGER NOT NULL DEFAULT 1,
+  current_version TEXT,
+  current_commit_sha TEXT,
+  current_config_digest TEXT,
+  attributes_json TEXT,
+  first_seen_at TEXT NOT NULL,
+  last_seen_at TEXT NOT NULL,
+  PRIMARY KEY (project_id, environment, id)
+);
+
+-- 2. Directed Operational Ontology Edges (Cycles Permitted; Handled via Tarjan SCC)
+CREATE TABLE IF NOT EXISTS ontology_edges (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  environment TEXT NOT NULL DEFAULT 'default',
+  id TEXT NOT NULL,
+  source_node_id TEXT NOT NULL,
+  predicate TEXT NOT NULL,
+  target_node_id TEXT NOT NULL,
+  relationship_certainty TEXT NOT NULL DEFAULT 'exact',
+  call_count INTEGER NOT NULL DEFAULT 0,
+  error_count INTEGER NOT NULL DEFAULT 0,
+  last_duration_ms DOUBLE PRECISION,
+  first_seen_at TEXT NOT NULL,
+  last_seen_at TEXT NOT NULL,
+  PRIMARY KEY (project_id, environment, id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_ontology_edges_source
+  ON ontology_edges (project_id, environment, source_node_id);
+
+CREATE INDEX IF NOT EXISTS idx_ontology_edges_target
+  ON ontology_edges (project_id, environment, target_node_id);
+
+-- 3. Explicit & Inferred Change Events
+CREATE TABLE IF NOT EXISTS change_events (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  environment TEXT NOT NULL DEFAULT 'default',
+  id TEXT NOT NULL,
+  target_node_id TEXT NOT NULL,
+  kind TEXT NOT NULL,
+  source TEXT NOT NULL,
+  relationship_certainty TEXT NOT NULL DEFAULT 'exact',
+  summary TEXT NOT NULL,
+  service_name TEXT,
+  version TEXT,
+  previous_version TEXT,
+  commit_sha TEXT,
+  previous_commit_sha TEXT,
+  config_digest TEXT,
+  repository_url TEXT,
+  pull_request_url TEXT,
+  actor_id TEXT,
+  caused_by_action_id TEXT,
+  agent_run_id TEXT,
+  touched_paths_json TEXT,
+  attributes_json TEXT,
+  occurred_at TEXT NOT NULL,
+  received_at TEXT NOT NULL,
+  expires_at TEXT,
+  PRIMARY KEY (project_id, environment, id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_change_events_node_time
+  ON change_events (project_id, environment, target_node_id, occurred_at DESC);
+
+-- 4. Durable Incident Investigations & Runner Lease Coordination
+CREATE TABLE IF NOT EXISTS incident_investigations (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  environment TEXT NOT NULL DEFAULT 'default',
+  id TEXT NOT NULL,
+  dedup_key TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'open',
+  target_node_id TEXT NOT NULL,
+  symptom_anchor_kind TEXT NOT NULL,
+  symptom_anchor_id TEXT NOT NULL,
+  symptom_fingerprint TEXT,
+  alert_count INTEGER NOT NULL DEFAULT 1,
+  latest_bundle_json TEXT,
+  lease_owner_id TEXT,
+  lease_expires_at TEXT,
+  dispatch_attempts INTEGER NOT NULL DEFAULT 0,
+  last_dispatched_at TEXT,
+  opened_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  resolved_at TEXT,
+  expires_at TEXT,
+  PRIMARY KEY (project_id, environment, id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_incidents_active_dedup
+  ON incident_investigations (project_id, environment, dedup_key, status);
+
+-- 5. Closed-Loop Investigation Findings (Separating Recovery vs Causal Proof)
+CREATE TABLE IF NOT EXISTS investigation_findings (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  environment TEXT NOT NULL DEFAULT 'default',
+  id TEXT NOT NULL,
+  incident_id TEXT,
+  fingerprint TEXT NOT NULL,
+  fingerprint_input_json TEXT NOT NULL,
+  title TEXT NOT NULL,
+  summary TEXT NOT NULL,
+  causal_proof_tier TEXT NOT NULL DEFAULT 'unverified_locus',
+  confidence DOUBLE PRECISION NOT NULL,
+  confidence_breakdown_json TEXT NOT NULL,
+  symptom_entity_kind TEXT NOT NULL,
+  symptom_entity_id TEXT NOT NULL,
+  fault_locus_entity_kind TEXT NOT NULL,
+  fault_locus_entity_id TEXT NOT NULL,
+  root_cause_entity_kind TEXT,
+  root_cause_entity_id TEXT,
+  contributing_change_id TEXT,
+  cohort_verdict TEXT,
+  contending_trace_id TEXT,
+  causal_chain_json TEXT NOT NULL,
+  cited_evidence_ids_json TEXT NOT NULL,
+  code_references_json TEXT,
+  counterfactual_proof_json TEXT,
+  recovery_status TEXT NOT NULL DEFAULT 'not_applicable',
+  causal_verification_status TEXT NOT NULL DEFAULT 'unverified',
+  watchdog_step INTEGER NOT NULL DEFAULT 0,
+  baseline_request_rate_per_min DOUBLE PRECISION,
+  next_watchdog_check_at TEXT,
+  actor_type TEXT NOT NULL,
+  actor_id TEXT NOT NULL,
+  agent_run_id TEXT,
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  expires_at TEXT,
+  PRIMARY KEY (project_id, environment, id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_findings_fingerprint
+  ON investigation_findings (project_id, environment, fingerprint, created_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_findings_watchdog
+  ON investigation_findings (project_id, recovery_status, next_watchdog_check_at);
+
+-- 6. Pre-Execution Remediation Proposals & Single-Use Grants
+CREATE TABLE IF NOT EXISTS remediation_proposals (
+  project_id TEXT NOT NULL DEFAULT 'default',
+  environment TEXT NOT NULL DEFAULT 'default',
+  id TEXT NOT NULL,
+  incident_id TEXT NOT NULL,
+  finding_id TEXT NOT NULL,
+  idempotency_key TEXT NOT NULL,
+  action_kind TEXT NOT NULL,
+  target_node_id TEXT NOT NULL,
+  expected_current_state_json TEXT NOT NULL,
+  desired_target_state_json TEXT NOT NULL,
+  requested_autonomy TEXT NOT NULL,
+  effective_autonomy TEXT NOT NULL,
+  policy_decision_reason TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'proposed',
+  grant_token_hash TEXT,
+  grant_expires_at TEXT,
+  approved_by_user_id TEXT,
+  approved_at TEXT,
+  claimed_by_runner_id TEXT,
+  claimed_at TEXT,
+  claim_expires_at TEXT,
+  remediation_agent_run_id TEXT,
+  resulting_change_id TEXT,
+  failure_reason TEXT,
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  expires_at TEXT,
+  PRIMARY KEY (project_id, environment, id),
+  UNIQUE (project_id, environment, idempotency_key)
+);

package/src/plugins/alerts-evaluator.ts

@@ -11,6 +11,7 @@ import {
 	type Logger,
 } from "../framework/logger";
 import { AlertsStore, compareValue } from "../lib/alerts-store";
+import { openOrDeduplicateIncident } from "../lib/autosre-remediation-gate";
 import { sqlDbFor } from "../lib/sql-db";
 import {
 	assertSafeWebhookUrl,
@@ -238,6 +239,7 @@ async function evaluateOneRule(
 	tracer: ChildSpanRunner | undefined,
 	ruleTimeoutMs: number,
 	delivery: WebhookDeliveryOptions,
+	onAlertFiring?: (rule: AlertRule) => Promise<void>,
 ): Promise<{ fired: number; resolved: number; redelivered: number }> {
 	const none = { fired: 0, resolved: 0, redelivered: 0 };
 	try {
@@ -296,6 +298,9 @@ async function evaluateOneRule(
 			// transition; otherwise the same-state branch below retries on the
 			// next tick without a second transition.
 			await store.transitionState(rule.id, rule.projectId, next, now);
+			if (next === "firing" && onAlertFiring) {
+				await onAlertFiring(rule).catch(() => undefined);
+			}
 			const ok = await deliver(next);
 			// Acknowledge before writing history: if the evaluation insert
 			// fails, the delivered flag is already durable and the next tick
@@ -364,6 +369,7 @@ export async function evaluateRuleBatch(
 		ruleTimeoutMs?: number;
 		allowPrivateWebhookTargets?: boolean;
 		hostLookup?: HostLookup;
+		onAlertFiring?: (rule: AlertRule) => Promise<void>;
 	},
 ): Promise<{
 	evaluated: number;
@@ -393,6 +399,7 @@ export async function evaluateRuleBatch(
 			options?.tracer,
 			ruleTimeoutMs,
 			delivery,
+			options?.onAlertFiring,
 		),
 	);
 	return {
@@ -421,7 +428,8 @@ export async function evaluateAllRules(
 	const logger = options?.logger ?? consoleLogger;
 	const tracer = options?.tracer;
 	const allowPrivateWebhookTargets = env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true";
-	const store = new AlertsStore(sqlDbFor(env), { allowPrivateWebhookTargets });
+	const db = sqlDbFor(env);
+	const store = new AlertsStore(db, { allowPrivateWebhookTargets });
 	const rules = await store.listEnabledRules();
 	return evaluateRuleBatch(store, rules, {
 		logger,
@@ -430,6 +438,14 @@ export async function evaluateAllRules(
 		ruleTimeoutMs: options?.ruleTimeoutMs,
 		allowPrivateWebhookTargets,
 		hostLookup: options?.hostLookup,
+		onAlertFiring: async (rule) => {
+			await openOrDeduplicateIncident(db, rule.projectId, {
+				environment: "default",
+				targetNodeId: `alert:${rule.id}`,
+				symptomAnchorKind: "alert",
+				symptomAnchorId: rule.id,
+			});
+		},
 	});
 }