@obsunified/collector 2.0.1 → 2.1.1

package/src/migrations-postgres/048_telemetry_spans_project_composite_pk.sql

@@ -0,0 +1,37 @@
+-- Migration 048: telemetry_spans composite primary key
+-- (project_id, trace_id, span_id)
+--
+-- Mirrors ../migrations/048_telemetry_spans_project_composite_pk.sql. 001
+-- keyed spans by (trace_id, span_id); 010 added project_id outside the key,
+-- so ingest's INSERT OR IGNORE (ON CONFLICT DO NOTHING here) silently
+-- dropped a span whose ids already existed under another project. Swaps the
+-- key for (project_id, trace_id, span_id) in place, keeping every row and
+-- index. The old key was unique, so the composite key cannot collide.
+--
+-- Idempotent: does nothing when the key is already composite.
+--
+-- Operational note: ADD PRIMARY KEY builds a new unique index under an
+-- ACCESS EXCLUSIVE lock. Retention keeps the table bounded, but on a busy
+-- database apply it in a quiet window.
+
+UPDATE telemetry_spans SET project_id = 'default' WHERE project_id IS NULL;
+ALTER TABLE telemetry_spans ALTER COLUMN project_id SET NOT NULL;
+
+DO $$
+DECLARE
+  pk_name TEXT;
+  pk_def TEXT;
+BEGIN
+  SELECT c.conname, pg_get_constraintdef(c.oid)
+    INTO pk_name, pk_def
+    FROM pg_constraint c
+   WHERE c.conrelid = 'telemetry_spans'::regclass AND c.contype = 'p';
+
+  IF pk_def IS DISTINCT FROM 'PRIMARY KEY (project_id, trace_id, span_id)' THEN
+    IF pk_name IS NOT NULL THEN
+      EXECUTE format('ALTER TABLE telemetry_spans DROP CONSTRAINT %I', pk_name);
+    END IF;
+    ALTER TABLE telemetry_spans ADD PRIMARY KEY (project_id, trace_id, span_id);
+  END IF;
+END
+$$;

package/src/migrations-postgres/049_alert_state_notified.sql

@@ -0,0 +1,10 @@
+-- Migration 049: alert_state.notified
+--
+-- Mirrors ../migrations/049_alert_state_notified.sql. INTEGER 0/1 like the
+-- sibling alert_evaluations.notified (not BOOLEAN), so the shared store SQL
+-- can test and set it the same way on both backends.
+--
+-- Backfill: existing rows default to 1 (delivered); the pre-049 evaluator
+-- only stored a transition after the channels accepted it. Idempotent.
+
+ALTER TABLE alert_state ADD COLUMN IF NOT EXISTS notified INTEGER NOT NULL DEFAULT 1;

package/src/migrations-postgres/README.md

@@ -32,9 +32,62 @@ remaining migrations are tracked in
 [`docs/postgres-migration.md`](../../../../docs/postgres-migration.md); they'll
 land alongside the first hosted-Postgres deployment.
 
-## What's NOT translated
+## Column types the stores depend on
 
-`009_projects_and_keys.sql` uses SQLite's `BLOB` type for the hashed API key.
-Postgres uses `BYTEA` and the column-name rewrite is in
-`009_projects_and_keys.sql` here (a non-mechanical translation; review carefully
-when porting forward).
+- `ingest_keys.key_hash` is `TEXT` on both backends. `ProjectsStore` binds the
+  SHA-256 hex digest as a string; an earlier Postgres version of `009` declared
+  the column `BYTEA`, which silently stored the ASCII bytes of the hex text and
+  made the `026` seed key unreachable. `041_ingest_key_hash_text.sql` converts
+  databases migrated with that version back to `TEXT` (fresh installs get
+  `TEXT` from `009` directly and `041` is a no-op). The D1 `041` file is an
+  intentional no-op that only keeps the numbering in parity.
+- Timestamp columns (`received_at`, `expires_at`, …) are `TEXT` holding
+  `Date#toISOString()` output. Postgres will not compare `text` to
+  `timestamptz`, so `postgresDialect` and `translateD1Sql` render every time
+  window as ISO text (`to_char(…, 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`).
+  `metric_exemplars` originally declared `received_at`/`expires_at` as
+  `TIMESTAMPTZ`, which made `MetricsStore.purgeExpired` fail with
+  `operator does not exist: timestamp with time zone < text`;
+  `043_metric_exemplars_text_timestamps.sql` converts those columns to UTC
+  ISO `TEXT` in place (fresh installs get `TEXT` from `035` and `043` is a
+  no-op; the D1 `043` is a numbering no-op).
+- `actions.started_at` is ISO `TEXT` and drives action-graph retention (the
+  spine has no `expires_at`). `044_actions_started_at_index.sql` adds
+  `idx_actions_started_at` on both backends so the hourly sweep reads expired
+  trees in index order. Deletes run per tree inside `PostgresAdapter.batch`
+  (one transaction) and re-check eligibility in every statement, but under
+  READ COMMITTED a concurrent insert into an already-expired tree can still
+  land between that tree's leaf deletes and its spine delete; see the header
+  of `src/lib/action-retention.ts`.
+- Row keys are project-scoped. `045`–`048` swap the single-column primary
+  keys for composite ones in place (`user_profiles (project_id, user_id)`;
+  `actions`, `agent_runs`, `tool_calls`, `retrieval_events`, `eval_results`,
+  `artifacts (project_id, id)`; `ai_span_payloads` and
+  `telemetry_spans (project_id, trace_id, span_id)`). Each file looks up the
+  current primary key by definition and is a no-op once the key is
+  composite; the D1 siblings rebuild the tables through `*_new` copies. The
+  stores upsert with `ON CONFLICT (project_id, …)`, so both backends must
+  carry these migrations before deploying collector 2.1.
+- `049` adds `alert_state.notified INTEGER` (0/1, like
+  `alert_evaluations.notified`): whether the current state's transition has
+  been delivered to the rule's channels. Existing rows backfill to 1. The
+  store falls back to the pre-049 statement when the column is missing, so
+  the collector keeps evaluating during a rolling deploy, but it cannot
+  retry undelivered notifications until 049 is applied.
+- Flags (`is_error`, `is_bot`, `payload_capture_enabled`) are `BOOLEAN` here
+  and `INTEGER 0/1` on D1. Shared SQL must test them as bare booleans
+  (`CASE WHEN is_error THEN 1 ELSE 0 END`, never `is_error = 1` or
+  `SUM(is_error)`). Hosts call `configurePostgresTypeParsers(pg.types)` so
+  BOOLEAN reads back as `0/1`, BIGINT/NUMERIC as numbers and JSON/JSONB as
+  text, matching what D1 returns.
+
+## Verifying against a real database
+
+`src/lib/sql-db-postgres.integration.test.ts` applies this directory into an
+empty scratch database, then exercises ingest-key auth, AI/usage ingest, the
+retention purge and every built-in analysis through `PostgresAdapter`:
+
+```bash
+OBS_PG_TEST_URL=postgres://obs:obs@127.0.0.1:5432/obs_scratch \
+  pnpm --filter @obsunified/collector exec vitest run src/lib/sql-db-postgres.integration.test.ts
+```

package/src/otlp/decode/body.test.ts

@@ -0,0 +1,76 @@
+import { gzipSync } from "node:zlib";
+import { Hono } from "hono";
+import { describe, expect, it } from "vitest";
+import { OtlpDecodeError, type ReadBodyLimits, readOtlpBody } from "./body";
+
+const MIB = 1024 * 1024;
+
+const read = async (
+	body: BodyInit,
+	headers: Record<string, string>,
+	limits: ReadBodyLimits = {},
+) => {
+	const app = new Hono().post("/t", async (c) => {
+		try {
+			const result = await readOtlpBody(c, limits);
+			return c.json({
+				bytes: result.bytes.byteLength,
+				wire: result.wireFormat,
+			});
+		} catch (err) {
+			if (err instanceof OtlpDecodeError)
+				return c.json({ error: err.message }, err.status);
+			throw err;
+		}
+	});
+	const res = await app.request("/t", { method: "POST", body, headers });
+	return {
+		status: res.status,
+		body: (await res.json()) as Record<string, unknown>,
+	};
+};
+
+describe("readOtlpBody limits", () => {
+	it("decompresses a normal gzip payload", async () => {
+		const json = JSON.stringify({ resourceSpans: [] });
+		const res = await read(gzipSync(json), {
+			"content-type": "application/json",
+			"content-encoding": "gzip",
+		});
+		expect(res).toEqual({
+			status: 200,
+			body: { bytes: json.length, wire: "json" },
+		});
+	});
+
+	it("refuses a gzip bomb once the decompressed cap is crossed", async () => {
+		// 8 MiB of zeros gzips to a few KiB; the reader must stop at 1 MiB.
+		const bomb = gzipSync(Buffer.alloc(8 * MIB));
+		expect(bomb.byteLength).toBeLessThan(64 * 1024);
+		const res = await read(
+			bomb,
+			{ "content-type": "application/x-protobuf", "content-encoding": "gzip" },
+			{ maxDecompressedBytes: MIB },
+		);
+		expect(res.status).toBe(413);
+		expect(res.body.error).toMatch(/Decompressed payload too large/);
+	});
+
+	it("refuses a raw body over the configured cap", async () => {
+		const res = await read(
+			"x".repeat(2048),
+			{ "content-type": "application/json" },
+			{ maxBodyBytes: 1024 },
+		);
+		expect(res.status).toBe(413);
+		expect(res.body.error).toMatch(/Payload too large \(max 1024 bytes\)/);
+	});
+
+	it("reports malformed gzip as 400, not 500", async () => {
+		const res = await read("definitely not gzip", {
+			"content-type": "application/json",
+			"content-encoding": "gzip",
+		});
+		expect(res.status).toBe(400);
+	});
+});

package/src/otlp/decode/body.ts

@@ -1,5 +1,10 @@
 import type { JsonValue } from "@bufbuild/protobuf";
 import type { Context } from "hono";
+import {
+	DEFAULT_MAX_BODY_BYTES,
+	DEFAULT_MAX_DECOMPRESSED_BYTES,
+	parseByteLimit,
+} from "../../framework/limits";
 
 export type OtlpWireFormat = "json" | "protobuf";
 
@@ -11,25 +16,50 @@ export interface ReadBodyResult {
 export class OtlpDecodeError extends Error {
 	constructor(
 		message: string,
-		public readonly status: 400 | 415,
+		public readonly status: 400 | 413 | 415,
 	) {
 		super(message);
 	}
 }
 
+export interface ReadBodyLimits {
+	/** Raw (compressed) body cap; defaults to `MAX_BODY_BYTES` / 10 MiB. */
+	maxBodyBytes?: number;
+	/** Gunzipped payload cap; defaults to 32 MiB. */
+	maxDecompressedBytes?: number;
+}
+
 /**
  * Reads and decompresses an OTLP request body, returning the raw bytes plus
  * the inferred wire format. Throws `OtlpDecodeError` for unsupported content-
- * types or malformed gzip.
+ * types, malformed gzip, or bodies over the raw / decompressed caps (413).
+ * The framework's body-limit middleware normally refuses oversized raw
+ * bodies first; the raw check here keeps the reader safe on its own, and
+ * the decompressed cap is what stops a gzip bomb.
  */
-export const readOtlpBody = async (c: Context): Promise<ReadBodyResult> => {
+export const readOtlpBody = async (
+	c: Context,
+	limits: ReadBodyLimits = {},
+): Promise<ReadBodyResult> => {
 	const contentType = (c.req.header("content-type") ?? "").toLowerCase();
 	const wireFormat = detectWireFormat(contentType);
+	const env = (c.env ?? {}) as { MAX_BODY_BYTES?: string };
+	const maxBodyBytes =
+		limits.maxBodyBytes ??
+		parseByteLimit(env.MAX_BODY_BYTES, DEFAULT_MAX_BODY_BYTES);
+	const maxDecompressedBytes =
+		limits.maxDecompressedBytes ?? DEFAULT_MAX_DECOMPRESSED_BYTES;
 
 	let body = await c.req.arrayBuffer();
+	if (body.byteLength > maxBodyBytes) {
+		throw new OtlpDecodeError(
+			`Payload too large (max ${maxBodyBytes} bytes)`,
+			413,
+		);
+	}
 	const encoding = (c.req.header("content-encoding") ?? "").toLowerCase();
 	if (encoding === "gzip") {
-		body = await gunzip(body);
+		body = await gunzip(body, maxDecompressedBytes);
 	} else if (encoding && encoding !== "identity") {
 		throw new OtlpDecodeError(`Unsupported content-encoding: ${encoding}`, 415);
 	}
@@ -46,12 +76,43 @@ const detectWireFormat = (contentType: string): OtlpWireFormat => {
 	);
 };
 
-const gunzip = async (input: ArrayBuffer): Promise<ArrayBuffer> => {
+/** Gunzip with a hard cap on output size; stops reading once exceeded. */
+const gunzip = async (
+	input: ArrayBuffer,
+	maxBytes: number,
+): Promise<ArrayBuffer> => {
 	const stream = new Response(input).body?.pipeThrough(
 		new DecompressionStream("gzip"),
 	);
 	if (!stream) throw new OtlpDecodeError("Empty gzip body", 400);
-	return new Response(stream).arrayBuffer();
+	const reader = stream.getReader();
+	const chunks: Uint8Array[] = [];
+	let total = 0;
+	try {
+		for (;;) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			total += value.byteLength;
+			if (total > maxBytes) {
+				await reader.cancel().catch(() => {});
+				throw new OtlpDecodeError(
+					`Decompressed payload too large (max ${maxBytes} bytes)`,
+					413,
+				);
+			}
+			chunks.push(value);
+		}
+	} catch (err) {
+		if (err instanceof OtlpDecodeError) throw err;
+		throw new OtlpDecodeError("Malformed gzip body", 400);
+	}
+	const out = new Uint8Array(total);
+	let offset = 0;
+	for (const chunk of chunks) {
+		out.set(chunk, offset);
+		offset += chunk.byteLength;
+	}
+	return out.buffer;
 };
 
 export const decodeJsonBody = (bytes: Uint8Array): JsonValue => {

package/src/plugins/_json.ts

@@ -0,0 +1,21 @@
+/**
+ * Parse a request body that must be a JSON object. `c.req.json()` happily
+ * returns `null`, arrays and primitives for bodies like `null` or `"x"`, and
+ * the first property access in the handler then threw a TypeError that
+ * surfaced as a 500. Returns `null` for anything that is not a plain object
+ * so routes can answer 400 uniformly.
+ */
+export async function readJsonObject<T extends object>(c: {
+	req: { json(): Promise<unknown> };
+}): Promise<T | null> {
+	let parsed: unknown;
+	try {
+		parsed = await c.req.json();
+	} catch {
+		return null;
+	}
+	if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+		return null;
+	}
+	return parsed as T;
+}

package/src/plugins/action-graph-processor.ts

@@ -907,8 +907,7 @@ export const actionGraphProcessorPlugin: CollectorPlugin = {
 							span_id, session_id, interaction_id, user_id, agent_run_id, step_id,
 							tool_call_id, prompt_version, model_name, provider, total_cost_usd, attrs_json
 						) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-						ON CONFLICT (id) DO UPDATE SET
-							project_id = excluded.project_id,
+						ON CONFLICT (project_id, id) DO UPDATE SET
 							root_action_id = excluded.root_action_id,
 							caused_by_action_id = excluded.caused_by_action_id,
 							actor_type = excluded.actor_type,
@@ -972,8 +971,7 @@ export const actionGraphProcessorPlugin: CollectorPlugin = {
 							id, project_id, agent_id, agent_name, agent_version, goal, outcome,
 							autonomy_level, status, error_message, total_cost_usd, total_duration_ms, metadata_json
 						) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-						ON CONFLICT (id) DO UPDATE SET
-							project_id = excluded.project_id,
+						ON CONFLICT (project_id, id) DO UPDATE SET
 							agent_id = excluded.agent_id,
 							agent_name = excluded.agent_name,
 							agent_version = excluded.agent_version,
@@ -1015,9 +1013,8 @@ export const actionGraphProcessorPlugin: CollectorPlugin = {
 							mcp_audit_json, mutation_before_json, mutation_after_json,
 							mutation_diff_json, mutation_artifact_id
 						) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-						ON CONFLICT (id) DO UPDATE SET
+						ON CONFLICT (project_id, id) DO UPDATE SET
 							action_id = excluded.action_id,
-							project_id = excluded.project_id,
 							tool_name = excluded.tool_name,
 							args_hash = excluded.args_hash,
 							result_hash = excluded.result_hash,
@@ -1062,9 +1059,8 @@ export const actionGraphProcessorPlugin: CollectorPlugin = {
 							id, action_id, project_id, retriever_name, query_hash, documents_json,
 							total_results, max_relevance_score, duration_ms
 						) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-						ON CONFLICT (id) DO UPDATE SET
+						ON CONFLICT (project_id, id) DO UPDATE SET
 							action_id = excluded.action_id,
-							project_id = excluded.project_id,
 							retriever_name = excluded.retriever_name,
 							query_hash = excluded.query_hash,
 							documents_json = excluded.documents_json,
@@ -1095,9 +1091,8 @@ export const actionGraphProcessorPlugin: CollectorPlugin = {
 							id, action_id, project_id, evaluator_name, evaluator_version,
 							score, passed, reasoning, rubric_json
 						) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-						ON CONFLICT (id) DO UPDATE SET
+						ON CONFLICT (project_id, id) DO UPDATE SET
 							action_id = excluded.action_id,
-							project_id = excluded.project_id,
 							evaluator_name = excluded.evaluator_name,
 							evaluator_version = excluded.evaluator_version,
 							score = excluded.score,
@@ -1128,9 +1123,8 @@ export const actionGraphProcessorPlugin: CollectorPlugin = {
 							id, action_id, project_id, artifact_name, artifact_type,
 							storage_ref, size_bytes, sha256_hash, content_preview
 						) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-						ON CONFLICT (id) DO UPDATE SET
+						ON CONFLICT (project_id, id) DO UPDATE SET
 							action_id = excluded.action_id,
-							project_id = excluded.project_id,
 							artifact_name = excluded.artifact_name,
 							artifact_type = excluded.artifact_type,
 							storage_ref = excluded.storage_ref,

package/src/plugins/ai-receiver.ts

@@ -15,18 +15,15 @@ import type { CollectorPlugin } from "../framework/collector";
 import { AIStore, type IngestEvaluation } from "../lib/ai-store";
 import { sqlDbFor } from "../lib/sql-db";
 import { getProjectId } from "./_context";
+import { readJsonObject } from "./_json";
 
 export const aiReceiverPlugin: CollectorPlugin = {
 	name: "ai-receiver",
 	register(app, _runtime) {
 		app.post("/v1/ai", async (c) => {
 			const projectId = getProjectId(c);
-			let payload: AICallPayload;
-			try {
-				payload = await c.req.json<AICallPayload>();
-			} catch {
-				return c.json({ error: "Invalid JSON body" }, 400);
-			}
+			const payload = await readJsonObject<AICallPayload>(c);
+			if (!payload) return c.json({ error: "Invalid JSON body" }, 400);
 			if (!payload.calls || !Array.isArray(payload.calls)) {
 				return c.json({ error: "Missing calls array" }, 400);
 			}
@@ -126,12 +123,8 @@ export const aiReceiverPlugin: CollectorPlugin = {
 		// one or both of score/label, and optional explanation + metadata.
 		app.post("/v1/ai/evaluations", async (c) => {
 			const projectId = getProjectId(c);
-			let payload: AIEvaluationPayload;
-			try {
-				payload = await c.req.json<AIEvaluationPayload>();
-			} catch {
-				return c.json({ error: "Invalid JSON body" }, 400);
-			}
+			const payload = await readJsonObject<AIEvaluationPayload>(c);
+			if (!payload) return c.json({ error: "Invalid JSON body" }, 400);
 			if (!payload.evaluations || !Array.isArray(payload.evaluations)) {
 				return c.json({ error: "Missing evaluations array" }, 400);
 			}

package/src/plugins/ai-span-payloads-processor.ts

@@ -130,8 +130,7 @@ export const aiSpanPayloadsProcessorPlugin: CollectorPlugin = {
 								input_json, output_json, input_hash, output_hash, session_id, user_id,
 								received_at, expires_at, interaction_id, action_id
 							) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-							ON CONFLICT(trace_id, span_id) DO UPDATE SET
-								project_id = excluded.project_id,
+							ON CONFLICT(project_id, trace_id, span_id) DO UPDATE SET
 								span_kind = excluded.span_kind,
 								input_json = excluded.input_json,
 								output_json = excluded.output_json,

package/src/plugins/alerts-evaluator-retention.test.ts

@@ -0,0 +1,170 @@
+/**
+ * Regression for the alert notification retry: the "still owed" decision
+ * lives on `alert_state.notified` (migration 049), not in alert_evaluations,
+ * which housekeeping purges after 720 hours. A long-lived firing rule whose
+ * evaluation history has been purged must not be re-notified every tick.
+ * Runs the real AlertsStore against the migrated SQLite schema; only the
+ * signal query and `fetch` are substituted.
+ */
+
+import type { AlertRule } from "@obsunified/types";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { AlertsStore } from "../lib/alerts-store";
+import {
+	ALERT_EVALUATIONS_MAX_AGE_HOURS,
+	purgeSecondaryRows,
+} from "../lib/housekeeping";
+import { ProjectsStore } from "../lib/projects-store";
+import { createMigratedSqliteDb } from "../lib/test-utils/node-sqlite-db";
+import { evaluateRuleBatch } from "./alerts-evaluator";
+
+type EvaluatorStore = Parameters<typeof evaluateRuleBatch>[0];
+
+const logger = { debug() {}, info() {}, warn() {}, error() {} };
+const HOUR = 3_600_000;
+
+/** Real store for state and evaluations; the signal value is injected. */
+const evaluatorOver = (
+	store: AlertsStore,
+	value: () => number,
+): EvaluatorStore => ({
+	listEnabledRules: () => store.listEnabledRules(),
+	evaluateRule: async () => value(),
+	getState: (id, project) => store.getState(id, project),
+	getAnalysisNarrative: (project, analysis) =>
+		store.getAnalysisNarrative(project, analysis),
+	transitionState: (id, project, next, now) =>
+		store.transitionState(id, project, next, now),
+	markNotified: (id, project, state, since) =>
+		store.markNotified(id, project, state, since),
+	recordEvaluation: (id, project, v, state, notified) =>
+		store.recordEvaluation(id, project, v, state, notified),
+});
+
+const setup = async () => {
+	const db = createMigratedSqliteDb();
+	const project = await new ProjectsStore(db).createProject({
+		name: "A",
+		slug: "a",
+	});
+	const store = new AlertsStore(db);
+	const rule = await store.createRule(project.id, {
+		name: "errors",
+		signal: "spans",
+		query: {},
+		threshold: 10,
+		windowMins: 5,
+		comparison: ">",
+		channels: [{ type: "webhook", url: "https://hooks.example.com/alert/x" }],
+		enabled: true,
+	});
+	const evaluations = () =>
+		db.count("alert_evaluations", "rule_id = ?", rule.id);
+	return { db, project, store, rule, evaluations };
+};
+
+const purgeAllEvaluations = async (
+	db: ReturnType<typeof createMigratedSqliteDb>,
+) =>
+	purgeSecondaryRows(db, {
+		retentionHours: 72,
+		now: () => Date.now() + (ALERT_EVALUATIONS_MAX_AGE_HOURS + 1) * HOUR,
+	});
+
+describe("alert notification retry survives evaluation retention", () => {
+	afterEach(() => {
+		vi.unstubAllGlobals();
+	});
+
+	it("does not re-send a delivered firing notification after its evaluations are purged", async () => {
+		const fetchMock = vi.fn(async () => new Response(null, { status: 200 }));
+		vi.stubGlobal("fetch", fetchMock);
+		const { db, project, store, rule, evaluations } = await setup();
+		const evaluator = evaluatorOver(store, () => 42);
+		const rules: AlertRule[] = [rule];
+
+		// Breach: state recorded, webhook accepted, flag set.
+		expect(await evaluateRuleBatch(evaluator, rules, { logger })).toEqual({
+			evaluated: 1,
+			fired: 1,
+			resolved: 0,
+			redelivered: 0,
+		});
+		expect(await store.getState(rule.id, project.id)).toMatchObject({
+			current_state: "firing",
+			notified: 1,
+		});
+		expect(fetchMock).toHaveBeenCalledTimes(1);
+
+		// Still breaching for longer than evaluation retention: history goes.
+		await evaluateRuleBatch(evaluator, rules, { logger });
+		expect(await evaluations()).toBe(2);
+		const purged = await purgeAllEvaluations(db);
+		expect(purged.alert_evaluations).toBe(2);
+		expect(await evaluations()).toBe(0);
+
+		// The flag on the state row is the source of truth: no resend.
+		for (let tick = 0; tick < 3; tick += 1) {
+			expect(await evaluateRuleBatch(evaluator, rules, { logger })).toEqual({
+				evaluated: 1,
+				fired: 0,
+				resolved: 0,
+				redelivered: 0,
+			});
+		}
+		expect(fetchMock).toHaveBeenCalledTimes(1);
+		expect(await store.getState(rule.id, project.id)).toMatchObject({
+			current_state: "firing",
+			notified: 1,
+		});
+	});
+
+	it("keeps retrying an undelivered notification across the purge until a channel accepts it", async () => {
+		const statuses = [503, 503, 200];
+		const fetchMock = vi.fn(
+			async () => new Response(null, { status: statuses.shift() ?? 200 }),
+		);
+		vi.stubGlobal("fetch", fetchMock);
+		const { db, project, store, rule } = await setup();
+		const evaluator = evaluatorOver(store, () => 42);
+		const rules: AlertRule[] = [rule];
+
+		await evaluateRuleBatch(evaluator, rules, { logger }); // 503
+		expect(await store.getState(rule.id, project.id)).toMatchObject({
+			current_state: "firing",
+			notified: 0,
+		});
+		await purgeAllEvaluations(db);
+
+		await evaluateRuleBatch(evaluator, rules, { logger }); // 503 again
+		expect(await evaluateRuleBatch(evaluator, rules, { logger })).toEqual({
+			evaluated: 1,
+			fired: 0,
+			resolved: 0,
+			redelivered: 1,
+		}); // 200
+		expect(fetchMock).toHaveBeenCalledTimes(3);
+		expect((await store.getState(rule.id, project.id))?.notified).toBe(1);
+
+		await evaluateRuleBatch(evaluator, rules, { logger });
+		expect(fetchMock).toHaveBeenCalledTimes(3);
+	});
+
+	it("listRules and getRule still expose the current state for the dashboard", async () => {
+		vi.stubGlobal(
+			"fetch",
+			vi.fn(async () => new Response(null, { status: 500 })),
+		);
+		const { project, store, rule } = await setup();
+		await evaluateRuleBatch(
+			evaluatorOver(store, () => 42),
+			[rule],
+			{ logger },
+		);
+		// Delivery failed, but the breach is visible.
+		expect((await store.getRule(rule.id, project.id))?.currentState).toBe(
+			"firing",
+		);
+		expect((await store.listRules(project.id))[0]?.currentState).toBe("firing");
+	});
+});