@obsunified/collector 2.0.1 → 2.2.0

package/src/lib/sql-db-postgres.integration.test.ts

@@ -0,0 +1,1225 @@
+/**
+ * Live-Postgres integration checks for the standalone Node collector path
+ * (RFC 0008 `PostgresAdapter`). Skipped unless `OBS_PG_TEST_URL` is set:
+ *
+ *   OBS_PG_TEST_URL=postgres://obs:obs@127.0.0.1:5432/obs_unified \
+ *     pnpm --filter @obsunified/collector exec vitest run \
+ *       src/lib/sql-db-postgres.integration.test.ts
+ *
+ * Point it at an EMPTY scratch database: the run applies
+ * `migrations-postgres/` into the connection's default schema, refuses to
+ * start if the schema already contains any table, and drops every table it
+ * created afterwards. (Session state such as `search_path` is deliberately not
+ * relied upon so the suite also runs against single-session servers like
+ * PGlite's socket bridge.)
+ *
+ * Each test pins a Postgres-only failure that D1 never surfaced: BYTEA vs
+ * hex `key_hash`, BOOLEAN flags, TEXT timestamps compared to
+ * `timestamptz` (including the `metric_exemplars` TIMESTAMPTZ columns that
+ * 043 converts), and SQLite built-ins in the shared analysis SQL.
+ */
+
+import { readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import type {
+	AICallRecord,
+	AlertRule,
+	StoredSpan,
+	UsageEventRecord,
+} from "@obsunified/types";
+import type { MiddlewareHandler } from "hono";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { deriveAnalysesForProject } from "../analyses/derive";
+import { INVESTIGATION_ANALYSES } from "../analyses/investigations";
+import { TIER0_ANALYSES } from "../analyses/tier0";
+import {
+	type CollectorRuntime,
+	createTelemetryCollectorApp,
+	type SpanProcessorPlugin,
+} from "../framework/collector";
+import type { CollectorEnv, CollectorRouteContext } from "../framework/env";
+import type { DecodedMetricPoint } from "../otlp/decode";
+import { aiSpanPayloadsProcessorPlugin } from "../plugins/ai-span-payloads-processor";
+import { identityReceiverPlugin } from "../plugins/identity-receiver";
+import { purgeExpiredActionGraph } from "./action-retention";
+import { ActionStore } from "./actions-store";
+import { getAICallsOverview, ingestAICallBatch } from "./ai-store/calls";
+import { purgeExpiredAIData } from "./ai-store/retention";
+import { getAISpansOverview } from "./ai-store/spans";
+import { AlertsStore } from "./alerts-store";
+import { runSqlAnalysis } from "./analyses-runner";
+import { sha256Hex } from "./hash";
+import { MetricsStore } from "./metrics-store";
+import { ProjectsStore } from "./projects-store";
+import {
+	configurePostgresTypeParsers,
+	PostgresAdapter,
+} from "./sql-db-postgres";
+import { ingestTelemetrySpans } from "./store/ingest";
+import {
+	actionRecord,
+	agentRunRecord,
+	countActionGraph,
+	FIXTURE_NOW_MS,
+	FIXTURE_RETENTION_HOURS,
+	FRESH_START,
+	seedEvalCase,
+	seedTree,
+} from "./test-utils/action-graph-fixtures";
+import { UsageStore } from "./usage-store";
+
+const databaseUrl = process.env.OBS_PG_TEST_URL;
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const migrationsDir = path.resolve(here, "../migrations-postgres");
+
+/** Plaintext lives in apps/web/.env.development.local; this is its SHA-256. */
+const DASHBOARD_SEED_HASH =
+	"6aa18dc1261b312e98b236d5a7dae698d1a90ecbe33d8024744063a066a35b9b";
+
+const VALID_STATUSES = new Set(["ok", "warn", "critical", "unknown"]);
+
+type PgModule = typeof import("pg");
+type PgPool = import("pg").Pool;
+type PgClient = import("pg").Client;
+
+const isoNow = () => new Date().toISOString();
+const isoIn = (ms: number) => new Date(Date.now() + ms).toISOString();
+
+const aiCall = (over: Partial<AICallRecord>): AICallRecord => ({
+	projectId: "default",
+	callId: `call-${Math.random().toString(36).slice(2)}`,
+	traceId: null,
+	spanId: null,
+	serviceName: "checkout",
+	modelName: "gpt-test",
+	provider: "openai",
+	callType: "chat",
+	requestJson: JSON.stringify({ messages: [] }),
+	responseJson: null,
+	promptTokens: 10,
+	completionTokens: 5,
+	totalCostUsd: 0.01,
+	latencyMs: 120,
+	isError: false,
+	errorMessage: null,
+	occurredAt: isoNow(),
+	receivedAt: isoNow(),
+	expiresAt: isoIn(60 * 60 * 1000),
+	...over,
+});
+
+const usageEvent = (over: Partial<UsageEventRecord>): UsageEventRecord => ({
+	projectId: "default",
+	eventId: `evt-${Math.random().toString(36).slice(2)}`,
+	sessionId: "sess-1",
+	visitorId: "visitor-1",
+	eventType: "page_view",
+	eventName: "page_view",
+	pagePath: "/",
+	pageTitle: "Home",
+	referrer: null,
+	severity: "info",
+	source: "web",
+	contextJson: JSON.stringify({ viewport: "1280x720" }),
+	propertiesJson: JSON.stringify({ loadTimeMs: 250 }),
+	userAgent: "Mozilla/5.0",
+	occurredAt: isoNow(),
+	receivedAt: isoNow(),
+	expiresAt: isoIn(60 * 60 * 1000),
+	country: "US",
+	browser: "Chrome",
+	os: "macOS",
+	deviceType: "desktop",
+	isBot: false,
+	utmSource: null,
+	utmMedium: null,
+	utmCampaign: null,
+	...over,
+});
+
+const metricPoint = (
+	over: Partial<DecodedMetricPoint>,
+): DecodedMetricPoint => ({
+	name: "http.server.duration",
+	description: null,
+	unit: "ms",
+	type: "histogram",
+	isMonotonic: null,
+	temporality: 2,
+	scopeName: "@opentelemetry/instrumentation-http",
+	scopeVersion: "1.0.0",
+	serviceName: "checkout",
+	resourceAttrsJson: JSON.stringify({ "service.name": "checkout" }),
+	attributesJson: JSON.stringify({ route: "/checkout" }),
+	identity: `metric-${Math.random().toString(36).slice(2)}`,
+	tsNs: "1800000000000000000",
+	startTsNs: null,
+	value: null,
+	count: 1,
+	sum: 42,
+	min: null,
+	max: null,
+	boundsJson: JSON.stringify([50]),
+	bucketCountsJson: JSON.stringify([1]),
+	extraJson: null,
+	exemplarsJson: null,
+	...over,
+});
+
+const exemplarJson = (traceId: string, tsNs: string) =>
+	JSON.stringify([{ value: 1, traceId, spanId: null, tsNs }]);
+
+/** Same `to_char` shape `postgresDialect.now()` and 043 render. */
+const ISO_TEXT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
+
+describe.skipIf(!databaseUrl)("PostgresAdapter against a live Postgres", () => {
+	const legacySchema = `obs_pg_test_legacy_${Date.now().toString(36)}`;
+	let pg: PgModule;
+	let pool: PgPool;
+	let db: PostgresAdapter;
+	let migrationFiles: string[] = [];
+	let preexistingTables = new Set<string>();
+
+	const adminClient = async (): Promise<PgClient> => {
+		const client = new pg.Client({ connectionString: databaseUrl });
+		await client.connect();
+		return client;
+	};
+
+	const listTables = async (client: PgClient): Promise<Set<string>> => {
+		const res = await client.query<{ table_name: string }>(
+			`SELECT table_name FROM information_schema.tables
+			 WHERE table_schema = current_schema() AND table_type = 'BASE TABLE'`,
+		);
+		return new Set(res.rows.map((r) => r.table_name));
+	};
+
+	beforeAll(async () => {
+		pg = (await import("pg")).default as unknown as PgModule;
+		configurePostgresTypeParsers(pg.types);
+
+		const admin = await adminClient();
+		try {
+			preexistingTables = await listTables(admin);
+			if (preexistingTables.size > 0) {
+				// Refuse any non-empty schema, not just one with collector tables,
+				// so the teardown below can never touch an unrelated database.
+				throw new Error(
+					`OBS_PG_TEST_URL must point at an empty scratch database (found ${preexistingTables.size} table(s): ${[...preexistingTables].sort().join(", ")})`,
+				);
+			}
+			await admin.query(`CREATE SCHEMA ${legacySchema}`);
+
+			// Apply every Postgres migration the same way
+			// apps/collector-node/scripts/migrate-pg.mjs does (one transaction
+			// per file).
+			migrationFiles = (await readdir(migrationsDir))
+				.filter((f) => f.endsWith(".sql"))
+				.sort();
+			for (const file of migrationFiles) {
+				const sql = await readFile(path.join(migrationsDir, file), "utf8");
+				await admin.query("BEGIN");
+				try {
+					await admin.query(sql);
+					await admin.query("COMMIT");
+				} catch (err) {
+					await admin.query("ROLLBACK");
+					throw new Error(`${file} failed: ${(err as Error).message}`);
+				}
+			}
+		} finally {
+			await admin.end();
+		}
+
+		// max: 1 keeps the run deterministic on single-session servers
+		// (PGlite's socket bridge) and costs nothing against real Postgres.
+		pool = new pg.Pool({ connectionString: databaseUrl, max: 1 });
+		db = new PostgresAdapter(pool);
+	}, 120_000);
+
+	afterAll(async () => {
+		await pool?.end();
+		if (!pg) return;
+		const admin = await adminClient();
+		try {
+			await admin.query(`DROP SCHEMA IF EXISTS ${legacySchema} CASCADE`);
+			const created = [...(await listTables(admin))].filter(
+				(t) => !preexistingTables.has(t),
+			);
+			for (const table of created) {
+				await admin.query(`DROP TABLE IF EXISTS "${table}" CASCADE`);
+			}
+		} finally {
+			await admin.end();
+		}
+	});
+
+	it("applies every migration and leaves ingest_keys.key_hash as TEXT", async () => {
+		expect(migrationFiles).toContain("041_ingest_key_hash_text.sql");
+		const row = await db
+			.prepare(
+				`SELECT data_type FROM information_schema.columns
+				 WHERE table_schema = current_schema() AND table_name = 'ingest_keys' AND column_name = 'key_hash'`,
+			)
+			.first<{ data_type: string }>();
+		expect(row?.data_type).toBe("text");
+	});
+
+	it("resolves the seeded obs-dashboard key by its hex digest", async () => {
+		const store = new ProjectsStore(db);
+		await expect(store.findByKeyHash(DASHBOARD_SEED_HASH)).resolves.toEqual({
+			projectId: "obs-dashboard",
+			keyId: "obs-dashboard-bootstrap",
+		});
+	});
+
+	it("bootstraps INGEST_KEY, mints and revokes keys through the hex digest", async () => {
+		const store = new ProjectsStore(db);
+		await store.bootstrapEnvKey("dev-ingest-key");
+		await store.bootstrapEnvKey("dev-ingest-key"); // idempotent
+		const bootstrapped = await store.findByKeyHash(
+			await sha256Hex("dev-ingest-key"),
+		);
+		expect(bootstrapped?.projectId).toBe("default");
+
+		const minted = await store.createKey("default", "ci");
+		const found = await store.findByKeyHash(await sha256Hex(minted.key));
+		expect(found).toEqual({ projectId: "default", keyId: minted.id });
+
+		expect(await store.revokeKey(minted.id, "other-project")).toBe(false);
+		expect(await store.revokeKey(minted.id, "default")).toBe(true);
+		expect(await store.findByKeyHash(await sha256Hex(minted.key))).toBeNull();
+
+		const listed = await store.listKeys("default");
+		expect(listed.map((k) => k.name).sort()).toEqual(["bootstrap", "ci"]);
+	});
+
+	it("migration 041 converts a legacy BYTEA key_hash column in place", async () => {
+		const legacy = await adminClient();
+		try {
+			await legacy.query(`SET search_path TO ${legacySchema}`);
+			// Shape of migrations-postgres/009 before the fix.
+			await legacy.query(`
+				CREATE TABLE projects (id TEXT PRIMARY KEY, name TEXT NOT NULL, slug TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL);
+				CREATE TABLE ingest_keys (
+					id TEXT PRIMARY KEY, project_id TEXT NOT NULL, key_hash BYTEA NOT NULL UNIQUE,
+					key_prefix TEXT NOT NULL, name TEXT NOT NULL, created_at TEXT NOT NULL, revoked_at TEXT,
+					FOREIGN KEY (project_id) REFERENCES projects (id));
+				CREATE INDEX idx_ingest_keys_hash ON ingest_keys (key_hash);
+				INSERT INTO projects VALUES ('default', 'Default', 'default', 'x'), ('obs-dashboard', 'Obs Dashboard', 'obs-dashboard', 'x');
+			`);
+			// 026 seeded the raw 32 digest bytes …
+			await legacy.query(
+				`INSERT INTO ingest_keys VALUES ('seed', 'obs-dashboard', decode($1, 'hex'), 'obs_dashboard_65', 'bootstrap', 'x', NULL)`,
+				[DASHBOARD_SEED_HASH],
+			);
+			// … while the collector bound the hex text, which BYTEA kept as 64 ASCII bytes.
+			const bootstrapHash = await sha256Hex("dev-ingest-key");
+			await legacy.query(
+				`INSERT INTO ingest_keys VALUES ('boot', 'default', $1, 'dev-ingest-key', 'bootstrap', 'x', NULL)`,
+				[bootstrapHash],
+			);
+			const before = await legacy.query(
+				`SELECT id, octet_length(key_hash) AS len FROM ingest_keys ORDER BY id`,
+			);
+			expect(before.rows).toEqual([
+				{ id: "boot", len: 64 },
+				{ id: "seed", len: 32 },
+			]);
+
+			const sql = await readFile(
+				path.join(migrationsDir, "041_ingest_key_hash_text.sql"),
+				"utf8",
+			);
+			await legacy.query(sql);
+			await legacy.query(sql); // re-running is a no-op
+
+			const type = await legacy.query(
+				`SELECT data_type FROM information_schema.columns
+				 WHERE table_schema = $1 AND table_name = 'ingest_keys' AND column_name = 'key_hash'`,
+				[legacySchema],
+			);
+			expect(type.rows[0]?.data_type).toBe("text");
+			const after = await legacy.query(
+				`SELECT id, key_hash FROM ingest_keys ORDER BY id`,
+			);
+			expect(after.rows).toEqual([
+				{ id: "boot", key_hash: bootstrapHash },
+				{ id: "seed", key_hash: DASHBOARD_SEED_HASH },
+			]);
+			const lookup = await legacy.query(
+				`SELECT id FROM ingest_keys WHERE key_hash = $1`,
+				[DASHBOARD_SEED_HASH],
+			);
+			expect(lookup.rows).toEqual([{ id: "seed" }]);
+		} finally {
+			// Single-session servers (PGlite) share GUCs across sockets, so put
+			// search_path back before the pooled adapter runs again.
+			await legacy.query("RESET search_path").catch(() => {});
+			await legacy.end();
+		}
+	});
+
+	it("creates metric_exemplars timestamps as TEXT and purges expired exemplars", async () => {
+		expect(migrationFiles).toContain(
+			"043_metric_exemplars_text_timestamps.sql",
+		);
+		const cols = await db
+			.prepare(
+				`SELECT column_name, data_type FROM information_schema.columns
+				 WHERE table_schema = current_schema() AND table_name = 'metric_exemplars'
+				   AND column_name IN ('received_at', 'expires_at')
+				 ORDER BY column_name`,
+			)
+			.all<{ column_name: string; data_type: string }>();
+		expect(cols.results).toEqual([
+			{ column_name: "expires_at", data_type: "text" },
+			{ column_name: "received_at", data_type: "text" },
+		]);
+
+		const metrics = new MetricsStore(db);
+		await metrics.ingestBatch({
+			projectId: "default",
+			receivedAt: isoNow(),
+			expiresAt: isoIn(-60_000),
+			points: [
+				metricPoint({ exemplarsJson: exemplarJson("trace-ex-expired", "1") }),
+			],
+		});
+		await metrics.ingestBatch({
+			projectId: "default",
+			receivedAt: isoNow(),
+			expiresAt: isoIn(3_600_000),
+			points: [
+				metricPoint({ exemplarsJson: exemplarJson("trace-ex-live", "2") }),
+			],
+		});
+
+		// One expired exemplar + its expired point.
+		expect(await metrics.purgeExpired()).toBe(2);
+		const exemplars = await db
+			.prepare(`SELECT trace_id FROM metric_exemplars ORDER BY trace_id`)
+			.all<{ trace_id: string }>();
+		expect(exemplars.results).toEqual([{ trace_id: "trace-ex-live" }]);
+		const points = await db
+			.prepare(`SELECT COUNT(*) AS n FROM metric_point`)
+			.first<{ n: number }>();
+		expect(points?.n).toBe(1);
+		const live = await metrics.exemplarsForTrace("default", "trace-ex-live");
+		expect(live.map((e) => e.traceId)).toEqual(["trace-ex-live"]);
+		expect(live[0]?.receivedAt).toMatch(ISO_TEXT);
+	});
+
+	it("migration 043 converts legacy TIMESTAMPTZ metric_exemplars columns in place", async () => {
+		// max: 1 pins SET search_path / timezone to the one client the
+		// adapter will reuse, on PGlite and on real Postgres alike.
+		const legacyPool = new pg.Pool({ connectionString: databaseUrl, max: 1 });
+		try {
+			await legacyPool.query(`SET search_path TO ${legacySchema}`);
+			// A non-UTC session zone must not leak into the converted text.
+			await legacyPool.query(`SET timezone TO 'America/New_York'`);
+			// Shape of migrations-postgres/035 before the fix (metric_series /
+			// metric_point trimmed to the columns the FK and purge touch).
+			await legacyPool.query(`
+				CREATE TABLE metric_series (id TEXT PRIMARY KEY);
+				CREATE TABLE metric_point (id TEXT PRIMARY KEY, expires_at TEXT NOT NULL);
+				CREATE TABLE metric_exemplars (
+					id TEXT PRIMARY KEY,
+					point_id TEXT NOT NULL REFERENCES metric_point(id),
+					series_id TEXT NOT NULL REFERENCES metric_series(id),
+					project_id TEXT NOT NULL,
+					metric_name TEXT NOT NULL,
+					service_name TEXT,
+					trace_id TEXT,
+					span_id TEXT,
+					ts_ns TEXT NOT NULL,
+					value DOUBLE PRECISION NOT NULL,
+					received_at TIMESTAMPTZ NOT NULL,
+					expires_at TIMESTAMPTZ NOT NULL
+				);
+				CREATE INDEX idx_metric_exemplars_expires ON metric_exemplars (expires_at);
+				INSERT INTO metric_series VALUES ('s1');
+				INSERT INTO metric_point VALUES
+					('p-expired', '2000-01-01T00:00:00.000Z'),
+					('p-live', '2999-01-01T00:00:00.000Z');
+			`);
+			// The collector bound Date#toISOString() text, which Postgres
+			// parsed into timestamptz. Include a non-UTC offset to prove the
+			// conversion normalises to UTC rather than echoing input.
+			const insert = `INSERT INTO metric_exemplars
+				(id, point_id, series_id, project_id, metric_name, service_name, trace_id, span_id, ts_ns, value, received_at, expires_at)
+				VALUES ($1, $2, 's1', 'default', 'http.server.duration', 'checkout', $3, NULL, '1', 1, $4::timestamptz, $5::timestamptz)`;
+			await legacyPool.query(insert, [
+				"ex-expired",
+				"p-expired",
+				"trace-legacy-expired",
+				"2000-01-01T00:00:00.000Z",
+				"2000-01-02T00:00:00.000Z",
+			]);
+			await legacyPool.query(insert, [
+				"ex-live",
+				"p-live",
+				"trace-legacy-live",
+				"2026-06-01T14:34:56.789+02:00",
+				"2999-01-01T00:00:00.000Z",
+			]);
+			const before = await legacyPool.query(
+				`SELECT data_type FROM information_schema.columns
+				 WHERE table_schema = $1 AND table_name = 'metric_exemplars' AND column_name = 'expires_at'`,
+				[legacySchema],
+			);
+			expect(before.rows[0]?.data_type).toBe("timestamp with time zone");
+
+			const legacyDb = new PostgresAdapter(legacyPool);
+			const metrics = new MetricsStore(legacyDb);
+			// The launch blocker: TEXT now() against TIMESTAMPTZ columns.
+			await expect(metrics.purgeExpired()).rejects.toThrow(
+				/operator does not exist: timestamp with time zone < text/,
+			);
+
+			const sql = await readFile(
+				path.join(migrationsDir, "043_metric_exemplars_text_timestamps.sql"),
+				"utf8",
+			);
+			await legacyPool.query(sql);
+			await legacyPool.query(sql); // re-running is a no-op
+
+			const types = await legacyPool.query<{
+				column_name: string;
+				data_type: string;
+			}>(
+				`SELECT column_name, data_type FROM information_schema.columns
+				 WHERE table_schema = $1 AND table_name = 'metric_exemplars'
+				   AND column_name IN ('received_at', 'expires_at')
+				 ORDER BY column_name`,
+				[legacySchema],
+			);
+			expect(types.rows).toEqual([
+				{ column_name: "expires_at", data_type: "text" },
+				{ column_name: "received_at", data_type: "text" },
+			]);
+			const converted = await legacyPool.query(
+				`SELECT id, received_at, expires_at FROM metric_exemplars ORDER BY id`,
+			);
+			expect(converted.rows).toEqual([
+				{
+					id: "ex-expired",
+					received_at: "2000-01-01T00:00:00.000Z",
+					expires_at: "2000-01-02T00:00:00.000Z",
+				},
+				{
+					id: "ex-live",
+					received_at: "2026-06-01T12:34:56.789Z",
+					expires_at: "2999-01-01T00:00:00.000Z",
+				},
+			]);
+			const indexes = await legacyPool.query<{ indexname: string }>(
+				`SELECT indexname FROM pg_indexes WHERE schemaname = $1 AND tablename = 'metric_exemplars'`,
+				[legacySchema],
+			);
+			expect(indexes.rows.map((r) => r.indexname)).toContain(
+				"idx_metric_exemplars_expires",
+			);
+
+			// Retention now runs through the real store: 1 exemplar + 1 point.
+			expect(await metrics.purgeExpired()).toBe(2);
+			const remaining = await legacyPool.query(
+				`SELECT id FROM metric_exemplars ORDER BY id`,
+			);
+			expect(remaining.rows).toEqual([{ id: "ex-live" }]);
+			const points = await legacyPool.query(
+				`SELECT id FROM metric_point ORDER BY id`,
+			);
+			expect(points.rows).toEqual([{ id: "p-live" }]);
+			const recent = await metrics.recentExemplars({ projectId: "default" });
+			expect(recent.map((e) => [e.traceId, e.receivedAt])).toEqual([
+				["trace-legacy-live", "2026-06-01T12:34:56.789Z"],
+			]);
+		} finally {
+			// Single-session servers (PGlite) share GUCs across sockets, so put
+			// search_path / timezone back before the pooled adapter runs again.
+			await legacyPool.query("RESET timezone").catch(() => {});
+			await legacyPool.query("RESET search_path").catch(() => {});
+			await legacyPool.end();
+		}
+	});
+
+	it("migration 044 creates idx_actions_started_at and is a no-op when re-applied", async () => {
+		expect(migrationFiles).toContain("044_actions_started_at_index.sql");
+		const indexed = () =>
+			db
+				.prepare(
+					`SELECT indexname FROM pg_indexes
+					 WHERE schemaname = current_schema() AND tablename = 'actions' AND indexname = 'idx_actions_started_at'`,
+				)
+				.first<{ indexname: string }>();
+		expect((await indexed())?.indexname).toBe("idx_actions_started_at");
+
+		// Upgrade path: an already-indexed database re-running 044 must not fail.
+		const admin = await adminClient();
+		try {
+			await admin.query(
+				await readFile(
+					path.join(migrationsDir, "044_actions_started_at_index.sql"),
+					"utf8",
+				),
+			);
+		} finally {
+			await admin.end();
+		}
+		expect((await indexed())?.indexname).toBe("idx_actions_started_at");
+	});
+
+	it("migrations 045–048 key profiles, action graph, payloads and spans by project and re-apply cleanly", async () => {
+		const expected: Record<string, string> = {
+			user_profiles: "PRIMARY KEY (project_id, user_id)",
+			actions: "PRIMARY KEY (project_id, id)",
+			agent_runs: "PRIMARY KEY (project_id, id)",
+			tool_calls: "PRIMARY KEY (project_id, id)",
+			retrieval_events: "PRIMARY KEY (project_id, id)",
+			eval_results: "PRIMARY KEY (project_id, id)",
+			artifacts: "PRIMARY KEY (project_id, id)",
+			ai_span_payloads: "PRIMARY KEY (project_id, trace_id, span_id)",
+			telemetry_spans: "PRIMARY KEY (project_id, trace_id, span_id)",
+		};
+		const primaryKeys = async () => {
+			const rows = await db
+				.prepare(
+					`SELECT c.conrelid::regclass::text AS table_name,
+					        pg_get_constraintdef(c.oid) AS pk
+					 FROM pg_constraint c
+					 JOIN pg_namespace n ON n.oid = c.connamespace
+					 WHERE c.contype = 'p' AND n.nspname = current_schema()`,
+				)
+				.all<{ table_name: string; pk: string }>();
+			return Object.fromEntries(
+				rows.results
+					.filter((r) => r.table_name in expected)
+					.map((r) => [r.table_name, r.pk]),
+			);
+		};
+		expect(await primaryKeys()).toEqual(expected);
+
+		// Upgrade path: re-running the four files on a converted database is
+		// a no-op (each looks the current key up by definition first).
+		const admin = await adminClient();
+		try {
+			for (const file of migrationFiles.filter(
+				(f) => f >= "045_" && f < "049_",
+			)) {
+				await admin.query(
+					await readFile(path.join(migrationsDir, file), "utf8"),
+				);
+			}
+		} finally {
+			await admin.end();
+		}
+		expect(await primaryKeys()).toEqual(expected);
+		const spanIndexes = await db
+			.prepare(
+				`SELECT indexname FROM pg_indexes
+				 WHERE schemaname = current_schema() AND tablename = 'telemetry_spans'
+				 ORDER BY indexname`,
+			)
+			.all<{ indexname: string }>();
+		expect(spanIndexes.results.map((r) => r.indexname)).toContain(
+			"idx_spans_project_received",
+		);
+	});
+
+	it("keeps colliding ids separate per project through the real write paths", async () => {
+		const cleanup = async () => {
+			for (const table of [
+				"actions",
+				"telemetry_spans",
+				"ai_span_payloads",
+				"user_profiles",
+			]) {
+				await db
+					.prepare(`DELETE FROM ${table} WHERE project_id IN (?, ?)`)
+					.bind("tenant-a", "tenant-b")
+					.run();
+			}
+		};
+		try {
+			const store = new ActionStore(db);
+			await store.upsertAction(
+				actionRecord({ id: "01SHARED", projectId: "tenant-a", name: "a run" }),
+			);
+			await store.upsertAction(
+				actionRecord({ id: "01SHARED", projectId: "tenant-b", name: "b run" }),
+			);
+			await store.upsertAction(
+				actionRecord({
+					id: "01SHARED",
+					projectId: "tenant-b",
+					name: "b run v2",
+				}),
+			);
+			expect((await store.getAction("tenant-a", "01SHARED"))?.name).toBe(
+				"a run",
+			);
+			expect((await store.getAction("tenant-b", "01SHARED"))?.name).toBe(
+				"b run v2",
+			);
+
+			const span = (projectId: string, spanName: string): StoredSpan => ({
+				projectId,
+				traceId: "trace-shared",
+				spanId: "span-shared",
+				parentSpanId: null,
+				traceState: null,
+				serviceName: "agent",
+				scopeName: null,
+				scopeVersion: null,
+				spanName,
+				spanKind: 1,
+				statusCode: 1,
+				statusMessage: null,
+				startTime: isoNow(),
+				endTime: isoNow(),
+				durationMs: 5,
+				attributesJson: JSON.stringify({
+					"openinference.span.kind": "LLM",
+					// An object: string payloads are stored verbatim, which JSONB rejects.
+					"ai.payload.input": { prompt: `${projectId} prompt` },
+				}),
+				droppedAttributesCount: 0,
+				resourceAttributesJson: "{}",
+				eventsJson: "[]",
+				droppedEventsCount: 0,
+				linksJson: "[]",
+				droppedLinksCount: 0,
+				receivedAt: isoNow(),
+				expiresAt: isoIn(60 * 60 * 1000),
+				sessionId: null,
+				interactionId: null,
+			});
+			// INSERT OR IGNORE → ON CONFLICT DO NOTHING used to drop tenant-b's span.
+			await ingestTelemetrySpans(db, [span("tenant-a", "a-llm")]);
+			await ingestTelemetrySpans(db, [span("tenant-b", "b-llm")]);
+			const names = await db
+				.prepare(
+					`SELECT project_id, span_name FROM telemetry_spans
+				 WHERE trace_id = ? AND span_id = ? ORDER BY project_id`,
+				)
+				.bind("trace-shared", "span-shared")
+				.all<{ project_id: string; span_name: string }>();
+			expect(names.results).toEqual([
+				{ project_id: "tenant-a", span_name: "a-llm" },
+				{ project_id: "tenant-b", span_name: "b-llm" },
+			]);
+
+			// Payload side table: the processor's upsert must stay per project.
+			const processors: SpanProcessorPlugin[] = [];
+			aiSpanPayloadsProcessorPlugin.register(
+				{} as Parameters<typeof aiSpanPayloadsProcessorPlugin.register>[0],
+				{
+					addSpanProcessor(p: SpanProcessorPlugin) {
+						processors.push(p);
+					},
+				} as unknown as CollectorRuntime,
+			);
+			const process = processors[0]?.process;
+			if (!process) throw new Error("span processor was not registered");
+			const ctx = {
+				env: { DB: db, OBS_PAYLOAD_CAPTURE_DEFAULT: "true" },
+				now: new Date(),
+				logger: console,
+			} as unknown as CollectorRouteContext;
+			await process([span("tenant-a", "a-llm")], ctx);
+			await process([span("tenant-b", "b-llm")], ctx);
+			const payloads = await db
+				.prepare(
+					`SELECT project_id, input_json FROM ai_span_payloads
+				 WHERE trace_id = ? AND span_id = ? ORDER BY project_id`,
+				)
+				.bind("trace-shared", "span-shared")
+				.all<{ project_id: string; input_json: string }>();
+			expect(
+				payloads.results.map((r) => [r.project_id, JSON.parse(r.input_json)]),
+			).toEqual([
+				["tenant-a", { prompt: "tenant-a prompt" }],
+				["tenant-b", { prompt: "tenant-b prompt" }],
+			]);
+			const aOverview = await getAISpansOverview(db, {
+				projectId: "tenant-a",
+				hours: 24,
+			});
+			expect(aOverview.spans.map((s) => s.spanName)).toEqual(["a-llm"]);
+
+			// /v1/identify: same userId in two projects → two profiles.
+			const setProject: MiddlewareHandler<{
+				Bindings: CollectorEnv;
+				Variables: { projectId: string };
+			}> = async (c, next) => {
+				const header = c.req.header("x-project-id");
+				if (header) c.set("projectId", header);
+				await next();
+			};
+			const app = createTelemetryCollectorApp({
+				plugins: [identityReceiverPlugin],
+				auth: { middleware: setProject },
+				dashboardAuth: { middleware: setProject, registerRoutes: () => {} },
+			});
+			const identify = (projectId: string, body: Record<string, unknown>) =>
+				app.request(
+					"/v1/identify",
+					{
+						method: "POST",
+						headers: {
+							"content-type": "application/json",
+							"x-project-id": projectId,
+						},
+						body: JSON.stringify(body),
+					},
+					{ DB: db as unknown as D1Database } as CollectorEnv,
+				);
+			const userId = "ada@example.com";
+			expect(
+				(
+					await identify("tenant-a", {
+						userId,
+						visitorId: "vis-a",
+						name: "Ada (a)",
+						properties: { plan: "a" },
+					})
+				).status,
+			).toBe(200);
+			expect(
+				(
+					await identify("tenant-b", {
+						userId,
+						visitorId: "vis-b",
+						name: "Ada (b)",
+					})
+				).status,
+			).toBe(200);
+			const profiles = await db
+				.prepare(
+					`SELECT project_id, visitor_id, name FROM user_profiles
+				 WHERE user_id = ? ORDER BY project_id`,
+				)
+				.bind(userId)
+				.all<{ project_id: string; visitor_id: string; name: string }>();
+			expect(profiles.results).toEqual([
+				{ project_id: "tenant-a", visitor_id: "vis-a", name: "Ada (a)" },
+				{ project_id: "tenant-b", visitor_id: "vis-b", name: "Ada (b)" },
+			]);
+		} finally {
+			// Leave no tenant rows behind for the purge / analysis tests below.
+			await cleanup();
+		}
+	});
+
+	it("purges expired action trees by TEXT started_at and keeps fresh, in-progress and eval-pinned trees", async () => {
+		const purge = (
+			over: {
+				maxActionsPerBatch?: number;
+				maxTreesPerBatch?: number;
+				maxBatches?: number;
+			} = {},
+		) =>
+			purgeExpiredActionGraph(db, {
+				retentionHours: FIXTURE_RETENTION_HOURS,
+				now: () => FIXTURE_NOW_MS,
+				...over,
+			});
+		const count = (table: string, column: string, value: string) =>
+			db
+				.prepare(`SELECT COUNT(*) AS n FROM ${table} WHERE ${column} = ?`)
+				.bind(value)
+				.first<{ n: number | string }>()
+				.then((r) => Number(r?.n ?? 0));
+
+		const expired = await seedTree(db, {
+			projectId: "pg-ret",
+			rootId: "pg-old",
+		});
+		const fresh = await seedTree(db, {
+			projectId: "pg-ret",
+			rootId: "pg-new",
+			rootStartedAt: FRESH_START,
+		});
+		await seedTree(db, {
+			projectId: "pg-ret",
+			rootId: "pg-hung",
+			childEndedAt: null,
+		});
+		await seedTree(db, {
+			projectId: "pg-ret",
+			rootId: "pg-running",
+			agentRunStatus: "running",
+		});
+		const pinned = await seedTree(db, {
+			projectId: "pg-ret",
+			rootId: "pg-pinned",
+		});
+		await seedEvalCase(db, {
+			id: "pg-case",
+			projectId: "pg-ret",
+			sourceActionId: pinned.childId,
+		});
+		await seedTree(db, { projectId: "pg-ret-other", rootId: "pg-other-old" });
+		// Three more expired two-action trees to exercise packing and batching.
+		for (let i = 0; i < 3; i += 1) {
+			await seedTree(db, {
+				projectId: "pg-ret",
+				rootId: `pg-bulk-${i}`,
+				withLeaves: false,
+			});
+		}
+		// An expired tree larger than the per-batch cap used below (5 actions).
+		const bigStore = new ActionStore(db);
+		for (let i = 0; i < 5; i += 1) {
+			await bigStore.upsertAction(
+				actionRecord({
+					id: i === 0 ? "pg-big" : `pg-big-${i}`,
+					projectId: "pg-ret",
+					rootActionId: "pg-big",
+					causedByActionId: i === 0 ? null : "pg-big",
+					agentRunId: "pg-big",
+				}),
+			);
+		}
+		await bigStore.upsertAgentRun(
+			agentRunRecord({ id: "pg-big", projectId: "pg-ret" }),
+		);
+
+		// Cap of 4 actions per batch, one batch: two whole 2-action trees go,
+		// nothing is split, and the 5-action tree is reported, not touched.
+		const first = await purge({ maxActionsPerBatch: 4, maxBatches: 1 });
+		expect(first).toMatchObject({
+			actions: 4,
+			trees: 2,
+			batches: 1,
+			truncated: true,
+			oversized_trees: 1,
+			oversized: [
+				{ project_id: "pg-ret", root_action_id: "pg-big", actions: 5 },
+			],
+		});
+		expect(await count("actions", "root_action_id", "pg-big")).toBe(5);
+		for (const root of [
+			"pg-old",
+			"pg-other-old",
+			"pg-bulk-0",
+			"pg-bulk-1",
+			"pg-bulk-2",
+		]) {
+			expect([0, 2]).toContain(await count("actions", "root_action_id", root));
+		}
+
+		const rest = await purge({ maxActionsPerBatch: 4 });
+		expect(rest).toMatchObject({ truncated: false, oversized_trees: 1 });
+		// 2 (pg-old) + 2 (pg-other-old) + 6 (bulk) spine rows in total.
+		expect(first.actions + rest.actions).toBe(10);
+		expect(first.trees + rest.trees).toBe(5);
+		expect(first.agent_runs + rest.agent_runs).toBe(5);
+		expect(first.tool_calls + rest.tool_calls).toBe(2);
+		expect(first.artifacts + rest.artifacts).toBe(2);
+
+		expect(await count("actions", "root_action_id", expired.rootId)).toBe(0);
+		expect(await count("tool_calls", "id", expired.toolCallId)).toBe(0);
+		expect(
+			await count("retrieval_events", "id", expired.retrievalEventId),
+		).toBe(0);
+		expect(await count("eval_results", "id", expired.evalResultId)).toBe(0);
+		expect(await count("artifacts", "id", expired.artifactId)).toBe(0);
+		expect(await count("agent_runs", "id", expired.rootId)).toBe(0);
+		expect(await count("actions", "root_action_id", "pg-other-old")).toBe(0);
+
+		expect(await count("actions", "root_action_id", fresh.rootId)).toBe(2);
+		expect(await count("tool_calls", "id", fresh.toolCallId)).toBe(1);
+		expect(await count("actions", "root_action_id", "pg-hung")).toBe(2);
+		expect(await count("actions", "root_action_id", "pg-running")).toBe(2);
+		expect(await count("agent_runs", "id", "pg-running")).toBe(1);
+		expect(await count("actions", "root_action_id", pinned.rootId)).toBe(2);
+		expect(await count("eval_cases", "id", "pg-case")).toBe(1);
+		expect(await count("actions", "root_action_id", "pg-big")).toBe(5);
+		expect(await count("agent_runs", "id", "pg-big")).toBe(1);
+
+		// With a cap that fits it, the oversized tree goes whole.
+		const big = await purge({ maxActionsPerBatch: 5 });
+		expect(big).toMatchObject({
+			actions: 5,
+			trees: 1,
+			agent_runs: 1,
+			oversized_trees: 0,
+		});
+		expect(await count("actions", "root_action_id", "pg-big")).toBe(0);
+		expect(await countActionGraph(db, "pg-ret")).toEqual({
+			actions: 8,
+			agent_runs: 4,
+			tool_calls: 4,
+			retrieval_events: 4,
+			eval_results: 4,
+			artifacts: 4,
+		});
+
+		// Idempotent: nothing left to purge.
+		expect(await purge()).toMatchObject({
+			actions: 0,
+			batches: 0,
+			truncated: false,
+		});
+	});
+
+	it("ingests AI calls and summarises BOOLEAN is_error inside a TEXT time window", async () => {
+		await ingestAICallBatch(db, [
+			aiCall({ callId: "ai-ok" }),
+			aiCall({ callId: "ai-err", isError: true, errorMessage: "boom" }),
+			aiCall({
+				callId: "ai-old",
+				receivedAt: new Date(Date.now() - 48 * 3600 * 1000).toISOString(),
+			}),
+		]);
+
+		const overview = await getAICallsOverview(db, {
+			projectId: "default",
+			hours: 24,
+		});
+		expect(overview.summary.totalCalls).toBe(2);
+		expect(overview.summary.errorCalls).toBe(1);
+		expect(overview.summary.totalPromptTokens).toBe(20);
+		expect(typeof overview.summary.totalCostUsd).toBe("number");
+		expect(overview.calls.map((c) => [c.callId, c.isError]).sort()).toEqual([
+			["ai-err", true],
+			["ai-ok", false],
+		]);
+		// JSONB columns must still come back as JSON text (JSONB normalises
+		// whitespace, so compare the parsed value).
+		expect(typeof overview.calls[0]?.requestJson).toBe("string");
+		expect(JSON.parse(overview.calls[0]?.requestJson ?? "null")).toEqual({
+			messages: [],
+		});
+
+		const errorsOnly = await getAICallsOverview(db, {
+			projectId: "default",
+			hours: 24,
+			isError: true,
+		});
+		expect(errorsOnly.calls.map((c) => c.callId)).toEqual(["ai-err"]);
+
+		const alerts = new AlertsStore(db);
+		const rule = {
+			id: "rule-ai-errors",
+			projectId: "default",
+			name: "AI errors",
+			signal: "ai",
+			query: { isError: true },
+			threshold: 0,
+			windowMins: 60,
+			comparison: "gt",
+			channels: [],
+			enabled: true,
+			createdAt: isoNow(),
+			updatedAt: isoNow(),
+		} as unknown as AlertRule;
+		expect(await alerts.evaluateRule(rule)).toBe(1);
+	});
+
+	it("migration 049 adds alert_state.notified with a delivered backfill, and the store drives it", async () => {
+		// Legacy shape: alert_state as created by 014, before 049.
+		const legacyPool = new pg.Pool({ connectionString: databaseUrl, max: 1 });
+		try {
+			await legacyPool.query(`SET search_path TO ${legacySchema}`);
+			await legacyPool.query(`
+				CREATE TABLE alert_state (
+					rule_id TEXT PRIMARY KEY,
+					project_id TEXT NOT NULL,
+					current_state TEXT NOT NULL,
+					last_state_change TEXT NOT NULL
+				);
+				INSERT INTO alert_state VALUES
+					('rule-legacy', 'default', 'firing', '2026-05-31T00:00:00.000Z');
+			`);
+			const sql = await readFile(
+				path.join(migrationsDir, "049_alert_state_notified.sql"),
+				"utf8",
+			);
+			await legacyPool.query(sql);
+			// Re-running is a no-op. Postgres answers the IF NOT EXISTS skip
+			// with a NOTICE, which PGlite's socket bridge does not relay
+			// cleanly; silence it on this session (real Postgres is unaffected).
+			await legacyPool.query(`SET client_min_messages TO WARNING`);
+			await legacyPool.query(sql);
+			const column = await legacyPool.query<{
+				data_type: string;
+				is_nullable: string;
+				column_default: string;
+			}>(
+				`SELECT data_type, is_nullable, column_default FROM information_schema.columns
+				 WHERE table_schema = $1 AND table_name = 'alert_state' AND column_name = 'notified'`,
+				[legacySchema],
+			);
+			expect(column.rows).toEqual([
+				{ data_type: "integer", is_nullable: "NO", column_default: "1" },
+			]);
+			const backfilled = await legacyPool.query<{ notified: number }>(
+				`SELECT notified FROM alert_state WHERE rule_id = 'rule-legacy'`,
+			);
+			expect(Number(backfilled.rows[0]?.notified)).toBe(1);
+		} finally {
+			// PGlite is one session for every socket: put search_path and the
+			// message level back before the pooled adapter runs again.
+			await legacyPool.query("RESET client_min_messages").catch(() => {});
+			await legacyPool.query("RESET search_path").catch(() => {});
+			await legacyPool.end();
+		}
+
+		// Live schema: transitions start undelivered; only the exact
+		// transition that was delivered is acknowledged.
+		const projects = new ProjectsStore(db);
+		const project = await projects.createProject({
+			name: "Alert state",
+			slug: `alert-state-${Date.now().toString(36)}`,
+		});
+		const alerts = new AlertsStore(db);
+		const rule = await alerts.createRule(project.id, {
+			name: "notified flag",
+			signal: "spans",
+			query: {},
+			threshold: 1,
+			windowMins: 5,
+			comparison: ">",
+			channels: [{ type: "webhook", url: "https://hooks.example.com/pg" }],
+			enabled: true,
+		});
+		const t1 = "2026-05-31T00:00:00.000Z";
+		await alerts.transitionState(rule.id, project.id, "firing", t1);
+		expect(await alerts.getState(rule.id, project.id)).toMatchObject({
+			current_state: "firing",
+			last_state_change: t1,
+			notified: 0,
+		});
+		expect(
+			await alerts.markNotified(rule.id, "other-project", "firing", t1),
+		).toBe(false);
+		expect(await alerts.markNotified(rule.id, project.id, "ok", t1)).toBe(
+			false,
+		);
+		expect(await alerts.markNotified(rule.id, project.id, "firing", t1)).toBe(
+			true,
+		);
+		expect((await alerts.getState(rule.id, project.id))?.notified).toBe(1);
+		expect(await alerts.markNotified(rule.id, project.id, "firing", t1)).toBe(
+			false,
+		);
+	});
+
+	it("purges expired AI data using the dialect's text-rendered now()", async () => {
+		await ingestAICallBatch(db, [
+			aiCall({ callId: "ai-expired", expiresAt: isoIn(-60_000) }),
+		]);
+		expect(await purgeExpiredAIData(db)).toBe(1);
+		const remaining = await db
+			.prepare(`SELECT call_id FROM ai_calls WHERE call_id = ?`)
+			.bind("ai-expired")
+			.first();
+		expect(remaining).toBeNull();
+	});
+
+	it("ingests usage events with BOOLEAN is_bot and reads them back as D1 flags", async () => {
+		const usage = new UsageStore(db);
+		await usage.ingest([
+			usageEvent({ eventId: "u-human" }),
+			usageEvent({ eventId: "u-bot", sessionId: "sess-bot", isBot: true }),
+		]);
+		const flags = await db
+			.prepare(`SELECT event_id, is_bot FROM usage_events ORDER BY event_id`)
+			.all<{ event_id: string; is_bot: unknown }>();
+		expect(flags.results).toEqual([
+			{ event_id: "u-bot", is_bot: 1 },
+			{ event_id: "u-human", is_bot: 0 },
+		]);
+
+		const overview = await usage.getOverview({
+			projectId: "default",
+			hours: 24,
+		});
+		// Same semantics as D1: bots are counted (not dropped) by getOverview.
+		expect(overview.botsFiltered).toBe(1);
+		expect(overview.summary.totalEvents).toBe(2);
+	});
+
+	it("runs every built-in and derived analysis definition", async () => {
+		// Seed two linked spans so Tier 1 derivation emits the service,
+		// edge, messaging-topic and AI analyses (julianday, json_extract …).
+		const now = isoNow();
+		const span = db.prepare(`
+			INSERT INTO telemetry_spans (project_id, trace_id, span_id, parent_span_id, service_name,
+				span_name, span_kind, status_code, start_time, end_time, duration_ms, attributes_json,
+				received_at, expires_at)
+			VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+		await db.batch([
+			span.bind(
+				"default",
+				"trace-1",
+				"span-a",
+				null,
+				"svc-a",
+				"publish orders",
+				4,
+				0,
+				now,
+				now,
+				12.5,
+				JSON.stringify({ "messaging.destination": "orders" }),
+				now,
+				isoIn(3_600_000),
+			),
+			db
+				.prepare(`
+			INSERT INTO telemetry_spans (project_id, trace_id, span_id, parent_span_id, service_name,
+				span_name, span_kind, status_code, start_time, end_time, duration_ms, attributes_json,
+				received_at, expires_at)
+			VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+				.bind(
+					"default",
+					"trace-1",
+					"span-b",
+					"span-a",
+					"svc-b",
+					"consume orders",
+					5,
+					2,
+					now,
+					now,
+					30,
+					JSON.stringify({
+						"messaging.destination": "orders",
+						"gen_ai.system": "openai",
+					}),
+					now,
+					isoIn(3_600_000),
+				),
+		]);
+
+		const derived = await deriveAnalysesForProject("default", db);
+		expect(derived.map((d) => d.id)).toEqual(
+			expect.arrayContaining(["ai_error_rate", "messaging_lag::orders"]),
+		);
+
+		const defs = [...TIER0_ANALYSES, ...INVESTIGATION_ANALYSES, ...derived];
+		for (const def of defs) {
+			const result = await runSqlAnalysis(def, {
+				db,
+				projectId: "default",
+				retentionHours: 24,
+			}).catch((err: Error) => {
+				throw new Error(`${def.id}: ${err.message}`);
+			});
+			expect(VALID_STATUSES.has(result.status), def.id).toBe(true);
+		}
+
+		const errorRate = await runSqlAnalysis(
+			TIER0_ANALYSES.find((d) => d.id === "overall_error_rate") as never,
+			{ db, projectId: "default", retentionHours: 24 },
+		);
+		expect(errorRate.primaryValue).toBe(0.5);
+		expect(Array.isArray(errorRate.payload.sparkline)).toBe(true);
+	});
+});