@obsunified/collector 2.3.0 → 2.4.4

package/src/lib/projects-store.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 import { sha256Hex } from "./hash";
 import { displayPrefix, ProjectsStore } from "./projects-store";
 import { createMigratedSqliteDb } from "./test-utils/node-sqlite-db";
@@ -60,3 +60,46 @@ describe("bootstrap key display prefix", () => {
 		db.close();
 	});
 });
+
+describe("ProjectsStore.bootstrapEnvKey rotation", () => {
+	it("warns (without revoking) when INGEST_KEY changes while the old bootstrap key is active", async () => {
+		const db = createMigratedSqliteDb();
+		const store = new ProjectsStore(db);
+		const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+		try {
+			await store.bootstrapEnvKey("first-bootstrap-key-0001");
+			await store.bootstrapEnvKey("first-bootstrap-key-0001"); // idempotent
+			expect(warn).not.toHaveBeenCalled();
+
+			await store.bootstrapEnvKey("second-bootstrap-key-0002");
+			expect(warn).toHaveBeenCalledTimes(1);
+			const message = String(warn.mock.calls[0]?.[0]);
+			expect(message).toContain("INGEST_KEY changed");
+			expect(message).toContain("revoke");
+			expect(message).not.toContain("first-bootstrap-key-0001");
+
+			// Nothing is auto-revoked: both keys still authenticate.
+			for (const key of [
+				"first-bootstrap-key-0001",
+				"second-bootstrap-key-0002",
+			]) {
+				expect(
+					(await store.findByKeyHash(await sha256Hex(key)))?.projectId,
+				).toBe("default");
+			}
+
+			// Once the old one is revoked, a restart with the new key is quiet.
+			const old = (await store.listKeys("default")).find(
+				(k) => k.name === "bootstrap" && k.keyPrefix.startsWith("first"),
+			);
+			expect(old).toBeDefined();
+			await store.revokeKey(old?.id ?? "", "default");
+			warn.mockClear();
+			await store.bootstrapEnvKey("second-bootstrap-key-0002");
+			expect(warn).not.toHaveBeenCalled();
+		} finally {
+			warn.mockRestore();
+			db.close();
+		}
+	});
+});

package/src/lib/projects-store.ts

@@ -215,6 +215,15 @@ export class ProjectsStore {
 			return;
 		}
 
+		// A different, still-active bootstrap key means INGEST_KEY was rotated.
+		// The old key is deliberately NOT revoked here: other instances (or
+		// clients mid-rollout) may still be using it. Tell the operator instead.
+		const previous = await this.db
+			.prepare(
+				`SELECT id, created_at FROM ingest_keys WHERE project_id = 'default' AND name = 'bootstrap' AND revoked_at IS NULL`,
+			)
+			.all<{ id: string; created_at: string }>();
+
 		const id = randomHex(16);
 		const createdAt = new Date().toISOString();
 		await this.db
@@ -223,5 +232,17 @@ export class ProjectsStore {
 			)
 			.bind(id, keyHash, keyPrefix, createdAt)
 			.run();
+
+		const stale = previous.results ?? [];
+		if (stale.length > 0) {
+			console.warn(
+				`[projects-store] INGEST_KEY changed: registered a new bootstrap ingest key (${keyPrefix}…) but ${stale.length} earlier bootstrap key(s) are still active and keep accepting ingest: ${stale
+					// Ids only: legacy rows may still hold a long key_prefix.
+					.map((row) => `id ${row.id} (created ${row.created_at})`)
+					.join(
+						", ",
+					)}. Once every client uses the new key, revoke the old one in the dashboard (Projects → default → Keys) or with DELETE /internal/projects/default/keys/<keyId>.`,
+			);
+		}
 	}
 }

package/src/lib/sql-db-postgres.integration.test.ts

@@ -1112,6 +1112,253 @@ describe.skipIf(!databaseUrl)("PostgresAdapter against a live Postgres", () => {
 		);
 	});
 
+	it("evaluates metrics alert rules and drives pendingSecs state (054-059)", async () => {
+		const columns = await db
+			.prepare(
+				`SELECT table_name, column_name FROM information_schema.columns
+				 WHERE table_schema = current_schema()
+				   AND ((table_name = 'alert_rules' AND column_name = 'pending_secs')
+				     OR (table_name = 'alert_state' AND column_name IN ('pending_since', 'pending_seen_at'))
+				     OR (table_name = 'alert_evaluations' AND column_name IN ('outcome', 'error_message')))
+				 ORDER BY table_name, column_name`,
+			)
+			.all<{ table_name: string; column_name: string }>();
+		expect(columns.results.map((c) => c.column_name)).toEqual([
+			"error_message",
+			"outcome",
+			"pending_secs",
+			"pending_seen_at",
+			"pending_since",
+		]);
+		const index = await db
+			.prepare(
+				`SELECT indexname FROM pg_indexes
+				 WHERE schemaname = current_schema() AND indexname = 'idx_metric_point_series_received'`,
+			)
+			.first<{ indexname: string }>();
+		expect(index?.indexname).toBe("idx_metric_point_series_received");
+
+		const project = await new ProjectsStore(db).createProject({
+			name: "Metric alerts",
+			slug: `metric-alerts-${Date.now().toString(36)}`,
+		});
+		const metrics = new MetricsStore(db);
+		const now = new Date();
+		const ago = (secs: number) => now.getTime() - secs * 1000;
+		const ingest = (secsAgo: number, points: DecodedMetricPoint[]) =>
+			metrics.ingestBatch({
+				projectId: project.id,
+				points,
+				receivedAt: new Date(ago(secsAgo)).toISOString(),
+				expiresAt: isoIn(60 * 60 * 1000),
+			});
+		const counter = (
+			name: string,
+			code: number,
+			secsAgo: number,
+			value: number,
+		) =>
+			metricPoint({
+				name,
+				type: "sum",
+				isMonotonic: true,
+				temporality: 2,
+				serviceName: "vbox",
+				resourceAttrsJson: JSON.stringify({ "service.name": "vbox" }),
+				attributesJson: JSON.stringify({ code }),
+				identity: `${name}|${code}`,
+				tsNs: (BigInt(ago(secsAgo)) * 1_000_000n).toString(),
+				startTsNs: (BigInt(ago(3600)) * 1_000_000n).toString(),
+				value,
+				count: null,
+				sum: null,
+				boundsJson: null,
+				bucketCountsJson: null,
+			});
+		// History before the 5-minute window for all three series, plus a
+		// series first seen inside it (code=503) whose first value counts.
+		await ingest(400, [
+			counter("calls_total", 200, 400, 90),
+			counter("calls_total", 500, 400, 1),
+			counter("calls_failed_total", 500, 400, 1),
+		]);
+		await ingest(240, [
+			counter("calls_total", 200, 240, 100),
+			counter("calls_total", 500, 240, 1),
+			counter("calls_failed_total", 500, 240, 1),
+			counter("calls_failed_total", 503, 240, 2),
+		]);
+		await ingest(60, [
+			// Counter reset on the 200 series: 100 → 4 counts 4.
+			counter("calls_total", 200, 60, 4),
+			counter("calls_total", 500, 60, 3),
+			counter("calls_failed_total", 500, 60, 3),
+		]);
+
+		const alerts = new AlertsStore(db);
+		const ratio = await alerts.createRule(project.id, {
+			name: "call error ratio",
+			signal: "metrics",
+			query: {
+				metric: "calls_failed_total",
+				attributes: { code: "500" },
+				aggregation: "increase",
+				denominator: { metric: "calls_total" },
+				minDenominator: 5,
+			},
+			threshold: 0.05,
+			windowMins: 5,
+			comparison: ">",
+			pendingSecs: 0,
+			channels: [{ type: "webhook", url: "https://hooks.example.com/pg" }],
+		});
+		// errors +2; attempts +4 (reset) +2 = 6 >= 5 → 2/6
+		expect(await alerts.evaluateRule(ratio, now)).toBeCloseTo(2 / 6);
+		// The new 503 series started an hour ago (before the window): its
+		// first point is a baseline, so only the 500 series' +2 counts.
+		const allErrors = await alerts.createRule(project.id, {
+			name: "all call errors",
+			signal: "metrics",
+			query: { metric: "calls_failed_total", aggregation: "increase" },
+			threshold: 0,
+			windowMins: 5,
+			comparison: ">",
+			channels: [{ type: "webhook", url: "https://hooks.example.com/pg" }],
+		});
+		expect(await alerts.evaluateRuleWithStatus(allErrors, now)).toEqual({
+			value: 2,
+			noData: false,
+		});
+		// `last` with a 30 s staleness bound: the newest points are 60 s old.
+		const gone = await alerts.createRule(project.id, {
+			name: "stale gauge",
+			signal: "metrics",
+			query: {
+				metric: "calls_total",
+				aggregation: "last",
+				staleSecs: 30,
+				noData: "keep",
+			},
+			threshold: 0,
+			windowMins: 5,
+			comparison: ">",
+			channels: [{ type: "webhook", url: "https://hooks.example.com/pg" }],
+		});
+		expect(await alerts.evaluateRuleWithStatus(gone, now)).toEqual({
+			value: 0,
+			noData: true,
+		});
+		await alerts.recordEvaluation(
+			gone.id,
+			project.id,
+			0,
+			"ok",
+			false,
+			"no_data",
+		);
+		expect(
+			(
+				await alerts.listEvaluations({
+					ruleId: gone.id,
+					projectId: project.id,
+					hours: 1,
+				})
+			).evaluations[0],
+		).toMatchObject({ outcome: "no_data", state: "ok" });
+		await alerts.recordEvaluation(
+			ratio.id,
+			project.id,
+			0,
+			"ok",
+			false,
+			"error",
+			"statement timeout on postgres://obs:secret@db/obs",
+		);
+		await alerts.transitionState(gone.id, project.id, "firing", isoNow());
+		const listed = new Map(
+			(await alerts.listRules(project.id)).map((r) => [r.id, r]),
+		);
+		expect(listed.get(ratio.id)).toMatchObject({
+			notificationDelivered: null,
+			lastEvaluation: {
+				outcome: "error",
+				value: 0,
+				error: "statement timeout on <url>",
+			},
+		});
+		expect(listed.get(gone.id)).toMatchObject({
+			currentState: "firing",
+			notificationDelivered: false,
+			lastEvaluation: { outcome: "no_data", error: null },
+		});
+		expect(listed.get(allErrors.id)?.lastEvaluation).toBeNull();
+
+		const stale = await alerts.createRule(project.id, {
+			name: "calls stopped",
+			signal: "metrics",
+			query: { metric: ["calls_total"], aggregation: "age_seconds" },
+			threshold: 60,
+			windowMins: 1,
+			comparison: ">=",
+			pendingSecs: 30,
+			channels: [{ type: "webhook", url: "https://hooks.example.com/pg" }],
+		});
+		expect(await alerts.evaluateRule(stale, now)).toBeCloseTo(60, 0);
+		const never = await alerts.createRule(project.id, {
+			name: "never reported",
+			signal: "metrics",
+			query: { metric: "not_a_metric", aggregation: "age_seconds" },
+			threshold: 60,
+			windowMins: 1,
+			comparison: ">=",
+			channels: [{ type: "webhook", url: "https://hooks.example.com/pg" }],
+		});
+		const neverValue = await alerts.evaluateRule(
+			never,
+			new Date(Date.parse(never.createdAt) + 90_000),
+		);
+		expect(neverValue).toBeCloseTo(90, 0);
+
+		// pending_since round-trips; a fresh state row is ok + delivered.
+		expect(stale.pendingSecs).toBe(30);
+		const since = new Date().toISOString();
+		await alerts.setPending(
+			stale.id,
+			project.id,
+			{ since, seenAt: since },
+			stale.createdAt,
+		);
+		expect(await alerts.getState(stale.id, project.id)).toMatchObject({
+			current_state: "ok",
+			last_state_change: stale.createdAt,
+			notified: 1,
+			pending_since: since,
+			pending_seen_at: since,
+		});
+		expect(
+			(await alerts.listRules(project.id)).find((r) => r.id === stale.id),
+		).toMatchObject({ pendingSecs: 30, pendingSince: since });
+		// An edit that changes evaluation resets the hold.
+		await alerts.updateRule(stale.id, project.id, { threshold: 61 });
+		expect(await alerts.getState(stale.id, project.id)).toMatchObject({
+			pending_since: null,
+			pending_seen_at: null,
+		});
+		await alerts.setPending(
+			stale.id,
+			project.id,
+			{ since, seenAt: since },
+			stale.createdAt,
+		);
+		await alerts.transitionState(stale.id, project.id, "firing", since);
+		await alerts.setPending(stale.id, project.id, null, stale.createdAt);
+		expect(await alerts.getState(stale.id, project.id)).toMatchObject({
+			current_state: "firing",
+			pending_since: null,
+			notified: 0,
+		});
+	});
+
 	it("purges expired AI data using the dialect's text-rendered now()", async () => {
 		await ingestAICallBatch(db, [
 			aiCall({ callId: "ai-expired", expiresAt: isoIn(-60_000) }),

package/src/lib/store/trace-queries.ts

@@ -33,7 +33,7 @@ export async function selectTraceCandidates(
 	if (options.search) {
 		const term = `%${options.search}%`;
 		whereClause +=
-			" AND (span_name LIKE ? OR status_message LIKE ? OR attributes_json LIKE ? OR events_json LIKE ?)";
+			" AND (span_name LIKE ? OR status_message LIKE ? OR CAST(attributes_json AS TEXT) LIKE ? OR CAST(events_json AS TEXT) LIKE ?)";
 		binds.push(term, term, term, term);
 	}
 

package/src/lib/webhook-url.test.ts

@@ -53,6 +53,41 @@ describe("assertSafeWebhookUrl", () => {
 		}
 	});
 
+	it("rejects IPv4 embedded in IPv6 (compatible, mapped, NAT64, 6to4)", () => {
+		for (const url of [
+			"http://[::7f00:1]/hook",
+			"http://[::127.0.0.1]/hook",
+			"http://[::a9fe:a9fe]/hook",
+			"http://[::ffff:7f00:1]/hook",
+			"http://[0:0:0:0:0:ffff:127.0.0.1]/hook",
+			"http://[64:ff9b::7f00:1]/hook",
+			"http://[64:ff9b::169.254.169.254]/hook",
+			"http://[2002:7f00:1::]/hook",
+			"http://[2002:a00:1::1]/hook",
+			"http://[::]/hook",
+		]) {
+			expect(() => assertSafeWebhookUrl(url), url).toThrow(WebhookUrlError);
+			// The documented opt-out still unlocks every one of them.
+			expect(
+				() => assertSafeWebhookUrl(url, { allowPrivateNetworks: true }),
+				url,
+			).not.toThrow();
+		}
+	});
+
+	it("keeps public IPv6 and public embedded IPv4 reachable", () => {
+		for (const url of [
+			"https://[2606:4700:4700::1111]/hook",
+			"https://[64:ff9b::808:808]/hook",
+			"https://[2002:808:808::1]/hook",
+			"https://[::ffff:8.8.8.8]/hook",
+			"https://198.20.0.1/hook",
+			"https://198.17.255.1/hook",
+		]) {
+			expect(() => assertSafeWebhookUrl(url), url).not.toThrow();
+		}
+	});
+
 	it("allows private hosts only with the explicit opt-in", () => {
 		const url = "http://alertmanager.internal:9093/api/v2/alerts";
 		expect(() => assertSafeWebhookUrl(url)).toThrow(WebhookUrlError);
@@ -107,6 +142,20 @@ describe("redactWebhookUrl", () => {
 		expect(
 			await resolveWebhookHost("v6.example", async () => ["fd00::1"]),
 		).toBe("private");
+		for (const answer of [
+			"::ffff:127.0.0.1",
+			"64:ff9b::a00:1",
+			"2002:a9fe:a9fe::",
+			"fe80::1%eth0",
+		]) {
+			expect(
+				await resolveWebhookHost("v6.example", async () => [answer]),
+				answer,
+			).toBe("private");
+		}
+		expect(
+			await resolveWebhookHost("v6.example", async () => ["2606:4700::1111"]),
+		).toBe("public");
 		// IP literals were judged already by the literal check.
 		expect(await resolveWebhookHost("10.0.0.8", lookup)).toBe("public");
 		// No answer is never "safe": the delivery fetch would resolve on its own.

package/src/lib/webhook-url.ts

@@ -38,25 +38,83 @@ const isPrivateIpv4 = (o: number[]): boolean =>
 	(o[0] === 192 && o[1] === 168) ||
 	o[0] >= 224; // multicast, reserved, broadcast
 
+/**
+ * Expand an IPv6 literal (no brackets, optional `%zone`) into its eight
+ * 16-bit groups. Accepts a trailing dotted IPv4 tail (`::127.0.0.1`).
+ * Returns null for anything that is not a well-formed address.
+ */
+const parseIpv6 = (literal: string): number[] | null => {
+	let text = literal.replace(/%.*$/, "");
+	const tail = /:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(text);
+	let tailGroups: number[] = [];
+	if (tail) {
+		const v4 = parseIpv4(tail[1]);
+		if (!v4) return null;
+		tailGroups = [(v4[0] << 8) | v4[1], (v4[2] << 8) | v4[3]];
+		text = text.slice(0, -tail[1].length);
+		// "::1.2.3.4" leaves "::", "64:ff9b::1.2.3.4" leaves "64:ff9b::",
+		// "::ffff:1.2.3.4" leaves "::ffff:"; drop the dangling separator
+		// unless it is part of "::".
+		if (text.endsWith(":") && !text.endsWith("::")) text = text.slice(0, -1);
+	}
+	const halves = text.split("::");
+	if (halves.length > 2) return null;
+	const toGroups = (part: string): number[] | null => {
+		if (part === "") return [];
+		const groups: number[] = [];
+		for (const g of part.split(":")) {
+			if (!/^[0-9a-f]{1,4}$/.test(g)) return null;
+			groups.push(Number.parseInt(g, 16));
+		}
+		return groups;
+	};
+	const head = toGroups(halves[0]);
+	const rest = halves.length === 2 ? toGroups(halves[1]) : [];
+	if (!head || !rest) return null;
+	const explicit = head.length + rest.length + tailGroups.length;
+	if (halves.length === 1) {
+		return explicit === 8 ? [...head, ...tailGroups] : null;
+	}
+	if (explicit > 7) return null;
+	return [
+		...head,
+		...new Array<number>(8 - explicit).fill(0),
+		...rest,
+		...tailGroups,
+	];
+};
+
+const embeddedIpv4 = (hi: number, lo: number): number[] => [
+	hi >> 8,
+	hi & 0xff,
+	lo >> 8,
+	lo & 0xff,
+];
+
 const isPrivateIpv6 = (host: string): boolean => {
 	// URL.hostname keeps the brackets for IPv6 literals.
 	const literal = host.replace(/^\[|\]$/g, "").toLowerCase();
-	if (literal === "::" || literal === "::1") return true;
-	// IPv4-mapped (::ffff:a.b.c.d). The WHATWG URL parser serializes the
-	// embedded address as two hex groups (::ffff:a00:1), so accept both.
-	const dotted = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(literal);
-	if (dotted) {
-		const v4 = parseIpv4(dotted[1]);
-		return v4 ? isPrivateIpv4(v4) : true;
+	const g = parseIpv6(literal);
+	if (!g) return true; // not a parseable address: fail closed
+	const zeroPrefix = (n: number) => g.slice(0, n).every((x) => x === 0);
+	// :: (unspecified), ::1 (loopback) and the deprecated IPv4-compatible
+	// ::a.b.c.d form (which the WHATWG URL parser serializes as ::7f00:1).
+	if (zeroPrefix(6)) {
+		if (g[6] === 0 && (g[7] === 0 || g[7] === 1)) return true;
+		return isPrivateIpv4(embeddedIpv4(g[6], g[7]));
+	}
+	// IPv4-mapped ::ffff:a.b.c.d (serialized as ::ffff:a00:1).
+	if (zeroPrefix(5) && g[5] === 0xffff) {
+		return isPrivateIpv4(embeddedIpv4(g[6], g[7]));
 	}
-	const hex = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(literal);
-	if (hex) {
-		const hi = Number.parseInt(hex[1], 16);
-		const lo = Number.parseInt(hex[2], 16);
-		return isPrivateIpv4([hi >> 8, hi & 0xff, lo >> 8, lo & 0xff]);
+	// NAT64 well-known prefix 64:ff9b::/96 — the gateway forwards to the
+	// embedded IPv4 address.
+	if (g[0] === 0x64 && g[1] === 0xff9b && g.slice(2, 6).every((x) => x === 0)) {
+		return isPrivateIpv4(embeddedIpv4(g[6], g[7]));
 	}
-	const first = Number.parseInt(literal.split(":")[0] || "0", 16);
-	if (Number.isNaN(first)) return true;
+	// 6to4 2002::/16 carries the IPv4 relay address in groups 1-2.
+	if (g[0] === 0x2002) return isPrivateIpv4(embeddedIpv4(g[1], g[2]));
+	const first = g[0];
 	if ((first & 0xfe00) === 0xfc00) return true; // fc00::/7 unique local
 	if ((first & 0xffc0) === 0xfe80) return true; // fe80::/10 link-local
 	if ((first & 0xff00) === 0xff00) return true; // ff00::/8 multicast

package/src/migrations/054_alert_rules_pending_secs.sql

@@ -0,0 +1,11 @@
+-- Migration 054: alert_rules.pending_secs
+--
+-- How long a rule's condition must hold on consecutive evaluations before
+-- the rule fires (Prometheus `for`). 0 keeps the existing behaviour: fire on
+-- the first evaluation whose value matches. The start of the pending period
+-- lives on alert_state (055).
+--
+-- Re-running fails with "duplicate column name", which the D1 runner treats
+-- as already applied.
+
+ALTER TABLE alert_rules ADD COLUMN pending_secs INTEGER NOT NULL DEFAULT 0;

package/src/migrations/055_alert_state_pending_since.sql

@@ -0,0 +1,12 @@
+-- Migration 055: alert_state.pending_since
+--
+-- ISO timestamp of the first evaluation, in the current unbroken run, whose
+-- value matched a rule that has a pending duration (054) and is not firing
+-- yet. NULL when the rule is not pending. The evaluator fires once
+-- `now - pending_since >= pending_secs` and clears it when the condition
+-- stops holding or the rule fires.
+--
+-- Re-running fails with "duplicate column name", which the D1 runner treats
+-- as already applied.
+
+ALTER TABLE alert_state ADD COLUMN pending_since TEXT;

package/src/migrations/056_metric_point_series_received.sql

@@ -0,0 +1,9 @@
+-- Migration 056: metric_point (series_id, received_at) index
+--
+-- `metrics` alert rules read each matching series' points by receive time:
+-- the points inside the rule window, and the newest point of the series for
+-- `age_seconds`. The existing (series_id, ts_ns) index orders by producer
+-- time; this one serves both reads without scanning the series' history.
+
+CREATE INDEX IF NOT EXISTS idx_metric_point_series_received
+  ON metric_point (series_id, received_at);

package/src/migrations/057_alert_evaluations_outcome.sql

@@ -0,0 +1,11 @@
+-- Migration 057: alert_evaluations.outcome
+--
+-- What an evaluation produced: 'value' (a value was compared), 'no_data'
+-- (a metrics rule with noData "keep" found no fresh series and left the
+-- state alone) or 'error' (the evaluation failed; the state was left alone).
+-- Existing rows are 'value'.
+--
+-- Re-running fails with "duplicate column name", which the D1 runner treats
+-- as already applied.
+
+ALTER TABLE alert_evaluations ADD COLUMN outcome TEXT NOT NULL DEFAULT 'value';

package/src/migrations/058_alert_state_pending_seen_at.sql

@@ -0,0 +1,12 @@
+-- Migration 058: alert_state.pending_seen_at
+--
+-- ISO time of the last evaluation that saw a pending rule's condition hold.
+-- The evaluator continues a pending period only when the next matching
+-- evaluation comes within the allowed gap of this one (and after the
+-- current collector process started); otherwise the hold starts over, as
+-- Prometheus resets `for` when evaluations stop.
+--
+-- Re-running fails with "duplicate column name", which the D1 runner treats
+-- as already applied.
+
+ALTER TABLE alert_state ADD COLUMN pending_seen_at TEXT;

package/src/migrations/059_alert_evaluations_error.sql

@@ -0,0 +1,11 @@
+-- Migration 059: alert_evaluations.error_message
+--
+-- Why an evaluation with outcome 'error' (057) failed: a timeout, a metrics
+-- query over its series or point cap, a database error. Truncated and with
+-- URLs replaced by the store, so no credentials or hook paths are kept.
+-- NULL for every other outcome.
+--
+-- Re-running fails with "duplicate column name", which the D1 runner treats
+-- as already applied.
+
+ALTER TABLE alert_evaluations ADD COLUMN error_message TEXT;

package/src/migrations-postgres/054_alert_rules_pending_secs.sql

@@ -0,0 +1,7 @@
+-- Migration 054: alert_rules.pending_secs
+--
+-- Mirrors ../migrations/054_alert_rules_pending_secs.sql. INTEGER like the
+-- sibling alert_rules.window_mins. Existing rules default to 0 (fire on the
+-- first matching evaluation, as before). Idempotent.
+
+ALTER TABLE alert_rules ADD COLUMN IF NOT EXISTS pending_secs INTEGER NOT NULL DEFAULT 0;

package/src/migrations-postgres/055_alert_state_pending_since.sql

@@ -0,0 +1,6 @@
+-- Migration 055: alert_state.pending_since
+--
+-- Mirrors ../migrations/055_alert_state_pending_since.sql. ISO TEXT like
+-- alert_state.last_state_change. Idempotent.
+
+ALTER TABLE alert_state ADD COLUMN IF NOT EXISTS pending_since TEXT;

package/src/migrations-postgres/056_metric_point_series_received.sql

@@ -0,0 +1,7 @@
+-- Migration 056: metric_point (series_id, received_at) index
+--
+-- Mirrors ../migrations/056_metric_point_series_received.sql. Serves the
+-- `metrics` alert evaluator's window and newest-point reads. Idempotent.
+
+CREATE INDEX IF NOT EXISTS idx_metric_point_series_received
+  ON metric_point (series_id, received_at);

package/src/migrations-postgres/057_alert_evaluations_outcome.sql

@@ -0,0 +1,6 @@
+-- Migration 057: alert_evaluations.outcome
+--
+-- Mirrors ../migrations/057_alert_evaluations_outcome.sql ('value',
+-- 'no_data' or 'error'; existing rows 'value'). Idempotent.
+
+ALTER TABLE alert_evaluations ADD COLUMN IF NOT EXISTS outcome TEXT NOT NULL DEFAULT 'value';

package/src/migrations-postgres/058_alert_state_pending_seen_at.sql

@@ -0,0 +1,6 @@
+-- Migration 058: alert_state.pending_seen_at
+--
+-- Mirrors ../migrations/058_alert_state_pending_seen_at.sql. ISO TEXT.
+-- Idempotent.
+
+ALTER TABLE alert_state ADD COLUMN IF NOT EXISTS pending_seen_at TEXT;

package/src/migrations-postgres/059_alert_evaluations_error.sql

@@ -0,0 +1,6 @@
+-- Migration 059: alert_evaluations.error_message
+--
+-- Mirrors ../migrations/059_alert_evaluations_error.sql. TEXT, NULL unless
+-- the outcome is 'error'. Idempotent.
+
+ALTER TABLE alert_evaluations ADD COLUMN IF NOT EXISTS error_message TEXT;

package/src/migrations-postgres/README.md

@@ -74,6 +74,19 @@ land alongside the first hosted-Postgres deployment.
   store falls back to the pre-049 statement when the column is missing, so
   the collector keeps evaluating during a rolling deploy, but it cannot
   retry undelivered notifications until 049 is applied.
+- `054` adds `alert_rules.pending_secs INTEGER NOT NULL DEFAULT 0` and `055`
+  `alert_state.pending_since TEXT` (ISO, NULL when not pending) for the
+  rule-level pending duration. Before 054 the store still writes rules
+  without a pending duration (legacy statement) and refuses `pendingSecs > 0`
+  with a message naming the migration. `056` adds
+  `idx_metric_point_series_received (series_id, received_at)` for the
+  `metrics` alert signal's window and newest-point reads.
+- `057` adds `alert_evaluations.outcome TEXT NOT NULL DEFAULT 'value'`
+  (`value`, `no_data`, `error`; the store writes the legacy row without it
+  before 057) and `058` `alert_state.pending_seen_at TEXT`, the last
+  evaluation that saw a pending condition hold (required by pending rules).
+- `059` adds `alert_evaluations.error_message TEXT` (sanitized reason for an
+  `error` outcome; the rules list reads it as `lastEvaluation.error`).
 - Flags (`is_error`, `is_bot`, `payload_capture_enabled`) are `BOOLEAN` here
   and `INTEGER 0/1` on D1. Shared SQL must test them as bare booleans
   (`CASE WHEN is_error THEN 1 ELSE 0 END`, never `is_error = 1` or