@obsunified/collector 2.2.3 → 2.4.3

package/src/lib/sql-db-postgres.integration.test.ts

@@ -1112,6 +1112,253 @@ describe.skipIf(!databaseUrl)("PostgresAdapter against a live Postgres", () => {
 		);
 	});
 
+	it("evaluates metrics alert rules and drives pendingSecs state (054-059)", async () => {
+		const columns = await db
+			.prepare(
+				`SELECT table_name, column_name FROM information_schema.columns
+				 WHERE table_schema = current_schema()
+				   AND ((table_name = 'alert_rules' AND column_name = 'pending_secs')
+				     OR (table_name = 'alert_state' AND column_name IN ('pending_since', 'pending_seen_at'))
+				     OR (table_name = 'alert_evaluations' AND column_name IN ('outcome', 'error_message')))
+				 ORDER BY table_name, column_name`,
+			)
+			.all<{ table_name: string; column_name: string }>();
+		expect(columns.results.map((c) => c.column_name)).toEqual([
+			"error_message",
+			"outcome",
+			"pending_secs",
+			"pending_seen_at",
+			"pending_since",
+		]);
+		const index = await db
+			.prepare(
+				`SELECT indexname FROM pg_indexes
+				 WHERE schemaname = current_schema() AND indexname = 'idx_metric_point_series_received'`,
+			)
+			.first<{ indexname: string }>();
+		expect(index?.indexname).toBe("idx_metric_point_series_received");
+
+		const project = await new ProjectsStore(db).createProject({
+			name: "Metric alerts",
+			slug: `metric-alerts-${Date.now().toString(36)}`,
+		});
+		const metrics = new MetricsStore(db);
+		const now = new Date();
+		const ago = (secs: number) => now.getTime() - secs * 1000;
+		const ingest = (secsAgo: number, points: DecodedMetricPoint[]) =>
+			metrics.ingestBatch({
+				projectId: project.id,
+				points,
+				receivedAt: new Date(ago(secsAgo)).toISOString(),
+				expiresAt: isoIn(60 * 60 * 1000),
+			});
+		const counter = (
+			name: string,
+			code: number,
+			secsAgo: number,
+			value: number,
+		) =>
+			metricPoint({
+				name,
+				type: "sum",
+				isMonotonic: true,
+				temporality: 2,
+				serviceName: "vbox",
+				resourceAttrsJson: JSON.stringify({ "service.name": "vbox" }),
+				attributesJson: JSON.stringify({ code }),
+				identity: `${name}|${code}`,
+				tsNs: (BigInt(ago(secsAgo)) * 1_000_000n).toString(),
+				startTsNs: (BigInt(ago(3600)) * 1_000_000n).toString(),
+				value,
+				count: null,
+				sum: null,
+				boundsJson: null,
+				bucketCountsJson: null,
+			});
+		// History before the 5-minute window for all three series, plus a
+		// series first seen inside it (code=503) whose first value counts.
+		await ingest(400, [
+			counter("calls_total", 200, 400, 90),
+			counter("calls_total", 500, 400, 1),
+			counter("calls_failed_total", 500, 400, 1),
+		]);
+		await ingest(240, [
+			counter("calls_total", 200, 240, 100),
+			counter("calls_total", 500, 240, 1),
+			counter("calls_failed_total", 500, 240, 1),
+			counter("calls_failed_total", 503, 240, 2),
+		]);
+		await ingest(60, [
+			// Counter reset on the 200 series: 100 → 4 counts 4.
+			counter("calls_total", 200, 60, 4),
+			counter("calls_total", 500, 60, 3),
+			counter("calls_failed_total", 500, 60, 3),
+		]);
+
+		const alerts = new AlertsStore(db);
+		const ratio = await alerts.createRule(project.id, {
+			name: "call error ratio",
+			signal: "metrics",
+			query: {
+				metric: "calls_failed_total",
+				attributes: { code: "500" },
+				aggregation: "increase",
+				denominator: { metric: "calls_total" },
+				minDenominator: 5,
+			},
+			threshold: 0.05,
+			windowMins: 5,
+			comparison: ">",
+			pendingSecs: 0,
+			channels: [{ type: "webhook", url: "https://hooks.example.com/pg" }],
+		});
+		// errors +2; attempts +4 (reset) +2 = 6 >= 5 → 2/6
+		expect(await alerts.evaluateRule(ratio, now)).toBeCloseTo(2 / 6);
+		// The new 503 series started an hour ago (before the window): its
+		// first point is a baseline, so only the 500 series' +2 counts.
+		const allErrors = await alerts.createRule(project.id, {
+			name: "all call errors",
+			signal: "metrics",
+			query: { metric: "calls_failed_total", aggregation: "increase" },
+			threshold: 0,
+			windowMins: 5,
+			comparison: ">",
+			channels: [{ type: "webhook", url: "https://hooks.example.com/pg" }],
+		});
+		expect(await alerts.evaluateRuleWithStatus(allErrors, now)).toEqual({
+			value: 2,
+			noData: false,
+		});
+		// `last` with a 30 s staleness bound: the newest points are 60 s old.
+		const gone = await alerts.createRule(project.id, {
+			name: "stale gauge",
+			signal: "metrics",
+			query: {
+				metric: "calls_total",
+				aggregation: "last",
+				staleSecs: 30,
+				noData: "keep",
+			},
+			threshold: 0,
+			windowMins: 5,
+			comparison: ">",
+			channels: [{ type: "webhook", url: "https://hooks.example.com/pg" }],
+		});
+		expect(await alerts.evaluateRuleWithStatus(gone, now)).toEqual({
+			value: 0,
+			noData: true,
+		});
+		await alerts.recordEvaluation(
+			gone.id,
+			project.id,
+			0,
+			"ok",
+			false,
+			"no_data",
+		);
+		expect(
+			(
+				await alerts.listEvaluations({
+					ruleId: gone.id,
+					projectId: project.id,
+					hours: 1,
+				})
+			).evaluations[0],
+		).toMatchObject({ outcome: "no_data", state: "ok" });
+		await alerts.recordEvaluation(
+			ratio.id,
+			project.id,
+			0,
+			"ok",
+			false,
+			"error",
+			"statement timeout on postgres://obs:secret@db/obs",
+		);
+		await alerts.transitionState(gone.id, project.id, "firing", isoNow());
+		const listed = new Map(
+			(await alerts.listRules(project.id)).map((r) => [r.id, r]),
+		);
+		expect(listed.get(ratio.id)).toMatchObject({
+			notificationDelivered: null,
+			lastEvaluation: {
+				outcome: "error",
+				value: 0,
+				error: "statement timeout on <url>",
+			},
+		});
+		expect(listed.get(gone.id)).toMatchObject({
+			currentState: "firing",
+			notificationDelivered: false,
+			lastEvaluation: { outcome: "no_data", error: null },
+		});
+		expect(listed.get(allErrors.id)?.lastEvaluation).toBeNull();
+
+		const stale = await alerts.createRule(project.id, {
+			name: "calls stopped",
+			signal: "metrics",
+			query: { metric: ["calls_total"], aggregation: "age_seconds" },
+			threshold: 60,
+			windowMins: 1,
+			comparison: ">=",
+			pendingSecs: 30,
+			channels: [{ type: "webhook", url: "https://hooks.example.com/pg" }],
+		});
+		expect(await alerts.evaluateRule(stale, now)).toBeCloseTo(60, 0);
+		const never = await alerts.createRule(project.id, {
+			name: "never reported",
+			signal: "metrics",
+			query: { metric: "not_a_metric", aggregation: "age_seconds" },
+			threshold: 60,
+			windowMins: 1,
+			comparison: ">=",
+			channels: [{ type: "webhook", url: "https://hooks.example.com/pg" }],
+		});
+		const neverValue = await alerts.evaluateRule(
+			never,
+			new Date(Date.parse(never.createdAt) + 90_000),
+		);
+		expect(neverValue).toBeCloseTo(90, 0);
+
+		// pending_since round-trips; a fresh state row is ok + delivered.
+		expect(stale.pendingSecs).toBe(30);
+		const since = new Date().toISOString();
+		await alerts.setPending(
+			stale.id,
+			project.id,
+			{ since, seenAt: since },
+			stale.createdAt,
+		);
+		expect(await alerts.getState(stale.id, project.id)).toMatchObject({
+			current_state: "ok",
+			last_state_change: stale.createdAt,
+			notified: 1,
+			pending_since: since,
+			pending_seen_at: since,
+		});
+		expect(
+			(await alerts.listRules(project.id)).find((r) => r.id === stale.id),
+		).toMatchObject({ pendingSecs: 30, pendingSince: since });
+		// An edit that changes evaluation resets the hold.
+		await alerts.updateRule(stale.id, project.id, { threshold: 61 });
+		expect(await alerts.getState(stale.id, project.id)).toMatchObject({
+			pending_since: null,
+			pending_seen_at: null,
+		});
+		await alerts.setPending(
+			stale.id,
+			project.id,
+			{ since, seenAt: since },
+			stale.createdAt,
+		);
+		await alerts.transitionState(stale.id, project.id, "firing", since);
+		await alerts.setPending(stale.id, project.id, null, stale.createdAt);
+		expect(await alerts.getState(stale.id, project.id)).toMatchObject({
+			current_state: "firing",
+			pending_since: null,
+			notified: 0,
+		});
+	});
+
 	it("purges expired AI data using the dialect's text-rendered now()", async () => {
 		await ingestAICallBatch(db, [
 			aiCall({ callId: "ai-expired", expiresAt: isoIn(-60_000) }),

package/src/lib/stage6.test.ts

@@ -201,7 +201,7 @@ describe("AlertsStore.listEvaluations evidence references", () => {
 				evidenceId: "alert:rule-1:rule",
 				entityKind: "alert",
 				entityId: "rule-1",
-				route: "#/alerts?alert=rule-1",
+				route: "#/alerts?id=rule-1",
 				source: "alerts.rule_evaluation_preview",
 			}),
 			expect.objectContaining({

package/src/lib/store/trace-queries.ts

@@ -33,7 +33,7 @@ export async function selectTraceCandidates(
 	if (options.search) {
 		const term = `%${options.search}%`;
 		whereClause +=
-			" AND (span_name LIKE ? OR status_message LIKE ? OR attributes_json LIKE ? OR events_json LIKE ?)";
+			" AND (span_name LIKE ? OR status_message LIKE ? OR CAST(attributes_json AS TEXT) LIKE ? OR CAST(events_json AS TEXT) LIKE ?)";
 		binds.push(term, term, term, term);
 	}
 

package/src/migrations/054_alert_rules_pending_secs.sql

@@ -0,0 +1,11 @@
+-- Migration 054: alert_rules.pending_secs
+--
+-- How long a rule's condition must hold on consecutive evaluations before
+-- the rule fires (Prometheus `for`). 0 keeps the existing behaviour: fire on
+-- the first evaluation whose value matches. The start of the pending period
+-- lives on alert_state (055).
+--
+-- Re-running fails with "duplicate column name", which the D1 runner treats
+-- as already applied.
+
+ALTER TABLE alert_rules ADD COLUMN pending_secs INTEGER NOT NULL DEFAULT 0;

package/src/migrations/055_alert_state_pending_since.sql

@@ -0,0 +1,12 @@
+-- Migration 055: alert_state.pending_since
+--
+-- ISO timestamp of the first evaluation, in the current unbroken run, whose
+-- value matched a rule that has a pending duration (054) and is not firing
+-- yet. NULL when the rule is not pending. The evaluator fires once
+-- `now - pending_since >= pending_secs` and clears it when the condition
+-- stops holding or the rule fires.
+--
+-- Re-running fails with "duplicate column name", which the D1 runner treats
+-- as already applied.
+
+ALTER TABLE alert_state ADD COLUMN pending_since TEXT;

package/src/migrations/056_metric_point_series_received.sql

@@ -0,0 +1,9 @@
+-- Migration 056: metric_point (series_id, received_at) index
+--
+-- `metrics` alert rules read each matching series' points by receive time:
+-- the points inside the rule window, and the newest point of the series for
+-- `age_seconds`. The existing (series_id, ts_ns) index orders by producer
+-- time; this one serves both reads without scanning the series' history.
+
+CREATE INDEX IF NOT EXISTS idx_metric_point_series_received
+  ON metric_point (series_id, received_at);

package/src/migrations/057_alert_evaluations_outcome.sql

@@ -0,0 +1,11 @@
+-- Migration 057: alert_evaluations.outcome
+--
+-- What an evaluation produced: 'value' (a value was compared), 'no_data'
+-- (a metrics rule with noData "keep" found no fresh series and left the
+-- state alone) or 'error' (the evaluation failed; the state was left alone).
+-- Existing rows are 'value'.
+--
+-- Re-running fails with "duplicate column name", which the D1 runner treats
+-- as already applied.
+
+ALTER TABLE alert_evaluations ADD COLUMN outcome TEXT NOT NULL DEFAULT 'value';

package/src/migrations/058_alert_state_pending_seen_at.sql

@@ -0,0 +1,12 @@
+-- Migration 058: alert_state.pending_seen_at
+--
+-- ISO time of the last evaluation that saw a pending rule's condition hold.
+-- The evaluator continues a pending period only when the next matching
+-- evaluation comes within the allowed gap of this one (and after the
+-- current collector process started); otherwise the hold starts over, as
+-- Prometheus resets `for` when evaluations stop.
+--
+-- Re-running fails with "duplicate column name", which the D1 runner treats
+-- as already applied.
+
+ALTER TABLE alert_state ADD COLUMN pending_seen_at TEXT;

package/src/migrations/059_alert_evaluations_error.sql

@@ -0,0 +1,11 @@
+-- Migration 059: alert_evaluations.error_message
+--
+-- Why an evaluation with outcome 'error' (057) failed: a timeout, a metrics
+-- query over its series or point cap, a database error. Truncated and with
+-- URLs replaced by the store, so no credentials or hook paths are kept.
+-- NULL for every other outcome.
+--
+-- Re-running fails with "duplicate column name", which the D1 runner treats
+-- as already applied.
+
+ALTER TABLE alert_evaluations ADD COLUMN error_message TEXT;

package/src/migrations/parity.test.ts

@@ -8,6 +8,7 @@
 
 import { readdirSync, readFileSync } from "node:fs";
 import path from "node:path";
+import { DatabaseSync } from "node:sqlite";
 import { fileURLToPath } from "node:url";
 import { describe, expect, it } from "vitest";
 
@@ -34,13 +35,74 @@ const RETENTION_TABLES = [
 	"remediation_proposals",
 ];
 
+export function checkMigrationFilePairings(
+	sqliteFiles: readonly string[],
+	postgresFiles: readonly string[],
+): string[] {
+	const errors: string[] = [];
+	const sqliteSet = new Set(sqliteFiles);
+	const pgSet = new Set(postgresFiles);
+	for (const f of sqliteFiles) {
+		if (!pgSet.has(f)) {
+			errors.push(`Missing Postgres migration counterpart for ${f}`);
+		}
+	}
+	for (const f of postgresFiles) {
+		if (!sqliteSet.has(f)) {
+			errors.push(`Missing SQLite migration counterpart for ${f}`);
+		}
+	}
+	const seenPrefixes = new Set<string>();
+	for (const f of sqliteFiles) {
+		const prefix = f.slice(0, 3);
+		if (seenPrefixes.has(prefix)) {
+			errors.push(`Duplicate migration prefix ${prefix} in ${f}`);
+		}
+		seenPrefixes.add(prefix);
+	}
+	return errors;
+}
+
+const extractCreatedTables = (sql: string): string[] => {
+	const names: string[] = [];
+	for (const m of sql.matchAll(
+		/CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?\s+([a-zA-Z0-9_]+)/gi,
+	)) {
+		const table = m[1].toLowerCase();
+		if (!table.endsWith("_new")) {
+			names.push(table);
+		}
+	}
+	return [...new Set(names)].sort();
+};
+
+const extractAddedColumns = (sql: string): string[] => {
+	const cols: string[] = [];
+	for (const m of sql.matchAll(
+		/ALTER\s+TABLE\s+([a-zA-Z0-9_]+)\s+ADD\s+COLUMN(?:\s+IF\s+NOT\s+EXISTS)?\s+([a-zA-Z0-9_]+)/gi,
+	)) {
+		cols.push(`${m[1].toLowerCase()}.${m[2].toLowerCase()}`);
+	}
+	return [...new Set(cols)].sort();
+};
+
 describe("migration parity", () => {
-	it("has the same numbered files on SQLite/D1 and Postgres", () => {
+	it("has the same numbered files on SQLite/D1 and Postgres and catches missing pairs", () => {
 		const sqlite = sqlFiles(sqliteDir);
 		const postgres = sqlFiles(postgresDir);
 		expect(postgres).toEqual(sqlite);
-		const prefixes = sqlite.map((f) => f.slice(0, 3));
-		expect(new Set(prefixes).size).toBe(prefixes.length);
+		expect(sqlite.length).toBeGreaterThanOrEqual(53);
+		expect(checkMigrationFilePairings(sqlite, postgres)).toEqual([]);
+
+		// Regression check: missing or duplicate migration files must be flagged.
+		expect(
+			checkMigrationFilePairings([...sqlite, "099_new_feature.sql"], postgres),
+		).toEqual([
+			"Missing Postgres migration counterpart for 099_new_feature.sql",
+		]);
+		expect(
+			checkMigrationFilePairings(sqlite, [...postgres, "099_pg_only.sql"]),
+		).toEqual(["Missing SQLite migration counterpart for 099_pg_only.sql"]);
 	});
 
 	it("declares retention timestamps as TEXT in every Postgres CREATE TABLE", () => {
@@ -65,4 +127,44 @@ describe("migration parity", () => {
 		}
 		expect(offenders).toEqual([]);
 	});
+
+	it("creates and alters the same tables and columns per migration file", () => {
+		const mismatches: string[] = [];
+		for (const file of sqlFiles(sqliteDir)) {
+			const sqliteSql = readFileSync(path.join(sqliteDir, file), "utf8");
+			const pgSql = readFileSync(path.join(postgresDir, file), "utf8");
+			const sTables = extractCreatedTables(sqliteSql);
+			const pTables = extractCreatedTables(pgSql);
+			if (JSON.stringify(sTables) !== JSON.stringify(pTables)) {
+				mismatches.push(
+					`${file} CREATE TABLE mismatch: sqlite=${JSON.stringify(sTables)} pg=${JSON.stringify(pTables)}`,
+				);
+			}
+			const sCols = extractAddedColumns(sqliteSql);
+			const pCols = extractAddedColumns(pgSql);
+			if (JSON.stringify(sCols) !== JSON.stringify(pCols)) {
+				mismatches.push(
+					`${file} ALTER TABLE ADD COLUMN mismatch: sqlite=${JSON.stringify(sCols)} pg=${JSON.stringify(pCols)}`,
+				);
+			}
+		}
+		expect(mismatches).toEqual([]);
+	});
+
+	it("applies all SQLite migrations cleanly in-memory without leftover staging tables", () => {
+		const db = new DatabaseSync(":memory:");
+		try {
+			for (const file of sqlFiles(sqliteDir)) {
+				db.exec(readFileSync(path.join(sqliteDir, file), "utf8"));
+			}
+			const leftover = db
+				.prepare(
+					"SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE '%_new'",
+				)
+				.all();
+			expect(leftover).toEqual([]);
+		} finally {
+			db.close();
+		}
+	});
 });

package/src/migrations-postgres/054_alert_rules_pending_secs.sql

@@ -0,0 +1,7 @@
+-- Migration 054: alert_rules.pending_secs
+--
+-- Mirrors ../migrations/054_alert_rules_pending_secs.sql. INTEGER like the
+-- sibling alert_rules.window_mins. Existing rules default to 0 (fire on the
+-- first matching evaluation, as before). Idempotent.
+
+ALTER TABLE alert_rules ADD COLUMN IF NOT EXISTS pending_secs INTEGER NOT NULL DEFAULT 0;

package/src/migrations-postgres/055_alert_state_pending_since.sql

@@ -0,0 +1,6 @@
+-- Migration 055: alert_state.pending_since
+--
+-- Mirrors ../migrations/055_alert_state_pending_since.sql. ISO TEXT like
+-- alert_state.last_state_change. Idempotent.
+
+ALTER TABLE alert_state ADD COLUMN IF NOT EXISTS pending_since TEXT;

package/src/migrations-postgres/056_metric_point_series_received.sql

@@ -0,0 +1,7 @@
+-- Migration 056: metric_point (series_id, received_at) index
+--
+-- Mirrors ../migrations/056_metric_point_series_received.sql. Serves the
+-- `metrics` alert evaluator's window and newest-point reads. Idempotent.
+
+CREATE INDEX IF NOT EXISTS idx_metric_point_series_received
+  ON metric_point (series_id, received_at);

package/src/migrations-postgres/057_alert_evaluations_outcome.sql

@@ -0,0 +1,6 @@
+-- Migration 057: alert_evaluations.outcome
+--
+-- Mirrors ../migrations/057_alert_evaluations_outcome.sql ('value',
+-- 'no_data' or 'error'; existing rows 'value'). Idempotent.
+
+ALTER TABLE alert_evaluations ADD COLUMN IF NOT EXISTS outcome TEXT NOT NULL DEFAULT 'value';

package/src/migrations-postgres/058_alert_state_pending_seen_at.sql

@@ -0,0 +1,6 @@
+-- Migration 058: alert_state.pending_seen_at
+--
+-- Mirrors ../migrations/058_alert_state_pending_seen_at.sql. ISO TEXT.
+-- Idempotent.
+
+ALTER TABLE alert_state ADD COLUMN IF NOT EXISTS pending_seen_at TEXT;

package/src/migrations-postgres/059_alert_evaluations_error.sql

@@ -0,0 +1,6 @@
+-- Migration 059: alert_evaluations.error_message
+--
+-- Mirrors ../migrations/059_alert_evaluations_error.sql. TEXT, NULL unless
+-- the outcome is 'error'. Idempotent.
+
+ALTER TABLE alert_evaluations ADD COLUMN IF NOT EXISTS error_message TEXT;

package/src/migrations-postgres/README.md

@@ -74,6 +74,19 @@ land alongside the first hosted-Postgres deployment.
   store falls back to the pre-049 statement when the column is missing, so
   the collector keeps evaluating during a rolling deploy, but it cannot
   retry undelivered notifications until 049 is applied.
+- `054` adds `alert_rules.pending_secs INTEGER NOT NULL DEFAULT 0` and `055`
+  `alert_state.pending_since TEXT` (ISO, NULL when not pending) for the
+  rule-level pending duration. Before 054 the store still writes rules
+  without a pending duration (legacy statement) and refuses `pendingSecs > 0`
+  with a message naming the migration. `056` adds
+  `idx_metric_point_series_received (series_id, received_at)` for the
+  `metrics` alert signal's window and newest-point reads.
+- `057` adds `alert_evaluations.outcome TEXT NOT NULL DEFAULT 'value'`
+  (`value`, `no_data`, `error`; the store writes the legacy row without it
+  before 057) and `058` `alert_state.pending_seen_at TEXT`, the last
+  evaluation that saw a pending condition hold (required by pending rules).
+- `059` adds `alert_evaluations.error_message TEXT` (sanitized reason for an
+  `error` outcome; the rules list reads it as `lastEvaluation.error`).
 - Flags (`is_error`, `is_bot`, `payload_capture_enabled`) are `BOOLEAN` here
   and `INTEGER 0/1` on D1. Shared SQL must test them as bare booleans
   (`CASE WHEN is_error THEN 1 ELSE 0 END`, never `is_error = 1` or

package/src/plugins/ai-receiver.ts

@@ -16,6 +16,7 @@ import { AIStore, type IngestEvaluation } from "../lib/ai-store";
 import { sqlDbFor } from "../lib/sql-db";
 import { getProjectId } from "./_context";
 import { readJsonObject } from "./_json";
+import { respondWithContract } from "./autosre-routes";
 
 export const aiReceiverPlugin: CollectorPlugin = {
 	name: "ai-receiver",
@@ -104,7 +105,11 @@ export const aiReceiverPlugin: CollectorPlugin = {
 			const store = new AIStore(sqlDbFor(c.env));
 			const call = await store.getAICall(projectId, c.req.param("callId"));
 			if (!call) return c.json({ error: "AI call not found" }, 404);
-			return c.json({ call, traceId: call.traceId, spanId: call.spanId });
+			return respondWithContract(c, "GET /internal/ai/calls/:callId", {
+				call,
+				traceId: call.traceId,
+				spanId: call.spanId,
+			});
 		});
 
 		// OpenInference-kind spans (LLM/TOOL/RETRIEVER/CHAIN/AGENT/...) joined