@obsunified/collector 2.0.1 → 2.2.0

package/src/plugins/alerts-evaluator.ts

@@ -11,7 +11,14 @@ import {
 	type Logger,
 } from "../framework/logger";
 import { AlertsStore, compareValue } from "../lib/alerts-store";
+import { openOrDeduplicateIncident } from "../lib/autosre-remediation-gate";
 import { sqlDbFor } from "../lib/sql-db";
+import {
+	assertSafeWebhookUrl,
+	type HostLookup,
+	redactWebhookUrl,
+	resolveWebhookHost,
+} from "../lib/webhook-url";
 
 const DEFAULT_RULE_CONCURRENCY = 5;
 const DEFAULT_RULE_TIMEOUT_MS = 10_000;
@@ -19,7 +26,22 @@ const DEFAULT_RULE_TIMEOUT_MS = 10_000;
 interface AlertEvaluatorStore {
 	listEnabledRules(): Promise<AlertRule[]>;
 	evaluateRule(rule: AlertRule): Promise<number>;
-	getState(ruleId: string): Promise<{ current_state: AlertState } | null>;
+	getState(
+		ruleId: string,
+		projectId: string,
+	): Promise<{
+		current_state: AlertState;
+		last_state_change: string;
+		/** 0 while the transition's notification is owed; absent pre-049 (= delivered). */
+		notified?: number;
+	} | null>;
+	/** Flip the exact transition `(state, lastStateChange)` to delivered. */
+	markNotified(
+		ruleId: string,
+		projectId: string,
+		state: AlertState,
+		lastStateChange: string,
+	): Promise<boolean>;
 	getAnalysisNarrative(
 		projectId: string,
 		analysisId: string,
@@ -65,12 +87,56 @@ interface WebhookPayload {
 	};
 }
 
+export interface WebhookDeliveryOptions {
+	/** Permit targets on loopback / private / link-local hosts. */
+	allowPrivateWebhookTargets?: boolean;
+	/**
+	 * Resolver used to refuse public names that point at private addresses.
+	 * Hosts that can resolve (Node: `dns.lookup`) pass one; Workers cannot
+	 * reach private networks from `fetch` and leave it unset.
+	 */
+	hostLookup?: HostLookup;
+}
+
 async function fireWebhook(
 	channel: AlertWebhookChannel,
 	payload: WebhookPayload,
 	logger: Logger,
 	tracer?: ChildSpanRunner,
+	delivery: WebhookDeliveryOptions = {},
 ): Promise<boolean> {
+	// Hook URLs carry secrets in the path (Slack, PagerDuty…): logs and span
+	// attributes only ever see the redacted form.
+	const target = redactWebhookUrl(channel.url);
+	// Re-check at fire time: rules created before the host guard existed
+	// (or with the opt-in later removed) must not reach internal addresses.
+	try {
+		const url = assertSafeWebhookUrl(channel.url, {
+			allowPrivateNetworks: delivery.allowPrivateWebhookTargets,
+		});
+		if (!delivery.allowPrivateWebhookTargets && delivery.hostLookup) {
+			const resolution = await resolveWebhookHost(
+				url.hostname,
+				delivery.hostLookup,
+			);
+			if (resolution === "private") {
+				throw new Error(
+					`channel.url host "${url.hostname}" resolves to a private address`,
+				);
+			}
+			if (resolution === "unresolvable") {
+				throw new Error(
+					`channel.url host "${url.hostname}" could not be resolved; refusing to deliver blind`,
+				);
+			}
+		}
+	} catch (err) {
+		logger.error("[alerts-evaluator] webhook target refused", {
+			url: target,
+			error: err instanceof Error ? err.message : String(err),
+		});
+		return false;
+	}
 	const exec = async (
 		span: { setAttribute(k: string, v: unknown): void } | null,
 	): Promise<boolean> => {
@@ -82,13 +148,24 @@ async function fireWebhook(
 					...(channel.headers ?? {}),
 				},
 				body: JSON.stringify(payload),
+				// Never follow redirects: a 3xx could re-aim the POST (and its
+				// headers) at an arbitrary host. A redirecting hook is a
+				// misconfiguration and counts as a failed delivery.
+				redirect: "manual",
 				signal: AbortSignal.timeout(5000),
 			});
 			if (span) span.setAttribute("http.response.status_code", response.status);
+			if (response.status >= 300 && response.status < 400) {
+				logger.error("[alerts-evaluator] webhook responded with a redirect", {
+					url: target,
+					status: response.status,
+				});
+				return false;
+			}
 			return response.ok;
 		} catch (err) {
 			logger.error("[alerts-evaluator] webhook delivery failed", {
-				url: channel.url,
+				url: target,
 				error: err instanceof Error ? err.message : String(err),
 			});
 			return false;
@@ -97,7 +174,7 @@ async function fireWebhook(
 	if (!tracer) return exec(null);
 	return tracer("webhook.alert", async (span) => exec(span), {
 		"http.method": "POST",
-		"http.url": channel.url,
+		"http.url": target,
 		"alert.state": payload.state,
 	});
 }
@@ -107,12 +184,13 @@ async function fireChannels(
 	payload: WebhookPayload,
 	logger: Logger,
 	tracer?: ChildSpanRunner,
+	delivery: WebhookDeliveryOptions = {},
 ): Promise<boolean> {
 	if (channels.length === 0) return true;
 	const results = await Promise.all(
 		channels
 			.filter((ch): ch is AlertWebhookChannel => ch.type === "webhook")
-			.map((ch) => fireWebhook(ch, payload, logger, tracer)),
+			.map((ch) => fireWebhook(ch, payload, logger, tracer, delivery)),
 	);
 	return results.every(Boolean);
 }
@@ -160,7 +238,10 @@ async function evaluateOneRule(
 	logger: Logger,
 	tracer: ChildSpanRunner | undefined,
 	ruleTimeoutMs: number,
-): Promise<{ fired: number; resolved: number }> {
+	delivery: WebhookDeliveryOptions,
+	onAlertFiring?: (rule: AlertRule) => Promise<void>,
+): Promise<{ fired: number; resolved: number; redelivered: number }> {
+	const none = { fired: 0, resolved: 0, redelivered: 0 };
 	try {
 		const value = await withTimeout(
 			store.evaluateRule(rule),
@@ -168,7 +249,7 @@ async function evaluateOneRule(
 			`rule evaluation timed out after ${ruleTimeoutMs}ms`,
 		);
 		const shouldFire = compareValue(value, rule.threshold, rule.comparison);
-		const priorState = await store.getState(rule.id);
+		const priorState = await store.getState(rule.id, rule.projectId);
 		const previous: AlertState = priorState?.current_state ?? "ok";
 		const next: AlertState = shouldFire ? "firing" : "ok";
 		const now = new Date().toISOString();
@@ -191,54 +272,81 @@ async function evaluateOneRule(
 					.catch(() => undefined)
 			: undefined;
 
-		if (previous === "ok" && next === "firing") {
-			const ok = await fireChannels(
+		const deliver = (state: AlertState) =>
+			fireChannels(
 				rule.channels,
 				{
 					rule: { id: rule.id, name: rule.name, signal: rule.signal },
 					value,
 					threshold: rule.threshold,
 					comparison: rule.comparison,
-					state: "firing",
+					state,
 					evaluatedAt: now,
 					projectId: rule.projectId,
 					analysis: analysisAttachment,
 				},
 				logger,
 				tracer,
+				delivery,
 			);
-			await store.transitionState(rule.id, rule.projectId, "firing", now);
-			await store.recordEvaluation(
-				rule.id,
-				rule.projectId,
-				value,
-				"firing",
-				ok,
-			);
-			return { fired: 1, resolved: 0 };
+
+		if (previous !== next) {
+			// The stored state is what the dashboard shows and must follow the
+			// signal, not the webhook: record the transition first (with its
+			// notification outstanding, `alert_state.notified = 0`), then
+			// notify. On accepted delivery the flag is flipped for exactly this
+			// transition; otherwise the same-state branch below retries on the
+			// next tick without a second transition.
+			await store.transitionState(rule.id, rule.projectId, next, now);
+			if (next === "firing" && onAlertFiring) {
+				await onAlertFiring(rule).catch(() => undefined);
+			}
+			const ok = await deliver(next);
+			// Acknowledge before writing history: if the evaluation insert
+			// fails, the delivered flag is already durable and the next tick
+			// will not send the same alert again.
+			if (ok) {
+				await store.markNotified(rule.id, rule.projectId, next, now);
+			}
+			await store.recordEvaluation(rule.id, rule.projectId, value, next, ok);
+			if (!ok) {
+				logger.warn(
+					"[alerts-evaluator] state recorded but delivery failed; will retry",
+					{ rule_id: rule.id, project_id: rule.projectId, state: next },
+				);
+			}
+			return next === "firing"
+				? { fired: 1, resolved: 0, redelivered: 0 }
+				: { fired: 0, resolved: 1, redelivered: 0 };
 		}
-		if (previous === "firing" && next === "ok") {
-			const ok = await fireChannels(
-				rule.channels,
-				{
-					rule: { id: rule.id, name: rule.name, signal: rule.signal },
-					value,
-					threshold: rule.threshold,
-					comparison: rule.comparison,
-					state: "ok",
-					evaluatedAt: now,
-					projectId: rule.projectId,
-					analysis: analysisAttachment,
-				},
-				logger,
-				tracer,
-			);
-			await store.transitionState(rule.id, rule.projectId, "ok", now);
-			await store.recordEvaluation(rule.id, rule.projectId, value, "ok", ok);
-			return { fired: 0, resolved: 1 };
+
+		// Same state. While the transition into it is still marked undelivered,
+		// retry the notification; the state itself is already correct. A row
+		// without the flag (schema before migration 049) counts as delivered.
+		const owed = priorState !== null && priorState.notified === 0;
+		if (owed) {
+			const ok = await deliver(previous);
+			if (ok) {
+				await store.markNotified(
+					rule.id,
+					rule.projectId,
+					previous,
+					priorState.last_state_change,
+				);
+			}
+			await store.recordEvaluation(rule.id, rule.projectId, value, next, ok);
+			if (!ok) {
+				logger.warn("[alerts-evaluator] delivery retry failed; will retry", {
+					rule_id: rule.id,
+					project_id: rule.projectId,
+					state: previous,
+				});
+				return none;
+			}
+			return { fired: 0, resolved: 0, redelivered: 1 };
 		}
 
-		// Same state — write an evaluation but do not fire channels.
+		// Same state, already notified — write an evaluation, do not fire.
 		await store.recordEvaluation(rule.id, rule.projectId, value, next, false);
 	} catch (err) {
 		logger.error("[alerts-evaluator] rule evaluation failed", {
@@ -248,7 +356,7 @@ async function evaluateOneRule(
 		});
 	}
 
-	return { fired: 0, resolved: 0 };
+	return none;
 }
 
 export async function evaluateRuleBatch(
@@ -259,13 +367,22 @@ export async function evaluateRuleBatch(
 		tracer?: ChildSpanRunner;
 		concurrency?: number;
 		ruleTimeoutMs?: number;
+		allowPrivateWebhookTargets?: boolean;
+		hostLookup?: HostLookup;
+		onAlertFiring?: (rule: AlertRule) => Promise<void>;
 	},
 ): Promise<{
 	evaluated: number;
 	fired: number;
 	resolved: number;
+	/** Notifications for an earlier transition that were delivered on this tick. */
+	redelivered: number;
 }> {
 	const logger = options?.logger ?? consoleLogger;
+	const delivery: WebhookDeliveryOptions = {
+		allowPrivateWebhookTargets: options?.allowPrivateWebhookTargets,
+		hostLookup: options?.hostLookup,
+	};
 	const concurrency = Math.max(
 		1,
 		Math.floor(options?.concurrency ?? DEFAULT_RULE_CONCURRENCY),
@@ -275,12 +392,21 @@ export async function evaluateRuleBatch(
 		Math.floor(options?.ruleTimeoutMs ?? DEFAULT_RULE_TIMEOUT_MS),
 	);
 	const results = await mapWithConcurrency(rules, concurrency, (rule) =>
-		evaluateOneRule(store, rule, logger, options?.tracer, ruleTimeoutMs),
+		evaluateOneRule(
+			store,
+			rule,
+			logger,
+			options?.tracer,
+			ruleTimeoutMs,
+			delivery,
+			options?.onAlertFiring,
+		),
 	);
 	return {
 		evaluated: rules.length,
 		fired: results.reduce((sum, result) => sum + result.fired, 0),
 		resolved: results.reduce((sum, result) => sum + result.resolved, 0),
+		redelivered: results.reduce((sum, result) => sum + result.redelivered, 0),
 	};
 }
 
@@ -291,21 +417,35 @@ export async function evaluateAllRules(
 		tracer?: ChildSpanRunner;
 		concurrency?: number;
 		ruleTimeoutMs?: number;
+		hostLookup?: HostLookup;
 	},
 ): Promise<{
 	evaluated: number;
 	fired: number;
 	resolved: number;
+	redelivered: number;
 }> {
 	const logger = options?.logger ?? consoleLogger;
 	const tracer = options?.tracer;
-	const store = new AlertsStore(sqlDbFor(env));
+	const allowPrivateWebhookTargets = env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true";
+	const db = sqlDbFor(env);
+	const store = new AlertsStore(db, { allowPrivateWebhookTargets });
 	const rules = await store.listEnabledRules();
 	return evaluateRuleBatch(store, rules, {
 		logger,
 		tracer,
 		concurrency: options?.concurrency,
 		ruleTimeoutMs: options?.ruleTimeoutMs,
+		allowPrivateWebhookTargets,
+		hostLookup: options?.hostLookup,
+		onAlertFiring: async (rule) => {
+			await openOrDeduplicateIncident(db, rule.projectId, {
+				environment: "default",
+				targetNodeId: `alert:${rule.id}`,
+				symptomAnchorKind: "alert",
+				symptomAnchorId: rule.id,
+			});
+		},
 	});
 }
 

package/src/plugins/alerts-routes.ts

@@ -6,12 +6,16 @@ import { alertEvidenceReferences } from "../lib/evidence-references";
 import { sqlDbFor } from "../lib/sql-db";
 import { getProjectId } from "./_context";
 
+const storeOptions = (env: { ALERT_WEBHOOK_ALLOW_PRIVATE?: string }) => ({
+	allowPrivateWebhookTargets: env.ALERT_WEBHOOK_ALLOW_PRIVATE === "true",
+});
+
 export const alertsRoutesPlugin: CollectorPlugin = {
 	name: "alerts-routes",
 	register(app) {
 		app.get("/internal/alerts/rules", async (c) => {
 			const projectId = getProjectId(c);
-			const store = new AlertsStore(sqlDbFor(c.env));
+			const store = new AlertsStore(sqlDbFor(c.env), storeOptions(c.env));
 			const rules = await store.listRules(projectId);
 			return c.json({ rules });
 		});
@@ -24,7 +28,7 @@ export const alertsRoutesPlugin: CollectorPlugin = {
 			} catch {
 				return c.json({ error: "Invalid JSON body" }, 400);
 			}
-			const store = new AlertsStore(sqlDbFor(c.env));
+			const store = new AlertsStore(sqlDbFor(c.env), storeOptions(c.env));
 			try {
 				const rule = await store.createRule(projectId, input);
 				return c.json({ rule }, 201);
@@ -43,7 +47,7 @@ export const alertsRoutesPlugin: CollectorPlugin = {
 			} catch {
 				return c.json({ error: "Invalid JSON body" }, 400);
 			}
-			const store = new AlertsStore(sqlDbFor(c.env));
+			const store = new AlertsStore(sqlDbFor(c.env), storeOptions(c.env));
 			try {
 				const rule = await store.updateRule(id, projectId, patch);
 				if (!rule) return c.json({ error: "Rule not found" }, 404);
@@ -57,7 +61,7 @@ export const alertsRoutesPlugin: CollectorPlugin = {
 		app.delete("/internal/alerts/rules/:id", async (c) => {
 			const projectId = getProjectId(c);
 			const id = c.req.param("id");
-			const store = new AlertsStore(sqlDbFor(c.env));
+			const store = new AlertsStore(sqlDbFor(c.env), storeOptions(c.env));
 			const deleted = await store.deleteRule(id, projectId);
 			if (!deleted) return c.json({ error: "Rule not found" }, 404);
 			return c.json({ success: true });
@@ -71,7 +75,7 @@ export const alertsRoutesPlugin: CollectorPlugin = {
 				1,
 				Math.min(720, Number.parseInt(c.req.query("hours") || "24", 10) || 24),
 			);
-			const store = new AlertsStore(sqlDbFor(c.env));
+			const store = new AlertsStore(sqlDbFor(c.env), storeOptions(c.env));
 			const response = await store.listEvaluations({
 				ruleId,
 				projectId,
@@ -83,7 +87,7 @@ export const alertsRoutesPlugin: CollectorPlugin = {
 		app.post("/internal/alerts/rules/:id/test", async (c) => {
 			const projectId = getProjectId(c);
 			const id = c.req.param("id");
-			const store = new AlertsStore(sqlDbFor(c.env));
+			const store = new AlertsStore(sqlDbFor(c.env), storeOptions(c.env));
 			const rule = await store.getRule(id, projectId);
 			if (!rule) return c.json({ error: "Rule not found" }, 404);
 			const value = await store.evaluateRule(rule);

package/src/plugins/ask-routes.ts

@@ -22,6 +22,7 @@ import { runAsk } from "../lib/ask";
 import type { LlmConfig } from "../lib/llm";
 import { sqlDbFor } from "../lib/sql-db";
 import { getProjectId } from "./_context";
+import { readJsonObject } from "./_json";
 
 const MAX_QUESTION_CHARS = 1000;
 
@@ -29,12 +30,10 @@ export const askRoutesPlugin: CollectorPlugin = {
 	name: "ask-routes",
 	register(app, runtime) {
 		app.post("/internal/ask", async (c) => {
-			let body: Partial<AskRequest> = {};
-			try {
-				body = (await c.req.json()) as Partial<AskRequest>;
-			} catch {
+			const body = await readJsonObject<Partial<AskRequest>>(c);
+			if (!body) {
 				return c.json(
-					{ error: "Bad Request", message: "body must be JSON" },
+					{ error: "Bad Request", message: "body must be a JSON object" },
 					400,
 				);
 			}