@obsunified/collector 2.0.1

package/src/plugins/ai-span-payloads-processor.ts

@@ -0,0 +1,180 @@
+/**
+ * Span processor that routes large AI payloads to the ai_span_payloads
+ * side table.
+ *
+ * For every span that has an `openinference.span.kind` attribute, we look
+ * for `ai.payload.input` / `ai.payload.output`, write them (plus the kind)
+ * to `ai_span_payloads`, and strip them from the span's attributes_json
+ * before it hits `telemetry_spans`.
+ *
+ * This keeps the hot spans table lean while preserving full replay data
+ * for AI calls.
+ */
+
+import type { JsonValue, StoredSpan } from "@obsunified/types";
+import {
+	ACTION_ID_KEY,
+	AI_PAYLOAD_INPUT_KEY,
+	AI_PAYLOAD_OUTPUT_KEY,
+	INTERACTION_ID_KEY,
+	isOpenInferenceSpanKind,
+	OPENINFERENCE_SPAN_KIND_KEY,
+	SESSION_ID_KEY,
+	USER_ID_KEY,
+} from "@obsunified/types/constants";
+import type { CollectorPlugin } from "../framework/collector";
+import { sha256Hex } from "../lib/hash";
+import { parseJsonRecord } from "../lib/json";
+import { isPayloadCaptureEnabled } from "../lib/payload-capture";
+import { sqlDbFor } from "../lib/sql-db";
+
+interface PayloadRow {
+	projectId: string;
+	traceId: string;
+	spanId: string;
+	spanKind: string;
+	inputJson: string | null;
+	outputJson: string | null;
+	inputHash: string | null;
+	outputHash: string | null;
+	sessionId: string | null;
+	userId: string | null;
+	interactionId: string | null;
+	actionId: string | null;
+	receivedAt: string;
+	expiresAt: string;
+}
+
+const asString = (value: JsonValue | undefined): string | null =>
+	typeof value === "string" && value.length > 0 ? value : null;
+
+const toJsonString = (value: unknown): string | null => {
+	if (value === undefined || value === null) return null;
+	if (typeof value === "string") return value;
+	try {
+		return JSON.stringify(value);
+	} catch {
+		return String(value);
+	}
+};
+
+export const aiSpanPayloadsProcessorPlugin: CollectorPlugin = {
+	name: "ai-span-payloads-processor",
+	register(_app, runtime) {
+		runtime.addSpanProcessor({
+			name: "ai-span-payloads-processor",
+			async process(spans, context) {
+				const db = sqlDbFor(context.env);
+				const rows: PayloadRow[] = [];
+				const captureByProject = new Map<string, boolean>();
+				const projectCaptureEnabled = async (projectId: string) => {
+					const cached = captureByProject.get(projectId);
+					if (cached !== undefined) return cached;
+					const enabled = await isPayloadCaptureEnabled(
+						db,
+						projectId,
+						context.env,
+					);
+					captureByProject.set(projectId, enabled);
+					return enabled;
+				};
+
+				const transformed = await Promise.all(
+					spans.map(async (span): Promise<StoredSpan> => {
+						const attrs = parseJsonRecord(span.attributesJson);
+						const kind = attrs[OPENINFERENCE_SPAN_KIND_KEY];
+						if (!isOpenInferenceSpanKind(kind)) return span;
+
+						const rawInput = attrs[AI_PAYLOAD_INPUT_KEY];
+						const rawOutput = attrs[AI_PAYLOAD_OUTPUT_KEY];
+						const hasInput = rawInput !== undefined;
+						const hasOutput = rawOutput !== undefined;
+						const inputJson = hasInput ? toJsonString(rawInput) : null;
+						const outputJson = hasOutput ? toJsonString(rawOutput) : null;
+						const capturePayloads = await projectCaptureEnabled(span.projectId);
+
+						// Always record the row for OpenInference spans so callers can
+						// join on (trace_id, span_id) even when payloads are absent.
+						rows.push({
+							projectId: span.projectId,
+							traceId: span.traceId,
+							spanId: span.spanId,
+							spanKind: kind,
+							inputJson: capturePayloads ? inputJson : null,
+							outputJson: capturePayloads ? outputJson : null,
+							inputHash: inputJson ? await sha256Hex(inputJson) : null,
+							outputHash: outputJson ? await sha256Hex(outputJson) : null,
+							sessionId: asString(attrs[SESSION_ID_KEY]),
+							userId: asString(attrs[USER_ID_KEY]),
+							interactionId: asString(attrs[INTERACTION_ID_KEY]),
+							actionId: asString(attrs[ACTION_ID_KEY]),
+							receivedAt: span.receivedAt,
+							expiresAt: span.expiresAt,
+						});
+
+						if (!hasInput && !hasOutput) return span;
+
+						// Strip payload attrs — they live in ai_span_payloads now.
+						const stripped = { ...attrs };
+						delete stripped[AI_PAYLOAD_INPUT_KEY];
+						delete stripped[AI_PAYLOAD_OUTPUT_KEY];
+						return { ...span, attributesJson: JSON.stringify(stripped) };
+					}),
+				);
+
+				if (rows.length > 0) {
+					try {
+						const stmt = db.prepare(
+							`INSERT INTO ai_span_payloads (
+								project_id, trace_id, span_id, span_kind,
+								input_json, output_json, input_hash, output_hash, session_id, user_id,
+								received_at, expires_at, interaction_id, action_id
+							) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+							ON CONFLICT(trace_id, span_id) DO UPDATE SET
+								project_id = excluded.project_id,
+								span_kind = excluded.span_kind,
+								input_json = excluded.input_json,
+								output_json = excluded.output_json,
+								input_hash = excluded.input_hash,
+								output_hash = excluded.output_hash,
+								session_id = excluded.session_id,
+								user_id = excluded.user_id,
+								received_at = excluded.received_at,
+								expires_at = excluded.expires_at,
+								interaction_id = excluded.interaction_id,
+								action_id = excluded.action_id`,
+						);
+						await db.batch(
+							rows.map((r) =>
+								stmt.bind(
+									r.projectId,
+									r.traceId,
+									r.spanId,
+									r.spanKind,
+									r.inputJson,
+									r.outputJson,
+									r.inputHash,
+									r.outputHash,
+									r.sessionId,
+									r.userId,
+									r.receivedAt,
+									r.expiresAt,
+									r.interactionId,
+									r.actionId,
+								),
+							),
+						);
+					} catch (err) {
+						// Don't fail the whole ingest on payload side-table issues —
+						// the spans themselves are still useful.
+						context.logger.error("[ai-span-payloads-processor] write failed", {
+							error: err instanceof Error ? err.message : String(err),
+						});
+					}
+				}
+
+				return transformed;
+			},
+		});
+	},
+};

package/src/plugins/alerts-evaluator.test.ts

@@ -0,0 +1,91 @@
+import type { AlertRule } from "@obsunified/types";
+import { describe, expect, it } from "vitest";
+import { evaluateRuleBatch } from "./alerts-evaluator";
+
+type EvaluatorStore = Parameters<typeof evaluateRuleBatch>[0];
+
+const makeRule = (id: string): AlertRule => ({
+	id,
+	projectId: "default",
+	name: `Rule ${id}`,
+	signal: "spans",
+	query: {},
+	threshold: 10,
+	windowMins: 5,
+	comparison: ">",
+	channels: [],
+	enabled: true,
+	createdAt: "2026-05-31T00:00:00.000Z",
+	updatedAt: "2026-05-31T00:00:00.000Z",
+	analysisId: null,
+	currentState: "ok",
+	lastStateChange: null,
+});
+
+const makeStore = (
+	overrides: Partial<EvaluatorStore> = {},
+): EvaluatorStore => ({
+	listEnabledRules: async () => [],
+	evaluateRule: async () => 0,
+	getState: async () => null,
+	getAnalysisNarrative: async () => null,
+	transitionState: async () => {},
+	recordEvaluation: async () => {},
+	...overrides,
+});
+
+describe("evaluateRuleBatch", () => {
+	it("evaluates rules with a bounded concurrency limit", async () => {
+		const rules = Array.from({ length: 6 }, (_, i) => makeRule(`r${i}`));
+		let active = 0;
+		let maxActive = 0;
+		const store = makeStore({
+			evaluateRule: async () => {
+				active += 1;
+				maxActive = Math.max(maxActive, active);
+				await new Promise((resolve) => setTimeout(resolve, 5));
+				active -= 1;
+				return 0;
+			},
+		});
+
+		const result = await evaluateRuleBatch(store, rules, { concurrency: 2 });
+
+		expect(result).toEqual({ evaluated: 6, fired: 0, resolved: 0 });
+		expect(maxActive).toBeLessThanOrEqual(2);
+	});
+
+	it("times out a stuck rule and continues the batch", async () => {
+		const errors: unknown[] = [];
+		let recorded = 0;
+		const store = makeStore({
+			evaluateRule: async (rule) => {
+				if (rule.id === "stuck") return new Promise<number>(() => {});
+				return 0;
+			},
+			recordEvaluation: async () => {
+				recorded += 1;
+			},
+		});
+
+		const result = await evaluateRuleBatch(
+			store,
+			[makeRule("stuck"), makeRule("ok")],
+			{
+				concurrency: 2,
+				ruleTimeoutMs: 1,
+				logger: {
+					debug: () => {},
+					info: () => {},
+					warn: () => {},
+					error: (_message, attrs) => errors.push(attrs),
+				},
+			},
+		);
+
+		expect(result).toEqual({ evaluated: 2, fired: 0, resolved: 0 });
+		expect(recorded).toBe(1);
+		expect(errors).toHaveLength(1);
+		expect(errors[0]).toMatchObject({ rule_id: "stuck" });
+	});
+});

package/src/plugins/alerts-evaluator.ts

@@ -0,0 +1,328 @@
+import type {
+	AlertChannel,
+	AlertRule,
+	AlertState,
+	AlertWebhookChannel,
+} from "@obsunified/types";
+import type { CollectorEnv } from "../framework/env";
+import {
+	type ChildSpanRunner,
+	consoleLogger,
+	type Logger,
+} from "../framework/logger";
+import { AlertsStore, compareValue } from "../lib/alerts-store";
+import { sqlDbFor } from "../lib/sql-db";
+
+const DEFAULT_RULE_CONCURRENCY = 5;
+const DEFAULT_RULE_TIMEOUT_MS = 10_000;
+
+interface AlertEvaluatorStore {
+	listEnabledRules(): Promise<AlertRule[]>;
+	evaluateRule(rule: AlertRule): Promise<number>;
+	getState(ruleId: string): Promise<{ current_state: AlertState } | null>;
+	getAnalysisNarrative(
+		projectId: string,
+		analysisId: string,
+	): Promise<{ narrative: string | null; status: string | null } | null>;
+	transitionState(
+		ruleId: string,
+		projectId: string,
+		newState: AlertState,
+		now: string,
+	): Promise<void>;
+	recordEvaluation(
+		ruleId: string,
+		projectId: string,
+		value: number,
+		state: AlertState,
+		notified: boolean,
+	): Promise<void>;
+}
+
+interface WebhookPayload {
+	rule: {
+		id: string;
+		name: string;
+		signal: string;
+	};
+	value: number;
+	threshold: number;
+	comparison: string;
+	state: AlertState;
+	evaluatedAt: string;
+	projectId: string;
+	/**
+	 * RFC 0002 Stage 6 — populated when the rule is bound to an Analysis
+	 * (`rule.analysisId` set). The narrative is the gated, LLM-generated
+	 * sentence the dashboard panel was last showing — what's actually
+	 * happening, in plain English. Webhooks consuming this can put the
+	 * narrative in the Slack/PagerDuty body directly.
+	 */
+	analysis?: {
+		id: string;
+		narrative: string | null;
+		status: string | null;
+	};
+}
+
+async function fireWebhook(
+	channel: AlertWebhookChannel,
+	payload: WebhookPayload,
+	logger: Logger,
+	tracer?: ChildSpanRunner,
+): Promise<boolean> {
+	const exec = async (
+		span: { setAttribute(k: string, v: unknown): void } | null,
+	): Promise<boolean> => {
+		try {
+			const response = await fetch(channel.url, {
+				method: "POST",
+				headers: {
+					"Content-Type": "application/json",
+					...(channel.headers ?? {}),
+				},
+				body: JSON.stringify(payload),
+				signal: AbortSignal.timeout(5000),
+			});
+			if (span) span.setAttribute("http.response.status_code", response.status);
+			return response.ok;
+		} catch (err) {
+			logger.error("[alerts-evaluator] webhook delivery failed", {
+				url: channel.url,
+				error: err instanceof Error ? err.message : String(err),
+			});
+			return false;
+		}
+	};
+	if (!tracer) return exec(null);
+	return tracer("webhook.alert", async (span) => exec(span), {
+		"http.method": "POST",
+		"http.url": channel.url,
+		"alert.state": payload.state,
+	});
+}
+
+async function fireChannels(
+	channels: AlertChannel[],
+	payload: WebhookPayload,
+	logger: Logger,
+	tracer?: ChildSpanRunner,
+): Promise<boolean> {
+	if (channels.length === 0) return true;
+	const results = await Promise.all(
+		channels
+			.filter((ch): ch is AlertWebhookChannel => ch.type === "webhook")
+			.map((ch) => fireWebhook(ch, payload, logger, tracer)),
+	);
+	return results.every(Boolean);
+}
+
+async function withTimeout<T>(
+	promise: Promise<T>,
+	ms: number,
+	message: string,
+): Promise<T> {
+	let timer: ReturnType<typeof setTimeout> | undefined;
+	try {
+		return await Promise.race([
+			promise,
+			new Promise<never>((_, reject) => {
+				timer = setTimeout(() => reject(new Error(message)), ms);
+			}),
+		]);
+	} finally {
+		if (timer) clearTimeout(timer);
+	}
+}
+
+async function mapWithConcurrency<T, R>(
+	items: T[],
+	concurrency: number,
+	mapper: (item: T) => Promise<R>,
+): Promise<R[]> {
+	const results: R[] = new Array(items.length);
+	let nextIndex = 0;
+	const workerCount = Math.min(concurrency, items.length);
+	await Promise.all(
+		Array.from({ length: workerCount }, async () => {
+			while (nextIndex < items.length) {
+				const current = nextIndex++;
+				results[current] = await mapper(items[current]);
+			}
+		}),
+	);
+	return results;
+}
+
+async function evaluateOneRule(
+	store: AlertEvaluatorStore,
+	rule: AlertRule,
+	logger: Logger,
+	tracer: ChildSpanRunner | undefined,
+	ruleTimeoutMs: number,
+): Promise<{ fired: number; resolved: number }> {
+	try {
+		const value = await withTimeout(
+			store.evaluateRule(rule),
+			ruleTimeoutMs,
+			`rule evaluation timed out after ${ruleTimeoutMs}ms`,
+		);
+		const shouldFire = compareValue(value, rule.threshold, rule.comparison);
+		const priorState = await store.getState(rule.id);
+		const previous: AlertState = priorState?.current_state ?? "ok";
+		const next: AlertState = shouldFire ? "firing" : "ok";
+		const now = new Date().toISOString();
+
+		// Stage 6: when the rule is bound to an analysis, pull the
+		// latest narrative + status so the webhook payload describes
+		// what's happening, not just "value > threshold".
+		const analysisAttachment = rule.analysisId
+			? await store
+					.getAnalysisNarrative(rule.projectId, rule.analysisId)
+					.then((n) =>
+						n
+							? {
+									id: rule.analysisId as string,
+									narrative: n.narrative,
+									status: n.status,
+								}
+							: undefined,
+					)
+					.catch(() => undefined)
+			: undefined;
+
+		if (previous === "ok" && next === "firing") {
+			const ok = await fireChannels(
+				rule.channels,
+				{
+					rule: { id: rule.id, name: rule.name, signal: rule.signal },
+					value,
+					threshold: rule.threshold,
+					comparison: rule.comparison,
+					state: "firing",
+					evaluatedAt: now,
+					projectId: rule.projectId,
+					analysis: analysisAttachment,
+				},
+				logger,
+				tracer,
+			);
+			await store.transitionState(rule.id, rule.projectId, "firing", now);
+			await store.recordEvaluation(
+				rule.id,
+				rule.projectId,
+				value,
+				"firing",
+				ok,
+			);
+			return { fired: 1, resolved: 0 };
+		}
+		if (previous === "firing" && next === "ok") {
+			const ok = await fireChannels(
+				rule.channels,
+				{
+					rule: { id: rule.id, name: rule.name, signal: rule.signal },
+					value,
+					threshold: rule.threshold,
+					comparison: rule.comparison,
+					state: "ok",
+					evaluatedAt: now,
+					projectId: rule.projectId,
+					analysis: analysisAttachment,
+				},
+				logger,
+				tracer,
+			);
+			await store.transitionState(rule.id, rule.projectId, "ok", now);
+			await store.recordEvaluation(rule.id, rule.projectId, value, "ok", ok);
+			return { fired: 0, resolved: 1 };
+		}
+
+		// Same state — write an evaluation but do not fire channels.
+		await store.recordEvaluation(rule.id, rule.projectId, value, next, false);
+	} catch (err) {
+		logger.error("[alerts-evaluator] rule evaluation failed", {
+			rule_id: rule.id,
+			project_id: rule.projectId,
+			error: err instanceof Error ? err.message : String(err),
+		});
+	}
+
+	return { fired: 0, resolved: 0 };
+}
+
+export async function evaluateRuleBatch(
+	store: AlertEvaluatorStore,
+	rules: AlertRule[],
+	options?: {
+		logger?: Logger;
+		tracer?: ChildSpanRunner;
+		concurrency?: number;
+		ruleTimeoutMs?: number;
+	},
+): Promise<{
+	evaluated: number;
+	fired: number;
+	resolved: number;
+}> {
+	const logger = options?.logger ?? consoleLogger;
+	const concurrency = Math.max(
+		1,
+		Math.floor(options?.concurrency ?? DEFAULT_RULE_CONCURRENCY),
+	);
+	const ruleTimeoutMs = Math.max(
+		1,
+		Math.floor(options?.ruleTimeoutMs ?? DEFAULT_RULE_TIMEOUT_MS),
+	);
+	const results = await mapWithConcurrency(rules, concurrency, (rule) =>
+		evaluateOneRule(store, rule, logger, options?.tracer, ruleTimeoutMs),
+	);
+	return {
+		evaluated: rules.length,
+		fired: results.reduce((sum, result) => sum + result.fired, 0),
+		resolved: results.reduce((sum, result) => sum + result.resolved, 0),
+	};
+}
+
+export async function evaluateAllRules(
+	env: CollectorEnv,
+	options?: {
+		logger?: Logger;
+		tracer?: ChildSpanRunner;
+		concurrency?: number;
+		ruleTimeoutMs?: number;
+	},
+): Promise<{
+	evaluated: number;
+	fired: number;
+	resolved: number;
+}> {
+	const logger = options?.logger ?? consoleLogger;
+	const tracer = options?.tracer;
+	const store = new AlertsStore(sqlDbFor(env));
+	const rules = await store.listEnabledRules();
+	return evaluateRuleBatch(store, rules, {
+		logger,
+		tracer,
+		concurrency: options?.concurrency,
+		ruleTimeoutMs: options?.ruleTimeoutMs,
+	});
+}
+
+/**
+ * Scheduled handler suitable for wrangler cron triggers. Dispatches on the
+ * cron expression so multiple triggers can share this handler.
+ */
+export function createAlertEvaluatorHandler() {
+	return {
+		async scheduled(
+			_event: { cron: string },
+			env: CollectorEnv,
+			ctx: {
+				waitUntil(promise: Promise<unknown>): void;
+			},
+		) {
+			ctx.waitUntil(evaluateAllRules(env));
+		},
+	};
+}

package/src/plugins/alerts-routes.ts

@@ -0,0 +1,101 @@
+import type { AlertRuleInput, AlertTestResponse } from "@obsunified/types";
+import { EVIDENCE_REFERENCE_CONTRACT } from "@obsunified/types";
+import type { CollectorPlugin } from "../framework/collector";
+import { AlertsStore, compareValue } from "../lib/alerts-store";
+import { alertEvidenceReferences } from "../lib/evidence-references";
+import { sqlDbFor } from "../lib/sql-db";
+import { getProjectId } from "./_context";
+
+export const alertsRoutesPlugin: CollectorPlugin = {
+	name: "alerts-routes",
+	register(app) {
+		app.get("/internal/alerts/rules", async (c) => {
+			const projectId = getProjectId(c);
+			const store = new AlertsStore(sqlDbFor(c.env));
+			const rules = await store.listRules(projectId);
+			return c.json({ rules });
+		});
+
+		app.post("/internal/alerts/rules", async (c) => {
+			const projectId = getProjectId(c);
+			let input: AlertRuleInput;
+			try {
+				input = await c.req.json<AlertRuleInput>();
+			} catch {
+				return c.json({ error: "Invalid JSON body" }, 400);
+			}
+			const store = new AlertsStore(sqlDbFor(c.env));
+			try {
+				const rule = await store.createRule(projectId, input);
+				return c.json({ rule }, 201);
+			} catch (err) {
+				const message = err instanceof Error ? err.message : String(err);
+				return c.json({ error: message }, 400);
+			}
+		});
+
+		app.patch("/internal/alerts/rules/:id", async (c) => {
+			const projectId = getProjectId(c);
+			const id = c.req.param("id");
+			let patch: Partial<AlertRuleInput>;
+			try {
+				patch = await c.req.json<Partial<AlertRuleInput>>();
+			} catch {
+				return c.json({ error: "Invalid JSON body" }, 400);
+			}
+			const store = new AlertsStore(sqlDbFor(c.env));
+			try {
+				const rule = await store.updateRule(id, projectId, patch);
+				if (!rule) return c.json({ error: "Rule not found" }, 404);
+				return c.json({ rule });
+			} catch (err) {
+				const message = err instanceof Error ? err.message : String(err);
+				return c.json({ error: message }, 400);
+			}
+		});
+
+		app.delete("/internal/alerts/rules/:id", async (c) => {
+			const projectId = getProjectId(c);
+			const id = c.req.param("id");
+			const store = new AlertsStore(sqlDbFor(c.env));
+			const deleted = await store.deleteRule(id, projectId);
+			if (!deleted) return c.json({ error: "Rule not found" }, 404);
+			return c.json({ success: true });
+		});
+
+		app.get("/internal/alerts/evaluations", async (c) => {
+			const projectId = getProjectId(c);
+			const ruleId = c.req.query("ruleId");
+			if (!ruleId) return c.json({ error: "ruleId is required" }, 400);
+			const hours = Math.max(
+				1,
+				Math.min(720, Number.parseInt(c.req.query("hours") || "24", 10) || 24),
+			);
+			const store = new AlertsStore(sqlDbFor(c.env));
+			const response = await store.listEvaluations({
+				ruleId,
+				projectId,
+				hours,
+			});
+			return c.json(response);
+		});
+
+		app.post("/internal/alerts/rules/:id/test", async (c) => {
+			const projectId = getProjectId(c);
+			const id = c.req.param("id");
+			const store = new AlertsStore(sqlDbFor(c.env));
+			const rule = await store.getRule(id, projectId);
+			if (!rule) return c.json({ error: "Rule not found" }, 404);
+			const value = await store.evaluateRule(rule);
+			const response: AlertTestResponse = {
+				value,
+				wouldFire: compareValue(value, rule.threshold, rule.comparison),
+				comparison: rule.comparison,
+				threshold: rule.threshold,
+				evidenceReferences: alertEvidenceReferences(rule),
+				evidenceContract: EVIDENCE_REFERENCE_CONTRACT,
+			};
+			return c.json(response);
+		});
+	},
+};