@obsunified/collector 2.3.0 → 2.4.3

package/src/lib/metric-alerts.ts

@@ -0,0 +1,640 @@
+/**
+ * `metrics` alert signal: validation and evaluation of an
+ * `AlertQueryMetrics` against `metric_series` / `metric_point`.
+ *
+ * Evaluation, per selector (numerator, then the optional denominator):
+ *
+ *   1. Series: `metric_series` rows of the project whose name is one of the
+ *      selector's names (and whose `service_name` matches, when given), kept
+ *      when every listed data-point / resource attribute equals the rule's
+ *      value. Attribute values are compared as strings, so `200` matches a
+ *      stored `"200"` and vice versa.
+ *   2. Points: the series' points whose `received_at` (the collector's clock,
+ *      as for every other alert signal) is inside the rule window, ordered by
+ *      producer time (`ts_ns`). The number read is `field` (`value`, or a
+ *      histogram/summary `count` / `sum`); points without it are skipped.
+ *   3. Staleness: for `last`, `min`, `max`, `avg` and `sum` a series takes
+ *      part only when its newest point is at most `staleSecs` (default 300)
+ *      old, so a series that stopped reporting (a renamed host, a replaced
+ *      pod) no longer adds its last value. `increase` / `rate` keep every
+ *      series with points in the window: growth that happened is real even
+ *      if the series stopped afterwards.
+ *   4. Each series is reduced on its own (`reduceSeries`), and the per-series
+ *      results are summed. A series with no usable point adds nothing.
+ *
+ * When no series takes part (in the numerator or the denominator) the
+ * result is "no data": the value is 0 and `noData` is true. The rule's
+ * `noData` policy decides what that means (`zero`: evaluate 0 as usual;
+ * `keep`: leave the alert state as it is).
+ *
+ * `age_seconds` ignores the window: it is the seconds since the newest
+ * `received_at` of any matching series. When no matching series has any
+ * stored point (never reported, or everything aged out of retention) it is
+ * the seconds since the rule was created, a finite number that grows until
+ * the threshold is crossed, so "never reported" fires after the same grace
+ * period as "stopped reporting".
+ *
+ * `age_seconds` is never "no data".
+ *
+ * Ratio: with a `denominator`, value = numerator / denominator when the
+ * denominator is > 0 and >= minDenominator (default 0), else 0.
+ *
+ * Cost bounds: at most `MAX_MATCHED_SERIES` series and `MAX_LOADED_POINTS`
+ * points per selector; past either the evaluation fails with an error (the
+ * evaluator records it as an `error` evaluation) rather than returning a
+ * value computed from part of the data. Metric rule windows are at most
+ * `MAX_METRIC_WINDOW_MINS`.
+ */
+
+import type {
+	AlertMetricAggregation,
+	AlertMetricAttributeValue,
+	AlertMetricField,
+	AlertMetricSelector,
+	AlertQueryMetrics,
+} from "@obsunified/types";
+import type { SqlDb } from "./sql-db";
+
+export const METRIC_AGGREGATIONS: readonly AlertMetricAggregation[] = [
+	"last",
+	"min",
+	"max",
+	"avg",
+	"sum",
+	"increase",
+	"rate",
+	"age_seconds",
+];
+
+export const METRIC_FIELDS: readonly AlertMetricField[] = [
+	"value",
+	"count",
+	"sum",
+];
+
+/** OTLP `AggregationTemporality`. */
+const TEMPORALITY_DELTA = 1;
+
+const MAX_METRIC_NAMES = 20;
+const MAX_ATTRIBUTE_KEYS = 32;
+/** A rule matching more series than this is refused at evaluation time. */
+export const MAX_MATCHED_SERIES = 1000;
+/** Points one selector may load per evaluation before it is refused. */
+export const MAX_LOADED_POINTS = 50_000;
+/** Longest window (`windowMins`) a metrics rule may use: one day. */
+export const MAX_METRIC_WINDOW_MINS = 1440;
+export const DEFAULT_STALE_SECS = 300;
+export const MAX_STALE_SECS = 86_400;
+export const METRIC_NO_DATA_POLICIES = ["zero", "keep"] as const;
+/** Keeps each statement under D1's ~100 bound-parameter limit. */
+const ID_CHUNK = 50;
+
+const SELECTOR_KEYS = [
+	"metric",
+	"attributes",
+	"resourceAttributes",
+	"serviceName",
+	"field",
+];
+const QUERY_KEYS = [
+	...SELECTOR_KEYS,
+	"aggregation",
+	"denominator",
+	"minDenominator",
+	"staleSecs",
+	"noData",
+];
+const DENOMINATOR_KEYS = [...SELECTOR_KEYS, "aggregation"];
+
+const isPlainObject = (v: unknown): v is Record<string, unknown> =>
+	typeof v === "object" && v !== null && !Array.isArray(v);
+
+const validateAttributes = (value: unknown, path: string): void => {
+	if (value === undefined) return;
+	if (!isPlainObject(value))
+		throw new Error(`${path} must be an object of attribute: value pairs`);
+	const entries = Object.entries(value);
+	if (entries.length > MAX_ATTRIBUTE_KEYS)
+		throw new Error(`${path} may list at most ${MAX_ATTRIBUTE_KEYS} keys`);
+	for (const [key, v] of entries) {
+		if (!key.trim()) throw new Error(`${path} has an empty attribute name`);
+		const ok =
+			typeof v === "string" ||
+			typeof v === "boolean" ||
+			(typeof v === "number" && Number.isFinite(v));
+		if (!ok)
+			throw new Error(
+				`${path}.${key} must be a string, finite number or boolean`,
+			);
+	}
+};
+
+const validateSelector = (
+	q: Record<string, unknown>,
+	path: string,
+	allowedKeys: string[],
+): void => {
+	for (const key of Object.keys(q)) {
+		if (!allowedKeys.includes(key))
+			throw new Error(
+				`${path}.${key} is not a metrics query field (allowed: ${allowedKeys.join(", ")})`,
+			);
+	}
+	const metric = q.metric;
+	if (typeof metric === "string") {
+		if (!metric.trim()) throw new Error(`${path}.metric must not be empty`);
+	} else if (Array.isArray(metric)) {
+		if (metric.length === 0)
+			throw new Error(`${path}.metric must name at least one metric`);
+		if (metric.length > MAX_METRIC_NAMES)
+			throw new Error(
+				`${path}.metric may name at most ${MAX_METRIC_NAMES} metrics`,
+			);
+		for (const m of metric) {
+			if (typeof m !== "string" || !m.trim())
+				throw new Error(`${path}.metric entries must be non-empty strings`);
+		}
+	} else {
+		throw new Error(
+			`${path}.metric is required: a metric name or a list of names`,
+		);
+	}
+	validateAttributes(q.attributes, `${path}.attributes`);
+	validateAttributes(q.resourceAttributes, `${path}.resourceAttributes`);
+	if (
+		q.serviceName !== undefined &&
+		(typeof q.serviceName !== "string" || !q.serviceName.trim())
+	)
+		throw new Error(`${path}.serviceName must be a non-empty string`);
+	if (
+		q.field !== undefined &&
+		!METRIC_FIELDS.includes(q.field as AlertMetricField)
+	)
+		throw new Error(
+			`${path}.field must be one of: ${METRIC_FIELDS.join(", ")}`,
+		);
+};
+
+/** Throws an Error with a user-facing message when `query` is malformed. */
+export function validateMetricQuery(query: unknown): void {
+	if (!isPlainObject(query))
+		throw new Error("query must be an object for the metrics signal");
+	validateSelector(query, "query", QUERY_KEYS);
+	const aggregation = query.aggregation;
+	if (!METRIC_AGGREGATIONS.includes(aggregation as AlertMetricAggregation))
+		throw new Error(
+			`query.aggregation must be one of: ${METRIC_AGGREGATIONS.join(", ")}`,
+		);
+	if (
+		query.staleSecs !== undefined &&
+		(!Number.isInteger(query.staleSecs) ||
+			(query.staleSecs as number) < 1 ||
+			(query.staleSecs as number) > MAX_STALE_SECS)
+	)
+		throw new Error(
+			`query.staleSecs must be an integer from 1 to ${MAX_STALE_SECS}`,
+		);
+	if (
+		query.noData !== undefined &&
+		!METRIC_NO_DATA_POLICIES.includes(query.noData as "zero" | "keep")
+	)
+		throw new Error(
+			`query.noData must be one of: ${METRIC_NO_DATA_POLICIES.join(", ")}`,
+		);
+	if (query.denominator === undefined) {
+		if (query.minDenominator !== undefined)
+			throw new Error("query.minDenominator requires query.denominator");
+		return;
+	}
+	if (aggregation === "age_seconds")
+		throw new Error("query.denominator cannot be used with age_seconds");
+	if (!isPlainObject(query.denominator))
+		throw new Error("query.denominator must be an object");
+	validateSelector(query.denominator, "query.denominator", DENOMINATOR_KEYS);
+	const denAgg = query.denominator.aggregation;
+	if (denAgg !== undefined) {
+		if (
+			denAgg === "age_seconds" ||
+			!METRIC_AGGREGATIONS.includes(denAgg as AlertMetricAggregation)
+		)
+			throw new Error(
+				`query.denominator.aggregation must be one of: ${METRIC_AGGREGATIONS.filter((a) => a !== "age_seconds").join(", ")}`,
+			);
+	}
+	if (
+		query.minDenominator !== undefined &&
+		(typeof query.minDenominator !== "number" ||
+			!Number.isFinite(query.minDenominator) ||
+			query.minDenominator < 0)
+	)
+		throw new Error("query.minDenominator must be a finite number >= 0");
+}
+
+// ── Pure reduction ─────────────────────────────────────────────────────────
+
+export interface MetricPointSample {
+	tsNs: string;
+	startTsNs: string | null;
+	value: number | null;
+	count: number | null;
+	sum: number | null;
+	receivedAt: string;
+}
+
+export interface SeriesReduceOptions {
+	/**
+	 * The series has no history before the window (its `metric_series` row
+	 * was created inside it): a cumulative counter's first point then counts
+	 * as growth from 0 instead of being the baseline.
+	 */
+	bornInWindow?: boolean;
+	aggregation: Exclude<AlertMetricAggregation, "age_seconds">;
+	field: AlertMetricField;
+	/** `metric_series.temporality`: 1 delta, 2 cumulative, null for gauges. */
+	temporality: number | null;
+	/** Start of the window in Unix nanoseconds. */
+	windowStartNs: bigint;
+	windowSecs: number;
+}
+
+const toBigInt = (ns: string | null): bigint | null => {
+	if (!ns) return null;
+	try {
+		return BigInt(ns);
+	} catch {
+		return null;
+	}
+};
+
+const readField = (p: MetricPointSample, field: AlertMetricField) => {
+	const raw = field === "count" ? p.count : field === "sum" ? p.sum : p.value;
+	const n = typeof raw === "string" ? Number(raw) : raw;
+	return typeof n === "number" && Number.isFinite(n) ? n : null;
+};
+
+/** Orders by producer time, then receive time (stable for equal ts_ns). */
+export const sortPoints = (points: MetricPointSample[]): MetricPointSample[] =>
+	[...points].sort((a, b) => {
+		const ta = toBigInt(a.tsNs) ?? 0n;
+		const tb = toBigInt(b.tsNs) ?? 0n;
+		if (ta !== tb) return ta < tb ? -1 : 1;
+		return a.receivedAt < b.receivedAt
+			? -1
+			: a.receivedAt > b.receivedAt
+				? 1
+				: 0;
+	});
+
+/**
+ * Counter growth over consecutive cumulative points (Prometheus `increase`
+ * without extrapolation). A drop, or a change of the point's start time,
+ * is a counter reset: the new value counts in full, as growth from 0. The
+ * first point is decided by its OTLP start time when it has one: a start
+ * inside the window means the counter began there, so its value counts from
+ * 0; a start before the window means part of the value grew before the
+ * window, so the point is the baseline (even for a series obs-unified first
+ * sees now: a long-running counter reaching a fresh database must not read
+ * as a burst). Only without a start time does `bornInWindow` decide (no
+ * history before the window: count from 0).
+ */
+const cumulativeIncrease = (
+	points: Array<{ v: number; start: bigint | null }>,
+	windowStartNs: bigint,
+	bornInWindow: boolean,
+): number => {
+	if (points.length === 0) return 0;
+	const first = points[0];
+	const countsFirst =
+		first.start !== null ? first.start >= windowStartNs : bornInWindow;
+	let total = countsFirst ? first.v : 0;
+	for (let i = 1; i < points.length; i++) {
+		const prev = points[i - 1];
+		const cur = points[i];
+		const restarted =
+			cur.start !== null && prev.start !== null && cur.start !== prev.start;
+		if (restarted || cur.v < prev.v) total += cur.v;
+		else total += cur.v - prev.v;
+	}
+	return total;
+};
+
+/**
+ * Reduces one series' window points to a number, or null when the series
+ * has no point carrying `field` in the window.
+ */
+export function reduceSeries(
+	points: MetricPointSample[],
+	opts: SeriesReduceOptions,
+): number | null {
+	const usable: Array<{ v: number; start: bigint | null }> = [];
+	for (const p of sortPoints(points)) {
+		const v = readField(p, opts.field);
+		if (v !== null) usable.push({ v, start: toBigInt(p.startTsNs) });
+	}
+	if (usable.length === 0) return null;
+	const values = usable.map((u) => u.v);
+	switch (opts.aggregation) {
+		case "last":
+			return values[values.length - 1];
+		case "min":
+			return values.reduce((a, b) => (b < a ? b : a));
+		case "max":
+			return values.reduce((a, b) => (b > a ? b : a));
+		case "sum":
+			return values.reduce((a, b) => a + b, 0);
+		case "avg":
+			return values.reduce((a, b) => a + b, 0) / values.length;
+		case "increase":
+		case "rate": {
+			const increase =
+				opts.temporality === TEMPORALITY_DELTA
+					? values.reduce((a, b) => a + b, 0)
+					: cumulativeIncrease(
+							usable,
+							opts.windowStartNs,
+							opts.bornInWindow ?? false,
+						);
+			return opts.aggregation === "rate"
+				? increase / opts.windowSecs
+				: increase;
+		}
+	}
+}
+
+/** numerator / denominator when denominator > 0 and >= minDenominator, else 0. */
+export const metricRatio = (
+	numerator: number,
+	denominator: number,
+	minDenominator: number,
+): number =>
+	denominator > 0 && denominator >= minDenominator
+		? numerator / denominator
+		: 0;
+
+// ── Database evaluation ────────────────────────────────────────────────────
+
+interface SeriesRow {
+	id: string;
+	temporality: number | string | null;
+	created_at: string;
+	attributes_json: unknown;
+	resource_attrs_json: unknown;
+}
+
+interface PointRow {
+	series_id: string;
+	ts_ns: string;
+	start_ts_ns: string | null;
+	value: number | null;
+	count: number | null;
+	sum: number | null;
+	received_at: string;
+}
+
+const parseAttrs = (raw: unknown): Record<string, unknown> => {
+	if (raw === null || raw === undefined) return {};
+	if (typeof raw === "string") {
+		try {
+			const parsed = JSON.parse(raw);
+			return isPlainObject(parsed) ? parsed : {};
+		} catch {
+			return {};
+		}
+	}
+	return isPlainObject(raw) ? raw : {};
+};
+
+const attrsMatch = (
+	stored: Record<string, unknown>,
+	wanted: Record<string, AlertMetricAttributeValue> | undefined,
+): boolean => {
+	if (!wanted) return true;
+	for (const [key, expected] of Object.entries(wanted)) {
+		const actual = stored[key];
+		if (actual === undefined || actual === null) return false;
+		if (typeof actual === "object") return false;
+		if (String(actual) !== String(expected)) return false;
+	}
+	return true;
+};
+
+const metricNames = (selector: AlertMetricSelector): string[] => [
+	...new Set(
+		(Array.isArray(selector.metric) ? selector.metric : [selector.metric]).map(
+			(m) => m.trim(),
+		),
+	),
+];
+
+const chunks = <T>(items: T[], size: number): T[][] => {
+	const out: T[][] = [];
+	for (let i = 0; i < items.length; i += size)
+		out.push(items.slice(i, i + size));
+	return out;
+};
+
+async function matchingSeries(
+	db: SqlDb,
+	projectId: string,
+	selector: AlertMetricSelector,
+): Promise<
+	Array<{ id: string; temporality: number | null; createdAt: string }>
+> {
+	const names = metricNames(selector);
+	let sql = `SELECT id, temporality, created_at, attributes_json, resource_attrs_json
+		FROM metric_series
+		WHERE project_id = ? AND name IN (${names.map(() => "?").join(", ")})`;
+	const binds: unknown[] = [projectId, ...names];
+	if (selector.serviceName) {
+		sql += ` AND service_name = ?`;
+		binds.push(selector.serviceName);
+	}
+	const rs = await db
+		.prepare(sql)
+		.bind(...binds)
+		.all<SeriesRow>();
+	const matched = (rs.results ?? []).filter(
+		(row) =>
+			attrsMatch(parseAttrs(row.attributes_json), selector.attributes) &&
+			attrsMatch(
+				parseAttrs(row.resource_attrs_json),
+				selector.resourceAttributes,
+			),
+	);
+	if (matched.length > MAX_MATCHED_SERIES)
+		throw new Error(
+			`metrics query matches ${matched.length} series (limit ${MAX_MATCHED_SERIES}); narrow it with attributes or serviceName`,
+		);
+	return matched.map((row) => ({
+		id: row.id,
+		createdAt: row.created_at,
+		temporality:
+			row.temporality === null || row.temporality === undefined
+				? null
+				: Number(row.temporality),
+	}));
+}
+
+export interface MetricEvalContext {
+	projectId: string;
+	windowMins: number;
+	/** The rule's `createdAt`; the reference for "never reported". */
+	ruleCreatedAt: string;
+	now: Date;
+	/** Staleness bound for gauge-style aggregations; default 300. */
+	staleSecs?: number;
+	/** Point cap per selector; default MAX_LOADED_POINTS (tests lower it). */
+	maxLoadedPoints?: number;
+}
+
+/** Aggregations whose series must be fresh (see step 3 above). */
+const STALENESS_APPLIES = new Set<AlertMetricAggregation>([
+	"last",
+	"min",
+	"max",
+	"avg",
+	"sum",
+]);
+
+export interface MetricEvaluation {
+	value: number;
+	/** No series took part; see the module comment. Never true for age_seconds. */
+	noData: boolean;
+}
+
+async function evaluateSelector(
+	db: SqlDb,
+	selector: AlertMetricSelector,
+	aggregation: Exclude<AlertMetricAggregation, "age_seconds">,
+	ctx: MetricEvalContext,
+): Promise<{ value: number; contributing: number }> {
+	const series = await matchingSeries(db, ctx.projectId, selector);
+	if (series.length === 0) return { value: 0, contributing: 0 };
+	const windowStartMs = ctx.now.getTime() - ctx.windowMins * 60_000;
+	const cutoff = new Date(windowStartMs).toISOString();
+	const bySeries = new Map<string, MetricPointSample[]>();
+	let loaded = 0;
+	for (const chunk of chunks(series, ID_CHUNK)) {
+		// One more row than the remaining budget tells "at the cap" from
+		// "over it" without loading the rest.
+		const cap = ctx.maxLoadedPoints ?? MAX_LOADED_POINTS;
+		const budget = cap - loaded;
+		const rs = await db
+			.prepare(
+				`SELECT series_id, ts_ns, start_ts_ns, value, count, sum, received_at
+				FROM metric_point
+				WHERE project_id = ? AND series_id IN (${chunk.map(() => "?").join(", ")})
+				  AND received_at >= ?
+				LIMIT ?`,
+			)
+			.bind(ctx.projectId, ...chunk.map((s) => s.id), cutoff, budget + 1)
+			.all<PointRow>();
+		const rows = rs.results ?? [];
+		loaded += rows.length;
+		if (loaded > cap)
+			throw new Error(
+				`metrics query loads more than ${cap} points in its window; shorten windowMins or narrow the query`,
+			);
+		for (const row of rows) {
+			const list = bySeries.get(row.series_id) ?? [];
+			list.push({
+				tsNs: String(row.ts_ns),
+				startTsNs: row.start_ts_ns === null ? null : String(row.start_ts_ns),
+				value: row.value,
+				count: row.count,
+				sum: row.sum,
+				receivedAt: row.received_at,
+			});
+			bySeries.set(row.series_id, list);
+		}
+	}
+	const field = selector.field ?? "value";
+	const windowStartNs = BigInt(windowStartMs) * 1_000_000n;
+	const freshCutoff = new Date(
+		ctx.now.getTime() - (ctx.staleSecs ?? DEFAULT_STALE_SECS) * 1000,
+	).toISOString();
+	let total = 0;
+	let contributing = 0;
+	for (const s of series) {
+		const points = bySeries.get(s.id) ?? [];
+		if (STALENESS_APPLIES.has(aggregation)) {
+			const newest = points.reduce<string | null>(
+				(max, p) => (max === null || p.receivedAt > max ? p.receivedAt : max),
+				null,
+			);
+			if (newest === null || newest < freshCutoff) continue;
+		}
+		const reduced = reduceSeries(points, {
+			aggregation,
+			field,
+			temporality: s.temporality,
+			windowStartNs,
+			windowSecs: ctx.windowMins * 60,
+			bornInWindow: s.createdAt >= cutoff,
+		});
+		if (reduced !== null) {
+			total += reduced;
+			contributing += 1;
+		}
+	}
+	return { value: total, contributing };
+}
+
+async function ageSeconds(
+	db: SqlDb,
+	selector: AlertMetricSelector,
+	ctx: MetricEvalContext,
+): Promise<number> {
+	const series = await matchingSeries(db, ctx.projectId, selector);
+	let newest: string | null = null;
+	for (const chunk of chunks(series, ID_CHUNK)) {
+		const row = await db
+			.prepare(
+				`SELECT MAX(received_at) AS newest FROM metric_point
+				WHERE project_id = ? AND series_id IN (${chunk.map(() => "?").join(", ")})`,
+			)
+			.bind(ctx.projectId, ...chunk.map((s) => s.id))
+			.first<{ newest: string | null }>();
+		if (row?.newest && (newest === null || row.newest > newest))
+			newest = row.newest;
+	}
+	const reference = Date.parse(newest ?? ctx.ruleCreatedAt);
+	if (!Number.isFinite(reference)) return 0;
+	return Math.max(0, (ctx.now.getTime() - reference) / 1000);
+}
+
+/** Evaluates a validated metrics query to a finite number. */
+export async function evaluateMetricQuery(
+	db: SqlDb,
+	query: AlertQueryMetrics,
+	ctx: MetricEvalContext,
+): Promise<MetricEvaluation> {
+	if (query.aggregation === "age_seconds")
+		return { value: await ageSeconds(db, query, ctx), noData: false };
+	const withStale = { ...ctx, staleSecs: query.staleSecs ?? ctx.staleSecs };
+	const numerator = await evaluateSelector(
+		db,
+		query,
+		query.aggregation,
+		withStale,
+	);
+	if (!query.denominator)
+		return {
+			value: numerator.value,
+			noData: numerator.contributing === 0,
+		};
+	const denominator = await evaluateSelector(
+		db,
+		query.denominator,
+		query.denominator.aggregation ?? query.aggregation,
+		withStale,
+	);
+	return {
+		value: metricRatio(
+			numerator.value,
+			denominator.value,
+			query.minDenominator ?? 0,
+		),
+		noData: numerator.contributing === 0 && denominator.contributing === 0,
+	};
+}

package/src/lib/metrics-store.ts

@@ -82,7 +82,11 @@ export class MetricsStore {
 		const { projectId, points, receivedAt, expiresAt } = opts;
 		if (points.length === 0) return;
 
-		const seriesByIdentity = await this.resolveSeries(projectId, points);
+		const seriesByIdentity = await this.resolveSeries(
+			projectId,
+			points,
+			receivedAt,
+		);
 
 		const pointStmt = this.db.prepare(`
 			INSERT INTO metric_point (
@@ -247,6 +251,7 @@ export class MetricsStore {
 	private async resolveSeries(
 		projectId: string,
 		points: DecodedMetricPoint[],
+		receivedAt: string,
 	): Promise<Map<string, SeriesRow>> {
 		const byIdentity = new Map<string, DecodedMetricPoint>();
 		for (const p of points) {
@@ -285,7 +290,9 @@ export class MetricsStore {
 		`);
 		const toInsert: SqlStatement[] = [];
 		const attemptedIds = new Set<string>();
-		const createdAt = new Date().toISOString();
+		// The batch's receive time: a series' created_at tells the metrics
+		// alert evaluator whether it has history before a window.
+		const createdAt = receivedAt;
 		for (const [identity, p] of byIdentity.entries()) {
 			if (existing.has(identity)) continue;
 			const id = crypto.randomUUID();