@intentius/chant-lexicon-prometheus 0.106.4 → 0.108.0

package/src/composites/red-alerts.test.ts

@@ -0,0 +1,182 @@
+/**
+ * `RedAlerts` over a spanmetrics connector's names: the expressions are the
+ * ones `RedDashboard` runs (both from the otel lexicon's
+ * `spanMetricsRedQueries()`), the rules pass the lexicon's checks, and over
+ * fixed series the alerts fire for the service that is failing or slow and
+ * for no other. `promtool check rules` runs when promtool is on PATH.
+ */
+import { describe, expect, test } from "vitest";
+import { spanMetricsNames, spanMetricsRedQueries, type SpanMetricsNames } from "@intentius/chant-lexicon-otel/metric-names";
+import { RedMetrics } from "@intentius/chant-lexicon-otel/composites/red-metrics";
+import { RedAlerts, redAlertRules, RED_ALERT_NAMES, type RedAlertsProps } from "./red-alerts";
+import { ruleGroupConfig } from "../rules";
+import { emitYaml } from "../build";
+import { RuleEvaluator, type Labels, type FiringAlert } from "../rule-eval";
+import { validateRuleFile } from "../validate-config";
+import { hasTool, promtoolCheckRules } from "../tools";
+import { isAlertingRuleConfig, type AlertingRuleConfig, type RuleGroupConfig } from "../model";
+
+const MIN = 60_000;
+const hasPromtool = hasTool(process.env.PROMTOOL ?? "promtool");
+const names = spanMetricsNames({ namespace: "shop", histogram: { unit: "s" } });
+
+function groupOf(props: Partial<RedAlertsProps> = {}): RuleGroupConfig {
+  return ruleGroupConfig(RedAlerts({ spanMetrics: names, ...props }).rules);
+}
+
+function alerts(group: RuleGroupConfig): AlertingRuleConfig[] {
+  return group.rules.filter(isAlertingRuleConfig);
+}
+
+/**
+ * Per-minute increments. Service `a` serves 100 spans a minute, 10 of them in
+ * error, half under 0.5s, 60 under 1s and the rest under 2.5s. Service `b`
+ * serves 100, none in error, all under 0.5s, and makes 50 failing client
+ * calls a minute, which the alerts don't count.
+ */
+function feed(n: SpanMetricsNames): (ev: RuleEvaluator, minute: number) => void {
+  const series: Array<{ labels: Labels; perMin: number }> = [];
+  const span = (service: string, kind: string, status: string) => ({
+    service_name: service,
+    span_name: "GET /",
+    span_kind: kind,
+    status_code: status,
+  });
+  const calls = n.calls.prometheus;
+  const buckets = `${n.duration!.prometheus}_bucket`;
+  series.push({ labels: { __name__: calls, ...span("a", "SPAN_KIND_SERVER", "STATUS_CODE_UNSET") }, perMin: 90 });
+  series.push({ labels: { __name__: calls, ...span("a", "SPAN_KIND_SERVER", "STATUS_CODE_ERROR") }, perMin: 10 });
+  series.push({ labels: { __name__: calls, ...span("b", "SPAN_KIND_SERVER", "STATUS_CODE_UNSET") }, perMin: 100 });
+  series.push({ labels: { __name__: calls, ...span("b", "SPAN_KIND_CLIENT", "STATUS_CODE_ERROR") }, perMin: 50 });
+  const bucket = (service: string, le: string, perMin: number) =>
+    series.push({ labels: { __name__: buckets, ...span(service, "SPAN_KIND_SERVER", "STATUS_CODE_UNSET"), le }, perMin });
+  for (const [le, a, b] of [
+    ["0.5", 50, 100],
+    ["1", 60, 100],
+    ["2.5", 100, 100],
+    ["+Inf", 100, 100],
+  ] as const) {
+    bucket("a", le, a);
+    bucket("b", le, b);
+  }
+  return (ev, minute) => {
+    for (const s of series) ev.add(s.labels, minute * MIN, s.perMin * minute);
+  };
+}
+
+function run(group: RuleGroupConfig, minutes = 15): Map<number, FiringAlert[]> {
+  const ev = new RuleEvaluator([group]);
+  const push = feed(names);
+  const firing = new Map<number, FiringAlert[]>();
+  for (let minute = 0; minute <= minutes; minute++) {
+    push(ev, minute);
+    firing.set(minute, ev.step(minute * MIN));
+  }
+  return firing;
+}
+
+const fired = (list: FiringAlert[] | undefined) => (list ?? []).map((a) => `${a.labels.alertname}{${a.labels.service_name}}`).sort();
+
+describe("RedAlerts defaults", () => {
+  const group = groupOf();
+
+  test("one group, red, with the error-ratio and latency alerts, warning, for 10m", () => {
+    expect(group.name).toBe("red");
+    expect(alerts(group).map((r) => [r.alert, r.for, r.labels?.severity])).toEqual([
+      [RED_ALERT_NAMES.errorRatio, "10m", "warning"],
+      [RED_ALERT_NAMES.latency, "10m", "warning"],
+    ]);
+  });
+
+  test("the expressions are spanMetricsRedQueries' over 5m, the same builder RedDashboard reads", () => {
+    const q = spanMetricsRedQueries(names, { range: "5m", quantiles: [0.95] });
+    const [err, lat] = alerts(group);
+    expect(err.expr).toBe(`${q.errorRatio} > 0.05`);
+    expect(lat.expr).toBe(`${q.duration[0].expr} > 1`);
+    expect(err.expr).toContain('span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"');
+  });
+
+  test("the rule file passes the lexicon's checks", () => {
+    expect(validateRuleFile({ groups: [group] })).toEqual([]);
+  });
+
+  test("fires for the failing, slow service only, once `for` has held", () => {
+    const firing = run(group);
+    expect(fired(firing.get(5))).toEqual([]);
+    expect(fired(firing.get(15))).toEqual([`${RED_ALERT_NAMES.errorRatio}{a}`, `${RED_ALERT_NAMES.latency}{a}`]);
+    const err = firing.get(15)!.find((a) => a.labels.alertname === RED_ALERT_NAMES.errorRatio)!;
+    expect(err.value).toBeCloseTo(0.1, 6);
+  });
+
+  test.skipIf(!hasPromtool)("promtool check rules passes", () => {
+    const result = promtoolCheckRules(emitYaml({ groups: [group] }));
+    expect(result.ran).toBe(true);
+    expect(result.ok, result.output).toBe(true);
+  });
+});
+
+describe("RedAlerts options", () => {
+  test("thresholds, quantile, severity, window and labels", () => {
+    const group = groupOf({
+      errorRatio: { threshold: 0.2, severity: "critical", annotations: { runbook_url: "https://runbooks.example.com/red" } },
+      latency: { quantile: 0.99, thresholdSeconds: 3, for: "5m" },
+      rateWindow: "10m",
+      name: "shop-red",
+      labels: { team: "shop" },
+    });
+    expect(group.name).toBe("shop-red");
+    expect(group.labels).toEqual({ team: "shop" });
+    const [err, lat] = alerts(group);
+    expect(err.expr.endsWith("> 0.2")).toBe(true);
+    expect(err.expr).toContain("[10m]");
+    expect(err.labels?.severity).toBe("critical");
+    expect(err.annotations?.runbook_url).toBe("https://runbooks.example.com/red");
+    expect(lat.expr.startsWith("histogram_quantile(0.99,")).toBe(true);
+    expect(lat.expr.endsWith("> 3")).toBe(true);
+    expect(lat.for).toBe("5m");
+    // 0.2 is above a's 0.1, and a's p99 (about 2.46s) is under 3s.
+    expect(fired(run(group).get(15))).toEqual([]);
+  });
+
+  test("a millisecond histogram gets the threshold in milliseconds", () => {
+    const ms = spanMetricsNames({});
+    const [, lat] = redAlertRules({ spanMetrics: ms, latency: { thresholdSeconds: 0.25 } });
+    expect(lat.expr).toContain("traces_span_metrics_duration_milliseconds_bucket");
+    expect(lat.expr.endsWith("> 250")).toBe(true);
+  });
+
+  test("minRate keeps quiet services quiet", () => {
+    const group = groupOf({ minRate: 10 });
+    expect(alerts(group)[0].expr).toContain("\nand on (service_name)\n");
+    expect(validateRuleFile({ groups: [group] })).toEqual([]);
+    // a sees 100 spans a minute, under 10 a second.
+    expect(fired(run(group).get(15))).toEqual([]);
+  });
+
+  test("either alert can be left out; without a histogram the latency alert is left out unless asked for", () => {
+    expect(alerts(groupOf({ latency: false })).map((r) => r.alert)).toEqual([RED_ALERT_NAMES.errorRatio]);
+    expect(alerts(groupOf({ errorRatio: false })).map((r) => r.alert)).toEqual([RED_ALERT_NAMES.latency]);
+    const noHistogram = spanMetricsNames({ histogram: { disable: true } });
+    expect(redAlertRules({ spanMetrics: noHistogram }).map((r) => r.alert)).toEqual([RED_ALERT_NAMES.errorRatio]);
+    expect(() => redAlertRules({ spanMetrics: noHistogram, latency: true })).toThrow(/RedAlerts: latency needs the connector's duration histogram/);
+    expect(() => RedAlerts({ spanMetrics: names, errorRatio: false, latency: false })).toThrow(/nothing to alert on/);
+  });
+
+  test("reads the names from RedMetrics' connector and exporter", () => {
+    const red = RedMetrics({ spanMetrics: { namespace: "edge" } });
+    const [err] = redAlertRules({ spanMetrics: red.spanMetrics, exporter: red.exporter });
+    expect(err.expr).toContain("edge_calls_total");
+  });
+
+  test("bad props are refused", () => {
+    expect(() => redAlertRules({} as RedAlertsProps)).toThrow(/spanMetrics is required/);
+    expect(() => groupOf({ errorRatio: { threshold: 1 } })).toThrow(/errorRatio.threshold/);
+    expect(() => groupOf({ latency: { quantile: 1 } })).toThrow(/latency.quantile/);
+    expect(() => groupOf({ latency: { thresholdSeconds: 0 } })).toThrow(/latency.thresholdSeconds/);
+    expect(() => groupOf({ rateWindow: "five" })).toThrow(/rateWindow/);
+    expect(() => groupOf({ groupBy: ["a-b"] })).toThrow(/groupBy/);
+    expect(() => groupOf({ minRate: 0 })).toThrow(/minRate/);
+    expect(() => groupOf({ name: "a b" })).toThrow(/name "a b"/);
+    expect(() => groupOf({ errorRatio: { for: "soon" } })).toThrow(/errorRatio.for/);
+  });
+});

package/src/composites/red-alerts.ts

@@ -0,0 +1,214 @@
+/**
+ * `RedAlerts`: error-ratio and latency alerts per service over the span
+ * metrics a `spanmetrics` connector emits.
+ *
+ * The names come from the connector's declaration through the otel lexicon's
+ * `spanMetricsNames()`, and the expressions from its
+ * `spanMetricsRedQueries()`, the same builder the grafana lexicon's
+ * `RedDashboard` runs its panels on. The alert and the panel a responder
+ * opens read one expression, with `rateWindow` in place of Grafana's
+ * `$__rate_interval`.
+ *
+ * Both alerts count server and consumer spans only by default, like the
+ * dashboard: a service's outgoing calls and internal spans don't dilute its
+ * error ratio. Both are on by default; the latency alert needs the
+ * connector's duration histogram and is left out without one unless asked
+ * for.
+ */
+
+import { Composite, type CompositeInstance } from "@intentius/chant/composite";
+import {
+  promNumber,
+  spanMetricsNames,
+  spanMetricsRedQueries,
+  type PrometheusNaming,
+  type SpanMetricsKind,
+  type SpanMetricsNames,
+} from "@intentius/chant-lexicon-otel/metric-names";
+import type { OTelComponent } from "@intentius/chant-lexicon-otel/define";
+import type { SpanMetricsConnectorConfig } from "@intentius/chant-lexicon-otel/components/connectors";
+import type { PrometheusExporterConfig } from "@intentius/chant-lexicon-otel/components/exporters";
+import { RuleGroup, type AlertingRule, type RuleGroupEntity } from "../rules";
+import type { LabelSet } from "../model";
+import { durationMs, isValidDuration } from "../duration";
+
+/** The fields both alerts take. */
+export interface RedAlertOptions {
+  /** How long the condition must hold (default `10m`). */
+  for?: string;
+  /** The `severity` label, which Alertmanager routes on (default `warning`). */
+  severity?: string;
+  /** More labels on the alert. */
+  labels?: LabelSet;
+  /** More annotations on the alert, e.g. `runbook_url`. */
+  annotations?: LabelSet;
+}
+
+export interface RedErrorRatioAlert extends RedAlertOptions {
+  /** The share of counted spans ending in error the alert fires above, between 0 and 1 (default 0.05). */
+  threshold?: number;
+}
+
+export interface RedLatencyAlert extends RedAlertOptions {
+  /** The duration quantile compared (default 0.95). */
+  quantile?: number;
+  /** The duration, in seconds, the quantile fires above (default 1). Converted to the histogram's unit. */
+  thresholdSeconds?: number;
+}
+
+export interface RedAlertsProps {
+  /** The `spanmetrics` connector the metrics come from (the otel lexicon's `RedMetrics` has it as `spanMetrics`), or the names `spanMetricsNames()` returned for it. */
+  spanMetrics: OTelComponent<"connector", "spanmetrics", SpanMetricsConnectorConfig> | SpanMetricsNames;
+  /** The `prometheus` exporter that serves them, when its `namespace` or `add_metric_suffixes` changes the names. */
+  exporter?: OTelComponent<"exporter", "prometheus", PrometheusExporterConfig> | PrometheusNaming;
+  /** Error ratio per service (default: on, above 0.05). `false` leaves it out. */
+  errorRatio?: boolean | RedErrorRatioAlert;
+  /** A duration quantile per service (default: on, p95 above 1s, when the connector has a histogram). `false` leaves it out. */
+  latency?: boolean | RedLatencyAlert;
+  /** The span kinds counted (default server and consumer spans); `[]` counts every kind. */
+  spanKinds?: SpanMetricsKind[];
+  /** The range every `rate` reads (default `5m`). */
+  rateWindow?: string;
+  /** Spans per second a service must see for its alerts to fire, so a handful of calls can't page (default: no floor). */
+  minRate?: number;
+  /** More Prometheus labels the alerts are split by besides the service, e.g. `deployment_environment`. They must be connector dimensions. */
+  groupBy?: string[];
+  /** The rule group's name (default `red`). */
+  name?: string;
+  /** Labels added to every rule, e.g. `team`. */
+  labels?: LabelSet;
+  /** Evaluation interval of the group (default: Prometheus's `evaluation_interval`). */
+  interval?: string;
+}
+
+export type RedAlertsMembers = {
+  /** The alerting rules. */
+  rules: RuleGroupEntity;
+};
+
+/** What `RedAlerts(...)` returns: its rule group, as `rules`. */
+export type RedAlertsInstance = CompositeInstance<RedAlertsMembers> & RedAlertsMembers;
+
+const LABEL = /^[A-Za-z_][A-Za-z0-9_]*$/;
+const GROUP_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
+
+/** The alert names `RedAlerts` writes. */
+export const RED_ALERT_NAMES = Object.freeze({ errorRatio: "ServiceErrorRatioHigh", latency: "ServiceLatencyHigh" });
+
+const DEFAULTS = { threshold: 0.05, quantile: 0.95, thresholdSeconds: 1, for: "10m", severity: "warning" };
+
+function fail(message: string): never {
+  throw new Error(`RedAlerts: ${message}`);
+}
+
+function isNames(x: unknown): x is SpanMetricsNames {
+  return typeof x === "object" && x !== null && "calls" in x && "labels" in x && "errorStatus" in x;
+}
+
+function options<T extends RedAlertOptions>(value: boolean | T | undefined): T | undefined {
+  if (value === false) return undefined;
+  return value === undefined || value === true ? ({} as T) : value;
+}
+
+function checkOptions(at: string, a: RedAlertOptions): void {
+  if (a.for !== undefined && !isValidDuration(a.for)) fail(`${at}.for "${a.for}" is not a Prometheus duration`);
+  if (a.severity !== undefined && (typeof a.severity !== "string" || a.severity === "")) fail(`${at}.severity must be a non-empty string`);
+}
+
+/** The alerting rules `RedAlerts` builds, without the group: for a group of your own. */
+export function redAlertRules(props: RedAlertsProps): AlertingRule[] {
+  if (!props || typeof props !== "object") fail("props are required");
+  if (!props.spanMetrics) fail("spanMetrics is required (a spanmetrics connector)");
+  const names = isNames(props.spanMetrics) ? props.spanMetrics : spanMetricsNames(props.spanMetrics, props.exporter);
+  const window = props.rateWindow ?? "5m";
+  if (!isValidDuration(window) || !durationMs(window)) fail(`rateWindow "${window}" is not a positive Prometheus duration`);
+  const groupBy = props.groupBy ?? [];
+  for (const l of groupBy) if (!LABEL.test(l)) fail(`groupBy label "${l}" is not a Prometheus label name`);
+  if (props.minRate !== undefined && !(typeof props.minRate === "number" && Number.isFinite(props.minRate) && props.minRate > 0)) {
+    fail(`minRate must be above 0, got ${String(props.minRate)}`);
+  }
+
+  const err = options(props.errorRatio);
+  // The latency alert is on by default only when there is a histogram to read.
+  const lat = props.latency === undefined && !names.duration ? undefined : options(props.latency);
+  if (lat && !names.duration) fail("latency needs the connector's duration histogram, and it is disabled");
+  if (err) {
+    checkOptions("errorRatio", err);
+    const t = err.threshold;
+    if (t !== undefined && !(typeof t === "number" && t > 0 && t < 1)) fail(`errorRatio.threshold must be above 0 and below 1, got ${String(t)}`);
+  }
+  if (lat) {
+    checkOptions("latency", lat);
+    const q = lat.quantile;
+    if (q !== undefined && !(typeof q === "number" && q > 0 && q < 1)) fail(`latency.quantile must be between 0 and 1, got ${String(q)}`);
+    const s = lat.thresholdSeconds;
+    if (s !== undefined && !(typeof s === "number" && Number.isFinite(s) && s > 0)) fail(`latency.thresholdSeconds must be above 0, got ${String(s)}`);
+  }
+
+  const quantile = lat?.quantile ?? DEFAULTS.quantile;
+  const q = spanMetricsRedQueries(names, { range: window, quantiles: [quantile], spanKinds: props.spanKinds, by: groupBy, owner: "RedAlerts" });
+  const by = [q.service, ...groupBy];
+  const floor = props.minRate !== undefined ? `\nand on (${by.join(", ")})\n${q.rate} > ${promNumber(props.minRate)}` : "";
+  const svc = `{{ $labels.${q.service} }}`;
+  const out: AlertingRule[] = [];
+  const build = (o: RedAlertOptions, alert: string, expr: string, summary: string, description: string) => {
+    out.push({
+      alert,
+      expr,
+      for: o.for ?? DEFAULTS.for,
+      labels: { ...(o.labels ?? {}), severity: o.severity ?? DEFAULTS.severity },
+      annotations: { summary, description, ...(o.annotations ?? {}) },
+    });
+  };
+
+  if (err) {
+    const threshold = err.threshold ?? DEFAULTS.threshold;
+    build(
+      err,
+      RED_ALERT_NAMES.errorRatio,
+      `${q.errorRatio} > ${promNumber(threshold)}${floor}`,
+      `Error ratio above ${promNumber(threshold * 100)}% for ${svc}`,
+      `{{ $value | humanizePercentage }} of ${svc} spans ended in error over the last ${window}.`,
+    );
+  }
+  if (lat && names.duration) {
+    const seconds = lat.thresholdSeconds ?? DEFAULTS.thresholdSeconds;
+    const inUnit = names.duration.unit === "s" ? seconds : seconds * 1000;
+    const qName = `p${promNumber(quantile * 100)}`;
+    const shown = names.duration.unit === "s" ? "{{ $value | humanizeDuration }}" : "{{ $value | printf \"%.0f\" }}ms";
+    build(
+      lat,
+      RED_ALERT_NAMES.latency,
+      `${q.duration[0].expr} > ${promNumber(inUnit)}${floor}`,
+      `${qName} duration above ${promNumber(seconds)}s for ${svc}`,
+      `${qName} duration of ${svc} is ${shown} over the last ${window}.`,
+    );
+  }
+  return out;
+}
+
+/**
+ * Error-ratio and latency alerts per service, from a `spanmetrics` connector.
+ *
+ * @example
+ * ```ts
+ * import { RedMetrics } from "@intentius/chant-lexicon-otel";
+ * import { RedAlerts } from "@intentius/chant-lexicon-prometheus";
+ *
+ * export const red = RedMetrics({ spanMetrics: { namespace: "shop", histogram: { unit: "s" } } });
+ * export const redAlerts = RedAlerts({ spanMetrics: red.spanMetrics, exporter: red.exporter, latency: { thresholdSeconds: 0.5 } });
+ * ```
+ */
+export const RedAlerts = Composite<RedAlertsProps, RedAlertsMembers>((props) => {
+  const rules = redAlertRules(props);
+  const name = props.name ?? "red";
+  if (!GROUP_NAME.test(name)) fail(`name "${name}" must be letters, digits, '.', '_' or '-'`);
+  if (rules.length === 0) fail("errorRatio and latency are both off, so there is nothing to alert on");
+  const group = new RuleGroup({
+    name,
+    ...(props.interval !== undefined ? { interval: props.interval } : {}),
+    ...(props.labels !== undefined ? { labels: props.labels } : {}),
+    rules,
+  });
+  return { rules: group };
+}, "RedAlerts");

package/src/composites/watchdog.test.ts

@@ -0,0 +1,85 @@
+/**
+ * `Watchdog`: the always-firing alert, its heartbeat route and receiver, and
+ * that the rule file and `alertmanager.yml` built with it pass the lexicon's
+ * checks once the route is nested under a root.
+ */
+import { describe, expect, test } from "vitest";
+import type { Declarable } from "@intentius/chant/declarable";
+import { Watchdog, WATCHDOG_URL_FILE } from "./watchdog";
+import { AlertRouting } from "./alert-routing";
+import { Receiver } from "../alertmanager";
+import { buildAlertmanagerConfig, buildRuleFile } from "../build";
+import { validateAlertmanagerConfig, validateRuleFile, validateSeverityRouting } from "../validate-config";
+import { RuleEvaluator } from "../rule-eval";
+import { ruleGroupConfig } from "../rules";
+
+const values = (...instances: Array<{ members: Record<string, unknown> }>) =>
+  instances.flatMap((i) => Object.values(i.members)) as Declarable[];
+
+describe("Watchdog defaults", () => {
+  const watchdog = Watchdog({});
+
+  test("one alert, Watchdog, on vector(1) with severity none, in group watchdog", () => {
+    const group = ruleGroupConfig(watchdog.rules);
+    expect(group.name).toBe("watchdog");
+    expect(group.rules).toHaveLength(1);
+    expect(group.rules[0]).toMatchObject({ alert: "Watchdog", expr: "vector(1)", labels: { severity: "none" } });
+    expect(validateRuleFile(buildRuleFile([watchdog.rules]).config)).toEqual([]);
+  });
+
+  test("it fires from the first evaluation", () => {
+    const ev = new RuleEvaluator([ruleGroupConfig(watchdog.rules)]);
+    expect(ev.step(0).map((a) => a.labels.alertname)).toEqual(["Watchdog"]);
+  });
+
+  test("the route matches the alert and sends it to a heartbeat webhook reading its URL from a file", () => {
+    expect(watchdog.route.props).toMatchObject({
+      matchers: ['alertname="Watchdog"', 'severity="none"'],
+      group_wait: "0s",
+      group_interval: "1m",
+      repeat_interval: "1m",
+    });
+    expect(watchdog.receiver?.props).toEqual({ name: "heartbeat", webhook_configs: [{ url_file: WATCHDOG_URL_FILE, send_resolved: false }] });
+  });
+
+  test("nested under AlertRouting, alertmanager.yml passes the checks and the watchdog route comes first", () => {
+    const routing = AlertRouting({ routes: [watchdog.route] });
+    const built = buildAlertmanagerConfig(values(watchdog, routing));
+    expect(built.warnings).toEqual([]);
+    expect(built.config.route?.routes?.[0]).toMatchObject({ receiver: "heartbeat", matchers: ['alertname="Watchdog"', 'severity="none"'] });
+    expect(validateAlertmanagerConfig(built.config)).toEqual([]);
+    expect(validateSeverityRouting([buildRuleFile([watchdog.rules]).config], built.config)).toEqual([]);
+  });
+
+  test("left unnested, it is a second root and the build says so", () => {
+    const routing = AlertRouting({});
+    expect(buildAlertmanagerConfig(values(watchdog, routing)).warnings.join("\n")).toMatch(/2 root Routes/);
+  });
+});
+
+describe("Watchdog options", () => {
+  test("a receiver of your own, by entity, props or name; and the alert's name, severity and repeat", () => {
+    const pd = new Receiver({ name: "pd-heartbeat", webhook_configs: [{ url_file: "/secrets/pd" }] });
+    const byEntity = Watchdog({ receiver: pd, alert: "DeadMansSwitch", severity: "heartbeat", repeatInterval: "5m" });
+    expect(byEntity.receiver).toBeUndefined();
+    expect(byEntity.route.props).toMatchObject({ receiver: pd, matchers: ['alertname="DeadMansSwitch"', 'severity="heartbeat"'], repeat_interval: "5m" });
+    expect(Watchdog({ receiver: "elsewhere" }).route.props.receiver).toBe("elsewhere");
+    expect(Watchdog({ receiver: { name: "hc", webhook_configs: [{ url_file: "/secrets/hc" }] } }).receiver?.props.name).toBe("hc");
+    expect(Watchdog({ urlFile: "/secrets/x" }).receiver?.props.webhook_configs?.[0].url_file).toBe("/secrets/x");
+  });
+
+  test("a repeat under a minute brings the group interval down with it (PROM223)", () => {
+    const fast = Watchdog({ repeatInterval: "30s" });
+    expect(fast.route.props).toMatchObject({ group_interval: "30s", repeat_interval: "30s" });
+    const built = buildAlertmanagerConfig(values(fast, AlertRouting({ routes: [fast.route] })));
+    expect(validateAlertmanagerConfig(built.config)).toEqual([]);
+  });
+
+  test("bad props are refused", () => {
+    expect(() => Watchdog({ alert: "dead man" })).toThrow(/Watchdog: alert "dead man"/);
+    expect(() => Watchdog({ repeatInterval: "often" })).toThrow(/repeatInterval/);
+    expect(() => Watchdog({ urlFile: "/x", receiver: "y" })).toThrow(/urlFile configures the default receiver/);
+    expect(() => Watchdog({ severity: "" })).toThrow(/severity/);
+    expect(() => Watchdog({ group: "a b" })).toThrow(/group "a b"/);
+  });
+});

package/src/composites/watchdog.ts

@@ -0,0 +1,142 @@
+/**
+ * `Watchdog`: an alert that always fires, routed to a heartbeat receiver.
+ *
+ * The alert's expression is `vector(1)`, so it fires from the first rule
+ * evaluation for as long as Prometheus evaluates rules and Alertmanager
+ * delivers notifications. The route sends it to a heartbeat service
+ * (healthchecks.io, Dead Man's Snitch, a PagerDuty or Opsgenie heartbeat),
+ * which raises its own alarm when the notifications stop: the whole alerting
+ * path, not just Prometheus, is down.
+ *
+ * The route is a child route, matched on the alert's name and severity. Put
+ * it first under the root route: `AlertRouting({ routes: [watchdog.route] })`,
+ * or the `routes` of a root `Route` of your own. Left unnested it is a second
+ * root, and the build warns.
+ *
+ * By default the receiver is a webhook that reads its URL from a file
+ * (`url_file`), since a heartbeat URL is a credential; mount the file into
+ * Alertmanager.
+ */
+
+import { Composite, type CompositeInstance } from "@intentius/chant/composite";
+import { Receiver, Route, isReceiver, type ReceiverEntity, type ReceiverProps, type RouteEntity } from "../alertmanager";
+import { RuleGroup, type RuleGroupEntity } from "../rules";
+import type { LabelSet } from "../model";
+import { durationMs, isValidDuration } from "../duration";
+
+export interface WatchdogProps {
+  /**
+   * Where the heartbeat goes: a declared `Receiver`, a receiver's props
+   * (the composite declares it), or the name of one declared elsewhere.
+   * Default: a `heartbeat` receiver with one webhook reading `urlFile`.
+   */
+  receiver?: ReceiverEntity | ReceiverProps | string;
+  /** The file the default receiver's webhook reads its URL from (default `/etc/alertmanager/secrets/heartbeat-url`). */
+  urlFile?: string;
+  /** The alert's name (default `Watchdog`). */
+  alert?: string;
+  /** The alert's `severity` label (default `none`): no severity route takes it. */
+  severity?: string;
+  /** How often the heartbeat is sent again while the alert fires (default `1m`). Keep it under the heartbeat service's period. */
+  repeatInterval?: string;
+  /** The rule group's name (default `watchdog`). */
+  group?: string;
+  /** More labels on the alert. */
+  labels?: LabelSet;
+  /** More annotations on the alert. */
+  annotations?: LabelSet;
+}
+
+export type WatchdogMembers = {
+  /** The always-firing alert. */
+  rules: RuleGroupEntity;
+  /** The child route that sends it to the heartbeat receiver. */
+  route: RouteEntity;
+  /** The heartbeat receiver, when this composite declares it. */
+  receiver?: ReceiverEntity;
+};
+
+export type WatchdogInstance = CompositeInstance<WatchdogMembers> & WatchdogMembers;
+
+const ALERT_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
+const GROUP_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
+
+/** The default heartbeat URL file. */
+export const WATCHDOG_URL_FILE = "/etc/alertmanager/secrets/heartbeat-url";
+
+function fail(message: string): never {
+  throw new Error(`Watchdog: ${message}`);
+}
+
+/**
+ * An always-firing alert and the route that sends it to a heartbeat receiver.
+ *
+ * @example
+ * ```ts
+ * import { AlertRouting, Watchdog } from "@intentius/chant-lexicon-prometheus";
+ *
+ * export const watchdog = Watchdog({ urlFile: "/etc/alertmanager/secrets/healthchecks-url" });
+ * export const routing = AlertRouting({ routes: [watchdog.route] });
+ * ```
+ */
+export const Watchdog = Composite<WatchdogProps, WatchdogMembers>((input) => {
+  const props = input ?? {};
+  const alert = props.alert ?? "Watchdog";
+  if (!ALERT_NAME.test(alert)) fail(`alert "${alert}" is not a valid alert name`);
+  const severity = props.severity ?? "none";
+  if (typeof severity !== "string" || severity === "") fail("severity must be a non-empty string");
+  const repeat = props.repeatInterval ?? "1m";
+  if (!isValidDuration(repeat) || !durationMs(repeat)) fail(`repeatInterval "${repeat}" is not a positive duration`);
+  // repeat_interval under group_interval is PROM223; a faster heartbeat brings the group interval down with it.
+  const groupInterval = (durationMs(repeat) ?? 0) < 60_000 ? repeat : "1m";
+  const groupName = props.group ?? "watchdog";
+  if (!GROUP_NAME.test(groupName)) fail(`group "${groupName}" must be letters, digits, '.', '_' or '-'`);
+  if (props.urlFile !== undefined && props.receiver !== undefined) fail("urlFile configures the default receiver; leave it out when receiver is set");
+  if (props.urlFile !== undefined && props.urlFile.trim() === "") fail("urlFile must be a path");
+
+  let receiver: ReceiverEntity | undefined;
+  let ref: ReceiverEntity | string;
+  if (typeof props.receiver === "string") {
+    if (props.receiver === "") fail("receiver must name a receiver");
+    ref = props.receiver;
+  } else if (isReceiver(props.receiver)) {
+    ref = props.receiver;
+  } else {
+    const p: ReceiverProps = props.receiver ?? {
+      name: "heartbeat",
+      webhook_configs: [{ url_file: props.urlFile ?? WATCHDOG_URL_FILE, send_resolved: false }],
+    };
+    if (!p.name) fail("receiver.name is required");
+    receiver = new Receiver(p);
+    ref = receiver;
+  }
+
+  const rules = new RuleGroup({
+    name: groupName,
+    rules: [
+      {
+        alert,
+        expr: "vector(1)",
+        labels: { ...(props.labels ?? {}), severity },
+        annotations: {
+          summary: "Always firing, to show the alerting pipeline works",
+          description:
+            "This alert fires all the time. A heartbeat service receives it every few minutes and raises its own alarm when it stops arriving, which means Prometheus, Alertmanager or the path between them is down.",
+          ...(props.annotations ?? {}),
+        },
+      },
+    ],
+  });
+  const route = new Route({
+    receiver: ref,
+    matchers: [`alertname=${JSON.stringify(alert)}`, `severity=${JSON.stringify(severity)}`],
+    group_by: ["alertname"],
+    group_wait: "0s",
+    group_interval: groupInterval,
+    repeat_interval: repeat,
+    continue: false,
+  });
+  const members: WatchdogMembers = { rules, route };
+  if (receiver) members.receiver = receiver;
+  return members;
+}, "Watchdog");