@intentius/chant-lexicon-prometheus 0.124.30 → 0.124.32

package/dist/composites/alert-routing.js

@@ -0,0 +1,210 @@
+/**
+ * `AlertRouting`: an Alertmanager routing tree by team and severity, the
+ * inhibit rules between severity levels, and the receivers.
+ *
+ * The root route sends to `receiver` (default: a `default` receiver with no
+ * integrations, which drops what reaches it). Below it, in order:
+ *
+ * 1. `routes`: child routes of your own, first, such as `Watchdog`'s.
+ * 2. One route per team, matched on the team label (`team` by default),
+ *    sending to the team's receiver; a team can send a severity level to
+ *    another receiver of its own (a pager for `critical`).
+ * 3. One route per severity level, for every alert no team route took.
+ *
+ * The default levels are the severities the lexicon's composites write:
+ * `critical` (with `Slo`'s `page`), `warning` (with `Slo`'s `ticket`, and
+ * `GenAiRules`' and `RedAlerts`' default `warning`) and `info`. Each level
+ * sends to the root receiver unless it names one, so a project routes
+ * `critical` to a pager and leaves the rest, and PROM202 finds every
+ * severity routed.
+ *
+ * While an alert of one level fires, the same alert at every lower level is
+ * held back (an inhibit rule per pair). "The same alert" is equal values of
+ * `inhibitEqual`, by default `alertname` and the labels the lexicon's
+ * composites tell their alerts apart by (`slo`, `service_name`) plus the team
+ * label: one SLO's page holds back that SLO's ticket, not another's.
+ */
+import { Composite } from "@intentius/chant/composite";
+import { InhibitRule, Receiver, Route, isReceiver, isRoute, } from "../alertmanager.js";
+import { isValidDuration } from "../duration.js";
+/** The default levels: the severities `Slo`, `GenAiRules` and `RedAlerts` write, most severe first. */
+export const ALERT_ROUTING_LEVELS = Object.freeze([
+    { name: "critical", severities: ["critical", "page"] },
+    { name: "warning", severities: ["warning", "ticket"] },
+    { name: "info", severities: ["info"] },
+]);
+const LEVEL_NAME = /^[A-Za-z][A-Za-z0-9_]*$/;
+const LABEL = /^[A-Za-z_][A-Za-z0-9_]*$/;
+function fail(message) {
+    throw new Error(`AlertRouting: ${message}`);
+}
+/** `severity="a"` for one value, `severity=~"a|b"` for several. */
+function severityMatcher(values) {
+    const escaped = values.map((v) => v.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
+    return values.length === 1 ? `severity=${JSON.stringify(values[0])}` : `severity=~${JSON.stringify(escaped.join("|"))}`;
+}
+function timing(t, at) {
+    const out = {};
+    for (const [key, field] of [
+        ["groupWait", "group_wait"],
+        ["groupInterval", "group_interval"],
+        ["repeatInterval", "repeat_interval"],
+    ]) {
+        const v = t[key];
+        if (v === undefined)
+            continue;
+        if (!isValidDuration(v))
+            fail(`${at}.${key} "${v}" is not a duration`);
+        out[field] = v;
+    }
+    return out;
+}
+/**
+ * A routing tree by team and severity, with inhibit rules between levels.
+ *
+ * @example
+ * ```ts
+ * import { AlertRouting, Watchdog } from "@intentius/chant-lexicon-prometheus";
+ *
+ * const pager = { name: "pager", pagerduty_configs: [{ routing_key_file: "/etc/alertmanager/secrets/pagerduty" }] };
+ * const chat = { name: "chat", slack_configs: [{ api_url_file: "/etc/alertmanager/secrets/slack", channel: "#alerts" }] };
+ * export const watchdog = Watchdog({});
+ * export const routing = AlertRouting({
+ *   receiver: chat,
+ *   levels: [{ name: "critical", severities: ["critical", "page"], receiver: pager }, { name: "warning", severities: ["warning", "ticket"] }],
+ *   routes: [watchdog.route],
+ * });
+ * ```
+ */
+export const AlertRouting = Composite((input) => {
+    const props = input ?? {};
+    const members = {};
+    const declared = new Map();
+    const external = new Map();
+    const receiverRef = (r, at) => {
+        if (typeof r === "string") {
+            if (r === "")
+                fail(`${at} must name a receiver`);
+            return r;
+        }
+        if (isReceiver(r)) {
+            const name = r.props.name;
+            const seen = declared.get(name)?.entity ?? external.get(name);
+            if (seen && seen !== r)
+                fail(`${at}: two receivers are named "${name}"`);
+            external.set(name, r);
+            return r;
+        }
+        if (!r || typeof r !== "object" || typeof r.name !== "string" || r.name === "")
+            fail(`${at}.name is required`);
+        const before = declared.get(r.name);
+        if (before) {
+            if (before.props !== r)
+                fail(`${at}: two receivers are named "${r.name}"; declare it once and pass the same object`);
+            return before.entity;
+        }
+        if (external.has(r.name))
+            fail(`${at}: two receivers are named "${r.name}"`);
+        const entity = new Receiver(r);
+        declared.set(r.name, { props: r, entity });
+        members[`receiver_${r.name.replace(/[^A-Za-z0-9_]/g, "_")}`] = entity;
+        return entity;
+    };
+    const teamLabel = props.teamLabel ?? "team";
+    if (!LABEL.test(teamLabel))
+        fail(`teamLabel "${teamLabel}" is not a label name`);
+    const groupBy = props.groupBy ?? ["alertname"];
+    for (const l of groupBy)
+        if (l !== "..." && !LABEL.test(l))
+            fail(`groupBy label "${l}" is not a label name`);
+    const equal = props.inhibitEqual ?? ["alertname", "slo", "service_name", teamLabel];
+    for (const l of equal)
+        if (!LABEL.test(l))
+            fail(`inhibitEqual label "${l}" is not a label name`);
+    const root = receiverRef(props.receiver ?? { name: "default" }, "receiver");
+    const levels = props.levels ?? ALERT_ROUTING_LEVELS;
+    if (!Array.isArray(levels))
+        fail("levels must be a list");
+    const levelNames = new Set();
+    const severities = new Map();
+    levels.forEach((l, i) => {
+        const at = `levels[${i}]`;
+        if (!l || typeof l.name !== "string" || !LEVEL_NAME.test(l.name))
+            fail(`${at}.name must be letters, digits and '_', starting with a letter`);
+        if (levelNames.has(l.name))
+            fail(`${at}: a second level is named "${l.name}"`);
+        levelNames.add(l.name);
+        if (!Array.isArray(l.severities) || l.severities.length === 0)
+            fail(`${at}.severities must name at least one severity`);
+        for (const s of l.severities) {
+            if (typeof s !== "string" || s === "")
+                fail(`${at}.severities must be non-empty strings`);
+            const other = severities.get(s);
+            if (other !== undefined)
+                fail(`${at}: severity "${s}" is already in level "${other}"`);
+            severities.set(s, l.name);
+        }
+    });
+    const children = [];
+    (props.routes ?? []).forEach((r, i) => {
+        if (!isRoute(r) && (typeof r !== "object" || r === null))
+            fail(`routes[${i}] must be a Route or route props`);
+        children.push(r);
+    });
+    const teams = props.teams ?? [];
+    const teamNames = new Set();
+    teams.forEach((t, i) => {
+        const at = `teams[${i}]`;
+        if (!t || typeof t.team !== "string" || t.team === "")
+            fail(`${at}.team is required`);
+        if (teamNames.has(t.team))
+            fail(`${at}: team "${t.team}" is listed twice`);
+        teamNames.add(t.team);
+        if (t.receiver === undefined)
+            fail(`${at}.receiver is required`);
+        const routes = [];
+        for (const [levelName, r] of Object.entries(t.levels ?? {})) {
+            const level = levels.find((l) => l.name === levelName);
+            if (!level)
+                fail(`${at}.levels names "${levelName}", which is not a level (${[...levelNames].join(", ")})`);
+            routes.push({ matchers: [severityMatcher(level.severities)], receiver: receiverRef(r, `${at}.levels.${levelName}`) });
+        }
+        children.push({
+            matchers: [`${teamLabel}=${JSON.stringify(t.team)}`],
+            receiver: receiverRef(t.receiver, `${at}.receiver`),
+            ...timing(t, at),
+            ...(routes.length > 0 ? { routes } : {}),
+        });
+    });
+    levels.forEach((l, i) => {
+        children.push({
+            matchers: [severityMatcher(l.severities)],
+            receiver: l.receiver !== undefined ? receiverRef(l.receiver, `levels[${i}].receiver`) : root,
+            ...timing(l, `levels[${i}]`),
+        });
+    });
+    const route = new Route({
+        receiver: root,
+        group_by: [...groupBy],
+        group_wait: "30s",
+        group_interval: "5m",
+        repeat_interval: "4h",
+        ...timing(props, "props"),
+        routes: children,
+    });
+    if (props.inhibit !== false) {
+        for (let hi = 0; hi < levels.length; hi++) {
+            for (let lo = hi + 1; lo < levels.length; lo++) {
+                const a = levels[hi];
+                const b = levels[lo];
+                members[`inhibit_${a.name}_${b.name}`] = new InhibitRule({
+                    name: `${a.name} holds back ${b.name}`,
+                    source_matchers: [severityMatcher(a.severities)],
+                    target_matchers: [severityMatcher(b.severities)],
+                    equal: [...equal],
+                });
+            }
+        }
+    }
+    return { route, ...members };
+}, "AlertRouting");

package/dist/composites/catalog.js

@@ -0,0 +1,344 @@
+export const compositeCatalog = [
+    {
+        "name": "AlertRouting",
+        "lexicon": "prometheus",
+        "description": "A routing tree by team and severity, with inhibit rules between levels.",
+        "bundles": [
+            "InhibitRule",
+            "Receiver",
+            "Route"
+        ],
+        "params": [
+            {
+                "name": "receiver",
+                "type": "AlertRoutingReceiver",
+                "required": false,
+                "description": "The root route's receiver, for what no child route takes (default: a `default` receiver with no integrations)."
+            },
+            {
+                "name": "levels",
+                "type": "AlertRoutingLevel[]",
+                "required": false,
+                "description": "Severity levels, most severe first (default `critical`, `warning`, `info`; see `ALERT_ROUTING_LEVELS`)."
+            },
+            {
+                "name": "teams",
+                "type": "AlertRoutingTeam[]",
+                "required": false,
+                "description": "Team routes, tried before the severity levels (default: none)."
+            },
+            {
+                "name": "teamLabel",
+                "type": "string",
+                "required": false,
+                "description": "The label team routes match on (default `team`)."
+            },
+            {
+                "name": "routes",
+                "type": "Array<RouteEntity | RouteProps>",
+                "required": false,
+                "description": "Child routes of your own, tried first, e.g. `[watchdog.route]`."
+            },
+            {
+                "name": "groupBy",
+                "type": "string[]",
+                "required": false,
+                "description": "Labels alerts are grouped by into one notification (default `[\"alertname\"]`)."
+            },
+            {
+                "name": "inhibit",
+                "type": "boolean",
+                "required": false,
+                "description": "Inhibit lower levels while a higher one fires (default: on)."
+            },
+            {
+                "name": "inhibitEqual",
+                "type": "string[]",
+                "required": false,
+                "description": "The labels that must match for one alert to hold back another (default `alertname`, `slo`, `service_name` and the team label)."
+            },
+            {
+                "name": "groupWait",
+                "type": "string",
+                "required": false,
+                "description": "How long to wait before the first notification of a new group."
+            },
+            {
+                "name": "groupInterval",
+                "type": "string",
+                "required": false,
+                "description": "How long to wait before notifying about new alerts in a group already notified."
+            },
+            {
+                "name": "repeatInterval",
+                "type": "string",
+                "required": false,
+                "description": "How long to wait before sending a notification again."
+            }
+        ]
+    },
+    {
+        "name": "GenAiRules",
+        "lexicon": "prometheus",
+        "description": "Recording rules and opt-in alerts for GenAI calls, from the otel preset's metrics.",
+        "bundles": [
+            "RuleGroup"
+        ],
+        "params": [
+            {
+                "name": "genAi",
+                "type": "GenAiMetrics | { metrics: GenAiMetrics }",
+                "required": true,
+                "description": "The preset's metrics: `genAiMetrics(options)` with the options the collector was built with, or `genAiComponents(options)`."
+            },
+            {
+                "name": "source",
+                "type": "\"client\" | \"spans\"",
+                "required": false,
+                "description": "Which metrics the model rules read (default `client` when the metrics include the conventions' client metrics, else `spans`)."
+            },
+            {
+                "name": "prices",
+                "type": "GenAiPrice[]",
+                "required": false,
+                "description": "Prices per provider and model."
+            },
+            {
+                "name": "alerts",
+                "type": "GenAiAlerting",
+                "required": false,
+                "description": "Opt-in alerts (default: none)."
+            },
+            {
+                "name": "name",
+                "type": "string",
+                "required": false,
+                "description": "The rule group's name (default `genai`)."
+            },
+            {
+                "name": "prefix",
+                "type": "string",
+                "required": false,
+                "description": "The first part of every recorded series name (default `gen_ai`)."
+            },
+            {
+                "name": "rateWindow",
+                "type": "string",
+                "required": false,
+                "description": "The range every `rate` reads (default `5m`)."
+            },
+            {
+                "name": "groupBy",
+                "type": "string[]",
+                "required": false,
+                "description": "More Prometheus labels every rule keeps, e.g. `job` or `service_name`."
+            },
+            {
+                "name": "labels",
+                "type": "LabelSet",
+                "required": false,
+                "description": "Labels added to every rule, e.g. `team`."
+            },
+            {
+                "name": "interval",
+                "type": "string",
+                "required": false,
+                "description": "Evaluation interval of the group (default: Prometheus's `evaluation_interval`)."
+            }
+        ]
+    },
+    {
+        "name": "RedAlerts",
+        "lexicon": "prometheus",
+        "description": "Error-ratio and latency alerts per service, from a `spanmetrics` connector.",
+        "bundles": [
+            "RuleGroup"
+        ],
+        "params": [
+            {
+                "name": "spanMetrics",
+                "type": "OTelComponent<\"connector\", \"spanmetrics\", SpanMetricsConnectorConfig> | SpanMetricsNames",
+                "required": true,
+                "description": "The `spanmetrics` connector the metrics come from (the otel lexicon's `RedMetrics` has it as `spanMetrics`), or the names `spanMetricsNames()` returned for it."
+            },
+            {
+                "name": "exporter",
+                "type": "OTelComponent<\"exporter\", \"prometheus\", PrometheusExporterConfig> | PrometheusNaming",
+                "required": false,
+                "description": "The `prometheus` exporter that serves them, when its `namespace` or `add_metric_suffixes` changes the names."
+            },
+            {
+                "name": "errorRatio",
+                "type": "boolean | RedErrorRatioAlert",
+                "required": false,
+                "description": "Error ratio per service (default: on, above 0.05)."
+            },
+            {
+                "name": "latency",
+                "type": "boolean | RedLatencyAlert",
+                "required": false,
+                "description": "A duration quantile per service (default: on, p95 above 1s, when the connector has a histogram)."
+            },
+            {
+                "name": "spanKinds",
+                "type": "SpanMetricsKind[]",
+                "required": false,
+                "description": "The span kinds counted (default server and consumer spans); `[]` counts every kind."
+            },
+            {
+                "name": "rateWindow",
+                "type": "string",
+                "required": false,
+                "description": "The range every `rate` reads (default `5m`)."
+            },
+            {
+                "name": "minRate",
+                "type": "number",
+                "required": false,
+                "description": "Spans per second a service must see for its alerts to fire, so a handful of calls can't page (default: no floor)."
+            },
+            {
+                "name": "groupBy",
+                "type": "string[]",
+                "required": false,
+                "description": "More Prometheus labels the alerts are split by besides the service, e.g. `deployment_environment`."
+            },
+            {
+                "name": "name",
+                "type": "string",
+                "required": false,
+                "description": "The rule group's name (default `red`)."
+            },
+            {
+                "name": "labels",
+                "type": "LabelSet",
+                "required": false,
+                "description": "Labels added to every rule, e.g. `team`."
+            },
+            {
+                "name": "interval",
+                "type": "string",
+                "required": false,
+                "description": "Evaluation interval of the group (default: Prometheus's `evaluation_interval`)."
+            }
+        ]
+    },
+    {
+        "name": "Slo",
+        "lexicon": "prometheus",
+        "description": "An SLO, built to recording rules and multiwindow multi-burn-rate alerts.",
+        "bundles": [
+            "RuleGroup"
+        ],
+        "params": [
+            {
+                "name": "name",
+                "type": "string",
+                "required": true,
+                "description": "The SLO's name, the `slo` label on every series and alert it builds and part of its group names."
+            },
+            {
+                "name": "objective",
+                "type": "number",
+                "required": true,
+                "description": "The target share of good events, strictly between 0 and 1, e.g. `0.995`."
+            },
+            {
+                "name": "window",
+                "type": "string",
+                "required": true,
+                "description": "The rolling window the objective holds over, a Prometheus duration, e.g. `28d` or `30d`."
+            },
+            {
+                "name": "sli",
+                "type": "SloSli",
+                "required": true,
+                "description": "The SLI: good (or error) events over total events, with `{{window}}` for the range."
+            },
+            {
+                "name": "description",
+                "type": "string",
+                "required": false,
+                "description": "A sentence for the alerts' description, e.g. what the SLO measures."
+            },
+            {
+                "name": "alerting",
+                "type": "SloAlerting",
+                "required": false,
+                "description": "Burn-rate alerts (default: both tiers with the Workbook's windows)."
+            },
+            {
+                "name": "labels",
+                "type": "LabelSet",
+                "required": false,
+                "description": "Labels added to every rule, e.g. `team` or `service`."
+            },
+            {
+                "name": "interval",
+                "type": "string",
+                "required": false,
+                "description": "Evaluation interval of the group (default: Prometheus's `evaluation_interval`)."
+            }
+        ]
+    },
+    {
+        "name": "Watchdog",
+        "lexicon": "prometheus",
+        "description": "An always-firing alert and the route that sends it to a heartbeat receiver.",
+        "bundles": [
+            "Receiver",
+            "Route",
+            "RuleGroup"
+        ],
+        "params": [
+            {
+                "name": "receiver",
+                "type": "ReceiverEntity | ReceiverProps | string",
+                "required": false,
+                "description": "Where the heartbeat goes: a declared `Receiver`, a receiver's props (the composite declares it), or the name of one declared elsewhere."
+            },
+            {
+                "name": "urlFile",
+                "type": "string",
+                "required": false,
+                "description": "The file the default receiver's webhook reads its URL from (default `/etc/alertmanager/secrets/heartbeat-url`)."
+            },
+            {
+                "name": "alert",
+                "type": "string",
+                "required": false,
+                "description": "The alert's name (default `Watchdog`)."
+            },
+            {
+                "name": "severity",
+                "type": "string",
+                "required": false,
+                "description": "The alert's `severity` label (default `none`): no severity route takes it."
+            },
+            {
+                "name": "repeatInterval",
+                "type": "string",
+                "required": false,
+                "description": "How often the heartbeat is sent again while the alert fires (default `1m`)."
+            },
+            {
+                "name": "group",
+                "type": "string",
+                "required": false,
+                "description": "The rule group's name (default `watchdog`)."
+            },
+            {
+                "name": "labels",
+                "type": "LabelSet",
+                "required": false,
+                "description": "More labels on the alert."
+            },
+            {
+                "name": "annotations",
+                "type": "LabelSet",
+                "required": false,
+                "description": "More annotations on the alert."
+            }
+        ]
+    }
+];