@intentius/chant-lexicon-prometheus 0.106.4 → 0.108.0

package/src/api/ruler.ts

@@ -0,0 +1,168 @@
+/**
+ * The ruler APIs (#3371): Mimir's, Cortex's and Loki's rule group config
+ * API, and the Prometheus-compatible `/api/v1/rules` every one of them (and
+ * a plain Prometheus) serves with each group's evaluation state.
+ *
+ * | Kind | Config API (YAML) | Evaluated rules (JSON) |
+ * |---|---|---|
+ * | `mimir` | `<prefix>/config/v1/rules[/{namespace}[/{group}]]` | `<prefix>/api/v1/rules` |
+ * | `cortex` | `/api/v1/rules[/{namespace}[/{group}]]` | `<prefix>/api/v1/rules` |
+ * | `loki` | `/loki/api/v1/rules[/{namespace}[/{group}]]` | `/prometheus/api/v1/rules` |
+ * | `prometheus` | none: rule files on disk | `/api/v1/rules` |
+ *
+ * `<prefix>` is `-http.prometheus-http-prefix`, `/prometheus` unless the
+ * profile says otherwise. Mimir does not register Cortex's `/api/v1/rules`
+ * config path. Every request carries the target's `X-Scope-OrgID`.
+ *
+ * The writes (`setGroup`, `deleteGroup`) are here for the ruler apply
+ * target (#3372), which reuses this module through
+ * `@intentius/chant-lexicon-prometheus/api`. Nothing here deletes a
+ * namespace: an apply bounded by the declared namespaces deletes groups one
+ * by one, and only inside {@link declaredNamespaces}.
+ */
+
+import { emitYaml } from "../build";
+import type { RuleGroupConfig } from "../model";
+import { loadPrometheusYaml } from "../import/parser";
+import type { RulerTarget } from "../config";
+import { PromApiError, type PromClient } from "./client";
+
+type Json = Record<string, unknown>;
+
+/** A rule group as the ruler's config API returns it: the rule-file shape, plus whatever the ruler adds (`source_tenants`, say). */
+export type RawRuleGroup = Json & { name: string };
+
+/** One rule as `/api/v1/rules` reports it. */
+export interface EvaluatedRule {
+  type: "alerting" | "recording";
+  name: string;
+  query: string;
+  /** Seconds. Alerting rules only. */
+  duration?: number;
+  /** Seconds. Alerting rules only. */
+  keepFiringFor?: number;
+  labels?: Record<string, string>;
+  annotations?: Record<string, string>;
+  health?: "ok" | "err" | "unknown" | string;
+  lastError?: string;
+  /** `inactive`, `pending` or `firing`. Alerting rules only. */
+  state?: string;
+  lastEvaluation?: string;
+  evaluationTime?: number;
+}
+
+/** One group as `/api/v1/rules` reports it. `file` is the rule file on a Prometheus, the namespace on a ruler. */
+export interface EvaluatedGroup {
+  name: string;
+  file: string;
+  /** Seconds. */
+  interval?: number;
+  limit?: number;
+  rules: EvaluatedRule[];
+  lastEvaluation?: string;
+  evaluationTime?: number;
+}
+
+function isObject(v: unknown): v is Json {
+  return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+
+const enc = encodeURIComponent;
+
+/** A bound ruler: the transport and the target it reaches. */
+export class RulerApi {
+  constructor(
+    readonly client: PromClient,
+    readonly target: RulerTarget,
+  ) {}
+
+  /** The base of the rule group config API, or undefined for a plain Prometheus, which has none. */
+  get configPath(): string | undefined {
+    switch (this.target.kind) {
+      case "mimir":
+        return `${this.target.prometheusPrefix}/config/v1/rules`;
+      case "cortex":
+        return "/api/v1/rules";
+      case "loki":
+        return "/loki/api/v1/rules";
+      case "prometheus":
+        return undefined;
+    }
+  }
+
+  /** The Prometheus-compatible rules endpoint, with evaluation state. */
+  get evaluatedPath(): string {
+    return `${this.target.prometheusPrefix}/api/v1/rules`;
+  }
+
+  private requireConfigPath(): string {
+    const p = this.configPath;
+    if (p === undefined) throw new Error(`a plain Prometheus (${this.target.source}) has no rule group API; its groups come from rule files`);
+    return p;
+  }
+
+  private parseNamespaces(text: string, path: string): Record<string, RawRuleGroup[]> {
+    const doc = loadPrometheusYaml(text);
+    if (!isObject(doc)) throw new PromApiError(502, "GET", path, "the answer is not a YAML mapping of namespaces to rule groups");
+    const out: Record<string, RawRuleGroup[]> = {};
+    for (const [ns, groups] of Object.entries(doc)) {
+      out[ns] = (Array.isArray(groups) ? groups : []).filter((g): g is RawRuleGroup => isObject(g) && typeof g.name === "string");
+    }
+    return out;
+  }
+
+  /** Every namespace the tenant has, with its groups. A 404 (Mimir: "no rule groups found") is none. */
+  async listNamespaces(): Promise<Record<string, RawRuleGroup[]>> {
+    const path = this.requireConfigPath();
+    const text = await this.client.getText(path);
+    return text === undefined ? {} : this.parseNamespaces(text, path);
+  }
+
+  /** One namespace's groups; undefined when the ruler has no such namespace. */
+  async readNamespace(namespace: string): Promise<RawRuleGroup[] | undefined> {
+    const path = `${this.requireConfigPath()}/${enc(namespace)}`;
+    const text = await this.client.getText(path);
+    if (text === undefined) return undefined;
+    return this.parseNamespaces(text, path)[namespace] ?? [];
+  }
+
+  /** The path {@link readGroup} reads, for `queried`. */
+  groupPath(namespace: string, group: string): string {
+    return `${this.requireConfigPath()}/${enc(namespace)}/${enc(group)}`;
+  }
+
+  /** One group; undefined when it is not there. */
+  async readGroup(namespace: string, group: string): Promise<RawRuleGroup | undefined> {
+    const path = this.groupPath(namespace, group);
+    const text = await this.client.getText(path);
+    if (text === undefined) return undefined;
+    const doc = loadPrometheusYaml(text);
+    if (!isObject(doc) || typeof doc.name !== "string") throw new PromApiError(502, "GET", path, "the answer is not a rule group");
+    return doc as RawRuleGroup;
+  }
+
+  /** Create or replace one group in `namespace` (202 on Mimir, Cortex and Loki). */
+  async setGroup(namespace: string, group: RuleGroupConfig | RawRuleGroup): Promise<void> {
+    await this.client.send("POST", `${this.requireConfigPath()}/${enc(namespace)}`, { text: emitYaml(group), contentType: "application/yaml" });
+  }
+
+  /** Delete one group from `namespace`. */
+  async deleteGroup(namespace: string, group: string): Promise<void> {
+    await this.client.send("DELETE", this.groupPath(namespace, group));
+  }
+
+  /** Every group the ruler evaluates, with its health. Read once per client. */
+  async evaluated(): Promise<EvaluatedGroup[]> {
+    return this.client.once("evaluated", async () => {
+      const path = this.evaluatedPath;
+      const body = await this.client.getJson<{ status?: string; data?: { groups?: unknown[] } }>(path);
+      if (body === undefined) throw new PromApiError(404, "GET", path, "the rules API is not served here");
+      if (body.status !== undefined && body.status !== "success") throw new PromApiError(502, "GET", path, `status ${body.status}`);
+      return (body.data?.groups ?? []).filter((g): g is EvaluatedGroup => isObject(g) && typeof g.name === "string").map((g) => ({
+        ...g,
+        file: typeof g.file === "string" ? g.file : "",
+        rules: Array.isArray(g.rules) ? g.rules.filter(isObject) as unknown as EvaluatedRule[] : [],
+      }));
+    });
+  }
+}

package/src/api/testdata/am-status-original.yml

@@ -0,0 +1,85 @@
+global:
+  resolve_timeout: 5m
+  http_config:
+    follow_redirects: true
+    enable_http2: true
+  smtp_from: am@example.com
+  smtp_hello: localhost
+  smtp_smarthost: smtp.example.com:587
+  smtp_require_tls: true
+  smtp_tls_config:
+    insecure_skip_verify: false
+  pagerduty_url: https://events.pagerduty.com/v2/enqueue
+  opsgenie_api_url: https://api.opsgenie.com/
+  wechat_api_url: https://qyapi.weixin.qq.com/cgi-bin/
+  victorops_api_url: https://alert.victorops.com/integrations/generic/20131114/alert/
+  telegram_api_url: https://api.telegram.org
+  webex_api_url: https://webexapis.com/v1/messages
+  rocketchat_api_url: https://open.rocket.chat/
+  slack_app_url: https://slack.com/api/chat.postMessage
+route:
+  receiver: team
+  group_by:
+  - alertname
+  continue: false
+  routes:
+  - receiver: pager
+    matchers:
+    - severity="page"
+    continue: false
+    mute_time_intervals:
+    - weekends
+inhibit_rules:
+- source_matchers:
+  - severity="page"
+  target_matchers:
+  - severity="ticket"
+  equal:
+  - alertname
+receivers:
+- name: team
+  webhook_configs:
+  - send_resolved: true
+    http_config:
+      follow_redirects: true
+      enable_http2: true
+    url: <secret>
+    url_file: ""
+    max_alerts: 0
+    timeout: 0
+- name: pager
+  email_configs:
+  - send_resolved: false
+    to: oncall@example.com
+    from: am@example.com
+    hello: localhost
+    smarthost: smtp.example.com:587
+    headers:
+      Subject: '{{ template "email.default.subject" . }}'
+    html: '{{ template "email.default.html" . }}'
+    require_tls: false
+    tls_config:
+      insecure_skip_verify: false
+  pagerduty_configs:
+  - send_resolved: true
+    http_config:
+      follow_redirects: true
+      enable_http2: true
+    routing_key: <secret>
+    url: https://events.pagerduty.com/v2/enqueue
+    client: '{{ template "pagerduty.default.client" . }}'
+    client_url: '{{ template "pagerduty.default.clientURL" . }}'
+    description: '{{ template "pagerduty.default.description" .}}'
+    details:
+      firing: '{{ .Alerts.Firing | toJson }}'
+      num_firing: '{{ .Alerts.Firing | len }}'
+      num_resolved: '{{ .Alerts.Resolved | len }}'
+      resolved: '{{ .Alerts.Resolved | toJson }}'
+      team: payments
+templates: []
+time_intervals:
+- name: weekends
+  time_intervals:
+  - weekdays:
+    - saturday
+    - sunday

package/src/composites/alert-routing.test.ts

@@ -0,0 +1,163 @@
+/**
+ * `AlertRouting`: the routing tree, the inhibit rules between levels and
+ * the receivers it declares, and that the severities `Slo`, `GenAiRules`,
+ * `RedAlerts` and `Watchdog` write all find a route (PROM202).
+ * `amtool check-config` runs when amtool is on PATH.
+ */
+import { describe, expect, test } from "vitest";
+import type { Declarable } from "@intentius/chant/declarable";
+import { genAiMetrics } from "@intentius/chant-lexicon-otel/genai";
+import { spanMetricsNames } from "@intentius/chant-lexicon-otel/metric-names";
+import { AlertRouting, ALERT_ROUTING_LEVELS, type AlertRoutingProps } from "./alert-routing";
+import { Slo } from "./slo";
+import { GenAiRules } from "./genai";
+import { RedAlerts } from "./red-alerts";
+import { Watchdog } from "./watchdog";
+import { Receiver, isInhibitRule, isReceiver } from "../alertmanager";
+import { alertmanagerYaml, buildAlertmanagerConfig, buildRuleFile } from "../build";
+import { validateAlertmanagerConfig, validateSeverityRouting } from "../validate-config";
+import { amtoolCheckConfig, hasTool } from "../tools";
+
+const hasAmtool = hasTool(process.env.AMTOOL ?? "amtool");
+
+const values = (...instances: Array<{ members: Record<string, unknown> }>) =>
+  instances.flatMap((i) => Object.values(i.members)) as Declarable[];
+
+function built(props: AlertRoutingProps = {}, ...more: Array<{ members: Record<string, unknown> }>) {
+  return buildAlertmanagerConfig(values(AlertRouting(props), ...more));
+}
+
+const pager = { name: "pager", pagerduty_configs: [{ routing_key_file: "/etc/alertmanager/secrets/pagerduty" }] };
+const chat = { name: "chat", slack_configs: [{ api_url_file: "/etc/alertmanager/secrets/slack", channel: "#alerts" }] };
+
+describe("AlertRouting defaults", () => {
+  const { config, warnings } = built();
+
+  test("a root route to a default receiver that drops, and one child route per level", () => {
+    expect(warnings).toEqual([]);
+    expect(config.receivers).toEqual([{ name: "default" }]);
+    expect(config.route).toMatchObject({ receiver: "default", group_by: ["alertname"], group_wait: "30s", group_interval: "5m", repeat_interval: "4h" });
+    expect(config.route?.routes).toEqual([
+      { receiver: "default", matchers: ['severity=~"critical|page"'] },
+      { receiver: "default", matchers: ['severity=~"warning|ticket"'] },
+      { receiver: "default", matchers: ['severity="info"'] },
+    ]);
+  });
+
+  test("each level holds back the levels below it, for the same alert", () => {
+    const equal = ["alertname", "slo", "service_name", "team"];
+    expect(config.inhibit_rules).toEqual([
+      { name: "critical holds back warning", source_matchers: ['severity=~"critical|page"'], target_matchers: ['severity=~"warning|ticket"'], equal },
+      { name: "critical holds back info", source_matchers: ['severity=~"critical|page"'], target_matchers: ['severity="info"'], equal },
+      { name: "warning holds back info", source_matchers: ['severity=~"warning|ticket"'], target_matchers: ['severity="info"'], equal },
+    ]);
+  });
+
+  test("passes the Alertmanager checks", () => {
+    expect(validateAlertmanagerConfig(config)).toEqual([]);
+  });
+
+  test("the default levels are the severities the lexicon's composites write", () => {
+    expect(ALERT_ROUTING_LEVELS.map((l) => l.severities)).toEqual([["critical", "page"], ["warning", "ticket"], ["info"]]);
+  });
+});
+
+describe("AlertRouting with the lexicon's alerting composites", () => {
+  const slo = Slo({
+    name: "checkout",
+    objective: 0.995,
+    window: "28d",
+    sli: {
+      good: 'sum(rate(traces_span_metrics_calls_total{span_name="checkout",status_code!="STATUS_CODE_ERROR"}[{{window}}]))',
+      total: 'sum(rate(traces_span_metrics_calls_total{span_name="checkout"}[{{window}}]))',
+    },
+    alerting: { page: { burnRates: "default" }, ticket: { burnRates: "default" } },
+  });
+  const genai = GenAiRules({ genAi: genAiMetrics(), alerts: { errorRatio: true, latency: { severity: "critical" } } });
+  const red = RedAlerts({ spanMetrics: spanMetricsNames({}) });
+  const watchdog = Watchdog({});
+  const rules = buildRuleFile([slo.rules, genai.rules, red.rules, watchdog.rules]).config;
+
+  test("every severity they write is routed (PROM202), with the watchdog first", () => {
+    const am = built({ receiver: chat, levels: [{ ...ALERT_ROUTING_LEVELS[0], receiver: pager }, ...ALERT_ROUTING_LEVELS.slice(1)], routes: [watchdog.route] }, watchdog);
+    expect(am.warnings).toEqual([]);
+    expect(validateSeverityRouting([rules], am.config)).toEqual([]);
+    expect(validateAlertmanagerConfig(am.config)).toEqual([]);
+    expect(am.config.route?.routes?.map((r) => r.receiver)).toEqual(["heartbeat", "pager", "chat", "chat"]);
+    expect(am.config.receivers?.map((r) => r.name)).toEqual(["chat", "heartbeat", "pager"]);
+  });
+
+  test.skipIf(!hasAmtool)("amtool check-config passes", () => {
+    const yaml = alertmanagerYaml(values(AlertRouting({ receiver: chat, routes: [watchdog.route] }), watchdog));
+    const r = amtoolCheckConfig(yaml);
+    expect(r.ran).toBe(true);
+    expect(r.ok, r.output).toBe(true);
+  });
+});
+
+describe("AlertRouting teams", () => {
+  test("a route per team before the levels, with a level of its own sent elsewhere", () => {
+    const dbPager = { name: "db-pager", pagerduty_configs: [{ routing_key_file: "/etc/alertmanager/secrets/db" }] };
+    const { config } = built({
+      receiver: chat,
+      teams: [{ team: "db", receiver: "db-chat", levels: { critical: dbPager }, repeatInterval: "1h" }],
+    });
+    expect(config.route?.routes?.[0]).toEqual({
+      receiver: "db-chat",
+      matchers: ['team="db"'],
+      repeat_interval: "1h",
+      routes: [{ receiver: "db-pager", matchers: ['severity=~"critical|page"'] }],
+    });
+    expect(config.route?.routes).toHaveLength(4);
+    expect(config.receivers?.map((r) => r.name)).toEqual(["chat", "db-pager"]);
+  });
+
+  test("teamLabel changes the label team routes match on, and inhibit equality with it", () => {
+    const { config } = built({ teamLabel: "owner", teams: [{ team: "web", receiver: chat }] });
+    expect(config.route?.routes?.[0].matchers).toEqual(['owner="web"']);
+    expect(config.inhibit_rules?.[0].equal).toEqual(["alertname", "slo", "service_name", "owner"]);
+  });
+});
+
+describe("AlertRouting options", () => {
+  test("levels of your own, timing, group_by, no inhibition", () => {
+    const routing = AlertRouting({
+      levels: [{ name: "sev1", severities: ["sev1"], receiver: pager, groupWait: "10s" }, { name: "sev2", severities: ["sev2"] }],
+      groupBy: ["alertname", "service_name"],
+      repeatInterval: "12h",
+      inhibit: false,
+    });
+    expect(Object.values(routing.members).some(isInhibitRule)).toBe(false);
+    expect(Object.keys(routing.members).sort()).toEqual(["receiver_default", "receiver_pager", "route"]);
+    const { config } = buildAlertmanagerConfig(values(routing));
+    expect(config.route).toMatchObject({ group_by: ["alertname", "service_name"], repeat_interval: "12h" });
+    expect(config.route?.routes).toEqual([
+      { receiver: "pager", matchers: ['severity="sev1"'], group_wait: "10s" },
+      { receiver: "default", matchers: ['severity="sev2"'] },
+    ]);
+    expect(validateAlertmanagerConfig(config)).toEqual([]);
+  });
+
+  test("a declared receiver is referenced, not declared again; one props object used twice is one receiver", () => {
+    const declared = new Receiver(chat);
+    const routing = AlertRouting({ receiver: declared, levels: [{ name: "a", severities: ["a"], receiver: pager }, { name: "b", severities: ["b"], receiver: pager }] });
+    const receivers = Object.values(routing.members).filter(isReceiver);
+    expect(receivers.map((r) => r.props.name)).toEqual(["pager"]);
+    expect(buildAlertmanagerConfig([declared, ...values(routing)]).config.receivers?.map((r) => r.name)).toEqual(["chat", "pager"]);
+  });
+
+  test("bad props are refused", () => {
+    expect(() => AlertRouting({ levels: [{ name: "a", severities: ["x"] }, { name: "b", severities: ["x"] }] })).toThrow(
+      /AlertRouting: levels\[1\]: severity "x" is already in level "a"/,
+    );
+    expect(() => AlertRouting({ levels: [{ name: "a", severities: [] }] })).toThrow(/at least one severity/);
+    expect(() => AlertRouting({ levels: [{ name: "a", severities: ["x"] }, { name: "a", severities: ["y"] }] })).toThrow(/second level/);
+    expect(() => AlertRouting({ receiver: { name: "x" }, levels: [{ name: "a", severities: ["a"], receiver: { name: "x" } }] })).toThrow(
+      /two receivers are named "x"/,
+    );
+    expect(() => AlertRouting({ teams: [{ team: "db", receiver: chat, levels: { sev9: pager } }] })).toThrow(/"sev9", which is not a level/);
+    expect(() => AlertRouting({ teams: [{ team: "db", receiver: chat }, { team: "db", receiver: chat }] })).toThrow(/listed twice/);
+    expect(() => AlertRouting({ teamLabel: "team-name" })).toThrow(/teamLabel/);
+    expect(() => AlertRouting({ groupWait: "soon" })).toThrow(/groupWait/);
+  });
+});