@intentius/chant-lexicon-prometheus 0.96.0

package/src/composites/catalog.ts

@@ -0,0 +1,65 @@
+// Generated by scripts/generate-composite-catalogs.ts (#2662) from this
+// lexicon's composite exports. Do not edit by hand: run
+// `npm run generate:composite-catalogs`. catalog.test.ts holds it to the exports.
+import type { CompositeEntry } from "@intentius/chant/lexicon";
+
+export const compositeCatalog: CompositeEntry[] = [
+  {
+    "name": "Slo",
+    "lexicon": "prometheus",
+    "description": "An SLO, built to recording rules and multiwindow multi-burn-rate alerts.",
+    "bundles": [
+      "RuleGroup"
+    ],
+    "params": [
+      {
+        "name": "name",
+        "type": "string",
+        "required": true,
+        "description": "The SLO's name, the `slo` label on every series and alert it builds and part of its group names."
+      },
+      {
+        "name": "objective",
+        "type": "number",
+        "required": true,
+        "description": "The target share of good events, strictly between 0 and 1, e.g. `0.995`."
+      },
+      {
+        "name": "window",
+        "type": "string",
+        "required": true,
+        "description": "The rolling window the objective holds over, a Prometheus duration, e.g. `28d` or `30d`."
+      },
+      {
+        "name": "sli",
+        "type": "SloSli",
+        "required": true,
+        "description": "The SLI: good (or error) events over total events, with `{{window}}` for the range."
+      },
+      {
+        "name": "description",
+        "type": "string",
+        "required": false,
+        "description": "A sentence for the alerts' description, e.g. what the SLO measures."
+      },
+      {
+        "name": "alerting",
+        "type": "SloAlerting",
+        "required": false,
+        "description": "Burn-rate alerts (default: both tiers with the Workbook's windows)."
+      },
+      {
+        "name": "labels",
+        "type": "LabelSet",
+        "required": false,
+        "description": "Labels added to every rule, e.g. `team` or `service`."
+      },
+      {
+        "name": "interval",
+        "type": "string",
+        "required": false,
+        "description": "Evaluation interval of the group (default: Prometheus's `evaluation_interval`)."
+      }
+    ]
+  }
+];

package/src/composites/composites.test.ts

@@ -0,0 +1,256 @@
+/**
+ * The `Slo` composite: validation, the rules it builds, and `sloMetrics()`.
+ * The burn-rate behaviour over synthetic series is in slo-burn.test.ts.
+ */
+import { describe, expect, test } from "vitest";
+import { load } from "js-yaml";
+import { Slo, sloMetrics, sloPropsProblem, type SloProps } from "./slo";
+import { ruleGroupConfig } from "../rules";
+import { ruleFileYaml } from "../build";
+import { validateRuleFile, validateSeverityRouting } from "../validate-config";
+import { hasTool, promtoolCheckRules } from "../tools";
+import type { AlertingRuleConfig, RecordingRuleConfig, RuleFileConfig } from "../model";
+
+const PROMTOOL = process.env.PROMTOOL ?? "promtool";
+const hasPromtool = hasTool(PROMTOOL);
+
+const orderAck: SloProps = {
+  name: "order-acknowledged",
+  objective: 0.995,
+  window: "28d",
+  sli: {
+    good: 'sum(rate(traces_span_metrics_calls_total{span_name="order.ack",status_code!="STATUS_CODE_ERROR"}[{{window}}]))',
+    total: 'sum(rate(traces_span_metrics_calls_total{span_name="order.ack"}[{{window}}]))',
+  },
+  alerting: { page: { burnRates: "default" }, ticket: { burnRates: "default" } },
+};
+
+type SloResult = ReturnType<typeof Slo>;
+
+function file(slo: SloResult): RuleFileConfig {
+  return load(ruleFileYaml([slo.rules])) as RuleFileConfig;
+}
+
+function recordings(slo: SloResult): RecordingRuleConfig[] {
+  return ruleGroupConfig(slo.rules).rules.filter((r): r is RecordingRuleConfig => "record" in r);
+}
+
+function alertsOf(slo: SloResult): AlertingRuleConfig[] {
+  return ruleGroupConfig(slo.rules).rules.filter((r): r is AlertingRuleConfig => "alert" in r);
+}
+
+describe("Slo validation", () => {
+  test.each([0, 1, -0.1, 1.5, Number.NaN])("rejects objective %s", (objective) => {
+    expect(() => Slo({ ...orderAck, objective })).toThrow(/objective must be strictly between 0 and 1/);
+  });
+
+  test.each(["", "28 days", "1.5d", "0", "30m1h"])("rejects window %j", (window) => {
+    expect(() => Slo({ ...orderAck, window })).toThrow(/window must be a positive Prometheus duration/);
+  });
+
+  test("rejects an SLI without the window placeholder, or one that does not parse", () => {
+    expect(() => Slo({ ...orderAck, sli: { good: "sum(rate(x[5m]))", total: "sum(rate(y[{{window}}]))" } })).toThrow(
+      /sli.good must contain \{\{window\}\}/,
+    );
+    expect(() => Slo({ ...orderAck, sli: { errors: "sum(rate(x[{{window}}])", total: "sum(rate(y[{{window}}]))" } })).toThrow(
+      /sli.errors is not valid PromQL/,
+    );
+    expect(() => Slo({ ...orderAck, sli: { total: "sum(rate(y[{{window}}]))" } as never })).toThrow(/good and total, or errors and total/);
+  });
+
+  test("rejects a long window longer than the SLO window, and a short window that is not shorter", () => {
+    expect(() => Slo({ ...orderAck, window: "2d" })).toThrow(/the long window 3d is longer than the SLO window 2d/);
+    expect(() =>
+      Slo({ ...orderAck, alerting: { ticket: false, page: { burnRates: [{ long: "1h", short: "1h", factor: 10 }] } } }),
+    ).toThrow(/must be shorter/);
+    expect(() =>
+      Slo({ ...orderAck, alerting: { ticket: false, page: { burnRates: [{ long: "1h", short: "5m", factor: 10, budgetConsumed: 0.02 }] } } }),
+    ).toThrow(/exactly one of budgetConsumed and factor/);
+  });
+
+  test("rejects a burn rate whose threshold is an error ratio of 1 or more", () => {
+    expect(() =>
+      Slo({ ...orderAck, objective: 0.9, alerting: { ticket: false, page: { burnRates: [{ long: "1h", short: "5m", factor: 10 }] } } }),
+    ).toThrow(/can never happen/);
+  });
+
+  test("rejects a name that cannot go in a label and a group name", () => {
+    expect(() => Slo({ ...orderAck, name: 'bad"name' })).toThrow(/name must be/);
+  });
+
+  test("sloPropsProblem accepts the issue's example", () => {
+    expect(sloPropsProblem(orderAck)).toBeUndefined();
+  });
+});
+
+describe("the rules an Slo builds", () => {
+  const slo = Slo(orderAck);
+
+  test("one group, clean under every PROM1xx check, and routable by severity", () => {
+    const f = file(slo);
+    expect(f.groups.map((g) => g.name)).toEqual(["slo-order-acknowledged"]);
+    expect(validateRuleFile(f)).toEqual([]);
+    const routes = { route: { receiver: "d", routes: [{ matchers: ['severity="page"'], receiver: "d" }, { matchers: ['severity="ticket"'], receiver: "d" }] }, receivers: [{ name: "d" }] };
+    expect(validateSeverityRouting([f], routes)).toEqual([]);
+    expect(validateSeverityRouting([f], { route: { receiver: "d", routes: [{ matchers: ['severity="page"'], receiver: "d" }] }, receivers: [{ name: "d" }] })).toHaveLength(1);
+  });
+
+  test("an error-ratio series per window, the window's own ratio, the objective and the budget left", () => {
+    const rec = recordings(slo);
+    expect(rec.map((r) => r.record)).toEqual([
+      "slo:sli_error:ratio_rate5m",
+      "slo:sli_error:ratio_rate30m",
+      "slo:sli_error:ratio_rate1h",
+      "slo:sli_error:ratio_rate2h",
+      "slo:sli_error:ratio_rate6h",
+      "slo:sli_error:ratio_rate1d",
+      "slo:sli_error:ratio_rate3d",
+      "slo:sli_error:ratio_rate28d",
+      "slo:objective:ratio",
+      "slo:error_budget:remaining",
+    ]);
+    for (const r of rec) expect(r.labels).toEqual({ slo: "order-acknowledged" });
+    expect(rec[2].expr).toContain('[1h]');
+    expect(rec[2].expr).not.toContain("{{window}}");
+    expect(rec[7].expr).toBe('avg_over_time(slo:sli_error:ratio_rate5m{slo="order-acknowledged"}[28d])');
+    expect(rec[9].expr).toBe('1 - (slo:sli_error:ratio_rate28d{slo="order-acknowledged"} / 0.005)');
+  });
+
+  test("one alert per window pair, factors scaled from 30 days to 28", () => {
+    const alerts = alertsOf(slo);
+    expect(alerts.map((a) => [a.alert, a.labels!.severity, a.labels!.long_window, a.labels!.short_window])).toEqual([
+      ["ErrorBudgetBurn", "page", "1h", "5m"],
+      ["ErrorBudgetBurn", "page", "6h", "30m"],
+      ["ErrorBudgetBurn", "ticket", "1d", "2h"],
+      ["ErrorBudgetBurn", "ticket", "3d", "6h"],
+    ]);
+    expect(alerts[0].expr).toContain('slo:sli_error:ratio_rate1h{slo="order-acknowledged"} > (13.44 * 0.005)');
+    expect(alerts[0].expr).toContain('slo:sli_error:ratio_rate5m{slo="order-acknowledged"} > (13.44 * 0.005)');
+    expect(alerts[0].annotations!.summary).toContain("13.44x");
+    expect(alerts.every((a) => a.for === undefined)).toBe(true);
+  });
+
+  test("a 30-day window keeps the Workbook's factors exactly", () => {
+    const m = sloMetrics({ ...orderAck, window: "30d" });
+    expect(m.burnRates.map((b) => b.factor)).toEqual([14.4, 6, 3, 1]);
+    expect(m.burnRates.map((b) => b.exhaustsIn)).toEqual(["2d2h", "5d", "1w3d", "4w2d"]);
+  });
+
+  test("tiers take their own severity, labels, annotations and for; alertName and group options apply", () => {
+    const custom = Slo({
+      ...orderAck,
+      name: "checkout",
+      labels: { team: "payments" },
+      interval: "30s",
+      alerting: {
+        alertName: "CheckoutBudgetBurn",
+        page: { severity: "critical", for: "2m", labels: { pager: "yes" }, annotations: { runbook_url: "https://runbooks/checkout" } },
+        ticket: false,
+      },
+    });
+    const g = ruleGroupConfig(custom.rules);
+    expect(g.interval).toBe("30s");
+    expect(g.labels).toEqual({ team: "payments" });
+    expect(alertsOf(custom)).toHaveLength(2);
+    const a = alertsOf(custom)[0];
+    expect(a.alert).toBe("CheckoutBudgetBurn");
+    expect(a.for).toBe("2m");
+    expect(a.labels).toMatchObject({ severity: "critical", pager: "yes", slo: "checkout" });
+    expect(a.annotations!.runbook_url).toBe("https://runbooks/checkout");
+    expect(sloMetrics(custom).windows).toEqual(["5m", "30m", "1h", "6h", "28d"]);
+  });
+
+  test("custom pairs: a factor is used as written, a budget share is scaled", () => {
+    const m = sloMetrics({
+      ...orderAck,
+      window: "7d",
+      alerting: { page: { burnRates: [{ long: "2h", short: "10m", factor: 10 }] }, ticket: { burnRates: [{ long: "1d", short: "2h", budgetConsumed: 0.2 }] } },
+    });
+    expect(m.burnRates.map((b) => b.factor)).toEqual([10, 1.4]);
+  });
+
+  test("errors/total SLIs record the ratio directly; no alerting records the window from the SLI", () => {
+    const direct = Slo({
+      ...orderAck,
+      name: "direct",
+      sli: { errors: "sum(rate(e_total[{{window}}]))", total: "sum(rate(t_total[{{window}}]))" },
+      alerting: { page: false, ticket: false },
+    });
+    expect(alertsOf(direct)).toEqual([]);
+    const rec = recordings(direct);
+    expect(rec.map((r) => r.record)).toEqual(["slo:sli_error:ratio_rate28d", "slo:objective:ratio", "slo:error_budget:remaining"]);
+    expect(rec[0].expr).toBe("(sum(rate(e_total[28d])))\n/\n(sum(rate(t_total[28d])))");
+    expect(sloMetrics(direct).group).toBe("slo-direct");
+    expect(sloMetrics(direct).burnRates).toEqual([]);
+  });
+
+  test("the same Slo renders inside the rule file promtool accepts", () => {
+    const r = promtoolCheckRules(ruleFileYaml([slo.rules]), PROMTOOL);
+    if (!hasPromtool) {
+      expect(r.ran).toBe(false);
+      return;
+    }
+    expect(r.output).toContain("SUCCESS");
+    expect(r.ok).toBe(true);
+  });
+});
+
+describe("sloMetrics", () => {
+  const slo = Slo(orderAck);
+
+  test("names every recorded series, the thresholds and the groups, for dashboards", () => {
+    const m = sloMetrics(slo);
+    expect(m).toMatchObject({
+      name: "order-acknowledged",
+      objective: 0.995,
+      errorBudget: 0.005,
+      window: "28d",
+      labels: { slo: "order-acknowledged" },
+      selector: '{slo="order-acknowledged"}',
+      windows: ["5m", "30m", "1h", "2h", "6h", "1d", "3d", "28d"],
+      windowErrorRatio: "slo:sli_error:ratio_rate28d",
+      errorBudgetRemaining: "slo:error_budget:remaining",
+      objectiveRatio: "slo:objective:ratio",
+      alertName: "ErrorBudgetBurn",
+      group: "slo-order-acknowledged",
+    });
+    expect(m.errorRatio["1h"]).toBe("slo:sli_error:ratio_rate1h");
+    expect(m.burnRates[0]).toEqual({
+      tier: "page",
+      severity: "page",
+      long: "1h",
+      short: "5m",
+      factor: 13.44,
+      threshold: 0.0672,
+      longRecord: "slo:sli_error:ratio_rate1h",
+      shortRecord: "slo:sli_error:ratio_rate5m",
+      alert: "ErrorBudgetBurn",
+      labels: { slo: "order-acknowledged", severity: "page", long_window: "1h", short_window: "5m" },
+      exhaustsIn: "2d2h",
+    });
+  });
+
+  test("every name it returns is recorded by the rules, so a dashboard never reads a missing series", () => {
+    const m = sloMetrics(slo);
+    const recorded = new Set(recordings(slo).map((r) => r.record));
+    for (const name of [...Object.values(m.errorRatio), m.windowErrorRatio, m.errorBudgetRemaining, m.objectiveRatio]) {
+      expect(recorded.has(name), name).toBe(true);
+    }
+    const alerts = alertsOf(slo);
+    m.burnRates.forEach((b, i) => {
+      expect(alerts[i].labels).toEqual(b.labels);
+      expect(alerts[i].expr).toContain(`${b.longRecord}${m.selector} >`);
+    });
+  });
+
+  test("reads the same from the instance, its group, or the props", () => {
+    expect(sloMetrics(slo.rules)).toEqual(sloMetrics(slo));
+    expect(sloMetrics(orderAck)).toEqual(sloMetrics(slo));
+  });
+
+  test("returns a copy, so a caller cannot change what the next caller reads", () => {
+    const m = sloMetrics(slo);
+    m.errorRatio["1h"] = "changed";
+    expect(sloMetrics(slo).errorRatio["1h"]).toBe("slo:sli_error:ratio_rate1h");
+  });
+});

package/src/composites/index.ts

@@ -0,0 +1,23 @@
+/**
+ * Prometheus lexicon composites: declarations that expand to rule groups.
+ */
+
+export {
+  Slo,
+  sloMetrics,
+  sloPropsProblem,
+  sliExprProblem,
+  DEFAULT_BURN_RATES,
+  SLO_WINDOW_PLACEHOLDER,
+} from "./slo";
+export type {
+  SloProps,
+  SloSli,
+  SloAlerting,
+  SloAlertTier,
+  BurnRateWindow,
+  SloMembers,
+  SloInstance,
+  SloMetrics,
+  SloBurnRate,
+} from "./slo";

package/src/composites/slo-burn.test.ts

@@ -0,0 +1,289 @@
+/**
+ * Burn-rate alerts over synthetic series: each window pair fires at its
+ * burn rate and not below it.
+ *
+ * A scenario is a request counter split into good (`code="200"`) and bad
+ * (`code="500"`) series, healthy for as long as the pair's long window and
+ * then failing at a constant multiple of the error budget. The rule file an
+ * `Slo` builds is evaluated over it step by step:
+ *
+ * - by the small evaluator in `rule-eval.ts`, which always runs, and
+ * - by `promtool test rules`, when promtool is on PATH (or named by
+ *   $PROMTOOL), against expectations the evaluator produced, so the two
+ *   engines are held to the same answer.
+ */
+import { describe, expect, test } from "vitest";
+import { dump } from "js-yaml";
+import { Slo, sloMetrics, type SloBurnRate, type SloProps } from "./slo";
+import { ruleGroupConfig } from "../rules";
+import { ruleFileYaml } from "../build";
+import { RuleEvaluator, type FiringAlert } from "../rule-eval";
+import { durationMs } from "../duration";
+import { hasTool, promtoolTestRules } from "../tools";
+import type { AlertingRuleConfig } from "../model";
+
+const PROMTOOL = process.env.PROMTOOL ?? "promtool";
+const hasPromtool = hasTool(PROMTOOL);
+const MIN = 60_000;
+/** Requests per minute. */
+const RATE = 1000;
+
+function props(window: string): SloProps {
+  return {
+    name: "checkout",
+    objective: 0.995,
+    window,
+    sli: {
+      good: 'sum(rate(requests_total{code!~"5.."}[{{window}}]))',
+      total: "sum(rate(requests_total[{{window}}]))",
+    },
+  };
+}
+
+interface Scenario {
+  /** Sample and evaluation interval, in minutes. */
+  step: number;
+  /** Healthy minutes before the burn starts. */
+  healthy: number;
+  /** Minutes of burn. */
+  burning: number;
+  /** Minutes of health after the burn. */
+  recovered?: number;
+  /** The burn rate: a multiple of the error budget. */
+  burn: number;
+}
+
+/** Bad requests in the minute ending at `minute`. */
+function badPerStep(s: Scenario, stepEnd: number, budget: number): number {
+  const burning = stepEnd > s.healthy && stepEnd <= s.healthy + s.burning;
+  return burning ? RATE * s.step * s.burn * budget : 0;
+}
+
+/** Firing alerts for one pair's labels at each evaluation, by minute. */
+function run(slo: ReturnType<typeof Slo>, s: Scenario): Map<number, FiringAlert[]> {
+  const m = sloMetrics(slo);
+  const ev = new RuleEvaluator([ruleGroupConfig(slo.rules)]);
+  const total = s.healthy + s.burning + (s.recovered ?? 0);
+  let good = 0;
+  let bad = 0;
+  const out = new Map<number, FiringAlert[]>();
+  for (let minute = 0; minute <= total; minute += s.step) {
+    if (minute > 0) {
+      const b = badPerStep(s, minute, m.errorBudget);
+      bad += b;
+      good += RATE * s.step - b;
+    }
+    ev.add({ __name__: "requests_total", code: "200" }, minute * MIN, good);
+    ev.add({ __name__: "requests_total", code: "500" }, minute * MIN, bad);
+    out.set(minute, ev.step(minute * MIN));
+  }
+  return out;
+}
+
+function isPair(a: FiringAlert, b: SloBurnRate): boolean {
+  return a.labels.alertname === b.alert && a.labels.long_window === b.long && a.labels.short_window === b.short;
+}
+
+/** The first minute the pair fires, or undefined. */
+function firstFiring(timeline: Map<number, FiringAlert[]>, b: SloBurnRate, from = 0): number | undefined {
+  for (const [minute, alerts] of timeline) if (minute >= from && alerts.some((a) => isPair(a, b))) return minute;
+  return undefined;
+}
+
+const minutes = (d: string) => durationMs(d)! / MIN;
+
+describe("each pair fires at its burn rate and not below it (rule-eval)", () => {
+  for (const window of ["30d", "28d"]) {
+    const slo = Slo(props(window));
+    const m = sloMetrics(slo);
+    for (const b of m.burnRates) {
+      const long = minutes(b.long);
+      const short = minutes(b.short);
+      // Five-minute steps for the ticket pairs keep days of samples quick.
+      const step = long >= 24 * 60 ? 5 : 1;
+
+      test(`${window} ${b.severity} ${b.long}/${b.short}: fires once the long window has burnt ${b.factor}x, then stops within the short window`, () => {
+        const s: Scenario = { step, healthy: long, burning: long, recovered: long, burn: b.factor * 1.25 };
+        const timeline = run(slo, s);
+        const fired = firstFiring(timeline, b);
+        // The long window's error ratio climbs linearly from the onset and crosses
+        // the threshold after long * factor / burn of it.
+        const expected = s.healthy + ((long - step) * b.factor) / s.burn;
+        expect(fired, "never fired").toBeDefined();
+        expect(Math.abs(fired! - expected)).toBeLessThanOrEqual(2 * step);
+        // Nothing before the onset.
+        expect(firstFiring(timeline, b)).toBeGreaterThan(s.healthy);
+        // Still firing at the end of the burn.
+        expect(timeline.get(s.healthy + s.burning)!.some((a) => isPair(a, b))).toBe(true);
+        // After the burn stops the long window is still above the threshold, but
+        // the short window drops below it, so the alert resets.
+        let stopped: number | undefined;
+        for (const [minute, alerts] of timeline) {
+          if (minute > s.healthy + s.burning && !alerts.some((a) => isPair(a, b))) {
+            stopped = minute;
+            break;
+          }
+        }
+        expect(stopped, "kept firing").toBeDefined();
+        expect(stopped! - (s.healthy + s.burning)).toBeLessThanOrEqual(short + step);
+      });
+
+      test(`${window} ${b.severity} ${b.long}/${b.short}: never fires at 0.9x its burn rate`, () => {
+        const s: Scenario = { step, healthy: long, burning: 2 * long, burn: b.factor * 0.9 };
+        const timeline = run(slo, s);
+        expect(firstFiring(timeline, b)).toBeUndefined();
+      });
+    }
+  }
+
+  test("a 28-day SLO pages at 14x, where a 30-day one does not: the factors scale with the window", () => {
+    for (const [window, fires] of [["28d", true], ["30d", false]] as const) {
+      const slo = Slo(props(window));
+      const b = sloMetrics(slo).burnRates[0];
+      const timeline = run(slo, { step: 1, healthy: 60, burning: 120, burn: 14 });
+      expect(firstFiring(timeline, b) !== undefined, window).toBe(fires);
+    }
+  });
+
+  test("error budget remaining is 1 minus the burn rate after a whole window of steady burn", () => {
+    const slo = Slo({ ...props("1d"), alerting: { page: { burnRates: [{ long: "1h", short: "5m", factor: 14.4 }] }, ticket: false } });
+    const m = sloMetrics(slo);
+    const ev = new RuleEvaluator([ruleGroupConfig(slo.rules)]);
+    let good = 0;
+    let bad = 0;
+    const burn = 0.5;
+    for (let minute = 0; minute <= 26 * 60; minute++) {
+      if (minute > 0) {
+        bad += RATE * burn * m.errorBudget;
+        good += RATE * (1 - burn * m.errorBudget);
+      }
+      ev.add({ __name__: "requests_total", code: "200" }, minute * MIN, good);
+      ev.add({ __name__: "requests_total", code: "500" }, minute * MIN, bad);
+      ev.step(minute * MIN);
+    }
+    const [remaining] = ev.query(`${m.errorBudgetRemaining}${m.selector}`, 26 * 60 * MIN);
+    expect(remaining.value).toBeCloseTo(1 - burn, 6);
+    const [objective] = ev.query(`${m.objectiveRatio}${m.selector}`, 26 * 60 * MIN);
+    expect(objective.value).toBe(0.995);
+  });
+});
+
+/** `a+bxn` segments for one counter, sample by sample. */
+function seriesValues(values: number[]): string {
+  // Written out as increments per segment of equal steps.
+  const parts: string[] = [];
+  let i = 0;
+  while (i < values.length) {
+    const start = values[i];
+    if (i + 1 >= values.length) {
+      parts.push(`${start}`);
+      break;
+    }
+    const inc = values[i + 1] - values[i];
+    let j = i + 1;
+    while (j + 1 < values.length && Math.abs(values[j + 1] - values[j] - inc) < 1e-9) j++;
+    parts.push(`${start}+${inc}x${j - i}`);
+    i = j + 1;
+  }
+  return parts.join(" ");
+}
+
+describe.skipIf(!hasPromtool)("each pair fires at its burn rate and not below it (promtool test rules)", () => {
+  const slo = Slo(props("30d"));
+  const m = sloMetrics(slo);
+  const alertRules = ruleGroupConfig(slo.rules).rules.filter((r): r is AlertingRuleConfig => "alert" in r);
+  const rulesYaml = ruleFileYaml([slo.rules]);
+
+  function testGroup(s: Scenario, checkAt: number[]) {
+    const good: number[] = [];
+    const bad: number[] = [];
+    let g = 0;
+    let e = 0;
+    const total = s.healthy + s.burning + (s.recovered ?? 0);
+    for (let minute = 0; minute <= total; minute += s.step) {
+      if (minute > 0) {
+        const b = badPerStep(s, minute, m.errorBudget);
+        e += b;
+        g += RATE * s.step - b;
+      }
+      good.push(g);
+      bad.push(e);
+    }
+    const timeline = run(slo, s);
+    return {
+      timeline,
+      group: {
+        interval: `${s.step}m`,
+        input_series: [
+          { series: 'requests_total{code="200"}', values: seriesValues(good) },
+          { series: 'requests_total{code="500"}', values: seriesValues(bad) },
+        ],
+        alert_rule_test: checkAt.map((minute) => ({
+          eval_time: `${minute}m`,
+          alertname: m.alertName,
+          exp_alerts: (timeline.get(minute) ?? []).map((a) => {
+            const rule = alertRules.find((r) => r.labels!.long_window === a.labels.long_window && r.labels!.short_window === a.labels.short_window)!;
+            const { alertname: _, ...labels } = a.labels;
+            return { exp_labels: labels, exp_annotations: rule.annotations };
+          }),
+        })),
+      },
+    };
+  }
+
+  for (const b of m.burnRates) {
+    const long = minutes(b.long);
+    const step = long >= 24 * 60 ? 5 : 1;
+    test(`${b.severity} ${b.long}/${b.short}`, () => {
+      const above: Scenario = { step, healthy: long, burning: long, burn: b.factor * 1.25 };
+      const fireAt = above.healthy + ((long - step) * b.factor) / above.burn;
+      const snap = (x: number) => Math.round(x / step) * step;
+      const aboveCheck = [snap(fireAt - 4 * step), snap(fireAt + 4 * step), above.healthy + long];
+      const upper = testGroup(above, aboveCheck);
+      // The evaluator's own claims, which promtool then confirms.
+      expect(upper.timeline.get(aboveCheck[0])!.some((a) => isPair(a, b))).toBe(false);
+      expect(upper.timeline.get(aboveCheck[1])!.some((a) => isPair(a, b))).toBe(true);
+      expect(upper.timeline.get(aboveCheck[2])!.some((a) => isPair(a, b))).toBe(true);
+
+      const below: Scenario = { step, healthy: long, burning: long, burn: b.factor * 0.9 };
+      const lower = testGroup(below, [below.healthy + long]);
+      expect(lower.timeline.get(below.healthy + long)!.some((a) => isPair(a, b))).toBe(false);
+
+      const testYaml = dump({ rule_files: ["rules.yml"], evaluation_interval: `${step}m`, tests: [upper.group, lower.group] }, { lineWidth: -1 });
+      const r = promtoolTestRules(rulesYaml, testYaml, PROMTOOL);
+      expect(r.ran).toBe(true);
+      expect(r.ok, r.output).toBe(true);
+    }, 120_000);
+  }
+});
+
+describe("rule-eval", () => {
+  test("throws on PromQL it does not implement instead of guessing", () => {
+    const ev = new RuleEvaluator([{ name: "g", rules: [{ record: "x", expr: "histogram_quantile(0.9, rate(a[5m]))" }] }]);
+    expect(() => ev.step(0)).toThrow(/not supported|needs a range/);
+  });
+
+  test("evaluates sums, ratios and set operators the way Prometheus does", () => {
+    const ev = new RuleEvaluator([
+      {
+        name: "g",
+        labels: { team: "a" },
+        rules: [
+          { record: "job:req:rate1m", expr: "sum by (job) (rate(req_total[2m]))" },
+          { alert: "High", expr: 'job:req:rate1m > 1 and on (job) job:req:rate1m{job="api"}', labels: { severity: "page" } },
+        ],
+      },
+    ]);
+    for (let minute = 0; minute <= 2; minute++) {
+      ev.add({ __name__: "req_total", job: "api", i: "1" }, minute * MIN, minute * 60 * 3);
+      ev.add({ __name__: "req_total", job: "api", i: "2" }, minute * MIN, minute * 60);
+      ev.add({ __name__: "req_total", job: "web", i: "1" }, minute * MIN, minute * 60 * 5);
+    }
+    const firing = ev.step(2 * MIN);
+    expect(ev.query("job:req:rate1m", 2 * MIN).map((e) => [e.labels.job, e.value])).toEqual([
+      ["api", 4],
+      ["web", 5],
+    ]);
+    expect(firing).toEqual([{ labels: { job: "api", team: "a", severity: "page", alertname: "High" }, value: 4 }]);
+  });
+});