@intentius/chant-lexicon-prometheus 0.106.3 → 0.107.1

package/src/op/activities/promtool.ts

@@ -0,0 +1,149 @@
+/**
+ * `promtool` and `amtool` as Op steps (#3369), over the functions in
+ * `../../tools.ts`.
+ *
+ * Those functions report `ran: false` when the binary is missing, so tests
+ * and CI can skip them. A step does not: a check that did not run must not
+ * read as a check that passed, so a missing binary fails the step, and so
+ * does a file the tool rejects, with the tool's output as the message.
+ *
+ * Paths are relative to the working directory. `promtoolTestRules` writes
+ * the rule file as `rules.yml` beside each test file, so a test file's
+ * `rule_files:` names `rules.yml`; the tests `sloRuleTests()` generates
+ * from `Slo` declarations do.
+ */
+
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { nonRetryableFailure } from "@intentius/chant/op";
+import { defaultExec, type ExecRunner } from "@intentius/chant-lexicon-otel/op/activities/exec";
+import {
+  amtoolCheckConfig as amtoolCheckConfigText,
+  promtoolCheckRules as promtoolCheckRulesText,
+  promtoolTestRules as promtoolTestRulesText,
+  type ToolResult,
+} from "../../tools";
+
+export interface ToolStepResult {
+  /** The files checked. */
+  files: string[];
+  ok: boolean;
+  output: string;
+}
+
+export interface PromtoolCheckRulesArgs {
+  /** One rule file or several, e.g. `dist/prometheus/rules.yml`. */
+  rules: string | string[];
+  /** The promtool binary. Default `$PROMTOOL`, else `promtool`. */
+  bin?: string;
+}
+
+export interface PromtoolTestRulesArgs {
+  /** The rule file under test, e.g. `dist/prometheus/rules.yml`. */
+  rules: string;
+  /** Unit-test files, each naming `rules.yml` in `rule_files:`. */
+  tests?: string | string[];
+  /** Unit-test documents given inline, such as the ones `sloRuleTests()` generates. */
+  testYaml?: string | string[];
+  /** The promtool binary. Default `$PROMTOOL`, else `promtool`. */
+  bin?: string;
+}
+
+export interface AmtoolCheckConfigArgs {
+  /** The Alertmanager config, e.g. `dist/prometheus/alertmanager.yml`. */
+  config: string;
+  /** The amtool binary. Default `$AMTOOL`, else `amtool`. */
+  bin?: string;
+}
+
+export interface AmtoolRoutesTestArgs {
+  /** The Alertmanager config. */
+  config: string;
+  /** The alert's labels. */
+  labels: Record<string, string>;
+  /** The receiver, or receivers in order, the labels must route to. */
+  expect: string | string[];
+  /** The amtool binary. Default `$AMTOOL`, else `amtool`. */
+  bin?: string;
+  /** Replaces the child process. For tests. */
+  _exec?: ExecRunner;
+}
+
+export interface AmtoolRoutesTestResult {
+  config: string;
+  labels: Record<string, string>;
+  /** The receivers amtool resolved, in order. */
+  receivers: string[];
+  ok: boolean;
+}
+
+const list = (v: string | string[] | undefined): string[] => (v === undefined ? [] : Array.isArray(v) ? v : [v]);
+const read = (path: string): string => readFileSync(resolve(process.cwd(), path), "utf8");
+
+function settle(tool: string, files: string[], r: ToolResult, bin: string): ToolStepResult {
+  if (!r.ran) throw new Error(`${bin} is not installed (or does not run): ${tool} did not check ${files.join(", ")}. Install it or set bin.`);
+  if (!r.ok) throw nonRetryableFailure(`${tool} rejected ${files.join(", ")}:\n${r.output.trim()}`, "PrometheusToolRejected");
+  return { files, ok: true, output: r.output.trim() };
+}
+
+const promtoolBin = (bin?: string) => bin ?? process.env.PROMTOOL ?? "promtool";
+const amtoolBin = (bin?: string) => bin ?? process.env.AMTOOL ?? "amtool";
+
+/** `promtool check rules` over each rule file. */
+export async function promtoolCheckRules(args: PromtoolCheckRulesArgs): Promise<ToolStepResult> {
+  const bin = promtoolBin(args.bin);
+  const files = list(args.rules);
+  if (files.length === 0) throw new Error("promtoolCheckRules: no rule file given");
+  const outputs: string[] = [];
+  for (const file of files) outputs.push(settle("promtool check rules", [file], promtoolCheckRulesText(read(file), bin), bin).output);
+  return { files, ok: true, output: outputs.join("\n") };
+}
+
+/** `promtool test rules` over the rule file, once per test file or inline test document. */
+export async function promtoolTestRules(args: PromtoolTestRulesArgs): Promise<ToolStepResult & { tests: number }> {
+  const bin = promtoolBin(args.bin);
+  const rules = read(args.rules);
+  const docs: Array<[string, string]> = [
+    ...list(args.tests).map((f): [string, string] => [f, read(f)]),
+    ...list(args.testYaml).map((y, i): [string, string] => [`testYaml[${i}]`, y]),
+  ];
+  if (docs.length === 0) throw new Error("promtoolTestRules: give tests or testYaml (or slos to the builder)");
+  const outputs: string[] = [];
+  for (const [name, yaml] of docs) outputs.push(settle("promtool test rules", [args.rules, name], promtoolTestRulesText(rules, yaml, bin), bin).output);
+  return { files: [args.rules, ...list(args.tests)], ok: true, output: outputs.join("\n"), tests: docs.length };
+}
+
+/** `amtool check-config` over the Alertmanager config. */
+export async function amtoolCheckConfig(args: AmtoolCheckConfigArgs): Promise<ToolStepResult> {
+  const bin = amtoolBin(args.bin);
+  return settle("amtool check-config", [args.config], amtoolCheckConfigText(read(args.config), bin), bin);
+}
+
+/**
+ * The receivers amtool printed: its last non-empty line, comma-separated
+ * (`oncall` or `oncall,tickets`).
+ */
+export function parseRoutesOutput(output: string): string[] {
+  const lines = output.split("\n").map((l) => l.trim()).filter((l) => l !== "" && !/^warning/i.test(l));
+  const last = lines[lines.length - 1] ?? "";
+  return last.split(",").map((s) => s.trim()).filter(Boolean);
+}
+
+/** `amtool config routes test`: the labels route to the expected receivers, in order. */
+export async function amtoolRoutesTest(args: AmtoolRoutesTestArgs): Promise<AmtoolRoutesTestResult> {
+  const exec = args._exec ?? defaultExec;
+  const bin = amtoolBin(args.bin);
+  const expected = list(args.expect);
+  const pairs = Object.entries(args.labels).map(([k, v]) => `${k}=${v}`);
+  const r = await exec(bin, ["config", "routes", "test", `--config.file=${resolve(process.cwd(), args.config)}`, ...pairs], { timeoutMs: 30_000 });
+  if (r.code === null) throw new Error(`could not run ${bin}: ${r.error ?? "not found"}. Install amtool or set bin.`);
+  if (r.code !== 0) throw nonRetryableFailure(`amtool config routes test failed (exit ${r.code}):\n${(r.stdout + r.stderr).trim()}`, "PrometheusToolRejected");
+  const receivers = parseRoutesOutput(r.stdout);
+  if (receivers.join(",") !== expected.join(",")) {
+    throw nonRetryableFailure(
+      `${pairs.join(" ")} routes to ${receivers.join(",") || "(none)"} in ${args.config}, expected ${expected.join(",")}`,
+      "PrometheusRouteMismatch",
+    );
+  }
+  return { config: args.config, labels: args.labels, receivers, ok: true };
+}

package/src/op/activities/rule-audit.test.ts

@@ -0,0 +1,89 @@
+import { describe, expect, test } from "vitest";
+import type { CommandRunner } from "@intentius/chant-lexicon-otel/op/activities/github";
+import { auditedSelectors, ruleAudit } from "./rule-audit";
+import type { ApiRuleGroup } from "./rules-loaded";
+
+const GROUPS: ApiRuleGroup[] = [
+  {
+    name: "slo-checkout",
+    rules: [
+      { name: "slo:sli_error:ratio_rate5m", type: "recording", health: "ok", query: 'sum(rate(requests_total{code=~"5.."}[5m])) / sum(rate(requests_total[5m]))' },
+      { name: "SloBurn", type: "alerting", health: "ok", query: 'slo:sli_error:ratio_rate5m{slo="checkout"} > 0.01' },
+    ],
+  },
+  {
+    name: "broken",
+    rules: [{ name: "Gone", type: "alerting", health: "err", lastError: "many-to-many matching not allowed", query: "absent(up{job=\"gone\"}) and ALERTS" }],
+  },
+];
+
+const NOW = new Date("2026-10-06T12:00:00Z");
+
+/** A Prometheus API stand-in. `empty` lists the selectors whose count query returns nothing. */
+function fakePrometheus(alerts: unknown[], empty: string[] = []) {
+  const queries: string[] = [];
+  const f = (async (input: string | URL | Request) => {
+    const url = new URL(String(input));
+    if (url.pathname === "/api/v1/rules") return Response.json({ status: "success", data: { groups: GROUPS } });
+    if (url.pathname === "/api/v1/alerts") return Response.json({ status: "success", data: { alerts } });
+    if (url.pathname === "/api/v1/query") {
+      const q = url.searchParams.get("query")!;
+      queries.push(q);
+      const isEmpty = empty.some((s) => q === `count(last_over_time(${s}[1h]))`);
+      return Response.json({ status: "success", data: { resultType: "vector", result: isEmpty ? [] : [{ metric: {}, value: [0, "3"] }] } });
+    }
+    return new Response("", { status: 404 });
+  }) as typeof fetch;
+  return { f, queries };
+}
+
+describe("ruleAudit", () => {
+  test("the selectors audited: distinct, without recorded series and ALERTS", () => {
+    expect(auditedSelectors(GROUPS)).toEqual(['requests_total', 'requests_total{code=~"5.."}', 'up{job="gone"}']);
+  });
+
+  test("rule errors, alerts past their thresholds, and selectors nothing emits", async () => {
+    const alerts = [
+      { labels: { alertname: "SloBurn", slo: "checkout" }, state: "pending", activeAt: "2026-10-06T09:00:00Z" },
+      { labels: { alertname: "SloBurn", slo: "search" }, state: "pending", activeAt: "2026-10-06T11:50:00Z" },
+      { labels: { alertname: "Gone" }, state: "firing", activeAt: "2026-10-04T12:00:00Z" },
+    ];
+    const { f, queries } = fakePrometheus(alerts, ['up{job="gone"}']);
+    const r = await ruleAudit({ url: "http://prom:9090", _fetch: f, _now: () => NOW });
+    expect(r.findings).toEqual([
+      { kind: "rule-error", subject: "broken", detail: "Gone: many-to-many matching not allowed" },
+      { kind: "pending-too-long", subject: 'SloBurn{slo="checkout"}', detail: "pending for 3.0h" },
+      { kind: "firing-too-long", subject: "Gone{}", detail: "firing for 48.0h" },
+      { kind: "selector-no-series", subject: 'up{job="gone"}', detail: "no series in the last 1h" },
+    ]);
+    expect(queries).toHaveLength(3);
+    expect(r.queried).toBe(3);
+    expect(r.summary).toContain("| selector-no-series |");
+  });
+
+  test("the selector budget caps the queries; the rest are counted unchecked", async () => {
+    const { f, queries } = fakePrometheus([]);
+    const r = await ruleAudit({ _fetch: f, selectorBudget: 1, _now: () => NOW });
+    expect(queries).toHaveLength(1);
+    expect(r.unchecked).toBe(2);
+    expect(r.summary).toContain("2 left unchecked by the budget");
+  });
+
+  test("issue mode keeps one issue current, and only when there are findings", async () => {
+    const calls: string[][] = [];
+    const run: CommandRunner = async (bin, args) => {
+      calls.push([bin, ...args]);
+      if (args[1] === "list") return JSON.stringify([{ number: 7, title: "prometheus: rule audit findings", url: "https://x/issues/7" }]);
+      return "";
+    };
+    const { f } = fakePrometheus([]);
+    const r = await ruleAudit({ _fetch: f, mode: "issue", _run: run, _now: () => NOW });
+    expect(r.issueUrl).toBe("https://x/issues/7");
+    expect(calls[1].slice(0, 4)).toEqual(["gh", "issue", "edit", "7"]);
+  });
+
+  test("an unreadable rules API fails the audit rather than reporting nothing", async () => {
+    const down = (async () => new Response("", { status: 502 })) as unknown as typeof fetch;
+    await expect(ruleAudit({ _fetch: down })).rejects.toThrow(/HTTP 502/);
+  });
+});

package/src/op/activities/rule-audit.ts

@@ -0,0 +1,168 @@
+/**
+ * `ruleAudit`: what a live Prometheus says about its rules (#3369), the
+ * activity behind `RuleAuditOp`.
+ *
+ * - groups with health errors: every rule `/api/v1/rules` reports with
+ *   `health: "err"`, with its `lastError`.
+ * - alerts pending or firing past a threshold, from `/api/v1/alerts`: a
+ *   pending alert older than `pendingFor` is one whose `for` never
+ *   completes (a flapping condition), and a firing one older than
+ *   `firingFor` is one nobody is acting on.
+ * - selectors no live target emits: each distinct vector selector in the
+ *   loaded rules' expressions, queried as
+ *   `count(last_over_time(<selector>[<lookback>]))`. An empty answer means
+ *   the rule reads a series nothing writes (a renamed metric, a dropped
+ *   label, a target that is gone). Selectors over series the rules record
+ *   themselves, and over `ALERTS`, are left out. One query per selector, at
+ *   most `selectorBudget` of them; the rest are counted as unchecked.
+ *
+ * `mode` is `report` (return the findings) or `issue` (keep one open issue
+ * current with them).
+ */
+
+import { defaultRunner, stickyIssue, type CommandRunner } from "@intentius/chant-lexicon-otel/op/activities/github";
+import { durationMs } from "../../duration";
+import { selectors } from "../../promql-analysis";
+import { fetchRuleGroups, prometheusUrl, type ApiRuleGroup } from "./rules-loaded";
+
+export type RuleAuditMode = "report" | "issue";
+
+export interface RuleAuditArgs {
+  /** Prometheus's base URL. Default `$PROMETHEUS_URL`, else `http://localhost:9090`. */
+  url?: string;
+  /** A pending alert older than this is a finding. Default `1h`. */
+  pendingFor?: string;
+  /** A firing alert older than this is a finding. Default `24h`. */
+  firingFor?: string;
+  /** At most this many selector queries. Default 50. */
+  selectorBudget?: number;
+  /** How far back a selector's series may last have been written. Default `1h`. */
+  lookback?: string;
+  mode?: RuleAuditMode;
+  /** The issue's title in `issue` mode. Default `prometheus: rule audit findings`. */
+  issueTitle?: string;
+  _fetch?: typeof fetch;
+  _run?: CommandRunner;
+  _now?: () => Date;
+}
+
+export type RuleAuditFindingKind = "rule-error" | "pending-too-long" | "firing-too-long" | "selector-no-series";
+
+export interface RuleAuditFinding {
+  kind: RuleAuditFindingKind;
+  /** The group, the alert, or the selector. */
+  subject: string;
+  detail: string;
+}
+
+export interface RuleAuditResult {
+  mode: RuleAuditMode;
+  findings: RuleAuditFinding[];
+  /** Selector queries made. */
+  queried: number;
+  /** Selectors left unchecked by the budget. */
+  unchecked: number;
+  summary: string;
+  issueUrl?: string;
+}
+
+interface ApiAlert {
+  labels: Record<string, string>;
+  state: string;
+  activeAt?: string;
+}
+
+async function getJson<T>(f: typeof fetch, url: string): Promise<T> {
+  const res = await f(url, { signal: AbortSignal.timeout(15_000) });
+  if (!res.ok) throw new Error(`GET ${url}: HTTP ${res.status}`);
+  const body = (await res.json()) as { status?: string; data?: T };
+  if (body.status !== "success" || body.data === undefined) throw new Error(`GET ${url}: status ${body.status ?? "missing"}`);
+  return body.data;
+}
+
+/** The distinct selectors the rules read, without the ones over recorded series and ALERTS. */
+export function auditedSelectors(groups: ApiRuleGroup[]): string[] {
+  const recorded = new Set<string>(["ALERTS", "ALERTS_FOR_STATE"]);
+  for (const g of groups) for (const r of g.rules ?? []) if (r.type === "recording") recorded.add(r.name);
+  const out = new Set<string>();
+  for (const g of groups) {
+    for (const r of g.rules ?? []) {
+      if (!r.query) continue;
+      for (const s of selectors(r.query)) {
+        if (s.name && recorded.has(s.name)) continue;
+        out.add(s.text.replace(/\s+/g, ""));
+      }
+    }
+  }
+  return [...out].sort();
+}
+
+function alertName(a: ApiAlert): string {
+  const { alertname, ...rest } = a.labels;
+  const extra = Object.entries(rest)
+    .sort(([x], [y]) => x.localeCompare(y))
+    .map(([k, v]) => `${k}="${v}"`)
+    .join(",");
+  return `${alertname ?? "(unnamed)"}{${extra}}`;
+}
+
+export function renderRuleAuditSummary(findings: RuleAuditFinding[], queried: number, unchecked: number): string {
+  let out = "## Prometheus rule audit\n\n";
+  if (findings.length === 0) out += "No findings.\n";
+  else {
+    out += "| Finding | Subject | Detail |\n|---|---|---|\n";
+    for (const f of findings) out += `| ${f.kind} | \`${f.subject.replace(/\|/g, "\\|")}\` | ${f.detail.replace(/\|/g, "\\|")} |\n`;
+  }
+  out += `\n${queried} selector(s) queried${unchecked > 0 ? `, ${unchecked} left unchecked by the budget` : ""}.\n`;
+  return out;
+}
+
+/** Audit a live Prometheus's rules, alerts and selectors. */
+export async function ruleAudit(args: RuleAuditArgs = {}): Promise<RuleAuditResult> {
+  const f = args._fetch ?? fetch;
+  const url = prometheusUrl(args.url);
+  const now = (args._now ?? (() => new Date()))().getTime();
+  const mode = args.mode ?? "report";
+  const findings: RuleAuditFinding[] = [];
+
+  const groups = await fetchRuleGroups(f, url);
+  for (const g of groups) {
+    const errored = (g.rules ?? []).filter((r) => r.health === "err");
+    for (const r of errored) findings.push({ kind: "rule-error", subject: g.name, detail: `${r.name}: ${r.lastError || "health err"}` });
+  }
+
+  const pendingMs = durationMs(args.pendingFor ?? "1h");
+  const firingMs = durationMs(args.firingFor ?? "24h");
+  if (pendingMs === undefined || firingMs === undefined) throw new Error("pendingFor and firingFor are Prometheus durations such as 1h");
+  const { alerts } = await getJson<{ alerts: ApiAlert[] }>(f, `${url}/api/v1/alerts`);
+  for (const a of alerts) {
+    const since = a.activeAt ? Date.parse(a.activeAt) : NaN;
+    if (Number.isNaN(since)) continue;
+    const age = now - since;
+    const hours = (age / 3_600_000).toFixed(1);
+    if (a.state === "pending" && age > pendingMs) findings.push({ kind: "pending-too-long", subject: alertName(a), detail: `pending for ${hours}h` });
+    if (a.state === "firing" && age > firingMs) findings.push({ kind: "firing-too-long", subject: alertName(a), detail: `firing for ${hours}h` });
+  }
+
+  const lookback = args.lookback ?? "1h";
+  if (durationMs(lookback) === undefined) throw new Error(`lookback ${JSON.stringify(lookback)} is not a Prometheus duration`);
+  const all = auditedSelectors(groups);
+  const budget = Math.max(0, args.selectorBudget ?? 50);
+  const checked = all.slice(0, budget);
+  for (const sel of checked) {
+    const query = `count(last_over_time(${sel}[${lookback}]))`;
+    try {
+      const data = await getJson<{ result: unknown[] }>(f, `${url}/api/v1/query?query=${encodeURIComponent(query)}`);
+      if (data.result.length === 0) findings.push({ kind: "selector-no-series", subject: sel, detail: `no series in the last ${lookback}` });
+    } catch {
+      // A query Prometheus refuses is not evidence the series is missing.
+    }
+  }
+
+  const summary = renderRuleAuditSummary(findings, checked.length, all.length - checked.length);
+  const result: RuleAuditResult = { mode, findings, queried: checked.length, unchecked: all.length - checked.length, summary };
+  if (mode === "issue" && findings.length > 0) {
+    result.issueUrl = await stickyIssue(args._run ?? defaultRunner, process.cwd(), args.issueTitle ?? "prometheus: rule audit findings", summary);
+  }
+  return result;
+}

package/src/op/activities/rules-loaded.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, test } from "vitest";
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { declaredGroups, groupVerdict, rulesLoadedObserve, type ApiRuleGroup } from "./rules-loaded";
+
+function rulesApi(groups: ApiRuleGroup[], extra: Record<string, unknown> = {}): typeof fetch {
+  return (async (input: string | URL | Request) => {
+    const url = String(input);
+    if (url.endsWith("/api/v1/rules")) return Response.json({ status: "success", data: { groups } });
+    if (extra[url] !== undefined) return Response.json(extra[url]);
+    return new Response("not found", { status: 404 });
+  }) as typeof fetch;
+}
+
+const LOADED: ApiRuleGroup[] = [
+  { name: "slo-checkout", file: "/etc/prometheus/rules.yml", rules: [{ name: "slo:sli_error:ratio_rate5m", type: "recording", health: "ok" }] },
+  {
+    name: "genai",
+    rules: [
+      { name: "genai:calls:rate5m", type: "recording", health: "ok" },
+      { name: "GenAiErrors", type: "alerting", health: "err", lastError: 'vector contains metrics with the same labelset after applying rule labels' },
+    ],
+  },
+];
+
+describe("rulesLoadedObserve", () => {
+  test("one resource per declared group: loaded and healthy, not loaded, or with a rule in error", async () => {
+    const r = await rulesLoadedObserve({ url: "http://prom:9090", groups: ["slo-checkout", "genai", "slo-search"], _fetch: rulesApi(LOADED) });
+    expect(r.resources).toEqual([
+      { name: "slo-checkout", status: "in-sync", detail: "1 rule(s) loaded" },
+      { name: "genai", status: "drifted", detail: "GenAiErrors: vector contains metrics with the same labelset after applying rule labels" },
+      { name: "slo-search", status: "drifted", detail: "group not loaded" },
+    ]);
+  });
+
+  test("the declared groups also come from rule files, without repeats", () => {
+    const dir = mkdtempSync(join(tmpdir(), "chant-rules-loaded-"));
+    writeFileSync(join(dir, "rules.yml"), "groups:\n  - name: slo-checkout\n    rules: []\n  - name: genai\n    rules: []\n");
+    expect(declaredGroups({ groups: ["genai", "extra"], rules: "rules.yml" }, dir)).toEqual(["genai", "extra", "slo-checkout"]);
+  });
+
+  test("an unreadable API is unknown for every group, never drifted", async () => {
+    const down = (async () => new Response("", { status: 503 })) as unknown as typeof fetch;
+    const r = await rulesLoadedObserve({ groups: ["a", "b"], _fetch: down });
+    expect(r.resources.map((x) => x.status)).toEqual(["unknown", "unknown"]);
+    expect(r.resources[0].detail).toMatch(/rules API unreadable: .*HTTP 503/);
+  });
+
+  test("no declared group is an error, not an empty observation", async () => {
+    await expect(rulesLoadedObserve({ _fetch: rulesApi([]) })).rejects.toThrow(/no groups declared/);
+  });
+
+  test("a group split across files is judged on all its rules", () => {
+    const split: ApiRuleGroup[] = [
+      { name: "g", rules: [{ name: "a", health: "ok" }] },
+      { name: "g", rules: [{ name: "b", health: "err", lastError: "" }] },
+    ];
+    expect(groupVerdict("g", split)).toEqual({ name: "g", status: "drifted", detail: "b: health err" });
+  });
+});

package/src/op/activities/rules-loaded.ts

@@ -0,0 +1,98 @@
+/**
+ * Declared rule groups as resources a `ConvergeOp` observes (#3369).
+ *
+ * `rulesLoadedObserve` reads `GET /api/v1/rules` and returns one resource
+ * per declared group:
+ *
+ * - `drifted` when Prometheus has not loaded the group (the rule file is not
+ *   mounted, did not parse, or the reload has not happened), or when a rule
+ *   in it has `health: "err"`, with the rule's `lastError` as the detail;
+ * - `in-sync` otherwise;
+ * - `unknown`, for every group, when the API cannot be read.
+ *
+ * The declared groups are `groups`, plus the groups of each rule file in
+ * `rules` (read on each tick, so a rebuilt file is what is observed).
+ * Prometheus is `url`, else `$PROMETHEUS_URL`, else `http://localhost:9090`.
+ */
+
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { load } from "js-yaml";
+
+export interface RulesLoadedObserveArgs {
+  /** Prometheus's base URL. */
+  url?: string;
+  /** Group names that must be loaded. */
+  groups?: string[];
+  /** Rule files whose groups must be loaded, relative to the working directory. */
+  rules?: string | string[];
+  /** Request timeout, ms. Default 10000. */
+  timeoutMs?: number;
+  /** Replaces fetch. For tests. */
+  _fetch?: typeof fetch;
+}
+
+export interface RulesLoadedObserveResult {
+  resources: { name: string; status: "in-sync" | "drifted" | "unknown"; detail: string }[];
+}
+
+/** One group as `/api/v1/rules` returns it. */
+export interface ApiRuleGroup {
+  name: string;
+  file?: string;
+  rules: Array<{ name: string; type?: string; query?: string; health?: string; lastError?: string; state?: string }>;
+}
+
+export function prometheusUrl(url?: string): string {
+  return (url ?? process.env.PROMETHEUS_URL ?? "http://localhost:9090").replace(/\/+$/, "");
+}
+
+/** The group names in a rule file's text. */
+export function ruleFileGroups(text: string): string[] {
+  const doc = load(text) as { groups?: Array<{ name?: unknown }> } | null;
+  return (doc?.groups ?? []).map((g) => g?.name).filter((n): n is string => typeof n === "string");
+}
+
+/** The declared groups, in order, without repeats. */
+export function declaredGroups(args: Pick<RulesLoadedObserveArgs, "groups" | "rules">, cwd = process.cwd()): string[] {
+  const out = [...(args.groups ?? [])];
+  const files = args.rules === undefined ? [] : Array.isArray(args.rules) ? args.rules : [args.rules];
+  for (const f of files) out.push(...ruleFileGroups(readFileSync(resolve(cwd, f), "utf8")));
+  return [...new Set(out)];
+}
+
+/** `GET /api/v1/rules`'s groups. Throws on anything but a successful answer. */
+export async function fetchRuleGroups(f: typeof fetch, url: string, timeoutMs = 10_000): Promise<ApiRuleGroup[]> {
+  const res = await f(`${url}/api/v1/rules`, { signal: AbortSignal.timeout(timeoutMs) });
+  if (!res.ok) throw new Error(`GET ${url}/api/v1/rules: HTTP ${res.status}`);
+  const body = (await res.json()) as { status?: string; data?: { groups?: ApiRuleGroup[] } };
+  if (body.status !== "success" || !Array.isArray(body.data?.groups)) throw new Error(`GET ${url}/api/v1/rules: status ${body.status ?? "missing"}`);
+  return body.data.groups;
+}
+
+/** The verdict for one declared group against the loaded groups. */
+export function groupVerdict(name: string, loaded: ApiRuleGroup[]): RulesLoadedObserveResult["resources"][number] {
+  const matches = loaded.filter((g) => g.name === name);
+  if (matches.length === 0) return { name, status: "drifted", detail: "group not loaded" };
+  const rules = matches.flatMap((g) => g.rules ?? []);
+  const errored = rules.filter((r) => r.health === "err");
+  if (errored.length > 0) {
+    return { name, status: "drifted", detail: errored.map((r) => `${r.name}: ${r.lastError || "health err"}`).join("; ") };
+  }
+  return { name, status: "in-sync", detail: `${rules.length} rule(s) loaded` };
+}
+
+/** Observe the declared rule groups: one resource per group. */
+export async function rulesLoadedObserve(args: RulesLoadedObserveArgs): Promise<RulesLoadedObserveResult> {
+  const names = declaredGroups(args);
+  if (names.length === 0) throw new Error("rulesLoadedObserve: no groups declared (give groups or rules)");
+  const url = prometheusUrl(args.url);
+  let loaded: ApiRuleGroup[];
+  try {
+    loaded = await fetchRuleGroups(args._fetch ?? fetch, url, args.timeoutMs);
+  } catch (err) {
+    const detail = `rules API unreadable: ${(err as Error).message}`;
+    return { resources: names.map((name) => ({ name, status: "unknown" as const, detail })) };
+  }
+  return { resources: names.map((name) => groupVerdict(name, loaded)) };
+}