@intentius/chant-lexicon-prometheus 0.106.4 → 0.108.0

package/src/import/alertmanager-live.ts

@@ -0,0 +1,291 @@
+/**
+ * Alertmanager's `/api/v2/status` config, back to the shape a person writes
+ * (#3371).
+ *
+ * `config.original` is the loaded config marshalled again, not the file
+ * (`Config.String()` in Alertmanager's `config/config.go`). Against
+ * Alertmanager v0.34.1 (`PROMETHEUS_PIN.alertmanager`) that adds three
+ * kinds of thing a file did not say:
+ *
+ * - `global:` with every default of `DefaultGlobalConfig()` written out:
+ *   `resolve_timeout: 5m`, the default `http_config`, `smtp_hello`,
+ *   `smtp_require_tls`, `smtp_tls_config` and the integration API URLs.
+ * - every receiver integration with the settings it inherited from `global:`
+ *   (`http_config`, an email config's `smarthost`, `from` and SMTP auth, a
+ *   Slack config's `api_url`, a PagerDuty config's `url`, and so on), as
+ *   `Config.UnmarshalYAML` copies them in.
+ * - every integration's own defaults (`DefaultEmailConfig`,
+ *   `DefaultSlackConfig`, `DefaultPagerdutyConfig`, ...) and the zero values
+ *   of fields marshalled without `omitempty` (`url_file: ""`,
+ *   `max_alerts: 0`, `continue: false`).
+ *
+ * {@link stripAlertmanagerDefaults} takes those out, so an import writes
+ * what a person would have declared. It drops a value only where it equals
+ * the default or the global it was copied from; a value someone set to the
+ * default is indistinguishable from one they left out, and goes too, which
+ * builds to the same config. `verbatim` skips it. Secrets read `<secret>`
+ * either way (see {@link maskedSecretPaths}).
+ */
+
+type Json = Record<string, unknown>;
+
+function isObject(v: unknown): v is Json {
+  return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+
+function canonical(v: unknown): unknown {
+  if (Array.isArray(v)) return v.map(canonical);
+  if (isObject(v)) return Object.fromEntries(Object.keys(v).sort().map((k) => [k, canonical(v[k])]));
+  return v;
+}
+
+function same(a: unknown, b: unknown): boolean {
+  return JSON.stringify(canonical(a)) === JSON.stringify(canonical(b));
+}
+
+/** Empty, false, zero or null: what a field marshalled without `omitempty` holds when nobody set it. */
+function isZero(v: unknown): boolean {
+  if (v === null || v === undefined || v === "" || v === false || v === 0) return true;
+  if (Array.isArray(v)) return v.length === 0;
+  if (isObject(v)) return Object.values(v).every(isZero);
+  return false;
+}
+
+/** `commoncfg.DefaultHTTPClientConfig`, marshalled. */
+const DEFAULT_HTTP_CONFIG = { follow_redirects: true, enable_http2: true };
+
+function isDefaultHttpConfig(v: unknown): boolean {
+  if (!isObject(v)) return false;
+  const rest = Object.fromEntries(Object.entries(v).filter(([k]) => !(k in DEFAULT_HTTP_CONFIG)));
+  return Object.entries(DEFAULT_HTTP_CONFIG).every(([k, d]) => v[k] === undefined || v[k] === d) && isZero(rest);
+}
+
+/** `DefaultGlobalConfig()` at the pin. `http_config` and `smtp_tls_config` are compared by {@link isDefaultHttpConfig} and {@link isZero}. */
+const GLOBAL_DEFAULTS: Json = {
+  resolve_timeout: "5m",
+  smtp_hello: "localhost",
+  smtp_require_tls: true,
+  pagerduty_url: "https://events.pagerduty.com/v2/enqueue",
+  opsgenie_api_url: "https://api.opsgenie.com/",
+  wechat_api_url: "https://qyapi.weixin.qq.com/cgi-bin/",
+  victorops_api_url: "https://alert.victorops.com/integrations/generic/20131114/alert/",
+  telegram_api_url: "https://api.telegram.org",
+  webex_api_url: "https://webexapis.com/v1/messages",
+  rocketchat_api_url: "https://open.rocket.chat/",
+  slack_app_url: "https://slack.com/api/chat.postMessage",
+};
+
+/** Fields `Config.UnmarshalYAML` copies from `global:` into each integration, integration field to global key. */
+const INHERITED: Record<string, Record<string, string>> = {
+  email_configs: {
+    tls_config: "smtp_tls_config",
+    smarthost: "smtp_smarthost",
+    from: "smtp_from",
+    hello: "smtp_hello",
+    auth_username: "smtp_auth_username",
+    auth_password: "smtp_auth_password",
+    auth_password_file: "smtp_auth_password_file",
+    auth_secret: "smtp_auth_secret",
+    auth_secret_file: "smtp_auth_secret_file",
+    auth_identity: "smtp_auth_identity",
+    require_tls: "smtp_require_tls",
+    force_implicit_tls: "smtp_force_implicit_tls",
+  },
+  slack_configs: {
+    app_url: "slack_app_url",
+    app_token: "slack_app_token",
+    app_token_file: "slack_app_token_file",
+    api_url: "slack_api_url",
+    api_url_file: "slack_api_url_file",
+  },
+  pagerduty_configs: { url: "pagerduty_url" },
+  opsgenie_configs: { api_url: "opsgenie_api_url", api_key: "opsgenie_api_key", api_key_file: "opsgenie_api_key_file" },
+  wechat_configs: { api_url: "wechat_api_url", api_secret: "wechat_api_secret", api_secret_file: "wechat_api_secret_file", corp_id: "wechat_api_corp_id" },
+  victorops_configs: { api_url: "victorops_api_url", api_key: "victorops_api_key", api_key_file: "victorops_api_key_file" },
+  telegram_configs: { api_url: "telegram_api_url", bot_token: "telegram_bot_token", bot_token_file: "telegram_bot_token_file" },
+  webex_configs: { api_url: "webex_api_url" },
+  rocketchat_configs: {
+    api_url: "rocketchat_api_url",
+    token: "rocketchat_token",
+    token_file: "rocketchat_token_file",
+    token_id: "rocketchat_token_id",
+    token_id_file: "rocketchat_token_id_file",
+  },
+  mattermost_configs: { webhook_url: "mattermost_webhook_url", webhook_url_file: "mattermost_webhook_url_file" },
+  jira_configs: { api_url: "jira_api_url" },
+};
+
+/** Each integration's `Default<Kind>Config` at the pin, the fields it sets. Kinds not listed keep what they read. */
+const INTEGRATION_DEFAULTS: Record<string, Json> = {
+  webhook_configs: { send_resolved: true },
+  email_configs: { send_resolved: false, html: '{{ template "email.default.html" . }}' },
+  slack_configs: {
+    send_resolved: false,
+    color: '{{ template "slack.default.color" . }}',
+    username: '{{ template "slack.default.username" . }}',
+    title: '{{ template "slack.default.title" . }}',
+    title_link: '{{ template "slack.default.titlelink" . }}',
+    icon_emoji: '{{ template "slack.default.iconemoji" . }}',
+    icon_url: '{{ template "slack.default.iconurl" . }}',
+    pretext: '{{ template "slack.default.pretext" . }}',
+    text: '{{ template "slack.default.text" . }}',
+    fallback: '{{ template "slack.default.fallback" . }}',
+    callback_id: '{{ template "slack.default.callbackid" . }}',
+    footer: '{{ template "slack.default.footer" . }}',
+  },
+  pagerduty_configs: {
+    send_resolved: true,
+    description: '{{ template "pagerduty.default.description" .}}',
+    client: '{{ template "pagerduty.default.client" . }}',
+    client_url: '{{ template "pagerduty.default.clientURL" . }}',
+  },
+  webex_configs: { send_resolved: true, message: '{{ template "webex.default.message" . }}' },
+  rocketchat_configs: {
+    send_resolved: false,
+    color: '{{ if eq .Status "firing" }}red{{ else }}green{{ end }}',
+    emoji: '{{ template "rocketchat.default.emoji" . }}',
+    icon_url: '{{ template "rocketchat.default.iconurl" . }}',
+    text: '{{ template "rocketchat.default.text" . }}',
+    title: '{{ template "rocketchat.default.title" . }}',
+    title_link: '{{ template "rocketchat.default.titlelink" . }}',
+  },
+  wechat_configs: {
+    send_resolved: false,
+    message: '{{ template "wechat.default.message" . }}',
+    to_user: '{{ template "wechat.default.to_user" . }}',
+    to_party: '{{ template "wechat.default.to_party" . }}',
+    to_tag: '{{ template "wechat.default.to_tag" . }}',
+    agent_id: '{{ template "wechat.default.agent_id" . }}',
+  },
+  victorops_configs: {
+    send_resolved: true,
+    message_type: "CRITICAL",
+    state_message: '{{ template "victorops.default.state_message" . }}',
+    entity_display_name: '{{ template "victorops.default.entity_display_name" . }}',
+    monitoring_tool: '{{ template "victorops.default.monitoring_tool" . }}',
+  },
+  pushover_configs: {
+    send_resolved: true,
+    title: '{{ template "pushover.default.title" . }}',
+    message: '{{ template "pushover.default.message" . }}',
+    url: '{{ template "pushover.default.url" . }}',
+    priority: '{{ if eq .Status "firing" }}2{{ else }}0{{ end }}',
+    retry: "1m",
+    expire: "1h",
+  },
+  sns_configs: { send_resolved: true, subject: '{{ template "sns.default.subject" . }}', message: '{{ template "sns.default.message" . }}' },
+};
+
+/** `DefaultEmailSubject`, which an email config's `headers` gain when they set no Subject. */
+const DEFAULT_EMAIL_SUBJECT = '{{ template "email.default.subject" . }}';
+
+/** `DefaultPagerdutyDetails`, merged into every PagerDuty config's `details`. */
+const DEFAULT_PAGERDUTY_DETAILS: Json = {
+  firing: "{{ .Alerts.Firing | toJson }}",
+  resolved: "{{ .Alerts.Resolved | toJson }}",
+  num_firing: "{{ .Alerts.Firing | len }}",
+  num_resolved: "{{ .Alerts.Resolved | len }}",
+};
+
+function stripIntegration(kind: string, raw: Json, global: Json): Json {
+  const out: Json = { ...raw };
+  const inherited = INHERITED[kind] ?? {};
+  const defaults = INTEGRATION_DEFAULTS[kind] ?? {};
+  for (const [field, globalKey] of Object.entries(inherited)) {
+    if (field in out && global[globalKey] !== undefined && same(out[field], global[globalKey])) delete out[field];
+    else if (field in out && global[globalKey] === undefined && isZero(out[field]) && field !== "require_tls") delete out[field];
+  }
+  if ("http_config" in out && (global.http_config !== undefined ? same(out.http_config, global.http_config) : isDefaultHttpConfig(out.http_config))) {
+    delete out.http_config;
+  }
+  for (const [field, d] of Object.entries(defaults)) {
+    if (field in out && same(out[field], d)) delete out[field];
+  }
+  if (kind === "email_configs" && isObject(out.headers)) {
+    const headers = { ...out.headers };
+    if (headers.Subject === DEFAULT_EMAIL_SUBJECT) delete headers.Subject;
+    if (Object.keys(headers).length === 0) delete out.headers;
+    else out.headers = headers;
+  }
+  if (kind === "pagerduty_configs" && isObject(out.details)) {
+    const details = Object.fromEntries(Object.entries(out.details).filter(([k, v]) => !same(v, DEFAULT_PAGERDUTY_DETAILS[k])));
+    if (Object.keys(details).length === 0) delete out.details;
+    else out.details = details;
+  }
+  // What is left at its zero value was marshalled without omitempty. A field with a known default or a global to
+  // inherit from survived the comparisons above because it differs (`send_resolved: false` on a webhook,
+  // `require_tls: false` under a global `true`), and `send_resolved` of a kind with no known default stays.
+  for (const [k, v] of Object.entries(out)) {
+    if (k === "send_resolved" || k in defaults) continue;
+    if (k in inherited && global[inherited[k]] !== undefined) continue;
+    if (isZero(v)) delete out[k];
+  }
+  return out;
+}
+
+function stripRoute(raw: unknown): unknown {
+  if (!isObject(raw)) return raw;
+  const out: Json = {};
+  for (const [k, v] of Object.entries(raw)) {
+    if (k === "routes" && Array.isArray(v)) {
+      if (v.length > 0) out.routes = v.map(stripRoute);
+      continue;
+    }
+    if (isZero(v)) continue;
+    out[k] = v;
+  }
+  return out;
+}
+
+function stripGlobal(raw: Json): Json {
+  const out: Json = {};
+  for (const [k, v] of Object.entries(raw)) {
+    if (k in GLOBAL_DEFAULTS && same(v, GLOBAL_DEFAULTS[k])) continue;
+    if (k === "http_config" && isDefaultHttpConfig(v)) continue;
+    if (k === "smtp_require_tls" && v === false) {
+      out[k] = v;
+      continue;
+    }
+    if (isZero(v)) continue;
+    out[k] = v;
+  }
+  return out;
+}
+
+/** The re-marshalled config with what Alertmanager added taken out (see the module comment). Pure; `doc` is not changed. */
+export function stripAlertmanagerDefaults(doc: Json): Json {
+  const global = isObject(doc.global) ? doc.global : {};
+  const out: Json = {};
+  for (const [section, value] of Object.entries(doc)) {
+    if (section === "global") {
+      const g = stripGlobal(global);
+      if (Object.keys(g).length > 0) out.global = g;
+    } else if (section === "receivers" && Array.isArray(value)) {
+      out.receivers = value.map((r) => {
+        if (!isObject(r)) return r;
+        const receiver: Json = {};
+        for (const [k, v] of Object.entries(r)) {
+          if (k.endsWith("_configs") && Array.isArray(v)) {
+            if (v.length > 0) receiver[k] = v.map((c) => (isObject(c) ? stripIntegration(k, c, global) : c));
+          } else if (k === "name" || !isZero(v)) receiver[k] = v;
+        }
+        return receiver;
+      });
+    } else if (section === "route") {
+      out.route = stripRoute(value);
+    } else if (section === "inhibit_rules" && Array.isArray(value)) {
+      if (value.length > 0) out.inhibit_rules = value.map((r) => (isObject(r) ? Object.fromEntries(Object.entries(r).filter(([, v]) => !isZero(v))) : r));
+    } else if (!isZero(value)) {
+      out[section] = value;
+    }
+  }
+  return out;
+}
+
+/** The paths that read `<secret>`, the mask Alertmanager writes over every secret it marshals. */
+export function maskedSecretPaths(doc: unknown, at = ""): string[] {
+  if (doc === "<secret>") return [at];
+  if (Array.isArray(doc)) return doc.flatMap((v, i) => maskedSecretPaths(v, `${at}[${i}]`));
+  if (isObject(doc)) return Object.entries(doc).flatMap(([k, v]) => maskedSecretPaths(v, at ? `${at}.${k}` : k));
+  return [];
+}

package/src/import/parser.ts

@@ -386,10 +386,27 @@ const PROM_VERSION = PROMETHEUS_PIN.prometheus.version;
 
 // ── entry points ────────────────────────────────────────────────────
 
+/** Load YAML the way the parser does: YAML 1.2 core types plus `<<` merge keys. */
+export function loadPrometheusYaml(content: string): unknown {
+  return content.trim() === "" ? {} : jsYaml.load(content, { schema: YAML_SCHEMA });
+}
+
+/** A rule file already loaded (live import builds one from a ruler's groups, ../export-resources.ts). */
+export function parseRuleFileDocument(doc: Record<string, unknown>): { file: RuleFileConfig; warnings: string[] } {
+  const warnings: string[] = [];
+  return { file: parseRuleFile(doc, warnings), warnings };
+}
+
+/** An `alertmanager.yml` already loaded (live import reads one from Alertmanager's API, ../export-resources.ts). */
+export function parseAlertmanagerDocument(doc: Record<string, unknown>): { config: AlertmanagerConfig; warnings: string[] } {
+  const warnings: string[] = [];
+  return { config: parseAlertmanager(doc, warnings), warnings };
+}
+
 /** Parse a rule file or `alertmanager.yml`, telling them apart by shape. */
 export function parsePrometheusYaml(content: string): ParsedPrometheusFile {
   const warnings: string[] = [];
-  const doc = content.trim() === "" ? {} : jsYaml.load(content, { schema: YAML_SCHEMA });
+  const doc = loadPrometheusYaml(content);
   if (!isPlainObject(doc)) {
     throw new Error("a Prometheus rule file, alertmanager.yml or prometheus.yml is a YAML mapping; this document is not one");
   }

package/src/import/roundtrip.test.ts

@@ -175,6 +175,7 @@ describe("YAML -> TypeScript -> YAML", () => {
       "getting-started/rules.yml",
       "k3d-stack/rules.yml",
       "k3d-stack/alertmanager.yml",
+      "observe-converge/rules.yml",
       "rules-from-data/rules.yml",
       "slo/rules.yml",
       "slo/alertmanager.yml",

package/src/index.ts

@@ -128,4 +128,41 @@ export {
   type GenAiBudget,
   type GenAiAlertInfo,
   type GenAiQuantileSeries,
+  RedAlerts,
+  redAlertRules,
+  RED_ALERT_NAMES,
+  type RedAlertsProps,
+  type RedAlertsMembers,
+  type RedAlertsInstance,
+  type RedAlertOptions,
+  type RedErrorRatioAlert,
+  type RedLatencyAlert,
+  Watchdog,
+  WATCHDOG_URL_FILE,
+  type WatchdogProps,
+  type WatchdogMembers,
+  type WatchdogInstance,
+  AlertRouting,
+  ALERT_ROUTING_LEVELS,
+  type AlertRoutingProps,
+  type AlertRoutingMembers,
+  type AlertRoutingInstance,
+  type AlertRoutingLevel,
+  type AlertRoutingTeam,
+  type AlertRoutingTiming,
+  type AlertRoutingReceiver,
 } from "./composites";
+
+// Config namespace (#3371): `prometheus.profiles.<env>` in chant.config.ts, the
+// ruler and Alertmanager each environment observes and imports from. The API
+// clients are `@intentius/chant-lexicon-prometheus/api`.
+export { prometheusConfigSchema, resolveRulerTarget, resolveAlertmanagerTarget, declaredNamespaces, namespaceOfGroup } from "./config";
+export type { PrometheusLexiconConfig, PrometheusProfile, RulerProfile, AlertmanagerProfile, RulerKind, AlertmanagerKind } from "./config";
+
+// Op steps (#3369). The activities resolve from
+// `@intentius/chant-lexicon-prometheus/op/activities`; the promtool and amtool
+// check builders share their names with the functions above, so they are at
+// `@intentius/chant-lexicon-prometheus/op/builders`.
+export { amtoolRoutesTest, alertmanagerSilence, alertmanagerUnsilence, rulesLoadedObserve, ruleAudit } from "./op/builders";
+export { RuleAuditOp, type RuleAuditOpConfig, type RuleAuditOpResources } from "./op/audit-op";
+export { sloRuleTests, type SloTestInput } from "./op/slo-rule-tests";

package/src/lint/post-synth/prom-helpers.ts

@@ -5,9 +5,9 @@
  *
  * Documents are recognized by shape, not by which lexicon emitted them, so a
  * rule file that reaches the output some other way (a hand-written YAML
- * sidecar) is checked too. `PrometheusRule` manifests are not read here;
- * their groups are the same `RuleGroup`s and are checked where they are
- * declared.
+ * sidecar) is checked too. `PrometheusRule` manifests are not read here:
+ * `chant build` hands each lexicon's checks only that lexicon's output, so the
+ * k8s lexicon's WK8606 runs `prometheusRuleDiagnostics` (below) over them.
  */
 
 import type { PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
@@ -135,3 +135,38 @@ export function routingDiagnostics(ctx: PostSynthContext): PostSynthDiagnostic[]
   const files = ruleFiles.map((r) => r.config);
   return alertmanager.flatMap(({ source, config }) => validateSeverityRouting(files, config).map((i) => toDiagnostic(i, source)));
 }
+
+/**
+ * The rule-file checks (PROM101-PROM107, PROM211, PROM213-PROM219) over the
+ * groups of every `PrometheusRule` in the build, whichever lexicon emitted
+ * it. `spec.groups` is a rule file's `groups`, so `validateRuleFile` reads it
+ * as is. Findings keep their PROM ids and name the PrometheusRule's namespace
+ * and name. PROM212 (runbook URLs) stays opt-in and is not run here.
+ */
+export function prometheusRuleDiagnostics(ctx: PostSynthContext): PostSynthDiagnostic[] {
+  const out: PostSynthDiagnostic[] = [];
+  for (const output of ctx.outputs.values()) {
+    const text = typeof output === "string" ? output : (output as SerializerResult).primary;
+    if (!text) continue;
+    let docs: unknown[];
+    try {
+      docs = loadAll(text);
+    } catch {
+      continue;
+    }
+    for (const raw of docs) {
+      const doc = raw as Record<string, any> | null;
+      if (!doc || typeof doc !== "object" || doc.kind !== "PrometheusRule") continue;
+      if (typeof doc.apiVersion !== "string" || !doc.apiVersion.startsWith("monitoring.coreos.com/")) continue;
+      const groups = doc.spec?.groups;
+      if (!Array.isArray(groups)) continue;
+      const name = String(doc.metadata?.name ?? "");
+      const source = `PrometheusRule ${doc.metadata?.namespace ? `${doc.metadata.namespace}/` : ""}${name}`;
+      for (const issue of validateRuleFile({ groups } as RuleFileConfig)) {
+        if (issue.code === "PROM212") continue;
+        out.push({ ...toDiagnostic(issue, source), ...(issue.subject ? {} : { entity: name }) });
+      }
+    }
+  }
+  return out;
+}

package/src/op/activities/index.ts

@@ -0,0 +1,31 @@
+/**
+ * prometheus Op activities (#3369), resolved by the core activity registry
+ * when a project's `chant.config.ts` lists the `prometheus` lexicon. Every
+ * function this module exports is an activity, bound by its export name, so
+ * the helpers behind them stay in their own files.
+ *
+ * - `promtoolCheckRules`, `promtoolTestRules`, `amtoolCheckConfig`: the
+ *   upstream tools over built files, failing the step when the tool is
+ *   missing or rejects the file.
+ * - `amtoolRoutesTest`: labels route to the expected receivers.
+ * - `alertmanagerSilence`, `alertmanagerUnsilence`: a silence around a
+ *   change, recorded so the final and `onFailure` phases can expire it.
+ * - `rulesLoadedObserve`: declared rule groups as the resources of a
+ *   `ConvergeOp({ observe })`.
+ * - `ruleAudit`: the step of `RuleAuditOp`.
+ */
+export { promtoolCheckRules, promtoolTestRules, amtoolCheckConfig, amtoolRoutesTest } from "./promtool";
+export type {
+  ToolStepResult,
+  PromtoolCheckRulesArgs,
+  PromtoolTestRulesArgs,
+  AmtoolCheckConfigArgs,
+  AmtoolRoutesTestArgs,
+  AmtoolRoutesTestResult,
+} from "./promtool";
+export { alertmanagerSilence, alertmanagerUnsilence } from "./silence";
+export type { AlertmanagerSilenceArgs, AlertmanagerSilenceResult, AlertmanagerUnsilenceArgs, AlertmanagerUnsilenceResult } from "./silence";
+export { rulesLoadedObserve } from "./rules-loaded";
+export type { RulesLoadedObserveArgs, RulesLoadedObserveResult } from "./rules-loaded";
+export { ruleAudit } from "./rule-audit";
+export type { RuleAuditArgs, RuleAuditResult, RuleAuditFinding, RuleAuditFindingKind, RuleAuditMode } from "./rule-audit";

package/src/op/activities/promtool.test.ts

@@ -0,0 +1,107 @@
+/**
+ * The promtool and amtool steps, against stand-in binaries: small shell
+ * scripts that answer `--version` and accept or reject a file by its
+ * content, run as the real tools would be (the step's own child process).
+ * The real tools are exercised by `tools.test.ts` and `slo-burn.test.ts`
+ * when they are on PATH.
+ */
+
+import { describe, expect, test } from "vitest";
+import { chmodSync, mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { ExecRunner } from "@intentius/chant-lexicon-otel/op/activities/exec";
+import { amtoolCheckConfig, amtoolRoutesTest, parseRoutesOutput, promtoolCheckRules, promtoolTestRules } from "./promtool";
+
+const dir = mkdtempSync(join(tmpdir(), "chant-promtool-step-"));
+
+function script(name: string, body: string): string {
+  const path = join(dir, name);
+  writeFileSync(path, `#!/bin/sh\n${body}\n`);
+  chmodSync(path, 0o755);
+  return path;
+}
+
+const PROMTOOL = script(
+  "promtool",
+  `case "$1" in
+  --version) echo "promtool, version 3.15.0"; exit 0;;
+  check) if grep -q BAD "$3"; then echo "FAILED: $3: bad rule" >&2; exit 1; fi; echo "SUCCESS: $3"; exit 0;;
+  test) if grep -q FAIL "$3"; then echo "FAILED: alert did not fire" >&2; exit 1; fi; grep -q "rule_files" "$3" || exit 3; test -f rules.yml || exit 4; echo "SUCCESS"; exit 0;;
+esac
+exit 2`,
+);
+const AMTOOL = script(
+  "amtool",
+  `case "$1" in
+  --version) echo "amtool, version 0.34.1"; exit 0;;
+  check-config) if grep -q BAD "$2"; then echo "FAILED" >&2; exit 1; fi; echo "SUCCESS"; exit 0;;
+esac
+exit 2`,
+);
+
+function file(name: string, text: string): string {
+  const path = join(dir, name);
+  writeFileSync(path, text);
+  return path;
+}
+
+const RULES = file("rules.yml", "groups:\n  - name: g\n    rules:\n      - record: x\n        expr: up\n");
+
+describe("promtool steps", () => {
+  test("check rules passes a good file and fails a bad one with the tool's output", async () => {
+    await expect(promtoolCheckRules({ rules: RULES, bin: PROMTOOL })).resolves.toMatchObject({ ok: true, files: [RULES] });
+    const bad = file("bad.yml", "BAD");
+    await expect(promtoolCheckRules({ rules: [RULES, bad], bin: PROMTOOL })).rejects.toThrow(/promtool check rules rejected .*bad\.yml.*bad rule/s);
+  });
+
+  test("a missing binary fails the step: a check that did not run is not a pass", async () => {
+    await expect(promtoolCheckRules({ rules: RULES, bin: join(dir, "no-such-promtool") })).rejects.toThrow(/is not installed/);
+  });
+
+  test("test rules runs once per test file and inline document, each beside rules.yml", async () => {
+    const t = file("t.test.yml", "rule_files: [rules.yml]\ntests: []\n");
+    const r = await promtoolTestRules({ rules: RULES, tests: t, testYaml: ["rule_files: [rules.yml]\ntests: []\n"], bin: PROMTOOL });
+    expect(r.tests).toBe(2);
+    await expect(promtoolTestRules({ rules: RULES, testYaml: "rule_files: [rules.yml]\n# FAIL\n", bin: PROMTOOL })).rejects.toThrow(/alert did not fire/);
+    await expect(promtoolTestRules({ rules: RULES, bin: PROMTOOL })).rejects.toThrow(/give tests or testYaml/);
+  });
+
+  test("amtool check-config", async () => {
+    await expect(amtoolCheckConfig({ config: file("am.yml", "route: {}\n"), bin: AMTOOL })).resolves.toMatchObject({ ok: true });
+    await expect(amtoolCheckConfig({ config: file("am-bad.yml", "BAD"), bin: AMTOOL })).rejects.toThrow(/amtool check-config rejected/);
+  });
+});
+
+describe("amtoolRoutesTest", () => {
+  test("reads the receivers amtool prints", () => {
+    expect(parseRoutesOutput("oncall\n")).toEqual(["oncall"]);
+    expect(parseRoutesOutput("oncall,tickets\n")).toEqual(["oncall", "tickets"]);
+    expect(parseRoutesOutput("")).toEqual([]);
+  });
+
+  const exec = (stdout: string, code = 0): { exec: ExecRunner; calls: string[][] } => {
+    const calls: string[][] = [];
+    return {
+      calls,
+      exec: async (bin, args) => {
+        calls.push([bin, ...args]);
+        return { code, stdout, stderr: "" };
+      },
+    };
+  };
+
+  test("passes when the labels route to the expected receivers", async () => {
+    const { exec: e, calls } = exec("oncall\n");
+    const r = await amtoolRoutesTest({ config: "/tmp/am.yml", labels: { severity: "page", slo: "checkout" }, expect: "oncall", bin: "amtool", _exec: e });
+    expect(r.receivers).toEqual(["oncall"]);
+    expect(calls[0]).toEqual(["amtool", "config", "routes", "test", "--config.file=/tmp/am.yml", "severity=page", "slo=checkout"]);
+  });
+
+  test("fails when they route elsewhere, naming both", async () => {
+    const { exec: e } = exec("tickets\n");
+    await expect(amtoolRoutesTest({ config: "/tmp/am.yml", labels: { severity: "page" }, expect: ["oncall"], _exec: e })).rejects.toThrow(
+      /severity=page routes to tickets .* expected oncall/,
+    );
+  });
+});