@remit/doctor 0.0.1

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@remit/doctor",
+  "version": "0.0.1",
+  "type": "module",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "default": "./src/index.ts"
+    }
+  },
+  "scripts": {
+    "start": "node --import tsx src/main.ts",
+    "check": "node --import tsx src/cli.ts",
+    "test:typecheck": "tsgo --noEmit",
+    "test:run": "node --import tsx --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-lines=95 --test 'src/**/*.test.ts'",
+    "test": "npm run test:typecheck && npm run test:run"
+  },
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/remit-mail/reader.git",
+    "directory": "packages/doctor"
+  }
+}

package/src/attempt.ts

@@ -0,0 +1,34 @@
+/**
+ * Turn a failure into a value.
+ *
+ * Used only where the design says a failure IS the signal: an endpoint that
+ * refuses the connection, a heartbeat directory that cannot be read. Those are
+ * facts the verdict has to carry, and a checker that propagates the first one
+ * reports nothing about everything else it looked at — which is the outcome
+ * "a signal that cannot be evaluated is degraded, never skipped" rules out.
+ *
+ * Everywhere else in this package, errors propagate.
+ */
+export type Attempt<T> =
+	| { readonly ok: true; readonly value: T }
+	| { readonly ok: false; readonly error: string };
+
+export const describeError = (error: unknown): string =>
+	error instanceof Error ? error.message : String(error);
+
+export const attempt = <T>(work: Promise<T>): Promise<Attempt<T>> =>
+	work.then(
+		(value) => ({ ok: true, value }) as const,
+		(error: unknown) => ({ ok: false, error: describeError(error) }) as const,
+	);
+
+/**
+ * `JSON.parse` in a promise, so a malformed document is a rejection a caller
+ * can `.catch()` rather than a synchronous throw needing a block try/catch.
+ * Same shape as the backend's own helper; written here because this package
+ * deliberately depends on nothing.
+ */
+export const safeJsonParse = <T>(raw: string): Promise<T> =>
+	new Promise((resolve) => {
+		resolve(JSON.parse(raw));
+	});

package/src/check.test.ts

@@ -0,0 +1,81 @@
+import assert from "node:assert/strict";
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, it } from "node:test";
+import { runCheck } from "./check.js";
+import { loadConfig } from "./config.js";
+
+const BODIES: Readonly<Record<string, string>> = {
+	backend: 'remit_account_sync_age_seconds{account_id="aaa"} 120\n',
+	queue: 'remit_queue_messages{queue="imap-sync-dlq",role="dead_letter"} 0\n',
+	"imap-worker": 'remit_imap_failures_total{operation="fetch",kind="auth"} 0\n',
+	"smtp-worker": 'remit_smtp_failures_total{kind="auth"} 0\n',
+};
+
+const stack = (overrides: Readonly<Record<string, string>> = {}) =>
+	(async (url: string) => {
+		const service = new URL(url).hostname;
+		const body = overrides[service] ?? BODIES[service];
+		if (body === undefined) throw new Error(`no such service: ${service}`);
+		return new Response(body, { status: 200 });
+	}) as unknown as typeof fetch;
+
+const heartbeatDir = async (services: readonly string[]): Promise<string> => {
+	const directory = await mkdtemp(join(tmpdir(), "remit-doctor-check-"));
+	for (const service of services) {
+		await writeFile(join(directory, `${service}.queue`), "now\n");
+	}
+	return directory;
+};
+
+describe("runCheck", () => {
+	it("reads both surfaces and produces one verdict", async () => {
+		const directory = await heartbeatDir([
+			"imap-worker",
+			"smtp-worker",
+			"account-worker",
+			"search-index-worker",
+		]);
+		const config = loadConfig({ DOCTOR_HEARTBEAT_DIR: directory });
+		const result = await runCheck(config, {}, new Date(), stack());
+		assert.equal(result.verdict, "healthy");
+	});
+
+	it("degrades on the heartbeat surface even when every scrape is clean", async () => {
+		const config = loadConfig({
+			DOCTOR_HEARTBEAT_DIR: await heartbeatDir(["imap-worker"]),
+		});
+		const result = await runCheck(config, {}, new Date(), stack());
+		assert.equal(result.verdict, "degraded");
+		assert.deepEqual(
+			result.reasons.map((reason) => reason.code),
+			["worker_heartbeat_stale"],
+		);
+	});
+
+	it("degrades on the metrics surface even when every heartbeat is fresh", async () => {
+		const config = loadConfig({
+			DOCTOR_HEARTBEAT_DIR: await heartbeatDir([
+				"imap-worker",
+				"smtp-worker",
+				"account-worker",
+				"search-index-worker",
+			]),
+		});
+		const result = await runCheck(
+			config,
+			{},
+			new Date(),
+			stack({
+				queue:
+					'remit_queue_messages{queue="imap-sync-dlq",role="dead_letter"} 4\n',
+			}),
+		);
+		assert.equal(result.verdict, "degraded");
+		assert.deepEqual(
+			result.reasons.map((reason) => reason.code),
+			["dead_letter_queue_not_empty"],
+		);
+	});
+});

package/src/check.ts

@@ -0,0 +1,36 @@
+import type { DoctorConfig } from "./config.js";
+import { readHeartbeats } from "./heartbeats.js";
+import { type Fetcher, scrapeAll } from "./scrape.js";
+import type { CounterState } from "./state.js";
+import { type CheckResult, evaluate } from "./verdict.js";
+
+/**
+ * One check: scrape the endpoints that carry a signal, read the heartbeat
+ * volume, and evaluate. The same function the loop runs on its interval and the
+ * exec seam runs on demand — the verdict is computed in one place and read
+ * three ways, at a shell, as an exit code, and as an alert.
+ */
+export const runCheck = async (
+	config: DoctorConfig,
+	previousCounters: Readonly<Record<string, CounterState>>,
+	now: Date = new Date(),
+	fetcher: Fetcher = fetch,
+): Promise<CheckResult> => {
+	const [scrapes, heartbeats] = await Promise.all([
+		scrapeAll(config.targets, config.scrapeTimeoutMs, fetcher),
+		readHeartbeats(
+			config.heartbeatDir,
+			config.heartbeatServices,
+			now.getTime(),
+		),
+	]);
+	return evaluate({
+		scrapes,
+		heartbeats,
+		previousCounters,
+		heartbeatMaxAgeSeconds: config.heartbeatMaxAgeSeconds,
+		syncAgeMaxSeconds: config.syncAgeMaxSeconds,
+		authFailureHoldSeconds: config.authFailureHoldSeconds,
+		now,
+	});
+};

package/src/cli.ts

@@ -0,0 +1,43 @@
+import { runCheck } from "./check.js";
+import { loadConfig } from "./config.js";
+import { describeError, log, setLogLevel } from "./log.js";
+import {
+	exitCodeFor,
+	NO_VERDICT_EXIT_CODE,
+	renderJson,
+	renderLines,
+} from "./report.js";
+import { readState } from "./state.js";
+
+/**
+ * The exec seam `remit doctor` drives (D4):
+ *
+ *   docker compose exec -T doctor node check.mjs [--json]
+ *
+ * Runs a fresh check and prints it. Fresh rather than the loop's last verdict,
+ * because `remit doctor` answers "is anything wrong now", and because the
+ * loop's verdict is the settled one the dwell rule announces, which is
+ * deliberately up to three checks behind the current state.
+ *
+ * It reads the loop's state file and never writes it. The auth-failure signal
+ * is a delta against the totals the loop last saw, so the seam needs that
+ * baseline; writing a new one would move the loop's own reference point and
+ * hide the next real increase.
+ */
+const json = process.argv.includes("--json");
+
+const check = async (): Promise<number> => {
+	const config = loadConfig();
+	setLogLevel(config.logLevel);
+	const state = await readState(config.stateDir);
+	const result = await runCheck(config, state.counters);
+	process.stdout.write(json ? renderJson(result) : renderLines(result));
+	return exitCodeFor(result);
+};
+
+process.exit(
+	await check().catch((error: unknown) => {
+		log.error({ error: describeError(error) }, "doctor: check could not run");
+		return NO_VERDICT_EXIT_CODE;
+	}),
+);

package/src/config.test.ts

@@ -0,0 +1,115 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { loadConfig, parseTargets } from "./config.js";
+
+describe("loadConfig", () => {
+	it("needs nothing set, and sends nothing when nothing is set", () => {
+		const config = loadConfig({});
+		assert.equal(config.webhookUrl, undefined);
+		assert.equal(config.deadManUrl, undefined);
+		assert.equal(config.dwellChecks, 3);
+		assert.equal(config.intervalMs, 60_000);
+		assert.deepEqual(
+			config.targets.map((target) => target.service),
+			["backend", "queue", "imap-worker", "smtp-worker"],
+		);
+		assert.deepEqual(config.heartbeatServices, [
+			"imap-worker",
+			"smtp-worker",
+			"account-worker",
+			"search-index-worker",
+		]);
+	});
+
+	it("refuses a webhook with no dead-man's switch, naming both variables", () => {
+		assert.throws(
+			() => loadConfig({ DOCTOR_WEBHOOK_URL: "https://hooks.example/x" }),
+			(error: Error) => {
+				assert.match(error.message, /DOCTOR_WEBHOOK_URL/);
+				assert.match(error.message, /DOCTOR_HEARTBEAT_URL/);
+				return true;
+			},
+		);
+	});
+
+	it("accepts a dead-man's switch with no webhook", () => {
+		const config = loadConfig({ DOCTOR_HEARTBEAT_URL: "https://hc.example/x" });
+		assert.equal(config.deadManUrl, "https://hc.example/x");
+	});
+
+	it("treats a blank value as unset, so an empty compose passthrough is not a webhook", () => {
+		const config = loadConfig({
+			DOCTOR_WEBHOOK_URL: "   ",
+			DOCTOR_HEARTBEAT_URL: "",
+		});
+		assert.equal(config.webhookUrl, undefined);
+		assert.equal(config.deadManUrl, undefined);
+	});
+
+	it("refuses a threshold that is not a positive number, by name", () => {
+		assert.throws(
+			() => loadConfig({ DOCTOR_SYNC_AGE_MAX_SECONDS: "soon" }),
+			/DOCTOR_SYNC_AGE_MAX_SECONDS/,
+		);
+		assert.throws(
+			() => loadConfig({ DOCTOR_DWELL_CHECKS: "0" }),
+			/DOCTOR_DWELL_CHECKS/,
+		);
+		assert.throws(
+			() => loadConfig({ DOCTOR_INTERVAL_SECONDS: "-5" }),
+			/DOCTOR_INTERVAL_SECONDS/,
+		);
+	});
+
+	it("reads the overrides an operator is expected to set", () => {
+		const config = loadConfig({
+			DOCTOR_INTERVAL_SECONDS: "15",
+			DOCTOR_DWELL_CHECKS: "2",
+			DOCTOR_SYNC_AGE_MAX_SECONDS: "600",
+			DOCTOR_HEARTBEAT_MAX_AGE_SECONDS: "120",
+			DOCTOR_HEARTBEAT_DIR: "/tmp/hb",
+			DOCTOR_HEARTBEAT_SERVICES: "imap-worker, smtp-worker",
+			DOCTOR_STATE_DIR: "/tmp/state",
+			DOCTOR_WEBHOOK_CONTENT_TYPE: "text/plain",
+			DOCTOR_WEBHOOK_TEMPLATE: "{{summary}}",
+			DOCTOR_HEARTBEAT_URL: "https://hc.example/x",
+			DOCTOR_WEBHOOK_URL: "https://ntfy.example/remit",
+		});
+		assert.equal(config.intervalMs, 15_000);
+		assert.equal(config.dwellChecks, 2);
+		assert.equal(config.syncAgeMaxSeconds, 600);
+		assert.equal(config.heartbeatMaxAgeSeconds, 120);
+		assert.equal(config.heartbeatDir, "/tmp/hb");
+		assert.deepEqual(config.heartbeatServices, ["imap-worker", "smtp-worker"]);
+		assert.equal(config.stateDir, "/tmp/state");
+		assert.equal(config.webhookContentType, "text/plain");
+		assert.equal(config.webhookTemplate, "{{summary}}");
+	});
+});
+
+describe("parseTargets", () => {
+	it("reads service=url pairs", () => {
+		assert.deepEqual(
+			parseTargets("a=http://a:1/metrics, b=http://b:2/metrics"),
+			[
+				{ service: "a", url: "http://a:1/metrics" },
+				{ service: "b", url: "http://b:2/metrics" },
+			],
+		);
+	});
+
+	it("keeps the whole url, colons and all", () => {
+		assert.deepEqual(parseTargets("a=http://a:9464/metrics"), [
+			{ service: "a", url: "http://a:9464/metrics" },
+		]);
+	});
+
+	it("refuses an entry with no service name", () => {
+		assert.throws(() => parseTargets("http://a/metrics"), /DOCTOR_TARGETS/);
+		assert.throws(() => parseTargets("=http://a/metrics"), /DOCTOR_TARGETS/);
+	});
+
+	it("reads an empty list as no targets", () => {
+		assert.deepEqual(loadConfig({ DOCTOR_TARGETS: " , " }).targets, []);
+	});
+});

package/src/config.ts

@@ -0,0 +1,241 @@
+/**
+ * Everything the checker reads from its environment, in one place, validated
+ * once at startup.
+ *
+ * The `doctor` service takes no `env_file`. It is the one container in the
+ * stack that opens an outbound connection to a third-party endpoint, so it
+ * holds the variables it needs and none of the deployment's secrets — the
+ * compose file passes each of these through by name.
+ */
+export type ContentType = string;
+
+export interface ScrapeTarget {
+	readonly service: string;
+	readonly url: string;
+}
+
+export interface DoctorConfig {
+	readonly intervalMs: number;
+	readonly targets: readonly ScrapeTarget[];
+	readonly scrapeTimeoutMs: number;
+	readonly heartbeatDir: string;
+	readonly heartbeatServices: readonly string[];
+	readonly heartbeatMaxAgeSeconds: number;
+	readonly syncAgeMaxSeconds: number;
+	readonly authFailureHoldSeconds: number;
+	readonly stateDir: string;
+	readonly dwellChecks: number;
+	readonly webhookUrl: string | undefined;
+	readonly webhookTemplate: string | undefined;
+	readonly webhookContentType: ContentType;
+	readonly deadManUrl: string | undefined;
+	readonly requestTimeoutMs: number;
+	readonly logLevel: string | undefined;
+}
+
+/**
+ * The four endpoints that carry a signal the verdict reads: dead-letter depth,
+ * per-account sync age, and the two authentication counters. `account-worker`
+ * and `search-index-worker` are absent on purpose — their liveness is a
+ * heartbeat file, and scraping a service to prove it is up duplicates a signal
+ * that already survives the metrics server failing.
+ */
+const DEFAULT_TARGETS: readonly ScrapeTarget[] = [
+	{ service: "backend", url: "http://backend:8080/metrics" },
+	{ service: "queue", url: "http://queue:9324/metrics" },
+	{ service: "imap-worker", url: "http://imap-worker:9464/metrics" },
+	{ service: "smtp-worker", url: "http://smtp-worker:9464/metrics" },
+];
+
+const DEFAULT_HEARTBEAT_SERVICES = [
+	"imap-worker",
+	"smtp-worker",
+	"account-worker",
+	"search-index-worker",
+];
+
+/** The same 420 s the workers' own compose healthcheck uses; one threshold. */
+const DEFAULT_HEARTBEAT_MAX_AGE_SECONDS = 420;
+
+/**
+ * Three hours. `remit_account_sync_age_seconds` sawtooths: the scheduler ticks
+ * hourly by default (`MAILBOX_SYNC_TICK_INTERVAL_SECONDS`) and a tick skips a
+ * mailbox stamped inside the freshness window, so a perfectly healthy account
+ * climbs past an hour every cycle. A threshold at or near the tick fires on
+ * accounts that are fine; raise this if you raised the tick.
+ */
+const DEFAULT_SYNC_AGE_MAX_SECONDS = 3 * 60 * 60;
+
+/**
+ * How long after the last authentication failure the condition still counts as
+ * failing. Three hours, for the same reason the sync-age threshold is three
+ * hours: authentication is retried on the sync tick, so the failures arrive in
+ * one burst per tick and the gaps between bursts are not recoveries.
+ *
+ * The signal is a counter delta, which is true for exactly one check. Without a
+ * hold the reason appears on one check in sixty and the three-check dwell never
+ * settles, so the one class of failure that never resolves itself would be the
+ * one that never alerts.
+ */
+const DEFAULT_AUTH_FAILURE_HOLD_SECONDS = 3 * 60 * 60;
+
+/** D8's number. Configurable so an operator can trade latency for quiet. */
+const DEFAULT_DWELL_CHECKS = 3;
+
+/**
+ * A dwell longer than this is indistinguishable from alerting being off, and a
+ * container that starts cleanly and never speaks is the worst way to learn that.
+ * An hour of agreeing checks is already far past any deliberate setting.
+ */
+const MAX_DWELL_CHECKS = 60;
+
+const DEFAULT_INTERVAL_SECONDS = 60;
+const DEFAULT_SCRAPE_TIMEOUT_SECONDS = 10;
+const DEFAULT_REQUEST_TIMEOUT_SECONDS = 10;
+
+const DEFAULT_CONTENT_TYPE = "application/json";
+
+export type Env = Record<string, string | undefined>;
+
+const text = (env: Env, name: string): string | undefined => {
+	const value = env[name]?.trim();
+	return value === undefined || value === "" ? undefined : value;
+};
+
+/**
+ * A number a human typed. An unparseable or out-of-range value is refused by
+ * name rather than silently replaced by the default: a threshold that quietly
+ * reverts is a check the operator believes is stricter than it is.
+ */
+const positiveNumber = (env: Env, name: string, fallback: number): number => {
+	const raw = text(env, name);
+	if (raw === undefined) return fallback;
+	const parsed = Number(raw);
+	if (!Number.isFinite(parsed) || parsed <= 0) {
+		throw new Error(`${name} must be a positive number, got: ${raw}`);
+	}
+	return parsed;
+};
+
+/**
+ * A count, not a measurement. `2.5` would persist a fractional run in the state
+ * file, and a very large value silently turns alerting off while the container
+ * starts cleanly and pings its dead-man forever — the failure mode with no
+ * symptom, which is the one this whole design exists to remove.
+ */
+const boundedCount = (
+	env: Env,
+	name: string,
+	fallback: number,
+	max: number,
+): number => {
+	const raw = text(env, name);
+	if (raw === undefined) return fallback;
+	const parsed = Number(raw);
+	if (!Number.isInteger(parsed) || parsed < 1 || parsed > max) {
+		throw new Error(
+			`${name} must be a whole number of checks between 1 and ${max}, got: ${raw}`,
+		);
+	}
+	return parsed;
+};
+
+/** `name=url,name=url`. Written out in full so nothing is inferred from a name. */
+export const parseTargets = (raw: string): ScrapeTarget[] =>
+	raw
+		.split(",")
+		.map((entry) => entry.trim())
+		.filter((entry) => entry !== "")
+		.map((entry) => {
+			const separator = entry.indexOf("=");
+			if (separator <= 0) {
+				throw new Error(
+					`DOCTOR_TARGETS entries are <service>=<url>, got: ${entry}`,
+				);
+			}
+			return {
+				service: entry.slice(0, separator).trim(),
+				url: entry.slice(separator + 1).trim(),
+			};
+		});
+
+/**
+ * D11. A webhook without a dead-man's switch is the half-configuration that
+ * looks armed and is not: if the VM dies nothing fires and the channel reads
+ * like a quiet week. It is refused at startup, by name, rather than warned
+ * about — a warning in a log nobody is watching is the same silence.
+ */
+export const loadConfig = (env: Env = process.env): DoctorConfig => {
+	const webhookUrl = text(env, "DOCTOR_WEBHOOK_URL");
+	const deadManUrl = text(env, "DOCTOR_HEARTBEAT_URL");
+	if (webhookUrl !== undefined && deadManUrl === undefined) {
+		throw new Error(
+			"DOCTOR_WEBHOOK_URL is set without DOCTOR_HEARTBEAT_URL. An alert channel " +
+				"nothing watches cannot tell a quiet week from a dead box: set " +
+				"DOCTOR_HEARTBEAT_URL to a dead-man's-switch URL (healthchecks.io, " +
+				"Cronitor, Uptime Kuma), or unset DOCTOR_WEBHOOK_URL.",
+		);
+	}
+
+	const targetsRaw = text(env, "DOCTOR_TARGETS");
+	const servicesRaw = text(env, "DOCTOR_HEARTBEAT_SERVICES");
+
+	return {
+		intervalMs:
+			positiveNumber(env, "DOCTOR_INTERVAL_SECONDS", DEFAULT_INTERVAL_SECONDS) *
+			1000,
+		targets:
+			targetsRaw === undefined ? DEFAULT_TARGETS : parseTargets(targetsRaw),
+		scrapeTimeoutMs:
+			positiveNumber(
+				env,
+				"DOCTOR_SCRAPE_TIMEOUT_SECONDS",
+				DEFAULT_SCRAPE_TIMEOUT_SECONDS,
+			) * 1000,
+		heartbeatDir: text(env, "DOCTOR_HEARTBEAT_DIR") ?? "/data/heartbeat",
+		heartbeatServices:
+			servicesRaw === undefined
+				? DEFAULT_HEARTBEAT_SERVICES
+				: servicesRaw
+						.split(",")
+						.map((name) => name.trim())
+						.filter((name) => name !== ""),
+		heartbeatMaxAgeSeconds: positiveNumber(
+			env,
+			"DOCTOR_HEARTBEAT_MAX_AGE_SECONDS",
+			DEFAULT_HEARTBEAT_MAX_AGE_SECONDS,
+		),
+		syncAgeMaxSeconds: positiveNumber(
+			env,
+			"DOCTOR_SYNC_AGE_MAX_SECONDS",
+			DEFAULT_SYNC_AGE_MAX_SECONDS,
+		),
+		authFailureHoldSeconds: positiveNumber(
+			env,
+			"DOCTOR_AUTH_FAILURE_HOLD_SECONDS",
+			DEFAULT_AUTH_FAILURE_HOLD_SECONDS,
+		),
+		stateDir: text(env, "DOCTOR_STATE_DIR") ?? "/data/doctor",
+		dwellChecks: boundedCount(
+			env,
+			"DOCTOR_DWELL_CHECKS",
+			DEFAULT_DWELL_CHECKS,
+			MAX_DWELL_CHECKS,
+		),
+		webhookUrl,
+		webhookTemplate: text(env, "DOCTOR_WEBHOOK_TEMPLATE"),
+		webhookContentType:
+			text(env, "DOCTOR_WEBHOOK_CONTENT_TYPE") ?? DEFAULT_CONTENT_TYPE,
+		deadManUrl,
+		// `DOCTOR_LOG_LEVEL`, because the compose service passes DOCTOR_* variables
+		// and nothing else — a plain LOG_LEVEL could never reach this container, so
+		// the per-check verdict line could never be turned on.
+		logLevel: text(env, "DOCTOR_LOG_LEVEL") ?? text(env, "LOG_LEVEL"),
+		requestTimeoutMs:
+			positiveNumber(
+				env,
+				"DOCTOR_REQUEST_TIMEOUT_SECONDS",
+				DEFAULT_REQUEST_TIMEOUT_SECONDS,
+			) * 1000,
+	};
+};

package/src/deadman.test.ts

@@ -0,0 +1,31 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { pingDeadMan } from "./deadman.js";
+
+describe("pingDeadMan", () => {
+	it("GETs the configured url and sends nothing else", async () => {
+		let seen: { url: string; init: RequestInit } | undefined;
+		await pingDeadMan("https://hc-ping.example/uuid", 1000, (async (
+			url: string,
+			init: RequestInit,
+		) => {
+			seen = { url, init };
+			return new Response("OK", { status: 200 });
+		}) as unknown as typeof fetch);
+		assert.equal(seen?.url, "https://hc-ping.example/uuid");
+		assert.equal(seen?.init.method, "GET");
+		assert.equal(seen?.init.body, undefined);
+	});
+
+	it("reports a rejected ping", async () => {
+		await assert.rejects(
+			pingDeadMan(
+				"https://hc-ping.example/uuid",
+				1000,
+				(async () =>
+					new Response("", { status: 404 })) as unknown as typeof fetch,
+			),
+			/HTTP 404/,
+		);
+	});
+});

package/src/deadman.ts

@@ -0,0 +1,34 @@
+import type { Fetcher } from "./scrape.js";
+
+/**
+ * D11. A ping on every completed check, whatever the verdict.
+ *
+ * If the VM is off, the disk is full, the network is gone or this container
+ * crashed, no alert fires — and an operator with only a webhook cannot tell
+ * that apart from a week with nothing wrong. That silent failure is the one
+ * this exists to remove, which is why the configuration refuses a webhook
+ * without a heartbeat rather than treating it as an upgrade.
+ *
+ * A check completes when it produces a verdict, including a `degraded` verdict
+ * produced from signals it could not read: a scrape failure degrades the
+ * verdict and still pings, because the checker is working. A check that throws
+ * before producing one does not ping.
+ *
+ * GET, not POST. It is the method healthchecks.io, Cronitor and Uptime Kuma's
+ * push monitor all accept, and Uptime Kuma accepts nothing else. Nothing is
+ * sent in the request beyond the URL the operator configured — the ping carries
+ * that the checker ran, not what it found.
+ */
+export const pingDeadMan = async (
+	url: string,
+	timeoutMs: number,
+	fetcher: Fetcher = fetch,
+): Promise<void> => {
+	const response = await fetcher(url, {
+		method: "GET",
+		signal: AbortSignal.timeout(timeoutMs),
+	});
+	if (!response.ok) {
+		throw new Error(`dead-man ping rejected: HTTP ${response.status}`);
+	}
+};