@remit/doctor 0.0.1

package/src/dwell.test.ts

@@ -0,0 +1,120 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { advance } from "./dwell.js";
+import { type DoctorState, initialState } from "./state.js";
+import type { Verdict } from "./verdict.js";
+
+const NOW = new Date("2026-07-27T10:00:00.000Z");
+
+/** Feed a run of verdicts through the machine and collect what it announced. */
+const run = (
+	verdicts: readonly Verdict[],
+	dwell = 3,
+	from: DoctorState = initialState,
+): { fired: (Verdict | undefined)[]; state: DoctorState } => {
+	let state = from;
+	const fired: (Verdict | undefined)[] = [];
+	for (const verdict of verdicts) {
+		const transition = advance(state, verdict, dwell, NOW);
+		state = transition.state;
+		fired.push(transition.fires);
+	}
+	return { fired, state };
+};
+
+describe("the dwell rule", () => {
+	it("says nothing on a healthy stack, however long it stays healthy", () => {
+		const { fired } = run([
+			"healthy",
+			"healthy",
+			"healthy",
+			"healthy",
+			"healthy",
+		]);
+		assert.deepEqual(fired, [
+			undefined,
+			undefined,
+			undefined,
+			undefined,
+			undefined,
+		]);
+	});
+
+	it("announces a degraded verdict once, on the third agreeing check", () => {
+		const { fired } = run(["degraded", "degraded", "degraded", "degraded"]);
+		assert.deepEqual(fired, [undefined, undefined, "degraded", undefined]);
+	});
+
+	it("announces the recovery once, three checks later", () => {
+		const { fired } = run([
+			"degraded",
+			"degraded",
+			"degraded",
+			"healthy",
+			"healthy",
+			"healthy",
+			"healthy",
+		]);
+		assert.deepEqual(fired, [
+			undefined,
+			undefined,
+			"degraded",
+			undefined,
+			undefined,
+			"healthy",
+			undefined,
+		]);
+	});
+
+	it("stays silent through a flap that never settles", () => {
+		const { fired } = run([
+			"degraded",
+			"healthy",
+			"degraded",
+			"healthy",
+			"degraded",
+			"healthy",
+			"degraded",
+			"healthy",
+		]);
+		assert.deepEqual(new Set(fired), new Set([undefined]));
+	});
+
+	it("stays silent when a degraded run is broken one check short", () => {
+		const { fired } = run(["degraded", "degraded", "healthy", "degraded"]);
+		assert.deepEqual(fired, [undefined, undefined, undefined, undefined]);
+	});
+
+	it("does not re-announce a condition already reported", () => {
+		const first = run(["degraded", "degraded", "degraded"]);
+		const second = run(
+			["degraded", "degraded", "degraded", "degraded"],
+			3,
+			first.state,
+		);
+		assert.deepEqual(new Set(second.fired), new Set([undefined]));
+	});
+
+	it("survives a restart without re-announcing, because the state is the input", () => {
+		const before = run(["degraded", "degraded", "degraded"]).state;
+		// A restart reloads exactly this state from the volume.
+		const after = run(["degraded", "degraded", "degraded"], 3, before);
+		assert.deepEqual(new Set(after.fired), new Set([undefined]));
+		assert.equal(after.state.firedVerdict, "degraded");
+	});
+
+	it("holds the run at the dwell count rather than counting up forever", () => {
+		const { state } = run(new Array(50).fill("healthy"));
+		assert.equal(state.candidateRuns, 3);
+	});
+
+	it("honours a shorter dwell", () => {
+		const { fired } = run(["degraded", "degraded"], 1);
+		assert.deepEqual(fired, ["degraded", undefined]);
+	});
+
+	it("stamps when it last decided", () => {
+		const { state } = run(["healthy"]);
+		assert.equal(state.updatedAt, NOW.toISOString());
+	});
+});

package/src/dwell.ts

@@ -0,0 +1,53 @@
+import type { DoctorState } from "./state.js";
+import type { Verdict } from "./verdict.js";
+
+export interface Transition {
+	readonly state: DoctorState;
+	/** The verdict to announce, or `undefined` for the silence that is normal. */
+	readonly fires: Verdict | undefined;
+}
+
+/**
+ * D8. A verdict is announced when it changes AND has held for `dwellChecks`
+ * consecutive checks. Never on an unchanged verdict, however long it persists.
+ *
+ * Transition-only firing without the dwell is the loudest possible response to
+ * a flapping signal: a verdict that oscillates every check sends two messages
+ * per cycle, which is worse than the periodic posting this rejects. A
+ * dead-letter message an operator replays and that fails again, and an account
+ * sitting on the sync-age threshold, both produce that shape.
+ *
+ * The cost is detection latency: at the default 60 s interval and three checks,
+ * an outage is announced up to three minutes after it starts and a recovery up
+ * to three minutes after it clears. Nobody acts inside three minutes on a
+ * mailbox that stopped syncing, and the dead-man's switch — which pings on
+ * every completed check, settled or not — is unaffected.
+ *
+ * `fires` is what the caller should send. It is the caller's job to record the
+ * returned state whether or not delivery succeeded: a transition is spent when
+ * it is decided, so a webhook that rejects the payload does not turn into an
+ * announcement on every subsequent check.
+ */
+export const advance = (
+	state: DoctorState,
+	verdict: Verdict,
+	dwellChecks: number,
+	now: Date,
+): Transition => {
+	const runs = state.candidateVerdict === verdict ? state.candidateRuns + 1 : 1;
+	const settled = runs >= dwellChecks;
+	const fires = settled && verdict !== state.firedVerdict ? verdict : undefined;
+	return {
+		state: {
+			...state,
+			candidateVerdict: verdict,
+			// Held at the dwell count rather than counting up forever: the run only
+			// ever answers "has it settled", and an unbounded integer in a file that
+			// lives for years is a number with nothing to say.
+			candidateRuns: Math.min(runs, dwellChecks),
+			firedVerdict: fires ?? state.firedVerdict,
+			updatedAt: now.toISOString(),
+		},
+		fires,
+	};
+};

package/src/heartbeats.test.ts

@@ -0,0 +1,68 @@
+import assert from "node:assert/strict";
+import { mkdtemp, utimes, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, it } from "node:test";
+import { readHeartbeats } from "./heartbeats.js";
+
+const SERVICES = ["imap-worker", "smtp-worker"];
+const NOW = Date.parse("2026-07-27T10:00:00.000Z");
+
+const withFiles = async (
+	files: Readonly<Record<string, number>>,
+): Promise<string> => {
+	const directory = await mkdtemp(join(tmpdir(), "remit-doctor-hb-"));
+	for (const [name, ageSeconds] of Object.entries(files)) {
+		const path = join(directory, name);
+		await writeFile(path, "2026-07-27T09:00:00.000Z\n");
+		const when = new Date(NOW - ageSeconds * 1000);
+		await utimes(path, when, when);
+	}
+	return directory;
+};
+
+describe("readHeartbeats", () => {
+	it("reports the age of a service's stalest loop, not its freshest", async () => {
+		const directory = await withFiles({
+			"imap-worker.imap-sync": 5,
+			"imap-worker.imap-flag-push": 900,
+			"smtp-worker.smtp-send": 10,
+		});
+		const readings = await readHeartbeats(directory, SERVICES, NOW);
+		assert.equal(readings[0].ageSeconds, 900);
+		assert.equal(readings[1].ageSeconds, 10);
+	});
+
+	it("reports a service with no file as unreadable, never as age zero", async () => {
+		const directory = await withFiles({ "imap-worker.imap-sync": 5 });
+		const [, smtp] = await readHeartbeats(directory, SERVICES, NOW);
+		assert.equal(smtp.ageSeconds, undefined);
+		assert.equal(smtp.error, "no heartbeat file");
+	});
+
+	it("reports every service as unreadable when the directory is not there", async () => {
+		const readings = await readHeartbeats(
+			"/nonexistent/heartbeat",
+			SERVICES,
+			NOW,
+		);
+		assert.deepEqual(
+			readings.map((reading) => reading.ageSeconds),
+			[undefined, undefined],
+		);
+		assert.match(readings[0].error ?? "", /cannot read/);
+	});
+
+	it("does not mistake one service's file for another's prefix", async () => {
+		const directory = await withFiles({
+			"imap-worker-extra.queue": 5,
+			"imap-worker.imap-sync": 20,
+		});
+		const [imap] = await readHeartbeats(directory, ["imap-worker"], NOW);
+		assert.equal(imap.ageSeconds, 20);
+	});
+
+	it("reads no services as no readings", async () => {
+		assert.deepEqual(await readHeartbeats(await withFiles({}), [], NOW), []);
+	});
+});

package/src/heartbeats.ts

@@ -0,0 +1,68 @@
+import { readdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+import { attempt } from "./attempt.js";
+
+/**
+ * The age of a worker's stalest poll loop, read off the shared `heartbeat`
+ * volume the workers write and this container mounts read-only (D1, D9).
+ *
+ * A worker runs one loop per queue and each rewrites its own
+ * `<service>.<queue>` file, so the oldest of a service's files is the signal:
+ * one loop wedged in a socket read while its siblings long-poll is exactly the
+ * failure a container-wide timestamp would hide.
+ */
+export interface HeartbeatReading {
+	readonly service: string;
+	/** Seconds since the stalest of this service's files was written. */
+	readonly ageSeconds: number | undefined;
+	/** Why no age could be read. `undefined` when one was. */
+	readonly error: string | undefined;
+}
+
+/**
+ * A service with no file at all reads as an error, never as age zero. The whole
+ * point of the volume is that the checker can look; a verdict computed from a
+ * directory it could not read, or from a worker that has written nothing since
+ * it started, must not come back healthy.
+ */
+export const readHeartbeats = async (
+	directory: string,
+	services: readonly string[],
+	now: number = Date.now(),
+): Promise<HeartbeatReading[]> => {
+	const listing = await attempt(readdir(directory));
+	if (!listing.ok) {
+		const message = `cannot read ${directory}: ${listing.error}`;
+		return services.map((service) => ({
+			service,
+			ageSeconds: undefined,
+			error: message,
+		}));
+	}
+
+	return Promise.all(
+		services.map(async (service) => {
+			const owned = listing.value.filter((name) =>
+				name.startsWith(`${service}.`),
+			);
+			if (owned.length === 0) {
+				return { service, ageSeconds: undefined, error: "no heartbeat file" };
+			}
+			const times = await attempt(
+				Promise.all(
+					owned.map(
+						async (name) => (await stat(join(directory, name))).mtimeMs,
+					),
+				),
+			);
+			if (!times.ok) {
+				return { service, ageSeconds: undefined, error: times.error };
+			}
+			return {
+				service,
+				ageSeconds: (now - Math.min(...times.value)) / 1000,
+				error: undefined,
+			};
+		}),
+	);
+};

package/src/index.ts

@@ -0,0 +1,18 @@
+export { runCheck } from "./check.js";
+export type { DoctorConfig, ScrapeTarget } from "./config.js";
+export { loadConfig } from "./config.js";
+export { pingDeadMan } from "./deadman.js";
+export { advance } from "./dwell.js";
+export { readHeartbeats } from "./heartbeats.js";
+export { parseMetrics } from "./prometheus.js";
+export {
+	exitCodeFor,
+	NO_VERDICT_EXIT_CODE,
+	renderJson,
+	renderLines,
+} from "./report.js";
+export type { DoctorState } from "./state.js";
+export { initialState, readState, writeState } from "./state.js";
+export type { CheckResult, Reason, ReasonCode, Verdict } from "./verdict.js";
+export { evaluate } from "./verdict.js";
+export { buildBody, postWebhook } from "./webhook.js";

package/src/log.ts

@@ -0,0 +1,71 @@
+/**
+ * The same JSON-lines field shape every other service writes (see
+ * deploy/vps/README.md, "Logs"), written by hand.
+ *
+ * The checker depends on nothing, which is what makes it the container most
+ * likely to still be running when the rest of the stack is not, and a logger is
+ * not the reason to change that. The queue sidecar makes the same trade for the
+ * same reason.
+ *
+ * Everything goes to stderr, including `info`. Stdout is the exec seam's
+ * output (see report.ts): a log line landing in it would corrupt what
+ * `remit doctor` parses.
+ *
+ * The threshold is set once at startup from `DOCTOR_LOG_LEVEL`, not read from
+ * the environment here. The compose service passes `DOCTOR_*` variables and
+ * nothing else — deliberately, so no application secret can arrive in this
+ * container — which means a plain `LOG_LEVEL` could never reach it and the
+ * per-check verdict line could never be turned on.
+ */
+export interface Log {
+	/** Routine traces — the per-check verdict line. Dropped at the default level. */
+	debug(fields: Record<string, unknown>, msg: string): void;
+	/** Startup, and an alert actually sent. Rare and worth a line. */
+	info(fields: Record<string, unknown>, msg: string): void;
+	error(fields: Record<string, unknown>, msg: string): void;
+}
+
+type Level = "debug" | "info" | "error";
+
+const service = process.env.REMIT_SERVICE_NAME ?? "doctor";
+
+const ORDER: readonly Level[] = ["debug", "info", "error"];
+
+const DEFAULT_THRESHOLD = ORDER.indexOf("info");
+
+let threshold = DEFAULT_THRESHOLD;
+
+/**
+ * A value that is not a level name leaves the threshold at `info` rather than
+ * silencing the container: a typo must not turn the log off.
+ */
+export const setLogLevel = (level: string | undefined): void => {
+	const wanted = ORDER.indexOf(level?.trim().toLowerCase() as Level);
+	threshold = wanted === -1 ? DEFAULT_THRESHOLD : wanted;
+};
+
+const write = (
+	level: Level,
+	fields: Record<string, unknown>,
+	msg: string,
+): void => {
+	if (ORDER.indexOf(level) < threshold) return;
+	process.stderr.write(
+		`${JSON.stringify({
+			level,
+			time: new Date().toISOString(),
+			service,
+			...fields,
+			msg,
+		})}\n`,
+	);
+};
+
+export const log: Log = {
+	debug: (fields, msg) => write("debug", fields, msg),
+	info: (fields, msg) => write("info", fields, msg),
+	error: (fields, msg) => write("error", fields, msg),
+};
+
+export const describeError = (error: unknown): string =>
+	error instanceof Error ? (error.stack ?? error.message) : String(error);

package/src/loop.test.ts

@@ -0,0 +1,301 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { loadConfig } from "./config.js";
+import type { Log } from "./log.js";
+import { type LoopDependencies, runLoop, runOnce, sleep } from "./loop.js";
+import { type DoctorState, initialState } from "./state.js";
+import type { CheckResult, Verdict } from "./verdict.js";
+
+const NOW = new Date("2026-07-27T10:00:00.000Z");
+
+const silent: Log = { debug: () => {}, info: () => {}, error: () => {} };
+
+const result = (verdict: Verdict): CheckResult => ({
+	verdict,
+	checkedAt: NOW.toISOString(),
+	summary: `remit is ${verdict}`,
+	reasons:
+		verdict === "healthy"
+			? []
+			: [
+					{
+						code: "dead_letter_queue_not_empty",
+						summary: "1 message is quarantined on 1 dead-letter queue (dlq)",
+						detail: undefined,
+					},
+				],
+	counters: {
+		"imap-worker:imap_auth_failures": { total: 1, lastRoseAt: null },
+	},
+});
+
+interface Recorder {
+	readonly deps: LoopDependencies;
+	readonly posted: Verdict[];
+	readonly pings: number[];
+	readonly saved: DoctorState[];
+}
+
+const recorder = (
+	verdicts: readonly Verdict[],
+	overrides: Partial<LoopDependencies> = {},
+): Recorder => {
+	const posted: Verdict[] = [];
+	const pings: number[] = [];
+	const saved: DoctorState[] = [];
+	let index = 0;
+	return {
+		posted,
+		pings,
+		saved,
+		deps: {
+			runCheck: async () =>
+				result(verdicts[Math.min(index++, verdicts.length - 1)]),
+			saveState: async (state) => {
+				saved.push(state);
+			},
+			postWebhook: async (check) => {
+				posted.push(check.verdict);
+				return { kind: "sent" as const };
+			},
+			pingDeadMan: async () => {
+				pings.push(pings.length);
+			},
+			now: () => NOW,
+			log: silent,
+			...overrides,
+		},
+	};
+};
+
+const configured = loadConfig({
+	DOCTOR_WEBHOOK_URL: "https://hooks.example/x",
+	DOCTOR_HEARTBEAT_URL: "https://hc.example/x",
+	DOCTOR_INTERVAL_SECONDS: "1",
+});
+
+describe("runOnce", () => {
+	it("sends nothing until the verdict settles, then sends it once", async () => {
+		const rec = recorder(["degraded", "degraded", "degraded", "degraded"]);
+		let state = initialState;
+		for (let turn = 0; turn < 4; turn += 1) {
+			state = await runOnce(rec.deps, configured, state);
+		}
+		assert.deepEqual(rec.posted, ["degraded"]);
+	});
+
+	it("pings the dead-man on every completed check, settled or not", async () => {
+		const rec = recorder(["healthy", "degraded", "degraded"]);
+		let state = initialState;
+		for (let turn = 0; turn < 3; turn += 1) {
+			state = await runOnce(rec.deps, configured, state);
+		}
+		assert.equal(rec.pings.length, 3);
+	});
+
+	it("pings even when the verdict is degraded because a scrape failed", async () => {
+		const rec = recorder(["degraded"]);
+		await runOnce(rec.deps, configured, initialState);
+		assert.equal(rec.pings.length, 1);
+	});
+
+	it("carries the counter baseline forward into the persisted state", async () => {
+		const rec = recorder(["healthy"]);
+		const state = await runOnce(rec.deps, configured, initialState);
+		assert.deepEqual(state.counters, {
+			"imap-worker:imap_auth_failures": { total: 1, lastRoseAt: null },
+		});
+	});
+
+	it("spends the transition on a payload the endpoint refused", async () => {
+		// A 4xx is the endpoint deciding about this body — a template written
+		// wrong, a revoked URL. Repeating it produces the same answer forever.
+		let attempts = 0;
+		const rec = recorder(new Array(6).fill("degraded"), {
+			postWebhook: async () => {
+				attempts += 1;
+				return { kind: "rejected" as const, detail: "HTTP 400" };
+			},
+		});
+		let state = initialState;
+		for (let turn = 0; turn < 6; turn += 1) {
+			state = await runOnce(rec.deps, configured, state);
+		}
+		assert.equal(attempts, 1);
+		assert.equal(state.firedVerdict, "degraded");
+		assert.equal(rec.pings.length, 6);
+	});
+
+	it("retries a transition the endpoint never received, on the next check", async () => {
+		// One transient 5xx on the firing check used to lose the outage alert
+		// outright: the dead-man's switch is a different URL at a different
+		// provider and keeps answering 200 while the webhook is down.
+		let attempts = 0;
+		const rec = recorder(new Array(6).fill("degraded"), {
+			postWebhook: async () => {
+				attempts += 1;
+				return attempts < 3
+					? { kind: "unreachable" as const, detail: "HTTP 503" }
+					: { kind: "sent" as const };
+			},
+		});
+		let state = initialState;
+		for (let turn = 0; turn < 6; turn += 1) {
+			state = await runOnce(rec.deps, configured, state);
+			if (turn < 2) assert.equal(state.firedVerdict, "healthy", `turn ${turn}`);
+		}
+		// Tried on the settling check and on each of the two after it, then landed
+		// and stopped.
+		assert.equal(attempts, 3);
+		assert.equal(state.firedVerdict, "degraded");
+	});
+
+	it("treats a thrown delivery as unreachable, not as a decision", async () => {
+		let attempts = 0;
+		const rec = recorder(new Array(5).fill("degraded"), {
+			postWebhook: async () => {
+				attempts += 1;
+				throw new Error("socket hang up");
+			},
+		});
+		let state = initialState;
+		for (let turn = 0; turn < 5; turn += 1) {
+			state = await runOnce(rec.deps, configured, state);
+		}
+		assert.equal(attempts, 3);
+		assert.equal(state.firedVerdict, "healthy");
+	});
+
+	it("keeps the settled run while retrying, so the retry is not another dwell away", async () => {
+		const rec = recorder(new Array(4).fill("degraded"), {
+			postWebhook: async () => ({
+				kind: "unreachable" as const,
+				detail: "HTTP 503",
+			}),
+		});
+		let state = initialState;
+		for (let turn = 0; turn < 4; turn += 1) {
+			state = await runOnce(rec.deps, configured, state);
+		}
+		assert.equal(state.candidateRuns, 3);
+	});
+
+	it("still pings when the webhook cannot be delivered", async () => {
+		const rec = recorder(["degraded", "degraded", "degraded"], {
+			postWebhook: async () => {
+				throw new Error("HTTP 400");
+			},
+		});
+		let state = initialState;
+		for (let turn = 0; turn < 3; turn += 1) {
+			state = await runOnce(rec.deps, configured, state);
+		}
+		assert.equal(rec.pings.length, 3);
+	});
+
+	it("keeps running when the dead-man ping fails", async () => {
+		const rec = recorder(["healthy"], {
+			pingDeadMan: async () => {
+				throw new Error("unreachable");
+			},
+		});
+		await assert.doesNotReject(runOnce(rec.deps, configured, initialState));
+	});
+
+	it("records the decision only after the endpoint has answered about it", async () => {
+		const order: string[] = [];
+		const rec = recorder(["degraded", "degraded", "degraded"], {
+			saveState: async () => {
+				order.push("save");
+			},
+			postWebhook: async () => {
+				order.push("post");
+				return { kind: "sent" as const };
+			},
+		});
+		let state = initialState;
+		for (let turn = 0; turn < 3; turn += 1) {
+			state = await runOnce(rec.deps, configured, state);
+		}
+		// The write follows the post, so a crash in the gap re-announces once.
+		// A duplicate is noise; a dropped outage alert is the failure this exists
+		// to prevent.
+		assert.deepEqual(order, ["save", "save", "post", "save"]);
+	});
+
+	it("sends nothing at all when no webhook is configured", async () => {
+		const rec = recorder(["degraded", "degraded", "degraded"]);
+		const quiet = loadConfig({});
+		let state = initialState;
+		for (let turn = 0; turn < 3; turn += 1) {
+			state = await runOnce(rec.deps, quiet, state);
+		}
+		assert.deepEqual(rec.posted, []);
+		assert.deepEqual(rec.pings, []);
+		// The verdict is still computed and recorded — `remit doctor` needs it.
+		assert.equal(state.firedVerdict, "degraded");
+	});
+});
+
+describe("runLoop", () => {
+	it("keeps going after a check that throws, and does not ping for it", async () => {
+		let calls = 0;
+		const rec = recorder(["healthy"], {
+			runCheck: async () => {
+				calls += 1;
+				if (calls === 1) throw new Error("scrape blew up");
+				return result("healthy");
+			},
+		});
+		const controller = new AbortController();
+		let turns = 0;
+		await runLoop({
+			config: configured,
+			initial: initialState,
+			signal: controller.signal,
+			deps: rec.deps,
+			sleep: async () => {
+				turns += 1;
+				if (turns >= 3) controller.abort();
+			},
+		});
+		assert.equal(calls, 3);
+		// Two of the three checks produced a verdict; the one that threw did not.
+		assert.equal(rec.pings.length, 2);
+	});
+
+	it("stops on abort", async () => {
+		const rec = recorder(["healthy"]);
+		const controller = new AbortController();
+		controller.abort();
+		const state = await runLoop({
+			config: configured,
+			initial: initialState,
+			signal: controller.signal,
+			deps: rec.deps,
+			sleep: async () => {},
+		});
+		assert.deepEqual(state, initialState);
+	});
+});
+
+describe("sleep", () => {
+	it("returns early on abort", async () => {
+		const controller = new AbortController();
+		const started = Date.now();
+		const waiting = sleep(60_000, controller.signal);
+		controller.abort();
+		await waiting;
+		assert.ok(Date.now() - started < 1000);
+	});
+
+	it("returns immediately when already aborted", async () => {
+		const controller = new AbortController();
+		controller.abort();
+		await sleep(60_000, controller.signal);
+	});
+
+	it("returns when the timer fires", async () => {
+		await sleep(1, new AbortController().signal);
+	});
+});