@remit/doctor 0.0.1

package/src/loop.ts

@@ -0,0 +1,154 @@
+import { attempt } from "./attempt.js";
+import type { DoctorConfig } from "./config.js";
+import { advance } from "./dwell.js";
+import type { Log } from "./log.js";
+import type { CounterState, DoctorState } from "./state.js";
+import type { CheckResult } from "./verdict.js";
+import type { Delivery } from "./webhook.js";
+
+export interface LoopDependencies {
+	readonly runCheck: (
+		counters: Readonly<Record<string, CounterState>>,
+	) => Promise<CheckResult>;
+	readonly saveState: (state: DoctorState) => Promise<void>;
+	readonly postWebhook: (result: CheckResult) => Promise<Delivery>;
+	readonly pingDeadMan: () => Promise<void>;
+	readonly now: () => Date;
+	readonly log: Log;
+}
+
+/**
+ * One turn of the loop: check, decide, announce a settled change, record, ping.
+ *
+ * A transition is spent when the endpoint has answered about it, not when it is
+ * decided. A payload an endpoint rejects (4xx) is a decision that repeating
+ * cannot change, so `firedVerdict` advances and the operator gets one error
+ * line. A payload that never arrived — a timeout, a refused connection, a 5xx —
+ * is not a decision, so `firedVerdict` is left where it was and the next check
+ * announces the same transition again. `candidateRuns` is already pinned at the
+ * dwell count, so the retry is immediate rather than another three checks away.
+ *
+ * The dead-man's switch cannot cover a lost delivery. It is a different URL at
+ * a different provider and it answers 200 all week while a webhook is down, so
+ * without the retry one transient 5xx on the firing check is an outage the
+ * operator is never told about, and the next thing they hear is the recovery.
+ *
+ * The state is written after the attempt, which means a crash in the gap
+ * between a delivered alert and the write re-announces once on restart. That is
+ * the cheaper of the two failures: a duplicate is noise, a dropped outage alert
+ * is the thing this exists to prevent.
+ *
+ * The dead-man ping is last and unconditional on the verdict. It reports that
+ * the checker is running, not what the checker found — a scrape failure
+ * degrades the verdict and still pings. A check that throws before producing a
+ * verdict does not reach it, which is the whole signal.
+ */
+export const runOnce = async (
+	deps: LoopDependencies,
+	config: DoctorConfig,
+	state: DoctorState,
+): Promise<DoctorState> => {
+	const result = await deps.runCheck(state.counters);
+	const transition = advance(
+		{ ...state, counters: result.counters },
+		result.verdict,
+		config.dwellChecks,
+		deps.now(),
+	);
+
+	deps.log.debug(
+		{
+			verdict: result.verdict,
+			reasons: result.reasons.map((reason) => reason.code),
+			settledRuns: transition.state.candidateRuns,
+			fires: transition.fires ?? null,
+		},
+		"doctor: check complete",
+	);
+
+	let next = transition.state;
+	const fired = transition.fires;
+	if (fired !== undefined && config.webhookUrl !== undefined) {
+		const delivery = await attempt(deps.postWebhook(result));
+		const outcome: Delivery = delivery.ok
+			? delivery.value
+			: { kind: "unreachable", detail: delivery.error };
+		if (outcome.kind === "sent") {
+			deps.log.info({ verdict: fired }, "doctor: alert sent");
+		} else if (outcome.kind === "rejected") {
+			deps.log.error(
+				{ verdict: fired, error: outcome.detail },
+				"doctor: the webhook refused this payload; the transition is spent",
+			);
+		} else {
+			// Roll the announcement back. Everything else the check learned —
+			// the counter baselines, the settled run — is kept.
+			next = { ...next, firedVerdict: state.firedVerdict };
+			deps.log.error(
+				{ verdict: fired, error: outcome.detail },
+				"doctor: the webhook could not be reached; retrying on the next check",
+			);
+		}
+	}
+
+	await deps.saveState(next);
+
+	if (config.deadManUrl !== undefined) {
+		const ping = await attempt(deps.pingDeadMan());
+		if (!ping.ok) {
+			deps.log.error({ error: ping.error }, "doctor: dead-man ping failed");
+		}
+	}
+
+	return next;
+};
+
+export interface RunLoopOptions {
+	readonly deps: LoopDependencies;
+	readonly config: DoctorConfig;
+	readonly initial: DoctorState;
+	readonly signal: AbortSignal;
+	readonly sleep: (ms: number, signal: AbortSignal) => Promise<void>;
+}
+
+/**
+ * Runs until aborted. A check that throws is logged and the loop continues: the
+ * container that reports the stack's failures must not be brought down by one
+ * of them, and a crash-looping checker is a dead-man's switch that fires for
+ * the wrong reason.
+ */
+export const runLoop = async (
+	options: RunLoopOptions,
+): Promise<DoctorState> => {
+	let state = options.initial;
+	while (!options.signal.aborted) {
+		const turn = await attempt(runOnce(options.deps, options.config, state));
+		if (turn.ok) {
+			state = turn.value;
+		} else {
+			options.deps.log.error(
+				{ error: turn.error },
+				"doctor: check failed to produce a verdict",
+			);
+		}
+		await options.sleep(options.config.intervalMs, options.signal);
+	}
+	return state;
+};
+
+export const sleep = (ms: number, signal: AbortSignal): Promise<void> =>
+	new Promise((resolve) => {
+		if (signal.aborted) {
+			resolve();
+			return;
+		}
+		const timer = setTimeout(() => {
+			signal.removeEventListener("abort", onAbort);
+			resolve();
+		}, ms);
+		const onAbort = () => {
+			clearTimeout(timer);
+			resolve();
+		};
+		signal.addEventListener("abort", onAbort, { once: true });
+	});

package/src/main.ts

@@ -0,0 +1,69 @@
+import { runCheck } from "./check.js";
+import { loadConfig } from "./config.js";
+import { pingDeadMan } from "./deadman.js";
+import { describeError, log, setLogLevel } from "./log.js";
+import { runLoop, sleep } from "./loop.js";
+import { readState, writeState } from "./state.js";
+import { postWebhook } from "./webhook.js";
+
+/**
+ * The `doctor` service (D9). A socket-free node process on the compose network:
+ * it scrapes the `/metrics` endpoints over that network and mounts the
+ * heartbeat volume read-only, and nothing else.
+ *
+ * It is its own container rather than a job inside the backend because an
+ * alerter inside the backend dies with the thing it is meant to report. It runs
+ * whether or not alerting is configured, because `remit doctor` execs into it.
+ */
+const config = await Promise.resolve()
+	.then(() => loadConfig())
+	.catch((error: unknown) => {
+		log.error({ error: describeError(error) }, "doctor: refusing to start");
+		return process.exit(1);
+	});
+
+setLogLevel(config.logLevel);
+
+const controller = new AbortController();
+for (const signal of ["SIGINT", "SIGTERM"] as const) {
+	process.on(signal, () => {
+		log.info({ signal }, "doctor: shutting down");
+		controller.abort();
+	});
+}
+
+log.info(
+	{
+		intervalSeconds: config.intervalMs / 1000,
+		dwellChecks: config.dwellChecks,
+		targets: config.targets.map((target) => target.service),
+		alerting: config.webhookUrl !== undefined,
+	},
+	"doctor: started",
+);
+
+await runLoop({
+	config,
+	initial: await readState(config.stateDir),
+	signal: controller.signal,
+	sleep,
+	deps: {
+		runCheck: (counters) => runCheck(config, counters),
+		saveState: (state) => writeState(config.stateDir, state),
+		postWebhook: (result) =>
+			postWebhook(
+				{
+					// Reached only when a transition fired, which requires the URL.
+					url: config.webhookUrl ?? "",
+					template: config.webhookTemplate,
+					contentType: config.webhookContentType,
+					timeoutMs: config.requestTimeoutMs,
+				},
+				result,
+			),
+		pingDeadMan: () =>
+			pingDeadMan(config.deadManUrl ?? "", config.requestTimeoutMs),
+		now: () => new Date(),
+		log,
+	},
+});

package/src/prometheus.test.ts

@@ -0,0 +1,105 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { parseMetrics, seriesNamed, sumOf } from "./prometheus.js";
+
+describe("parseMetrics", () => {
+	it("reads a labelled sample", () => {
+		const samples = parseMetrics(
+			'# HELP remit_queue_messages Messages.\n# TYPE remit_queue_messages gauge\nremit_queue_messages{queue="imap-sync",role="work"} 3\n',
+		);
+		assert.deepEqual(samples, [
+			{
+				name: "remit_queue_messages",
+				labels: { queue: "imap-sync", role: "work" },
+				value: 3,
+			},
+		]);
+	});
+
+	it("reads a sample with no labels", () => {
+		assert.deepEqual(parseMetrics("remit_search_index_backlog_rows 0\n"), [
+			{ name: "remit_search_index_backlog_rows", labels: {}, value: 0 },
+		]);
+	});
+
+	it("drops comments and blank lines without dropping the response", () => {
+		const samples = parseMetrics("# HELP x y\n\n  \nx 1\n");
+		assert.equal(samples.length, 1);
+	});
+
+	it("skips a line it does not recognise rather than refusing the response", () => {
+		const samples = parseMetrics("not a sample at all\nx 1\n");
+		assert.deepEqual(
+			samples.map((sample) => sample.name),
+			["x"],
+		);
+	});
+
+	it("unescapes a quote, a backslash and a newline in a label value", () => {
+		const [sample] = parseMetrics(
+			'x{a="he said \\"hi\\"",b="c:\\\\path",c="one\\ntwo"} 1\n',
+		);
+		assert.equal(sample.labels.a, 'he said "hi"');
+		assert.equal(sample.labels.b, "c:\\path");
+		assert.equal(sample.labels.c, "one\ntwo");
+	});
+
+	it("keeps an unknown escape verbatim, both characters", () => {
+		const [sample] = parseMetrics('x{a="tab\\there"} 1\n');
+		assert.equal(sample.labels.a, "tab\\there");
+	});
+
+	it("splits on the comma between pairs, not one inside a value", () => {
+		const [sample] = parseMetrics('x{a="one, two",b="three"} 1\n');
+		assert.deepEqual(sample.labels, { a: "one, two", b: "three" });
+	});
+
+	it("reads a quoted comma at the end of a value", () => {
+		const [sample] = parseMetrics('x{a="ends\\",",b="two"} 1\n');
+		assert.deepEqual(sample.labels, { a: 'ends",', b: "two" });
+	});
+
+	it("reads infinities and NaN", () => {
+		const samples = parseMetrics("a +Inf\nb -Inf\nc NaN\n");
+		assert.equal(samples[0].value, Number.POSITIVE_INFINITY);
+		assert.equal(samples[1].value, Number.NEGATIVE_INFINITY);
+		assert.ok(Number.isNaN(samples[2].value));
+	});
+
+	it("ignores a trailing timestamp", () => {
+		const [sample] = parseMetrics("x 5 1700000000000\n");
+		assert.equal(sample.value, 5);
+	});
+
+	it("reads an exponent-notation value", () => {
+		const [sample] = parseMetrics("x 1.5e+03\n");
+		assert.equal(sample.value, 1500);
+	});
+
+	it("tolerates an empty label block", () => {
+		const [sample] = parseMetrics("x{} 2\n");
+		assert.deepEqual(sample.labels, {});
+	});
+
+	it("stops on a malformed label block instead of throwing", () => {
+		assert.deepEqual(parseMetrics("x{a} 2\n")[0].labels, {});
+		assert.deepEqual(parseMetrics('x{a=b"} 2\n')[0].labels, {});
+	});
+});
+
+describe("seriesNamed and sumOf", () => {
+	const samples = parseMetrics('a{k="1"} 2\na{k="2"} 3\nb 9\nc +Inf\n');
+
+	it("selects one series", () => {
+		assert.equal(seriesNamed(samples, "a").length, 2);
+	});
+
+	it("sums a series and reads an absent one as zero", () => {
+		assert.equal(sumOf(samples, "a"), 5);
+		assert.equal(sumOf(samples, "missing"), 0);
+	});
+
+	it("does not let a non-finite sample poison a sum", () => {
+		assert.equal(sumOf(samples, "c"), 0);
+	});
+});

package/src/prometheus.ts

@@ -0,0 +1,136 @@
+/**
+ * The subset of the Prometheus text exposition format this checker reads back.
+ *
+ * Hand-written rather than a dependency: the checker is the container that has
+ * to keep working when the rest of the stack does not, and the whole grammar it
+ * needs is a name, an optional label set and a float. prom-client renders the
+ * format and does not parse it, so a parser is either sixty lines here or a
+ * third-party package in the one image whose job is to outlive failures.
+ */
+export interface Sample {
+	readonly name: string;
+	readonly labels: Readonly<Record<string, string>>;
+	readonly value: number;
+}
+
+const UNESCAPE: Readonly<Record<string, string>> = {
+	n: "\n",
+	'"': '"',
+	"\\": "\\",
+};
+
+/**
+ * Label values escape `\`, `"` and a newline, and nothing else — a `\t` in the
+ * wire format is a literal backslash followed by `t`, so an unknown escape
+ * keeps both characters rather than swallowing the backslash.
+ */
+const unescapeLabelValue = (raw: string): string => {
+	let out = "";
+	for (let index = 0; index < raw.length; index += 1) {
+		const char = raw[index];
+		if (char !== "\\" || index === raw.length - 1) {
+			out += char;
+			continue;
+		}
+		const next = raw[index + 1];
+		const decoded = UNESCAPE[next];
+		out += decoded ?? `\\${next}`;
+		index += 1;
+	}
+	return out;
+};
+
+/**
+ * Split a label block on the commas that separate pairs, ignoring any comma
+ * inside a quoted value. Written as a scan rather than a regular expression
+ * because a value may contain `",` verbatim once unescaped.
+ */
+const parseLabels = (block: string): Record<string, string> => {
+	const labels: Record<string, string> = {};
+	let index = 0;
+	while (index < block.length) {
+		const equals = block.indexOf("=", index);
+		if (equals === -1) break;
+		const name = block.slice(index, equals).trim();
+		const openQuote = block.indexOf('"', equals);
+		if (openQuote === -1) break;
+		let cursor = openQuote + 1;
+		let raw = "";
+		while (cursor < block.length) {
+			const char = block[cursor];
+			if (char === "\\") {
+				raw += block.slice(cursor, cursor + 2);
+				cursor += 2;
+				continue;
+			}
+			if (char === '"') break;
+			raw += char;
+			cursor += 1;
+		}
+		// Only when the value was actually closed. A truncated block is garbage,
+		// and recording a label whose value is "whatever was left" invents a
+		// series the exporter never rendered.
+		if (name !== "" && block[cursor] === '"') {
+			labels[name] = unescapeLabelValue(raw);
+		}
+		const comma = block.indexOf(",", cursor);
+		if (comma === -1) break;
+		index = comma + 1;
+	}
+	return labels;
+};
+
+/**
+ * The value, or `undefined` when the token is not one. `NaN` is a legitimate
+ * sample value and parses; a word that merely happens to sit where a value goes
+ * does not, so a line of prose cannot become a series.
+ */
+const parseValue = (raw: string): number | undefined => {
+	const token = raw.trim().split(/\s+/)[0] ?? "";
+	if (token === "+Inf") return Number.POSITIVE_INFINITY;
+	if (token === "-Inf") return Number.NEGATIVE_INFINITY;
+	if (token === "NaN") return Number.NaN;
+	const parsed = Number(token);
+	return token === "" || Number.isNaN(parsed) ? undefined : parsed;
+};
+
+const SERIES = /^([A-Za-z_:][A-Za-z0-9_:]*)(\{(.*)\})?[ \t]+(.+)$/;
+
+/**
+ * Every sample in an exposition response, in the order it was rendered.
+ * `# HELP`/`# TYPE` lines and blanks are dropped; a line that is not a sample
+ * is skipped rather than throwing, because a scraper that refuses a whole
+ * response over one unfamiliar line reports a healthy service as unreachable.
+ */
+export const parseMetrics = (body: string): Sample[] => {
+	const samples: Sample[] = [];
+	for (const line of body.split("\n")) {
+		const trimmed = line.trim();
+		if (trimmed === "" || trimmed.startsWith("#")) continue;
+		const match = SERIES.exec(trimmed);
+		if (!match) continue;
+		const [, name, , labelBlock, raw] = match;
+		const value = parseValue(raw);
+		if (value === undefined) continue;
+		samples.push({
+			name,
+			labels: labelBlock === undefined ? {} : parseLabels(labelBlock),
+			value,
+		});
+	}
+	return samples;
+};
+
+/** Every sample of one series, across every scraped target. */
+export const seriesNamed = (
+	samples: readonly Sample[],
+	name: string,
+): Sample[] => samples.filter((sample) => sample.name === name);
+
+/** The sum of one series, `0` when nothing exported it. */
+export const sumOf = (samples: readonly Sample[], name: string): number =>
+	seriesNamed(samples, name).reduce(
+		(total, sample) =>
+			total + (Number.isFinite(sample.value) ? sample.value : 0),
+		0,
+	);

package/src/report.test.ts

@@ -0,0 +1,165 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { exitCodeFor, renderJson, renderLines } from "./report.js";
+import type { CheckResult } from "./verdict.js";
+
+const degraded: CheckResult = {
+	verdict: "degraded",
+	checkedAt: "2026-07-27T10:00:00.000Z",
+	summary: "remit is degraded",
+	reasons: [
+		{
+			code: "account_sync_stalled",
+			summary: "1 of 3 accounts have not completed a sync in over 3h",
+			detail: "0f8a: 40000s",
+		},
+		{
+			code: "dead_letter_queue_not_empty",
+			summary:
+				"2 messages are quarantined on 1 dead-letter queue (imap-sync-dlq)",
+			detail: undefined,
+		},
+	],
+	counters: {},
+};
+
+const healthy: CheckResult = {
+	verdict: "healthy",
+	checkedAt: "2026-07-27T10:00:00.000Z",
+	summary: "remit is healthy",
+	reasons: [],
+	counters: {},
+};
+
+/** The parse the wrapper does: first token is the key, the rest is the value. */
+const parseLines = (out: string): [string, string][] =>
+	out
+		.split("\n")
+		.filter((line) => line !== "")
+		.map((line) => {
+			const space = line.indexOf(" ");
+			return [line.slice(0, space), line.slice(space + 1)] as [string, string];
+		});
+
+describe("the line format", () => {
+	it("opens with the verdict, the timestamp and the headline", () => {
+		const records = parseLines(renderLines(degraded));
+		assert.deepEqual(records.slice(0, 3), [
+			["verdict", "degraded"],
+			["checked-at", "2026-07-27T10:00:00.000Z"],
+			["summary", "remit is degraded"],
+		]);
+	});
+
+	it("carries one record per reason, then the details", () => {
+		const records = parseLines(renderLines(degraded));
+		assert.deepEqual(
+			records.filter(([key]) => key === "reason").map(([, value]) => value),
+			[
+				"account_sync_stalled 1 of 3 accounts have not completed a sync in over 3h",
+				"dead_letter_queue_not_empty 2 messages are quarantined on 1 dead-letter queue (imap-sync-dlq)",
+			],
+		);
+		assert.deepEqual(
+			records.filter(([key]) => key === "detail").map(([, value]) => value),
+			["account_sync_stalled 0f8a: 40000s"],
+		);
+	});
+
+	it("uses a closed key vocabulary, so an unknown key is a version skew and not a value", () => {
+		const keys = new Set(parseLines(renderLines(degraded)).map(([key]) => key));
+		assert.deepEqual([...keys].sort(), [
+			"checked-at",
+			"detail",
+			"reason",
+			"summary",
+			"verdict",
+		]);
+	});
+
+	it("puts no reason records in a healthy report", () => {
+		const records = parseLines(renderLines(healthy));
+		assert.equal(records.length, 3);
+	});
+
+	it("never wraps a record, so one line is always one record", () => {
+		for (const line of renderLines(degraded).trimEnd().split("\n")) {
+			assert.ok(line.length > 0);
+			assert.ok(!line.includes("\n"));
+		}
+	});
+
+	// A queue name can hold a newline: the exposition format escapes it and the
+	// parser decodes it back. Split across two lines, a caller reading by
+	// position takes the remainder as a record with a garbage key and silently
+	// drops half the reason. The JSON body escapes its way out of this; a line
+	// format cannot.
+	it("keeps a newline in a value from splitting the record", () => {
+		const nasty: CheckResult = {
+			...degraded,
+			reasons: [
+				{
+					code: "dead_letter_queue_not_empty",
+					summary:
+						'2 messages are quarantined on 1 dead-letter queue (bad\nname"x)',
+					detail: "first\r\nsecond",
+				},
+			],
+		};
+		const out = renderLines(nasty);
+		assert.equal(out.trimEnd().split("\n").length, 5);
+		const records = parseLines(out);
+		assert.deepEqual(
+			records.filter(([key]) => key === "reason").map(([, value]) => value),
+			[
+				'dead_letter_queue_not_empty 2 messages are quarantined on 1 dead-letter queue (bad name"x)',
+			],
+		);
+		assert.deepEqual(
+			records.filter(([key]) => key === "detail").map(([, value]) => value),
+			["dead_letter_queue_not_empty first second"],
+		);
+	});
+
+	it("collapses every C0 control character, not only the newline", () => {
+		const out = renderLines({
+			...degraded,
+			reasons: [
+				{
+					code: "scrape_failed",
+					summary: "a\u0000b\u001bc\u007fd",
+					detail: undefined,
+				},
+			],
+		});
+		assert.equal(out.trimEnd().split("\n").length, 4);
+		assert.match(out, /reason scrape_failed a b c d/);
+	});
+});
+
+describe("the json format", () => {
+	it("parses, and carries the same verdict and reasons", () => {
+		const parsed = JSON.parse(renderJson(degraded)) as {
+			verdict: string;
+			reasons: { code: string; summary: string; detail: string | null }[];
+		};
+		assert.equal(parsed.verdict, "degraded");
+		assert.deepEqual(
+			parsed.reasons.map((reason) => reason.code),
+			["account_sync_stalled", "dead_letter_queue_not_empty"],
+		);
+		assert.equal(parsed.reasons[1].detail, null);
+	});
+
+	it("renders a healthy verdict as an empty reason list, not an absent key", () => {
+		const parsed = JSON.parse(renderJson(healthy)) as { reasons: unknown[] };
+		assert.deepEqual(parsed.reasons, []);
+	});
+});
+
+describe("exit codes", () => {
+	it("is zero on healthy and non-zero on degraded, so cron can use it directly", () => {
+		assert.equal(exitCodeFor(healthy), 0);
+		assert.equal(exitCodeFor(degraded), 1);
+	});
+});