@remit/doctor 0.0.1

package/src/report.ts

@@ -0,0 +1,106 @@
+import type { CheckResult } from "./verdict.js";
+
+/**
+ * The exec seam (D4). `remit doctor` is a POSIX shell script whose only tool is
+ * `docker`, so it does not scrape anything itself — it runs
+ * `docker compose exec -T doctor node check.mjs` and formats what comes back.
+ *
+ * Two renderings, because the wrapper needs both and can build neither from the
+ * other: a line format `while read -r key rest` parses in shell without a JSON
+ * parser, and a JSON object for `remit doctor --json` to pass straight through.
+ *
+ * The line format is a contract. Keys are a closed vocabulary, one record per
+ * line, the key is the first space-delimited token and the value is the rest of
+ * the line verbatim. New keys may be added; existing keys do not change meaning.
+ *
+ *   verdict     healthy | degraded
+ *   checked-at  ISO 8601 UTC
+ *   summary     one-line headline, no reason detail
+ *   reason      <code> <summary>   — zero or more, stable order
+ *   detail      <code> <detail>    — zero or more, only for reasons that have one
+ *
+ * `reason` summaries carry counts, service names and queue names (D10).
+ * `detail` carries the account ids behind them and is printed here because this
+ * output never leaves the box; nothing in the alert path reads it.
+ */
+
+/**
+ * One record is one line, whatever the value contains.
+ *
+ * A queue name may hold a newline — the exposition format escapes it, and the
+ * parser decodes it back into a real one — and a caught error's message may
+ * hold anything. Either would split a record in two, and a caller parsing by
+ * position reads the remainder as a record with a garbage key: half a reason
+ * silently dropped, with nothing to say it happened. The JSON rendering and the
+ * webhook body escape their way out of this; a line format cannot, so the
+ * control characters are collapsed before they get in.
+ */
+const DELETE = 0x7f;
+const LAST_CONTROL = 0x1f;
+
+const isControl = (code: number): boolean =>
+	code <= LAST_CONTROL || code === DELETE;
+
+// Written as a code-point scan rather than a character class: a regular
+// expression carrying literal control characters is exactly the pattern the
+// lint rule exists to catch, and spelling the range out reads better anyway.
+const oneLine = (value: string): string => {
+	let out = "";
+	let pending = false;
+	for (const character of value) {
+		if (isControl(character.charCodeAt(0))) {
+			pending = out !== "";
+			continue;
+		}
+		if (pending) {
+			out += " ";
+			pending = false;
+		}
+		out += character;
+	}
+	return out;
+};
+
+export const renderLines = (result: CheckResult): string => {
+	const lines = [
+		`verdict ${result.verdict}`,
+		`checked-at ${result.checkedAt}`,
+		`summary ${oneLine(result.summary)}`,
+	];
+	for (const reason of result.reasons) {
+		lines.push(`reason ${reason.code} ${oneLine(reason.summary)}`);
+	}
+	for (const reason of result.reasons) {
+		if (reason.detail !== undefined) {
+			lines.push(`detail ${reason.code} ${oneLine(reason.detail)}`);
+		}
+	}
+	return `${lines.join("\n")}\n`;
+};
+
+export const renderJson = (result: CheckResult): string =>
+	`${JSON.stringify(
+		{
+			verdict: result.verdict,
+			checkedAt: result.checkedAt,
+			summary: result.summary,
+			reasons: result.reasons.map((reason) => ({
+				code: reason.code,
+				summary: reason.summary,
+				detail: reason.detail ?? null,
+			})),
+		},
+		null,
+		2,
+	)}\n`;
+
+/**
+ * 0 healthy, 1 degraded. A crash before a verdict exists exits 2 — both are
+ * non-zero, so a cron job or an external monitor can use the command directly,
+ * and a caller that wants to distinguish "something is wrong with the stack"
+ * from "the checker could not answer" can.
+ */
+export const exitCodeFor = (result: CheckResult): number =>
+	result.verdict === "healthy" ? 0 : 1;
+
+export const NO_VERDICT_EXIT_CODE = 2;

package/src/scrape.test.ts

@@ -0,0 +1,71 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { scrapeAll, scrapeTarget } from "./scrape.js";
+
+const ok = (body: string) =>
+	(async () => new Response(body, { status: 200 })) as unknown as typeof fetch;
+
+describe("scrapeTarget", () => {
+	it("parses a 200 response", async () => {
+		const result = await scrapeTarget(
+			{ service: "queue", url: "http://queue:9324/metrics" },
+			1000,
+			ok('remit_queue_messages{queue="a",role="work"} 1\n'),
+		);
+		assert.equal(result.error, undefined);
+		assert.equal(result.samples.length, 1);
+	});
+
+	it("reads a non-2xx as a failure, not as an empty set of signals", async () => {
+		const result = await scrapeTarget(
+			{ service: "backend", url: "http://backend:8080/metrics" },
+			1000,
+			(async () =>
+				new Response("collection failed", {
+					status: 500,
+				})) as unknown as typeof fetch,
+		);
+		assert.equal(result.error, "HTTP 500");
+		assert.deepEqual(result.samples, []);
+	});
+
+	it("reads a refused connection as a failure rather than throwing", async () => {
+		const result = await scrapeTarget(
+			{ service: "backend", url: "http://backend:8080/metrics" },
+			1000,
+			(async () => {
+				throw new Error("connect ECONNREFUSED");
+			}) as unknown as typeof fetch,
+		);
+		assert.match(result.error ?? "", /ECONNREFUSED/);
+	});
+
+	it("describes a thrown non-Error", async () => {
+		const result = await scrapeTarget(
+			{ service: "backend", url: "http://backend:8080/metrics" },
+			1000,
+			(async () => {
+				throw "nope";
+			}) as unknown as typeof fetch,
+		);
+		assert.equal(result.error, "nope");
+	});
+});
+
+describe("scrapeAll", () => {
+	it("one refused endpoint does not cost the readings of the others", async () => {
+		const results = await scrapeAll(
+			[
+				{ service: "backend", url: "http://backend:8080/metrics" },
+				{ service: "queue", url: "http://queue:9324/metrics" },
+			],
+			1000,
+			(async (url: string) => {
+				if (url.includes("backend")) throw new Error("down");
+				return new Response("x 1\n", { status: 200 });
+			}) as unknown as typeof fetch,
+		);
+		assert.equal(results[0].error, "down");
+		assert.equal(results[1].samples.length, 1);
+	});
+});

package/src/scrape.ts

@@ -0,0 +1,63 @@
+import { attempt } from "./attempt.js";
+import type { ScrapeTarget } from "./config.js";
+import { parseMetrics, type Sample } from "./prometheus.js";
+
+/**
+ * One target's scrape. A failure is a value, not a throw: an unreachable
+ * endpoint is a signal the verdict has to carry, and a checker that aborts on
+ * the first refused connection reports nothing about the services that did
+ * answer.
+ */
+export interface ScrapeResult {
+	readonly service: string;
+	readonly samples: readonly Sample[];
+	readonly error: string | undefined;
+}
+
+export type Fetcher = typeof fetch;
+
+export const scrapeTarget = async (
+	target: ScrapeTarget,
+	timeoutMs: number,
+	fetcher: Fetcher = fetch,
+): Promise<ScrapeResult> => {
+	const response = await attempt(
+		fetcher(target.url, {
+			signal: AbortSignal.timeout(timeoutMs),
+			headers: { accept: "text/plain" },
+		}),
+	);
+	if (!response.ok) {
+		return { service: target.service, samples: [], error: response.error };
+	}
+	if (!response.value.ok) {
+		return {
+			service: target.service,
+			samples: [],
+			error: `HTTP ${response.value.status}`,
+		};
+	}
+	const body = await attempt(response.value.text());
+	if (!body.ok) {
+		return { service: target.service, samples: [], error: body.error };
+	}
+	return {
+		service: target.service,
+		samples: parseMetrics(body.value),
+		error: undefined,
+	};
+};
+
+/**
+ * Every target at once. They are independent endpoints on the same network and
+ * the slowest is bounded by the scrape timeout, so a sequential pass would only
+ * make the whole check as slow as the sum of its stalls.
+ */
+export const scrapeAll = async (
+	targets: readonly ScrapeTarget[],
+	timeoutMs: number,
+	fetcher: Fetcher = fetch,
+): Promise<ScrapeResult[]> =>
+	Promise.all(
+		targets.map((target) => scrapeTarget(target, timeoutMs, fetcher)),
+	);

package/src/state.test.ts

@@ -0,0 +1,111 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, it } from "node:test";
+import {
+	type DoctorState,
+	initialState,
+	parseState,
+	readState,
+	STATE_VERSION,
+	stateFile,
+	writeState,
+} from "./state.js";
+
+const temporaryDir = () => mkdtemp(join(tmpdir(), "remit-doctor-state-"));
+
+const state: DoctorState = {
+	version: STATE_VERSION,
+	firedVerdict: "degraded",
+	candidateVerdict: "degraded",
+	candidateRuns: 3,
+	counters: {
+		"imap-worker:imap_auth_failures": { total: 4, lastRoseAt: 1785142027000 },
+	},
+	updatedAt: "2026-07-27T10:00:00.000Z",
+};
+
+describe("parseState", () => {
+	it("round-trips what writeState writes", async () => {
+		assert.deepEqual(await parseState(JSON.stringify(state)), state);
+	});
+
+	it("falls back to the baseline rather than refusing to start", async () => {
+		assert.deepEqual(await parseState("not json at all"), initialState);
+		assert.deepEqual(await parseState("null"), initialState);
+		assert.deepEqual(await parseState('"a string"'), initialState);
+		assert.deepEqual(await parseState('{"version":3}'), initialState);
+		assert.deepEqual(
+			await parseState('{"version":2,"firedVerdict":"broken"}'),
+			initialState,
+		);
+		assert.deepEqual(
+			await parseState(
+				'{"version":2,"firedVerdict":"healthy","candidateVerdict":"healthy","candidateRuns":"three","counters":{}}',
+			),
+			initialState,
+		);
+		assert.deepEqual(
+			await parseState(
+				'{"version":2,"firedVerdict":"healthy","candidateVerdict":"healthy","candidateRuns":1,"counters":{"a":"b"}}',
+			),
+			initialState,
+		);
+	});
+
+	it("assumes a stack nobody has checked is healthy, so no install announces itself", () => {
+		assert.equal(initialState.firedVerdict, "healthy");
+	});
+
+	it("resets a state file written by the version before the counter timestamps", async () => {
+		// v1 stored `counters` as bare totals, which cannot say when a counter last
+		// rose. Reading one as a baseline would leave the auth signal unable to
+		// hold, so the file is replaced; the cost is at most one repeated alert.
+		assert.deepEqual(
+			await parseState(
+				'{"version":1,"firedVerdict":"degraded","candidateVerdict":"degraded","candidateRuns":3,"counters":{"imap-worker:imap_auth_failures":4}}',
+			),
+			initialState,
+		);
+	});
+
+	it("tolerates a missing updatedAt", async () => {
+		const parsed = await parseState(
+			'{"version":2,"firedVerdict":"healthy","candidateVerdict":"healthy","candidateRuns":1,"counters":{}}',
+		);
+		assert.equal(parsed.updatedAt, undefined);
+	});
+});
+
+describe("readState and writeState", () => {
+	it("persists across a read", async () => {
+		const directory = await temporaryDir();
+		await writeState(directory, state);
+		assert.deepEqual(await readState(directory), state);
+	});
+
+	it("creates its directory", async () => {
+		const directory = join(await temporaryDir(), "nested", "deeper");
+		await writeState(directory, state);
+		assert.deepEqual(await readState(directory), state);
+	});
+
+	it("reads a directory with no state file as a fresh install", async () => {
+		assert.deepEqual(await readState(await temporaryDir()), initialState);
+	});
+
+	it("reads a truncated file as a fresh install rather than throwing", async () => {
+		const directory = await temporaryDir();
+		await writeFile(stateFile(directory), '{"version":1,"fired');
+		assert.deepEqual(await readState(directory), initialState);
+	});
+
+	it("renames into place, so a kill mid-write cannot truncate the live file", async () => {
+		const directory = await temporaryDir();
+		await writeState(directory, state);
+		await writeState(directory, { ...state, candidateRuns: 1 });
+		const raw = await readFile(stateFile(directory), "utf8");
+		assert.equal(JSON.parse(raw).candidateRuns, 1);
+	});
+});

package/src/state.ts

@@ -0,0 +1,127 @@
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { attempt, safeJsonParse } from "./attempt.js";
+import type { Verdict } from "./verdict.js";
+
+/**
+ * What the checker has to remember across a restart: which verdict it last
+ * announced, how long the current one has held, and the counter totals a delta
+ * is measured against.
+ *
+ * Without the first field a container restart re-announces a condition already
+ * reported, which is precisely the noise D8 exists to remove — and a checker
+ * with `restart: unless-stopped` restarts on every `remit update`.
+ */
+export const STATE_VERSION = 2;
+
+/**
+ * A counter's last observed total, and when it last went up.
+ *
+ * The total is the baseline a delta is measured against. `lastRoseAt` is what
+ * makes the signal a condition rather than an instant: authentication failures
+ * arrive in one burst per sync tick, so the quiet hour between two bursts is
+ * not a recovery, and a reason that is true for one check in sixty can never
+ * satisfy a three-check dwell.
+ */
+export interface CounterState {
+	readonly total: number;
+	/** Epoch milliseconds, or `null` when it has not risen since first seen. */
+	readonly lastRoseAt: number | null;
+}
+
+export interface DoctorState {
+	readonly version: typeof STATE_VERSION;
+	/** The last verdict actually sent. Never null: a fresh install is healthy. */
+	readonly firedVerdict: Verdict;
+	/** The verdict the run below is counting. */
+	readonly candidateVerdict: Verdict;
+	/** Consecutive checks that have agreed on `candidateVerdict`. */
+	readonly candidateRuns: number;
+	readonly counters: Readonly<Record<string, CounterState>>;
+	readonly updatedAt: string | undefined;
+}
+
+/**
+ * A stack that has never been checked is assumed healthy rather than unknown.
+ * Treating it as unknown makes the first settled `healthy` a transition, so
+ * every install would announce itself to the operator's channel on boot, and
+ * the one message people learn to ignore is the one that arrives when nothing
+ * is wrong.
+ */
+export const initialState: DoctorState = {
+	version: STATE_VERSION,
+	firedVerdict: "healthy",
+	candidateVerdict: "healthy",
+	candidateRuns: 0,
+	counters: {},
+	updatedAt: undefined,
+};
+
+const isVerdict = (value: unknown): value is Verdict =>
+	value === "healthy" || value === "degraded";
+
+const isCounterState = (value: unknown): value is CounterState =>
+	typeof value === "object" &&
+	value !== null &&
+	typeof (value as CounterState).total === "number" &&
+	(typeof (value as CounterState).lastRoseAt === "number" ||
+		(value as CounterState).lastRoseAt === null);
+
+const isCounters = (value: unknown): value is Record<string, CounterState> =>
+	typeof value === "object" &&
+	value !== null &&
+	!Array.isArray(value) &&
+	Object.values(value).every(isCounterState);
+
+/**
+ * A state file this version does not recognise is replaced by the baseline, not
+ * refused. The alternative is a container that will not start because of its
+ * own bookkeeping, and the cost of the reset is at most one repeated alert.
+ */
+export const parseState = async (raw: string): Promise<DoctorState> => {
+	const parsed = await safeJsonParse<unknown>(raw).catch(() => undefined);
+	if (typeof parsed !== "object" || parsed === null) return initialState;
+	const candidate = parsed as Record<string, unknown>;
+	if (
+		candidate.version !== STATE_VERSION ||
+		!isVerdict(candidate.firedVerdict) ||
+		!isVerdict(candidate.candidateVerdict) ||
+		typeof candidate.candidateRuns !== "number" ||
+		!isCounters(candidate.counters)
+	) {
+		return initialState;
+	}
+	return {
+		version: STATE_VERSION,
+		firedVerdict: candidate.firedVerdict,
+		candidateVerdict: candidate.candidateVerdict,
+		candidateRuns: candidate.candidateRuns,
+		counters: candidate.counters,
+		updatedAt:
+			typeof candidate.updatedAt === "string" ? candidate.updatedAt : undefined,
+	};
+};
+
+export const stateFile = (directory: string): string =>
+	join(directory, "state.json");
+
+export const readState = async (directory: string): Promise<DoctorState> => {
+	const raw = await attempt(readFile(stateFile(directory), "utf8"));
+	return raw.ok ? parseState(raw.value) : initialState;
+};
+
+/**
+ * Write then rename. A checker killed mid-write must come back to the previous
+ * state rather than to a truncated file it would read as a fresh install and
+ * re-announce from.
+ */
+export const writeState = async (
+	directory: string,
+	state: DoctorState,
+): Promise<void> => {
+	await mkdir(directory, { recursive: true });
+	const target = stateFile(directory);
+	const temporary = `${target}.tmp`;
+	await writeFile(temporary, `${JSON.stringify(state, null, "\t")}\n`);
+	await rename(temporary, target);
+};