@remit/doctor 0.0.2 → 0.0.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/doctor",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/check.test.ts

@@ -21,6 +21,31 @@ const stack = (overrides: Readonly<Record<string, string>> = {}) =>
 		return new Response(body, { status: 200 });
 	}) as unknown as typeof fetch;
 
+/**
+ * The stack plus a tunnel agent, recording every URL the check asks for. A
+ * `status` of `undefined` is an agent that is not there to answer at all.
+ */
+const withTunnel = (
+	inner: typeof fetch,
+	status: number | undefined,
+	seen: string[],
+) =>
+	(async (url: string, init?: RequestInit) => {
+		seen.push(url);
+		if (new URL(url).hostname !== "tunnel") return inner(url, init);
+		if (status === undefined) {
+			throw new Error("connect ECONNREFUSED 172.18.0.9:2000");
+		}
+		return new Response("", { status });
+	}) as unknown as typeof fetch;
+
+const ALL_WORKERS = [
+	"imap-worker",
+	"smtp-worker",
+	"account-worker",
+	"search-index-worker",
+];
+
 const heartbeatDir = async (services: readonly string[]): Promise<string> => {
 	const directory = await mkdtemp(join(tmpdir(), "remit-doctor-check-"));
 	for (const service of services) {
@@ -79,3 +104,79 @@ describe("runCheck", () => {
 		);
 	});
 });
+
+describe("runCheck in tunnel mode", () => {
+	const tunnelConfig = async () =>
+		loadConfig({
+			DOCTOR_HEARTBEAT_DIR: await heartbeatDir(ALL_WORKERS),
+			DOCTOR_TLS_MODE: "tunnel",
+		});
+
+	it("degrades when the readiness endpoint does not answer 200", async () => {
+		const seen: string[] = [];
+		const result = await runCheck(
+			await tunnelConfig(),
+			{},
+			new Date(),
+			withTunnel(stack(), 503, seen),
+		);
+		assert.equal(result.verdict, "degraded");
+		assert.deepEqual(
+			result.reasons.map((reason) => reason.code),
+			["tunnel_disconnected"],
+		);
+		assert.ok(seen.includes("http://tunnel:2000/ready"));
+	});
+
+	it("degrades when the agent does not answer at all", async () => {
+		const result = await runCheck(
+			await tunnelConfig(),
+			{},
+			new Date(),
+			withTunnel(stack(), undefined, []),
+		);
+		assert.deepEqual(
+			result.reasons.map((reason) => reason.code),
+			["tunnel_disconnected"],
+		);
+	});
+
+	it("stays silent while the readiness endpoint answers 200", async () => {
+		const result = await runCheck(
+			await tunnelConfig(),
+			{},
+			new Date(),
+			withTunnel(stack(), 200, []),
+		);
+		assert.equal(result.verdict, "healthy");
+	});
+
+	it("asks the endpoint the deployment named", async () => {
+		const seen: string[] = [];
+		await runCheck(
+			loadConfig({
+				DOCTOR_HEARTBEAT_DIR: await heartbeatDir(ALL_WORKERS),
+				DOCTOR_TLS_MODE: "tunnel",
+				DOCTOR_TUNNEL_READY_URL: "http://tunnel:2000/healthz",
+			}),
+			{},
+			new Date(),
+			withTunnel(stack(), 200, seen),
+		);
+		assert.ok(seen.includes("http://tunnel:2000/healthz"));
+	});
+
+	it("never looks for a tunnel on a deployment that does not serve through one", async () => {
+		const seen: string[] = [];
+		const result = await runCheck(
+			loadConfig({ DOCTOR_HEARTBEAT_DIR: await heartbeatDir(ALL_WORKERS) }),
+			{},
+			new Date(),
+			// The agent is absent, as it is on every other mode. Probing it anyway
+			// would be the check degrading a healthy deployment.
+			withTunnel(stack(), undefined, seen),
+		);
+		assert.equal(result.verdict, "healthy");
+		assert.ok(!seen.some((url) => new URL(url).hostname === "tunnel"));
+	});
+});

package/src/check.ts

@@ -1,9 +1,23 @@
-import type { DoctorConfig } from "./config.js";
+import { type DoctorConfig, TUNNEL_TLS_MODE } from "./config.js";
 import { readHeartbeats } from "./heartbeats.js";
 import { type Fetcher, scrapeAll } from "./scrape.js";
 import type { CounterState } from "./state.js";
+import { probeTunnel, type TunnelReading } from "./tunnel.js";
 import { type CheckResult, evaluate } from "./verdict.js";
 
+/**
+ * The tunnel is only a signal on a deployment that serves through one. A probe
+ * on any other mode would report a service that is not in the stack, which is
+ * the check being degraded by its own configuration.
+ */
+const readTunnel = (
+	config: DoctorConfig,
+	fetcher: Fetcher,
+): Promise<TunnelReading | undefined> =>
+	config.tlsMode === TUNNEL_TLS_MODE
+		? probeTunnel(config.tunnelReadyUrl, config.scrapeTimeoutMs, fetcher)
+		: Promise.resolve(undefined);
+
 /**
  * One check: scrape the endpoints that carry a signal, read the heartbeat
  * volume, and evaluate. The same function the loop runs on its interval and the
@@ -16,17 +30,19 @@ export const runCheck = async (
 	now: Date = new Date(),
 	fetcher: Fetcher = fetch,
 ): Promise<CheckResult> => {
-	const [scrapes, heartbeats] = await Promise.all([
+	const [scrapes, heartbeats, tunnel] = await Promise.all([
 		scrapeAll(config.targets, config.scrapeTimeoutMs, fetcher),
 		readHeartbeats(
 			config.heartbeatDir,
 			config.heartbeatServices,
 			now.getTime(),
 		),
+		readTunnel(config, fetcher),
 	]);
 	return evaluate({
 		scrapes,
 		heartbeats,
+		tunnel,
 		previousCounters,
 		heartbeatMaxAgeSeconds: config.heartbeatMaxAgeSeconds,
 		syncAgeMaxSeconds: config.syncAgeMaxSeconds,

package/src/config.test.ts

@@ -19,6 +19,17 @@ describe("loadConfig", () => {
 			"account-worker",
 			"search-index-worker",
 		]);
+		assert.equal(config.tlsMode, "off");
+		assert.equal(config.tunnelReadyUrl, "http://tunnel:2000/ready");
+	});
+
+	it("takes the deployment's serving mode and the edge's readiness endpoint", () => {
+		const config = loadConfig({
+			DOCTOR_TLS_MODE: "tunnel",
+			DOCTOR_TUNNEL_READY_URL: "http://edge:9000/healthz",
+		});
+		assert.equal(config.tlsMode, "tunnel");
+		assert.equal(config.tunnelReadyUrl, "http://edge:9000/healthz");
 	});
 
 	it("refuses a webhook with no dead-man's switch, naming both variables", () => {

package/src/config.ts

@@ -31,6 +31,16 @@ export interface DoctorConfig {
 	readonly deadManUrl: string | undefined;
 	readonly requestTimeoutMs: number;
 	readonly logLevel: string | undefined;
+	/**
+	 * The deployment's `TLS_MODE`, verbatim: the compose service hands it
+	 * through as `DOCTOR_TLS_MODE`, so which signals apply stays a property of
+	 * how the deployment serves rather than a second value that can disagree
+	 * with it. The mode is carried, not a predicate derived from it: the reason
+	 * the tunnel signal exists is the mode, and a `tunnelEnabled` boolean here
+	 * would put that derivation somewhere no other consumer can see.
+	 */
+	readonly tlsMode: string;
+	readonly tunnelReadyUrl: string;
 }
 
 /**
@@ -58,26 +68,35 @@ const DEFAULT_HEARTBEAT_SERVICES = [
 const DEFAULT_HEARTBEAT_MAX_AGE_SECONDS = 420;
 
 /**
- * Three hours. `remit_account_sync_age_seconds` sawtooths: the scheduler ticks
- * hourly by default (`MAILBOX_SYNC_TICK_INTERVAL_SECONDS`) and a tick skips a
- * mailbox stamped inside the freshness window, so a perfectly healthy account
- * climbs past an hour every cycle. A threshold at or near the tick fires on
- * accounts that are fine; raise this if you raised the tick.
+ * One hour. `remit_account_sync_age_seconds` sawtooths, and the height of the
+ * tooth is the scheduler's OFFLINE interval — the age at which an account
+ * becomes due — plus one TICK of sampling lag before a tick picks it up, plus
+ * the round itself. At the standalone stack's 15-minute offline interval and
+ * 5-minute tick (deploy/vps/docker-compose.sqlite.yml) a healthy account peaks
+ * around 25 minutes, so an hour clears it with room for several consecutive
+ * failed rounds and still names a genuinely stalled account inside the hour.
+ *
+ * A threshold below that peak fires on accounts that are fine, which is the
+ * failure that matters most here: this is the signal that says mail stopped
+ * arriving, and a signal that cries wolf is one an operator learns to ignore.
+ * Raise this if you raised either scheduler interval.
  */
-const DEFAULT_SYNC_AGE_MAX_SECONDS = 3 * 60 * 60;
+const DEFAULT_SYNC_AGE_MAX_SECONDS = 60 * 60;
 
 /**
  * How long after the last authentication failure the condition still counts as
- * failing. Three hours, for the same reason the sync-age threshold is three
- * hours: authentication is retried on the sync tick, so the failures arrive in
- * one burst per tick and the gaps between bursts are not recoveries.
+ * failing. One hour, held equal to the sync-age threshold: authentication is
+ * retried on the sync cycle, so the failures arrive in one burst per cycle and
+ * the gaps between bursts are not recoveries, and two windows of the same width
+ * mean a deployment that fixes its password gets one recovery message rather
+ * than two.
  *
  * The signal is a counter delta, which is true for exactly one check. Without a
  * hold the reason appears on one check in sixty and the three-check dwell never
  * settles, so the one class of failure that never resolves itself would be the
  * one that never alerts.
  */
-const DEFAULT_AUTH_FAILURE_HOLD_SECONDS = 3 * 60 * 60;
+const DEFAULT_AUTH_FAILURE_HOLD_SECONDS = 60 * 60;
 
 /** D8's number. Configurable so an operator can trade latency for quiet. */
 const DEFAULT_DWELL_CHECKS = 3;
@@ -95,6 +114,18 @@ const DEFAULT_REQUEST_TIMEOUT_SECONDS = 10;
 
 const DEFAULT_CONTENT_TYPE = "application/json";
 
+/** The mode under which the tunnel signal exists at all. */
+export const TUNNEL_TLS_MODE = "tunnel";
+
+const DEFAULT_TLS_MODE = "off";
+
+/**
+ * `cloudflared`'s readiness endpoint, on the metrics port the compose service
+ * gives it. Overridable because the provider contract is generic: another edge
+ * supplies its own image, credential variable and readiness endpoint.
+ */
+const DEFAULT_TUNNEL_READY_URL = "http://tunnel:2000/ready";
+
 export type Env = Record<string, string | undefined>;
 
 const text = (env: Env, name: string): string | undefined => {
@@ -237,5 +268,8 @@ export const loadConfig = (env: Env = process.env): DoctorConfig => {
 				"DOCTOR_REQUEST_TIMEOUT_SECONDS",
 				DEFAULT_REQUEST_TIMEOUT_SECONDS,
 			) * 1000,
+		tlsMode: text(env, "DOCTOR_TLS_MODE") ?? DEFAULT_TLS_MODE,
+		tunnelReadyUrl:
+			text(env, "DOCTOR_TUNNEL_READY_URL") ?? DEFAULT_TUNNEL_READY_URL,
 	};
 };

package/src/tunnel.test.ts

@@ -0,0 +1,44 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { probeTunnel } from "./tunnel.js";
+
+const READY_URL = "http://tunnel:2000/ready";
+
+const answering = (status: number) =>
+	(async () => new Response("", { status })) as unknown as typeof fetch;
+
+const refusing = (message: string) =>
+	(async () => {
+		throw new Error(message);
+	}) as unknown as typeof fetch;
+
+describe("probeTunnel", () => {
+	it("reads a connected agent off its 200", async () => {
+		const reading = await probeTunnel(READY_URL, 1000, answering(200));
+		assert.equal(reading.error, undefined);
+	});
+
+	it("reads a disconnected agent off anything else, and says the status", async () => {
+		const reading = await probeTunnel(READY_URL, 1000, answering(503));
+		assert.equal(reading.error, "HTTP 503");
+	});
+
+	it("turns a refused connection into a reading rather than a throw", async () => {
+		const reading = await probeTunnel(
+			READY_URL,
+			1000,
+			refusing("connect ECONNREFUSED 172.18.0.9:2000"),
+		);
+		assert.match(reading.error ?? "", /ECONNREFUSED/);
+	});
+
+	it("asks the URL it was given", async () => {
+		const seen: string[] = [];
+		const recording = (async (url: string) => {
+			seen.push(url);
+			return new Response("", { status: 200 });
+		}) as unknown as typeof fetch;
+		await probeTunnel("http://edge:9000/healthz", 1000, recording);
+		assert.deepEqual(seen, ["http://edge:9000/healthz"]);
+	});
+});

package/src/tunnel.ts

@@ -0,0 +1,31 @@
+import { attempt } from "./attempt.js";
+import type { Fetcher } from "./scrape.js";
+
+/**
+ * Whether the reverse-tunnel agent still holds a connection to the provider's
+ * edge (D12 of the tunnel design). `cloudflared` answers 200 on its metrics
+ * port's `/ready` only while at least one edge connection is established, so a
+ * refused connection, a timeout and a non-200 all say the same thing: nothing
+ * on the internet can reach this deployment right now.
+ *
+ * A failure is a value, for the reason every other reading here is one — the
+ * condition is what the verdict exists to report, and a throw would take the
+ * rest of the check down with it.
+ */
+export interface TunnelReading {
+	/** Why the readiness endpoint did not answer 200. `undefined` when it did. */
+	readonly error: string | undefined;
+}
+
+export const probeTunnel = async (
+	url: string,
+	timeoutMs: number,
+	fetcher: Fetcher = fetch,
+): Promise<TunnelReading> => {
+	const response = await attempt(
+		fetcher(url, { signal: AbortSignal.timeout(timeoutMs) }),
+	);
+	if (!response.ok) return { error: response.error };
+	if (!response.value.ok) return { error: `HTTP ${response.value.status}` };
+	return { error: undefined };
+};

package/src/verdict.test.ts

@@ -55,6 +55,7 @@ const counter = (total: number, roseMsAgo = 4 * 60 * 60 * 1000) => ({
 const input = (overrides: Partial<VerdictInput> = {}): VerdictInput => ({
 	scrapes: HEALTHY_SCRAPES,
 	heartbeats: HEALTHY_HEARTBEATS,
+	tunnel: undefined,
 	previousCounters: {},
 	heartbeatMaxAgeSeconds: 420,
 	syncAgeMaxSeconds: 10_800,
@@ -444,6 +445,38 @@ describe("evaluate", () => {
 		assert.equal(result.verdict, "healthy");
 	});
 
+	it("degrades when the tunnel is not connected, while everything else reads clean", () => {
+		const result = evaluate(input({ tunnel: { error: "HTTP 503" } }));
+		assert.equal(result.verdict, "degraded");
+		assert.deepEqual(
+			result.reasons.map((reason) => reason.code),
+			["tunnel_disconnected"],
+		);
+		assert.match(result.reasons[0].summary, /not connected to its edge/);
+		assert.equal(result.reasons[0].detail, "HTTP 503");
+	});
+
+	it("stays healthy when the tunnel answered", () => {
+		const result = evaluate(input({ tunnel: { error: undefined } }));
+		assert.equal(result.verdict, "healthy");
+	});
+
+	it("puts the tunnel ahead of the conditions a dropped tunnel does not cause", () => {
+		const result = evaluate(
+			input({
+				tunnel: { error: "connect ECONNREFUSED" },
+				heartbeats: [
+					{ service: "imap-worker", ageSeconds: 900, error: undefined },
+					...HEALTHY_HEARTBEATS.slice(1),
+				],
+			}),
+		);
+		assert.deepEqual(
+			result.reasons.map((reason) => reason.code),
+			["tunnel_disconnected", "worker_heartbeat_stale"],
+		);
+	});
+
 	it("keeps every address, subject and folder name out of every summary", () => {
 		const result = evaluate(
 			input({
@@ -504,6 +537,7 @@ describe("no reason summary may carry a value D10 forbids", () => {
 					{ service: "imap-worker", ageSeconds: undefined, error: SENTINEL },
 					...HEALTHY_HEARTBEATS.slice(1),
 				],
+				tunnel: { error: SENTINEL },
 				previousCounters: {
 					"imap-worker:imap_auth_failures": counter(0),
 					"smtp-worker:smtp_auth_failures": counter(0),
@@ -517,6 +551,7 @@ describe("no reason summary may carry a value D10 forbids", () => {
 			"account_sync_stalled",
 			"dead_letter_queue_not_empty",
 			"mail_auth_failing",
+			"tunnel_disconnected",
 			"worker_heartbeat_stale",
 		]);
 		for (const reason of result.reasons) {

package/src/verdict.ts

@@ -2,6 +2,7 @@ import type { HeartbeatReading } from "./heartbeats.js";
 import { seriesNamed } from "./prometheus.js";
 import type { ScrapeResult } from "./scrape.js";
 import type { CounterState } from "./state.js";
+import type { TunnelReading } from "./tunnel.js";
 
 export type Verdict = "healthy" | "degraded";
 
@@ -11,7 +12,8 @@ export type ReasonCode =
 	| "dead_letter_queue_not_empty"
 	| "account_sync_stalled"
 	| "mail_auth_failing"
-	| "signal_missing";
+	| "signal_missing"
+	| "tunnel_disconnected";
 
 /**
  * One thing that is wrong.
@@ -68,6 +70,13 @@ export interface VerdictInput extends VerdictThresholds {
 	readonly scrapes: readonly ScrapeResult[];
 	readonly heartbeats: readonly HeartbeatReading[];
 	readonly previousCounters: Readonly<Record<string, CounterState>>;
+	/**
+	 * The tunnel's readiness, or `undefined` on a deployment that does not serve
+	 * through one. Absent is not-applicable here, unlike every other signal,
+	 * where absent is degraded: a deployment with no tunnel has no tunnel to be
+	 * disconnected from.
+	 */
+	readonly tunnel: TunnelReading | undefined;
 	readonly now: Date;
 }
 
@@ -319,9 +328,32 @@ const missingSeries = (
 	};
 };
 
+/**
+ * The stack can be entirely healthy and still be serving nobody: in `tunnel`
+ * mode the only route in is the agent's connection to the edge, and when it
+ * drops every other signal here stays green while the browser gets the
+ * provider's error page.
+ *
+ * The checker's own two outbound calls — the webhook and the dead-man ping —
+ * dial straight out and do not pass through the agent, so this is the one
+ * reason that is still delivered while the condition it reports holds.
+ */
+const disconnectedTunnel = (
+	tunnel: TunnelReading | undefined,
+): Reason | undefined => {
+	if (tunnel === undefined || tunnel.error === undefined) return undefined;
+	return {
+		code: "tunnel_disconnected",
+		summary:
+			"the tunnel agent is not connected to its edge, so the public address serves nobody",
+		detail: tunnel.error,
+	};
+};
+
 const ORDER: readonly ReasonCode[] = [
 	"scrape_failed",
 	"signal_missing",
+	"tunnel_disconnected",
 	"worker_heartbeat_stale",
 	"account_sync_stalled",
 	"mail_auth_failing",
@@ -360,6 +392,7 @@ export const evaluate = (input: VerdictInput): CheckResult => {
 	const found = [
 		scrapeFailures(input.scrapes),
 		missingSeries(input.scrapes),
+		disconnectedTunnel(input.tunnel),
 		staleHeartbeats(input.heartbeats, input.heartbeatMaxAgeSeconds),
 		stalledSync(input.scrapes, input.syncAgeMaxSeconds),
 		authFailures(counters, input.authFailureHoldSeconds, now),