@remit/logger-lambda 0.0.9 → 0.0.11

package/src/metrics.test.ts

@@ -0,0 +1,405 @@
+import assert from "node:assert/strict";
+import type { AddressInfo } from "node:net";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import {
+	createMetricsRequestListener,
+	DEFAULT_METRICS_PORT,
+	metricsContentType,
+	onScrape,
+	queueNameFromEventSource,
+	recordHandlerOutcome,
+	recordImapFailure,
+	recordQueueEvent,
+	recordSmtpFailure,
+	registry,
+	renderMetrics,
+	resetMetrics,
+	setAccountSyncAges,
+	startMetricsServer,
+} from "./metrics.js";
+
+/** A `#{name}{labels} value` line, matched on the labels a test cares about. */
+const sample = (text: string, name: string, labels: string[] = []): number => {
+	const line = text
+		.split("\n")
+		.find(
+			(candidate) =>
+				candidate.startsWith(`${name}{`) ||
+				(labels.length === 0 && candidate.startsWith(`${name} `)),
+		);
+	const matching = text
+		.split("\n")
+		.filter(
+			(candidate) =>
+				candidate.startsWith(name) &&
+				labels.every((label) => candidate.includes(label)),
+		);
+	const chosen = labels.length > 0 ? matching[0] : line;
+	assert.ok(
+		chosen,
+		`expected a sample for ${name}${labels.join("")} in:\n${text}`,
+	);
+	return Number(chosen.slice(chosen.lastIndexOf(" ") + 1));
+};
+
+const get = async (
+	port: number,
+	path: string,
+): Promise<{ status: number; contentType: string; body: string }> => {
+	const response = await fetch(`http://127.0.0.1:${port}${path}`);
+	return {
+		status: response.status,
+		contentType: response.headers.get("content-type") ?? "",
+		body: await response.text(),
+	};
+};
+
+describe("metrics registry", () => {
+	beforeEach(() => resetMetrics());
+
+	it("renders exposition text with a HELP and TYPE line per metric", async () => {
+		recordHandlerOutcome({
+			handler: "imap-worker-body",
+			outcome: "success",
+			durationMs: 1500,
+		});
+		const text = await renderMetrics();
+		assert.match(text, /^# HELP remit_handler_duration_seconds .+$/m);
+		assert.match(text, /^# TYPE remit_handler_duration_seconds histogram$/m);
+	});
+
+	it("records handler duration in seconds, by target and outcome", async () => {
+		recordHandlerOutcome({
+			handler: "smtp-worker",
+			outcome: "failure",
+			durationMs: 2500,
+		});
+		const text = await renderMetrics();
+		assert.equal(
+			sample(text, "remit_handler_duration_seconds_sum", [
+				'handler="smtp-worker"',
+				'outcome="failure"',
+			]),
+			2.5,
+		);
+		assert.equal(
+			sample(text, "remit_handler_duration_seconds_count", [
+				'handler="smtp-worker"',
+				'outcome="failure"',
+			]),
+			1,
+		);
+	});
+
+	it("records per-message work by queue, event type and outcome", async () => {
+		recordQueueEvent({
+			queue: "remit-body",
+			eventType: "SYNC_MESSAGE_BODY",
+			outcome: "success",
+			durationMs: 1000,
+		});
+		recordQueueEvent({
+			queue: "remit-body",
+			eventType: "SYNC_MESSAGE_BODY",
+			outcome: "failure",
+			durationMs: 500,
+		});
+		const text = await renderMetrics();
+		assert.equal(
+			sample(text, "remit_queue_event_duration_seconds_count", [
+				'queue="remit-body"',
+				'event_type="SYNC_MESSAGE_BODY"',
+				'outcome="success"',
+			]),
+			1,
+		);
+		assert.equal(
+			sample(text, "remit_queue_event_duration_seconds_sum", [
+				'outcome="failure"',
+			]),
+			0.5,
+		);
+	});
+
+	it("counts IMAP authentication failures apart from other kinds", async () => {
+		recordImapFailure("SYNC_MESSAGES", "auth");
+		recordImapFailure("SYNC_MESSAGES", "auth");
+		recordImapFailure("SYNC_MESSAGES", "network");
+		const text = await renderMetrics();
+		assert.equal(
+			sample(text, "remit_imap_failures_total", [
+				'operation="SYNC_MESSAGES"',
+				'kind="auth"',
+			]),
+			2,
+		);
+		assert.equal(
+			sample(text, "remit_imap_failures_total", ['kind="network"']),
+			1,
+		);
+	});
+
+	it("counts SMTP authentication failures apart from other kinds", async () => {
+		recordSmtpFailure("auth");
+		recordSmtpFailure("other");
+		const text = await renderMetrics();
+		assert.equal(sample(text, "remit_smtp_failures_total", ['kind="auth"']), 1);
+		assert.equal(
+			sample(text, "remit_smtp_failures_total", ['kind="other"']),
+			1,
+		);
+	});
+
+	it("labels sync age by account id and never by address", async () => {
+		setAccountSyncAges([
+			{ accountId: "acct-1", ageSeconds: 42 },
+			{ accountId: "acct-2", ageSeconds: 900 },
+		]);
+		const text = await renderMetrics();
+		assert.equal(
+			sample(text, "remit_account_sync_age_seconds", ['account_id="acct-1"']),
+			42,
+		);
+		assert.equal(
+			sample(text, "remit_account_sync_age_seconds", ['account_id="acct-2"']),
+			900,
+		);
+		assert.ok(!text.includes("@"));
+	});
+
+	it("drops the sync age series of an account that is gone", async () => {
+		setAccountSyncAges([{ accountId: "acct-1", ageSeconds: 42 }]);
+		setAccountSyncAges([{ accountId: "acct-2", ageSeconds: 7 }]);
+		const text = await renderMetrics();
+		assert.ok(!text.includes('account_id="acct-1"'));
+		assert.equal(
+			sample(text, "remit_account_sync_age_seconds", ['account_id="acct-2"']),
+			7,
+		);
+	});
+
+	it("declares no unlabelled series, so no process renders a value it cannot know", async () => {
+		const text = await renderMetrics();
+		const samples = text
+			.split("\n")
+			.filter((line) => line.length > 0 && !line.startsWith("#"));
+		assert.deepEqual(
+			samples,
+			[],
+			"a fresh registry must render nothing until something records",
+		);
+	});
+
+	it("runs registered collectors before rendering", async () => {
+		let ran = 0;
+		onScrape(async () => {
+			ran += 1;
+			recordSmtpFailure("other");
+		});
+		const text = await renderMetrics();
+		assert.equal(ran, 1);
+		assert.equal(
+			sample(text, "remit_smtp_failures_total", ['kind="other"']),
+			1,
+		);
+	});
+
+	it("serialises overlapping scrapes so neither sees the other's reset", async () => {
+		// The sync-age gauge clears itself before repopulating. Two scrapes that
+		// interleave would let one render from the gauge the other just emptied.
+		let renders = 0;
+		onScrape(async () => {
+			setAccountSyncAges([{ accountId: "acct-1", ageSeconds: 1 }]);
+			await new Promise((resolve) => setImmediate(resolve));
+			renders += 1;
+		});
+		const [first, second] = await Promise.all([
+			renderMetrics(),
+			renderMetrics(),
+		]);
+		assert.equal(renders, 2);
+		for (const text of [first, second]) {
+			assert.equal(
+				sample(text, "remit_account_sync_age_seconds", ['account_id="acct-1"']),
+				1,
+			);
+		}
+	});
+
+	it("recovers after a scrape that failed", async () => {
+		let fail = true;
+		onScrape(async () => {
+			if (fail) {
+				fail = false;
+				throw new Error("transient");
+			}
+			recordSmtpFailure("auth");
+		});
+		await assert.rejects(renderMetrics(), /transient/);
+		const text = await renderMetrics();
+		assert.equal(sample(text, "remit_smtp_failures_total", ['kind="auth"']), 1);
+	});
+
+	it("fails the render when a collector cannot evaluate its signal", async () => {
+		onScrape(async () => {
+			throw new Error("database is gone");
+		});
+		await assert.rejects(renderMetrics(), /database is gone/);
+	});
+});
+
+describe("queueNameFromEventSource", () => {
+	it("reads the queue name from a real SQS ARN", () => {
+		assert.equal(
+			queueNameFromEventSource("arn:aws:sqs:eu-west-1:123456789012:remit-body"),
+			"remit-body",
+		);
+	});
+
+	it("reads the queue name from the sidecar queue URL the poller passes", () => {
+		assert.equal(
+			queueNameFromEventSource(
+				"http://queue:9324/000000000000/remit-flags.fifo",
+			),
+			"remit-flags.fifo",
+		);
+	});
+
+	it("ignores a query string", () => {
+		assert.equal(
+			queueNameFromEventSource("http://queue:9324/0/remit-smtp?x=1"),
+			"remit-smtp",
+		);
+	});
+
+	it("reports unknown rather than guessing when the source is absent", () => {
+		assert.equal(queueNameFromEventSource(undefined), "unknown");
+	});
+});
+
+describe("the /metrics endpoint", () => {
+	const servers: { close: (cb: () => void) => void }[] = [];
+
+	const listen = async (): Promise<number> => {
+		const server = startMetricsServer({ port: 0, host: "127.0.0.1" });
+		servers.push(server);
+		await new Promise<void>((resolve) => server.once("listening", resolve));
+		return (server.address() as AddressInfo).port;
+	};
+
+	beforeEach(() => resetMetrics());
+
+	afterEach(async () => {
+		while (servers.length > 0) {
+			const server = servers.pop();
+			await new Promise<void>((resolve) => server?.close(() => resolve()));
+		}
+	});
+
+	it("serves Prometheus text on GET /metrics", async () => {
+		recordSmtpFailure("auth");
+		const port = await listen();
+		const response = await get(port, "/metrics");
+		assert.equal(response.status, 200);
+		assert.equal(response.contentType, metricsContentType);
+		assert.match(response.contentType, /^text\/plain/);
+		assert.match(response.body, /^# TYPE remit_smtp_failures_total counter$/m);
+		assert.match(
+			response.body,
+			/^remit_smtp_failures_total\{kind="auth"\} 1$/m,
+		);
+	});
+
+	it("serves /metrics with a query string attached", async () => {
+		const port = await listen();
+		assert.equal((await get(port, "/metrics?debug=1")).status, 200);
+	});
+
+	it("serves nothing else — no health route, no other path", async () => {
+		const port = await listen();
+		assert.equal((await get(port, "/health")).status, 404);
+		assert.equal((await get(port, "/")).status, 404);
+	});
+
+	it("refuses any method other than GET", async () => {
+		const port = await listen();
+		const response = await fetch(`http://127.0.0.1:${port}/metrics`, {
+			method: "POST",
+		});
+		assert.equal(response.status, 404);
+	});
+
+	it("answers 500 when a signal cannot be evaluated", async () => {
+		onScrape(async () => {
+			throw new Error("queue store unreachable");
+		});
+		const port = await listen();
+		const response = await get(port, "/metrics");
+		assert.equal(response.status, 500);
+		assert.match(response.body, /queue store unreachable/);
+	});
+
+	it("defaults to the documented port when none is configured", () => {
+		assert.equal(DEFAULT_METRICS_PORT, 9464);
+	});
+
+	it("survives a port it cannot bind instead of taking the process down", async () => {
+		const held = await listen();
+		const reported: Error[] = [];
+		const server = startMetricsServer({
+			port: held,
+			host: "127.0.0.1",
+			onError: (error) => reported.push(error),
+		});
+		servers.push(server);
+		await new Promise((resolve) => setTimeout(resolve, 50));
+		assert.equal(reported.length, 1);
+		assert.match(String(reported[0]), /EADDRINUSE/);
+		// The port that was already bound still answers: nothing died.
+		assert.equal((await get(held, "/metrics")).status, 200);
+	});
+
+	it("reports a bind failure as one JSON line on stderr by default", async () => {
+		const held = await listen();
+		const written: string[] = [];
+		const restore = process.stderr.write.bind(process.stderr);
+		process.stderr.write = ((chunk: string) => {
+			written.push(String(chunk));
+			return true;
+		}) as typeof process.stderr.write;
+		const server = startMetricsServer({ port: held, host: "127.0.0.1" });
+		servers.push(server);
+		await new Promise((resolve) => setTimeout(resolve, 50));
+		process.stderr.write = restore;
+
+		assert.equal(written.length, 1);
+		const line = JSON.parse(written[0]) as {
+			level: string;
+			msg: string;
+			error: string;
+		};
+		assert.equal(line.level, "error");
+		assert.match(line.msg, /serving no metrics/);
+		assert.match(line.error, /EADDRINUSE/);
+	});
+
+	it("reads the host from METRICS_HOST", async () => {
+		process.env.METRICS_HOST = "127.0.0.1";
+		const server = startMetricsServer({ port: 0 });
+		servers.push(server);
+		await new Promise<void>((resolve) => server.once("listening", resolve));
+		delete process.env.METRICS_HOST;
+		assert.equal((server.address() as AddressInfo).address, "127.0.0.1");
+	});
+
+	it("reads the port from METRICS_PORT", async () => {
+		const listener = createMetricsRequestListener();
+		assert.equal(typeof listener, "function");
+		process.env.METRICS_PORT = "0";
+		const server = startMetricsServer({ host: "127.0.0.1" });
+		servers.push(server);
+		await new Promise<void>((resolve) => server.once("listening", resolve));
+		delete process.env.METRICS_PORT;
+		assert.ok((server.address() as AddressInfo).port > 0);
+	});
+});

package/src/metrics.ts

@@ -0,0 +1,261 @@
+import {
+	createServer,
+	type IncomingMessage,
+	type Server,
+	type ServerResponse,
+} from "node:http";
+import { Counter, Gauge, Histogram, Registry } from "prom-client";
+
+/**
+ * The process-wide metric registry every service renders at `/metrics`
+ * (docs/design/standalone-observability.md D2/D5). Powertools' `Metrics`
+ * emitted CloudWatch Embedded Metric Format, which no configuration turns into
+ * a scrape endpoint; this registry is what the endpoint renders.
+ *
+ * The exported recorders are the only way a series enters it. The registry
+ * itself is exported for a service that owns a signal of its own shape — the
+ * queue sidecar's per-queue depth gauges — so the whole process still renders
+ * from one registry.
+ */
+export const registry = new Registry();
+
+/** Content type of the exposition format `renderMetrics` produces. */
+export const metricsContentType = registry.contentType;
+
+/** The port every service that has no listener of its own serves `/metrics` on. */
+export const DEFAULT_METRICS_PORT = 9464;
+
+// A mail sync spans three orders of magnitude: a flag push is milliseconds, a
+// body fetch or an indexing round runs to the queue's 300 s visibility timeout.
+const DURATION_BUCKETS_SECONDS = [0.05, 0.25, 1, 5, 15, 60, 300];
+
+export type HandlerOutcome = "success" | "failure";
+
+/**
+ * Authentication is its own kind because it is the one class that never
+ * resolves itself: an expired grant or a changed password fails identically
+ * forever, and it is the most common way a self-hosted mailbox goes quiet.
+ */
+export type MailFailureKind = "auth" | "network" | "other";
+
+const handlerDurationSeconds = new Histogram({
+	name: "remit_handler_duration_seconds",
+	help: "Queue handler invocation duration, by poller target and outcome.",
+	labelNames: ["handler", "outcome"],
+	buckets: DURATION_BUCKETS_SECONDS,
+	registers: [registry],
+});
+
+const queueEventDurationSeconds = new Histogram({
+	name: "remit_queue_event_duration_seconds",
+	help: "Per-message work duration, by queue, event type and outcome.",
+	labelNames: ["queue", "event_type", "outcome"],
+	buckets: DURATION_BUCKETS_SECONDS,
+	registers: [registry],
+});
+
+const imapFailuresTotal = new Counter({
+	name: "remit_imap_failures_total",
+	help: "IMAP operation failures, by operation and failure kind.",
+	labelNames: ["operation", "kind"],
+	registers: [registry],
+});
+
+const smtpFailuresTotal = new Counter({
+	name: "remit_smtp_failures_total",
+	help: "SMTP send failures, by failure kind.",
+	labelNames: ["kind"],
+	registers: [registry],
+});
+
+const accountSyncAgeSeconds = new Gauge({
+	name: "remit_account_sync_age_seconds",
+	help: "Seconds since this account last completed a message-sync round.",
+	labelNames: ["account_id"],
+	registers: [registry],
+});
+
+/** One queue handler invocation — a whole SQS batch, from `withTelemetry`. */
+export const recordHandlerOutcome = (input: {
+	handler: string;
+	outcome: HandlerOutcome;
+	durationMs: number;
+}): void => {
+	handlerDurationSeconds.observe(
+		{ handler: input.handler, outcome: input.outcome },
+		input.durationMs / 1000,
+	);
+};
+
+/** One message's work, where the worker knows which event it just ran. */
+export const recordQueueEvent = (input: {
+	queue: string;
+	eventType: string;
+	outcome: HandlerOutcome;
+	durationMs: number;
+}): void => {
+	queueEventDurationSeconds.observe(
+		{ queue: input.queue, event_type: input.eventType, outcome: input.outcome },
+		input.durationMs / 1000,
+	);
+};
+
+export const recordImapFailure = (
+	operation: string,
+	kind: MailFailureKind,
+	count = 1,
+): void => {
+	imapFailuresTotal.inc({ operation, kind }, count);
+};
+
+export const recordSmtpFailure = (kind: MailFailureKind): void => {
+	smtpFailuresTotal.inc({ kind });
+};
+
+/**
+ * The whole per-account series, replaced. Accounts come and go, and a gauge
+ * that only ever gains label sets would keep reporting a deleted account's last
+ * known age forever.
+ */
+export const setAccountSyncAges = (
+	ages: readonly { accountId: string; ageSeconds: number }[],
+): void => {
+	accountSyncAgeSeconds.reset();
+	for (const { accountId, ageSeconds } of ages) {
+		accountSyncAgeSeconds.set({ account_id: accountId }, ageSeconds);
+	}
+};
+
+type ScrapeCollector = () => Promise<void>;
+
+const scrapeCollectors: ScrapeCollector[] = [];
+
+/**
+ * Register work that has to run before a scrape is answered — the gauges whose
+ * value is a database read, not something a handler counted. A collector that
+ * throws fails the scrape, which is the honest answer: a signal that cannot be
+ * evaluated must not render as a healthy number.
+ */
+export const onScrape = (collector: ScrapeCollector): void => {
+	scrapeCollectors.push(collector);
+};
+
+// Scrapes are serialised. Both whole-series gauges clear themselves before
+// repopulating, so two overlapping scrapes — a scraper's retry, or a Prometheus
+// poll landing on top of a `remit doctor` exec — can render one response from a
+// gauge the other has just emptied, dropping an account or a queue from it. A
+// series an alert evaluates on absence must not flap for that reason.
+let inFlightScrape: Promise<string> = Promise.resolve("");
+
+const collectAndRender = async (): Promise<string> => {
+	for (const collect of scrapeCollectors) {
+		await collect();
+	}
+	return registry.metrics();
+};
+
+export const renderMetrics = (): Promise<string> => {
+	inFlightScrape = inFlightScrape.then(collectAndRender, collectAndRender);
+	return inFlightScrape;
+};
+
+/** Drops every recorded sample and every registered collector — test use only. */
+export const resetMetrics = (): void => {
+	registry.resetMetrics();
+	scrapeCollectors.length = 0;
+	inFlightScrape = Promise.resolve("");
+};
+
+/**
+ * Derive a queue name from an SQS record's `eventSourceARN`. Lambda delivers a
+ * real ARN (`arn:aws:sqs:<region>:<account>:<name>`); the standalone poller sets
+ * the queue URL it received from (`http://queue:9324/<account>/<name>`). The
+ * name is the last segment either way.
+ */
+export const queueNameFromEventSource = (
+	eventSourceArn: string | undefined,
+): string => {
+	const withoutQuery = (eventSourceArn ?? "").split("?")[0];
+	const segments = withoutQuery.split(/[/:]/).filter(Boolean);
+	return segments.at(-1) ?? "unknown";
+};
+
+const METRICS_PATH = "/metrics";
+
+/**
+ * `GET /metrics` and nothing else. No health route (worker liveness is a
+ * heartbeat file, not a request path), no other method, no other path.
+ */
+export type MetricsRequestListener = (
+	req: IncomingMessage,
+	res: ServerResponse,
+) => void;
+
+export const createMetricsRequestListener =
+	(): MetricsRequestListener => (req, res) => {
+		const path = (req.url ?? "").split("?")[0];
+		if (req.method !== "GET" || path !== METRICS_PATH) {
+			res.writeHead(404, { "content-type": "text/plain" });
+			res.end("not found\n");
+			return;
+		}
+		renderMetrics()
+			.then((body) => {
+				res.writeHead(200, { "content-type": metricsContentType });
+				res.end(body);
+			})
+			.catch((error: unknown) => {
+				res.writeHead(500, { "content-type": "text/plain" });
+				res.end(`metrics collection failed: ${String(error)}\n`);
+			});
+	};
+
+export interface MetricsServerOptions {
+	readonly port?: number;
+	readonly host?: string;
+	/** Where a bind or socket failure is reported. Defaults to a stderr JSON line. */
+	readonly onError?: (error: Error) => void;
+}
+
+// This module is imported by the logger, so it cannot import the logger back.
+// One JSON object on stderr matches the shape every service writes anyway.
+const reportToStderr = (error: Error): void => {
+	process.stderr.write(
+		`${JSON.stringify({
+			level: "error",
+			time: new Date().toISOString(),
+			msg: "metrics server unavailable; this service is serving no metrics",
+			error: error.message,
+		})}\n`,
+	);
+};
+
+/**
+ * The listener D2 adds to each worker image, for `/metrics` alone. Bound to the
+ * compose network and never published to the host: the only host ports in the
+ * deployment stay caddy's 80 and 443.
+ *
+ * A port it cannot bind is reported and then dropped. `listen` resolves on a
+ * later tick, by which time the poll loop is already running, so an unhandled
+ * `'error'` event here would kill a worker in the middle of syncing mail —
+ * 9464 is the OpenTelemetry Prometheus exporter's default, so a collector on
+ * the same host is a real trigger. An observability endpoint must never be able
+ * to stop mail from arriving; absent metrics are the correct failure.
+ *
+ * Unreferenced from the event loop, so it never keeps a worker alive past the
+ * end of its poll loop.
+ */
+export const startMetricsServer = (
+	options: MetricsServerOptions = {},
+): Server => {
+	const port =
+		options.port ??
+		Number(process.env.METRICS_PORT ?? String(DEFAULT_METRICS_PORT));
+	const host = options.host ?? process.env.METRICS_HOST ?? "0.0.0.0";
+	const onError = options.onError ?? reportToStderr;
+	const server = createServer(createMetricsRequestListener());
+	server.unref();
+	server.on("error", onError);
+	server.listen(port, host);
+	return server;
+};