@remit/logger-lambda 0.0.10 → 0.0.11

package/README.md

@@ -28,6 +28,40 @@ export const handler = async (event: SQSEvent) => {
 };
 ```
 
+## Metrics
+
+`@remit/logger-lambda/metrics` owns the process-wide Prometheus registry every
+service renders at `/metrics`, and the recorders that write to it. See
+[docs/design/standalone-observability.md](../../docs/design/standalone-observability.md)
+for the signal set and why it is pulled rather than pushed.
+
+```typescript
+import {
+  onScrape,
+  recordImapFailure,
+  startMetricsServer,
+} from "@remit/logger-lambda/metrics";
+
+// A service with no listener of its own gets one for /metrics alone. A port it
+// cannot bind is reported and then dropped: the process keeps doing its work
+// and serves no metrics.
+startMetricsServer();
+
+// Counters and histograms are written where the work happens.
+recordImapFailure("SYNC_MESSAGES", "auth");
+
+// A gauge whose value is a read is collected when a scrape arrives. A collector
+// that throws fails the scrape rather than rendering a stale or zero value.
+onScrape(async () => setBacklog(await countUndrainedRows()));
+```
+
+A signal only one service can answer for is declared by that service against the
+exported `registry` — see `search-index-worker/src/metrics.ts` and
+`queue-sidecar/src/metrics.ts`. Declaring it here would render it in every
+process that imports this module, including the four that cannot know its value.
+
+`withTelemetry` records handler duration and outcome into the same registry.
+
 Both argument orders work: `(bindings, message)` and `(message, bindings)`.
 Bindings land at the top level of the line, never nested. A value under `error`
 that is an `Error` is expanded to `type`, `message` and `stack`; anything else is
@@ -58,3 +92,5 @@ one adds to the bindings of the one it runs inside.
 | -------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------ |
 | `LOG_LEVEL`          | `info`  | `trace`, `debug`, `info`, `warn`, `error`, `fatal` or `silent`. An unrecognised value logs one warning and falls back to the default. |
 | `REMIT_SERVICE_NAME` | `remit` | The `service` field on every line. Stamped into each service bundle at build time by `npm-scripts/docker-bundle.mjs`.                 |
+| `METRICS_PORT`       | `9464`  | Port `startMetricsServer` binds. |
+| `METRICS_HOST`       | `0.0.0.0` | Interface `startMetricsServer` binds. Set `127.0.0.1` when the service runs as a host process rather than a container. |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/logger-lambda",
-  "version": "0.0.10",
+  "version": "0.0.11",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -8,6 +8,10 @@
     ".": {
       "types": "./src/index.ts",
       "default": "./src/index.ts"
+    },
+    "./metrics": {
+      "types": "./src/metrics.ts",
+      "default": "./src/metrics.ts"
     }
   },
   "scripts": {
@@ -17,9 +21,9 @@
     "build": "npm run test:typecheck"
   },
   "dependencies": {
-    "@aws-lambda-powertools/metrics": "^2",
     "@types/aws-lambda": "*",
-    "pino": "^10"
+    "pino": "^10",
+    "prom-client": "^15.1.3"
   },
   "license": "MIT",
   "publishConfig": {

package/src/index.ts

@@ -1,2 +1,2 @@
-export { MetricUnit } from "@aws-lambda-powertools/metrics";
 export * from "./logger.js";
+export * from "./metrics.js";

package/src/logger.test.ts

@@ -4,36 +4,17 @@ import type { Context } from "aws-lambda";
 
 // The logging half of this package is covered by log-output.test.ts against its
 // real stdout output; this file is the telemetry wrapper, so the logger is
-// silenced and only the metric calls are observed.
+// silenced and only what reaches the metric registry is observed.
 process.env.LOG_LEVEL = "silent";
 
-const addMetric = mock.fn();
-const publishStoredMetrics = mock.fn();
-const captureColdStartMetric = mock.fn();
-
-class MockMetrics {
-	addMetric = addMetric;
-	publishStoredMetrics = publishStoredMetrics;
-	captureColdStartMetric = captureColdStartMetric;
-}
-
-mock.module("@aws-lambda-powertools/metrics", {
-	namedExports: {
-		Metrics: MockMetrics,
-		MetricUnit: { Count: "Count", Milliseconds: "Milliseconds" },
-	},
-});
-
-const { metrics, withTelemetry } = await import("./logger.js");
-const { Metrics } = await import("@aws-lambda-powertools/metrics");
+const { withTelemetry } = await import("./logger.js");
+const { registry, resetMetrics } = await import("./metrics.js");
 
 type Recorded = { mock: { calls: { arguments: unknown[] }[] } };
 
 const calls = (fn: Recorded): unknown[][] =>
 	fn.mock.calls.map((call) => call.arguments);
 
-const recorded = [addMetric, publishStoredMetrics, captureColdStartMetric];
-
 const makeContext = (): Context =>
 	({
 		awsRequestId: "test-request-id",
@@ -50,14 +31,34 @@ const makeContext = (): Context =>
 		succeed: () => {},
 	}) as unknown as Context;
 
-describe("remit-logger-lambda", () => {
-	beforeEach(() => {
-		for (const fn of recorded) fn.mock.resetCalls();
-	});
+type HistogramValue = {
+	metricName?: string;
+	labels: Record<string, string | number>;
+	value: number;
+};
+
+const handlerAggregate = async (
+	suffix: string,
+	labels: Record<string, string>,
+): Promise<number> => {
+	const metric = registry.getSingleMetric("remit_handler_duration_seconds");
+	assert.ok(metric, "expected the handler duration histogram to be registered");
+	const { values } = (await metric.get()) as { values: HistogramValue[] };
+	const match = values.find(
+		(value) =>
+			value.metricName === `remit_handler_duration_seconds_${suffix}` &&
+			value.labels.handler === labels.handler &&
+			value.labels.outcome === labels.outcome,
+	);
+	assert.ok(
+		match,
+		`expected a _${suffix} sample for ${JSON.stringify(labels)}`,
+	);
+	return match.value;
+};
 
-	it("exports metrics as a Metrics instance", () => {
-		assert.ok(metrics instanceof Metrics);
-	});
+describe("remit-logger-lambda", () => {
+	beforeEach(() => resetMetrics());
 
 	it("withTelemetry calls the handler and returns its result", async () => {
 		const handler = mock.fn(async () => "hello");
@@ -75,34 +76,20 @@ describe("remit-logger-lambda", () => {
 		await assert.rejects(wrapped({}, makeContext()), /boom/);
 	});
 
-	it("withTelemetry publishes metrics in finally even on error", async () => {
+	it("withTelemetry records a failed invocation against the registry", async () => {
 		const handler = mock.fn(async () => {
 			throw new Error("fail");
 		});
-		const wrapped = withTelemetry(handler);
-		await assert.rejects(wrapped({}, makeContext()));
-		assert.ok(calls(publishStoredMetrics).length > 0);
-	});
-
-	it("withTelemetry emits errorCount on handler failure", async () => {
-		const handler = mock.fn(async () => {
-			throw new Error("fail");
-		});
-		const wrapped = withTelemetry(handler);
-		await assert.rejects(wrapped({}, makeContext()));
-		assert.deepEqual(calls(addMetric), [["errorCount", "Count", 1]]);
+		await assert.rejects(withTelemetry(handler)({}, makeContext()));
+		const labels = { handler: "test-function", outcome: "failure" };
+		assert.equal(await handlerAggregate("count", labels), 1);
 	});
 
-	it("withTelemetry emits invocationCount and invocationLatency on success", async () => {
+	it("withTelemetry records a successful invocation and its duration", async () => {
 		const handler = mock.fn(async () => 42);
-		const wrapped = withTelemetry(handler);
-		await wrapped({}, makeContext());
-		const [countCall, latencyCall] = calls(addMetric);
-		assert.deepEqual(countCall, ["invocationCount", "Count", 1]);
-		assert.deepEqual(latencyCall.slice(0, 2), [
-			"invocationLatency",
-			"Milliseconds",
-		]);
-		assert.equal(typeof latencyCall[2], "number");
+		await withTelemetry(handler)({}, makeContext());
+		const labels = { handler: "test-function", outcome: "success" };
+		assert.equal(await handlerAggregate("count", labels), 1);
+		assert.equal(typeof (await handlerAggregate("sum", labels)), "number");
 	});
 });

package/src/logger.ts

@@ -1,7 +1,7 @@
 import { AsyncLocalStorage } from "node:async_hooks";
-import { Metrics, MetricUnit } from "@aws-lambda-powertools/metrics";
 import type { Context } from "aws-lambda";
 import { pino, stdSerializers } from "pino";
+import { recordHandlerOutcome } from "./metrics.js";
 
 type LogBindings = Record<string, unknown>;
 
@@ -157,11 +157,6 @@ const createAdapter = (target: PinoLogger, persistent: LogBindings): Logger => {
 
 export const logger: Logger = createAdapter(root, {});
 
-export const metrics = new Metrics({
-	namespace: process.env.POWERTOOLS_METRICS_NAMESPACE ?? "Remit",
-	serviceName: process.env.POWERTOOLS_SERVICE_NAME ?? "remit",
-});
-
 export const createLogger = (): Logger => createAdapter(root, {});
 
 export const withTelemetry = <TEvent, TResult>(
@@ -172,24 +167,24 @@ export const withTelemetry = <TEvent, TResult>(
 			functionName: context.functionName,
 		});
 
-		metrics.captureColdStartMetric();
-
 		const start = Date.now();
 
 		try {
 			const result = await handler(event, context);
-			const duration = Date.now() - start;
-
-			metrics.addMetric("invocationCount", MetricUnit.Count, 1);
-			metrics.addMetric("invocationLatency", MetricUnit.Milliseconds, duration);
-
+			recordHandlerOutcome({
+				handler: context.functionName,
+				outcome: "success",
+				durationMs: Date.now() - start,
+			});
 			return result;
 		} catch (err) {
-			metrics.addMetric("errorCount", MetricUnit.Count, 1);
+			recordHandlerOutcome({
+				handler: context.functionName,
+				outcome: "failure",
+				durationMs: Date.now() - start,
+			});
 			logger.error("Lambda invocation failed", { error: err });
 			throw err;
-		} finally {
-			metrics.publishStoredMetrics();
 		}
 	};
 };

package/src/metrics.test.ts

@@ -0,0 +1,405 @@
+import assert from "node:assert/strict";
+import type { AddressInfo } from "node:net";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import {
+	createMetricsRequestListener,
+	DEFAULT_METRICS_PORT,
+	metricsContentType,
+	onScrape,
+	queueNameFromEventSource,
+	recordHandlerOutcome,
+	recordImapFailure,
+	recordQueueEvent,
+	recordSmtpFailure,
+	registry,
+	renderMetrics,
+	resetMetrics,
+	setAccountSyncAges,
+	startMetricsServer,
+} from "./metrics.js";
+
+/** A `#{name}{labels} value` line, matched on the labels a test cares about. */
+const sample = (text: string, name: string, labels: string[] = []): number => {
+	const line = text
+		.split("\n")
+		.find(
+			(candidate) =>
+				candidate.startsWith(`${name}{`) ||
+				(labels.length === 0 && candidate.startsWith(`${name} `)),
+		);
+	const matching = text
+		.split("\n")
+		.filter(
+			(candidate) =>
+				candidate.startsWith(name) &&
+				labels.every((label) => candidate.includes(label)),
+		);
+	const chosen = labels.length > 0 ? matching[0] : line;
+	assert.ok(
+		chosen,
+		`expected a sample for ${name}${labels.join("")} in:\n${text}`,
+	);
+	return Number(chosen.slice(chosen.lastIndexOf(" ") + 1));
+};
+
+const get = async (
+	port: number,
+	path: string,
+): Promise<{ status: number; contentType: string; body: string }> => {
+	const response = await fetch(`http://127.0.0.1:${port}${path}`);
+	return {
+		status: response.status,
+		contentType: response.headers.get("content-type") ?? "",
+		body: await response.text(),
+	};
+};
+
+describe("metrics registry", () => {
+	beforeEach(() => resetMetrics());
+
+	it("renders exposition text with a HELP and TYPE line per metric", async () => {
+		recordHandlerOutcome({
+			handler: "imap-worker-body",
+			outcome: "success",
+			durationMs: 1500,
+		});
+		const text = await renderMetrics();
+		assert.match(text, /^# HELP remit_handler_duration_seconds .+$/m);
+		assert.match(text, /^# TYPE remit_handler_duration_seconds histogram$/m);
+	});
+
+	it("records handler duration in seconds, by target and outcome", async () => {
+		recordHandlerOutcome({
+			handler: "smtp-worker",
+			outcome: "failure",
+			durationMs: 2500,
+		});
+		const text = await renderMetrics();
+		assert.equal(
+			sample(text, "remit_handler_duration_seconds_sum", [
+				'handler="smtp-worker"',
+				'outcome="failure"',
+			]),
+			2.5,
+		);
+		assert.equal(
+			sample(text, "remit_handler_duration_seconds_count", [
+				'handler="smtp-worker"',
+				'outcome="failure"',
+			]),
+			1,
+		);
+	});
+
+	it("records per-message work by queue, event type and outcome", async () => {
+		recordQueueEvent({
+			queue: "remit-body",
+			eventType: "SYNC_MESSAGE_BODY",
+			outcome: "success",
+			durationMs: 1000,
+		});
+		recordQueueEvent({
+			queue: "remit-body",
+			eventType: "SYNC_MESSAGE_BODY",
+			outcome: "failure",
+			durationMs: 500,
+		});
+		const text = await renderMetrics();
+		assert.equal(
+			sample(text, "remit_queue_event_duration_seconds_count", [
+				'queue="remit-body"',
+				'event_type="SYNC_MESSAGE_BODY"',
+				'outcome="success"',
+			]),
+			1,
+		);
+		assert.equal(
+			sample(text, "remit_queue_event_duration_seconds_sum", [
+				'outcome="failure"',
+			]),
+			0.5,
+		);
+	});
+
+	it("counts IMAP authentication failures apart from other kinds", async () => {
+		recordImapFailure("SYNC_MESSAGES", "auth");
+		recordImapFailure("SYNC_MESSAGES", "auth");
+		recordImapFailure("SYNC_MESSAGES", "network");
+		const text = await renderMetrics();
+		assert.equal(
+			sample(text, "remit_imap_failures_total", [
+				'operation="SYNC_MESSAGES"',
+				'kind="auth"',
+			]),
+			2,
+		);
+		assert.equal(
+			sample(text, "remit_imap_failures_total", ['kind="network"']),
+			1,
+		);
+	});
+
+	it("counts SMTP authentication failures apart from other kinds", async () => {
+		recordSmtpFailure("auth");
+		recordSmtpFailure("other");
+		const text = await renderMetrics();
+		assert.equal(sample(text, "remit_smtp_failures_total", ['kind="auth"']), 1);
+		assert.equal(
+			sample(text, "remit_smtp_failures_total", ['kind="other"']),
+			1,
+		);
+	});
+
+	it("labels sync age by account id and never by address", async () => {
+		setAccountSyncAges([
+			{ accountId: "acct-1", ageSeconds: 42 },
+			{ accountId: "acct-2", ageSeconds: 900 },
+		]);
+		const text = await renderMetrics();
+		assert.equal(
+			sample(text, "remit_account_sync_age_seconds", ['account_id="acct-1"']),
+			42,
+		);
+		assert.equal(
+			sample(text, "remit_account_sync_age_seconds", ['account_id="acct-2"']),
+			900,
+		);
+		assert.ok(!text.includes("@"));
+	});
+
+	it("drops the sync age series of an account that is gone", async () => {
+		setAccountSyncAges([{ accountId: "acct-1", ageSeconds: 42 }]);
+		setAccountSyncAges([{ accountId: "acct-2", ageSeconds: 7 }]);
+		const text = await renderMetrics();
+		assert.ok(!text.includes('account_id="acct-1"'));
+		assert.equal(
+			sample(text, "remit_account_sync_age_seconds", ['account_id="acct-2"']),
+			7,
+		);
+	});
+
+	it("declares no unlabelled series, so no process renders a value it cannot know", async () => {
+		const text = await renderMetrics();
+		const samples = text
+			.split("\n")
+			.filter((line) => line.length > 0 && !line.startsWith("#"));
+		assert.deepEqual(
+			samples,
+			[],
+			"a fresh registry must render nothing until something records",
+		);
+	});
+
+	it("runs registered collectors before rendering", async () => {
+		let ran = 0;
+		onScrape(async () => {
+			ran += 1;
+			recordSmtpFailure("other");
+		});
+		const text = await renderMetrics();
+		assert.equal(ran, 1);
+		assert.equal(
+			sample(text, "remit_smtp_failures_total", ['kind="other"']),
+			1,
+		);
+	});
+
+	it("serialises overlapping scrapes so neither sees the other's reset", async () => {
+		// The sync-age gauge clears itself before repopulating. Two scrapes that
+		// interleave would let one render from the gauge the other just emptied.
+		let renders = 0;
+		onScrape(async () => {
+			setAccountSyncAges([{ accountId: "acct-1", ageSeconds: 1 }]);
+			await new Promise((resolve) => setImmediate(resolve));
+			renders += 1;
+		});
+		const [first, second] = await Promise.all([
+			renderMetrics(),
+			renderMetrics(),
+		]);
+		assert.equal(renders, 2);
+		for (const text of [first, second]) {
+			assert.equal(
+				sample(text, "remit_account_sync_age_seconds", ['account_id="acct-1"']),
+				1,
+			);
+		}
+	});
+
+	it("recovers after a scrape that failed", async () => {
+		let fail = true;
+		onScrape(async () => {
+			if (fail) {
+				fail = false;
+				throw new Error("transient");
+			}
+			recordSmtpFailure("auth");
+		});
+		await assert.rejects(renderMetrics(), /transient/);
+		const text = await renderMetrics();
+		assert.equal(sample(text, "remit_smtp_failures_total", ['kind="auth"']), 1);
+	});
+
+	it("fails the render when a collector cannot evaluate its signal", async () => {
+		onScrape(async () => {
+			throw new Error("database is gone");
+		});
+		await assert.rejects(renderMetrics(), /database is gone/);
+	});
+});
+
+describe("queueNameFromEventSource", () => {
+	it("reads the queue name from a real SQS ARN", () => {
+		assert.equal(
+			queueNameFromEventSource("arn:aws:sqs:eu-west-1:123456789012:remit-body"),
+			"remit-body",
+		);
+	});
+
+	it("reads the queue name from the sidecar queue URL the poller passes", () => {
+		assert.equal(
+			queueNameFromEventSource(
+				"http://queue:9324/000000000000/remit-flags.fifo",
+			),
+			"remit-flags.fifo",
+		);
+	});
+
+	it("ignores a query string", () => {
+		assert.equal(
+			queueNameFromEventSource("http://queue:9324/0/remit-smtp?x=1"),
+			"remit-smtp",
+		);
+	});
+
+	it("reports unknown rather than guessing when the source is absent", () => {
+		assert.equal(queueNameFromEventSource(undefined), "unknown");
+	});
+});
+
+describe("the /metrics endpoint", () => {
+	const servers: { close: (cb: () => void) => void }[] = [];
+
+	const listen = async (): Promise<number> => {
+		const server = startMetricsServer({ port: 0, host: "127.0.0.1" });
+		servers.push(server);
+		await new Promise<void>((resolve) => server.once("listening", resolve));
+		return (server.address() as AddressInfo).port;
+	};
+
+	beforeEach(() => resetMetrics());
+
+	afterEach(async () => {
+		while (servers.length > 0) {
+			const server = servers.pop();
+			await new Promise<void>((resolve) => server?.close(() => resolve()));
+		}
+	});
+
+	it("serves Prometheus text on GET /metrics", async () => {
+		recordSmtpFailure("auth");
+		const port = await listen();
+		const response = await get(port, "/metrics");
+		assert.equal(response.status, 200);
+		assert.equal(response.contentType, metricsContentType);
+		assert.match(response.contentType, /^text\/plain/);
+		assert.match(response.body, /^# TYPE remit_smtp_failures_total counter$/m);
+		assert.match(
+			response.body,
+			/^remit_smtp_failures_total\{kind="auth"\} 1$/m,
+		);
+	});
+
+	it("serves /metrics with a query string attached", async () => {
+		const port = await listen();
+		assert.equal((await get(port, "/metrics?debug=1")).status, 200);
+	});
+
+	it("serves nothing else — no health route, no other path", async () => {
+		const port = await listen();
+		assert.equal((await get(port, "/health")).status, 404);
+		assert.equal((await get(port, "/")).status, 404);
+	});
+
+	it("refuses any method other than GET", async () => {
+		const port = await listen();
+		const response = await fetch(`http://127.0.0.1:${port}/metrics`, {
+			method: "POST",
+		});
+		assert.equal(response.status, 404);
+	});
+
+	it("answers 500 when a signal cannot be evaluated", async () => {
+		onScrape(async () => {
+			throw new Error("queue store unreachable");
+		});
+		const port = await listen();
+		const response = await get(port, "/metrics");
+		assert.equal(response.status, 500);
+		assert.match(response.body, /queue store unreachable/);
+	});
+
+	it("defaults to the documented port when none is configured", () => {
+		assert.equal(DEFAULT_METRICS_PORT, 9464);
+	});
+
+	it("survives a port it cannot bind instead of taking the process down", async () => {
+		const held = await listen();
+		const reported: Error[] = [];
+		const server = startMetricsServer({
+			port: held,
+			host: "127.0.0.1",
+			onError: (error) => reported.push(error),
+		});
+		servers.push(server);
+		await new Promise((resolve) => setTimeout(resolve, 50));
+		assert.equal(reported.length, 1);
+		assert.match(String(reported[0]), /EADDRINUSE/);
+		// The port that was already bound still answers: nothing died.
+		assert.equal((await get(held, "/metrics")).status, 200);
+	});
+
+	it("reports a bind failure as one JSON line on stderr by default", async () => {
+		const held = await listen();
+		const written: string[] = [];
+		const restore = process.stderr.write.bind(process.stderr);
+		process.stderr.write = ((chunk: string) => {
+			written.push(String(chunk));
+			return true;
+		}) as typeof process.stderr.write;
+		const server = startMetricsServer({ port: held, host: "127.0.0.1" });
+		servers.push(server);
+		await new Promise((resolve) => setTimeout(resolve, 50));
+		process.stderr.write = restore;
+
+		assert.equal(written.length, 1);
+		const line = JSON.parse(written[0]) as {
+			level: string;
+			msg: string;
+			error: string;
+		};
+		assert.equal(line.level, "error");
+		assert.match(line.msg, /serving no metrics/);
+		assert.match(line.error, /EADDRINUSE/);
+	});
+
+	it("reads the host from METRICS_HOST", async () => {
+		process.env.METRICS_HOST = "127.0.0.1";
+		const server = startMetricsServer({ port: 0 });
+		servers.push(server);
+		await new Promise<void>((resolve) => server.once("listening", resolve));
+		delete process.env.METRICS_HOST;
+		assert.equal((server.address() as AddressInfo).address, "127.0.0.1");
+	});
+
+	it("reads the port from METRICS_PORT", async () => {
+		const listener = createMetricsRequestListener();
+		assert.equal(typeof listener, "function");
+		process.env.METRICS_PORT = "0";
+		const server = startMetricsServer({ host: "127.0.0.1" });
+		servers.push(server);
+		await new Promise<void>((resolve) => server.once("listening", resolve));
+		delete process.env.METRICS_PORT;
+		assert.ok((server.address() as AddressInfo).port > 0);
+	});
+});

package/src/metrics.ts

@@ -0,0 +1,261 @@
+import {
+	createServer,
+	type IncomingMessage,
+	type Server,
+	type ServerResponse,
+} from "node:http";
+import { Counter, Gauge, Histogram, Registry } from "prom-client";
+
+/**
+ * The process-wide metric registry every service renders at `/metrics`
+ * (docs/design/standalone-observability.md D2/D5). Powertools' `Metrics`
+ * emitted CloudWatch Embedded Metric Format, which no configuration turns into
+ * a scrape endpoint; this registry is what the endpoint renders.
+ *
+ * The exported recorders are the only way a series enters it. The registry
+ * itself is exported for a service that owns a signal of its own shape — the
+ * queue sidecar's per-queue depth gauges — so the whole process still renders
+ * from one registry.
+ */
+export const registry = new Registry();
+
+/** Content type of the exposition format `renderMetrics` produces. */
+export const metricsContentType = registry.contentType;
+
+/** The port every service that has no listener of its own serves `/metrics` on. */
+export const DEFAULT_METRICS_PORT = 9464;
+
+// A mail sync spans three orders of magnitude: a flag push is milliseconds, a
+// body fetch or an indexing round runs to the queue's 300 s visibility timeout.
+const DURATION_BUCKETS_SECONDS = [0.05, 0.25, 1, 5, 15, 60, 300];
+
+export type HandlerOutcome = "success" | "failure";
+
+/**
+ * Authentication is its own kind because it is the one class that never
+ * resolves itself: an expired grant or a changed password fails identically
+ * forever, and it is the most common way a self-hosted mailbox goes quiet.
+ */
+export type MailFailureKind = "auth" | "network" | "other";
+
+const handlerDurationSeconds = new Histogram({
+	name: "remit_handler_duration_seconds",
+	help: "Queue handler invocation duration, by poller target and outcome.",
+	labelNames: ["handler", "outcome"],
+	buckets: DURATION_BUCKETS_SECONDS,
+	registers: [registry],
+});
+
+const queueEventDurationSeconds = new Histogram({
+	name: "remit_queue_event_duration_seconds",
+	help: "Per-message work duration, by queue, event type and outcome.",
+	labelNames: ["queue", "event_type", "outcome"],
+	buckets: DURATION_BUCKETS_SECONDS,
+	registers: [registry],
+});
+
+const imapFailuresTotal = new Counter({
+	name: "remit_imap_failures_total",
+	help: "IMAP operation failures, by operation and failure kind.",
+	labelNames: ["operation", "kind"],
+	registers: [registry],
+});
+
+const smtpFailuresTotal = new Counter({
+	name: "remit_smtp_failures_total",
+	help: "SMTP send failures, by failure kind.",
+	labelNames: ["kind"],
+	registers: [registry],
+});
+
+const accountSyncAgeSeconds = new Gauge({
+	name: "remit_account_sync_age_seconds",
+	help: "Seconds since this account last completed a message-sync round.",
+	labelNames: ["account_id"],
+	registers: [registry],
+});
+
+/** One queue handler invocation — a whole SQS batch, from `withTelemetry`. */
+export const recordHandlerOutcome = (input: {
+	handler: string;
+	outcome: HandlerOutcome;
+	durationMs: number;
+}): void => {
+	handlerDurationSeconds.observe(
+		{ handler: input.handler, outcome: input.outcome },
+		input.durationMs / 1000,
+	);
+};
+
+/** One message's work, where the worker knows which event it just ran. */
+export const recordQueueEvent = (input: {
+	queue: string;
+	eventType: string;
+	outcome: HandlerOutcome;
+	durationMs: number;
+}): void => {
+	queueEventDurationSeconds.observe(
+		{ queue: input.queue, event_type: input.eventType, outcome: input.outcome },
+		input.durationMs / 1000,
+	);
+};
+
+export const recordImapFailure = (
+	operation: string,
+	kind: MailFailureKind,
+	count = 1,
+): void => {
+	imapFailuresTotal.inc({ operation, kind }, count);
+};
+
+export const recordSmtpFailure = (kind: MailFailureKind): void => {
+	smtpFailuresTotal.inc({ kind });
+};
+
+/**
+ * The whole per-account series, replaced. Accounts come and go, and a gauge
+ * that only ever gains label sets would keep reporting a deleted account's last
+ * known age forever.
+ */
+export const setAccountSyncAges = (
+	ages: readonly { accountId: string; ageSeconds: number }[],
+): void => {
+	accountSyncAgeSeconds.reset();
+	for (const { accountId, ageSeconds } of ages) {
+		accountSyncAgeSeconds.set({ account_id: accountId }, ageSeconds);
+	}
+};
+
+type ScrapeCollector = () => Promise<void>;
+
+const scrapeCollectors: ScrapeCollector[] = [];
+
+/**
+ * Register work that has to run before a scrape is answered — the gauges whose
+ * value is a database read, not something a handler counted. A collector that
+ * throws fails the scrape, which is the honest answer: a signal that cannot be
+ * evaluated must not render as a healthy number.
+ */
+export const onScrape = (collector: ScrapeCollector): void => {
+	scrapeCollectors.push(collector);
+};
+
+// Scrapes are serialised. Both whole-series gauges clear themselves before
+// repopulating, so two overlapping scrapes — a scraper's retry, or a Prometheus
+// poll landing on top of a `remit doctor` exec — can render one response from a
+// gauge the other has just emptied, dropping an account or a queue from it. A
+// series an alert evaluates on absence must not flap for that reason.
+let inFlightScrape: Promise<string> = Promise.resolve("");
+
+const collectAndRender = async (): Promise<string> => {
+	for (const collect of scrapeCollectors) {
+		await collect();
+	}
+	return registry.metrics();
+};
+
+export const renderMetrics = (): Promise<string> => {
+	inFlightScrape = inFlightScrape.then(collectAndRender, collectAndRender);
+	return inFlightScrape;
+};
+
+/** Drops every recorded sample and every registered collector — test use only. */
+export const resetMetrics = (): void => {
+	registry.resetMetrics();
+	scrapeCollectors.length = 0;
+	inFlightScrape = Promise.resolve("");
+};
+
+/**
+ * Derive a queue name from an SQS record's `eventSourceARN`. Lambda delivers a
+ * real ARN (`arn:aws:sqs:<region>:<account>:<name>`); the standalone poller sets
+ * the queue URL it received from (`http://queue:9324/<account>/<name>`). The
+ * name is the last segment either way.
+ */
+export const queueNameFromEventSource = (
+	eventSourceArn: string | undefined,
+): string => {
+	const withoutQuery = (eventSourceArn ?? "").split("?")[0];
+	const segments = withoutQuery.split(/[/:]/).filter(Boolean);
+	return segments.at(-1) ?? "unknown";
+};
+
+const METRICS_PATH = "/metrics";
+
+/**
+ * `GET /metrics` and nothing else. No health route (worker liveness is a
+ * heartbeat file, not a request path), no other method, no other path.
+ */
+export type MetricsRequestListener = (
+	req: IncomingMessage,
+	res: ServerResponse,
+) => void;
+
+export const createMetricsRequestListener =
+	(): MetricsRequestListener => (req, res) => {
+		const path = (req.url ?? "").split("?")[0];
+		if (req.method !== "GET" || path !== METRICS_PATH) {
+			res.writeHead(404, { "content-type": "text/plain" });
+			res.end("not found\n");
+			return;
+		}
+		renderMetrics()
+			.then((body) => {
+				res.writeHead(200, { "content-type": metricsContentType });
+				res.end(body);
+			})
+			.catch((error: unknown) => {
+				res.writeHead(500, { "content-type": "text/plain" });
+				res.end(`metrics collection failed: ${String(error)}\n`);
+			});
+	};
+
+export interface MetricsServerOptions {
+	readonly port?: number;
+	readonly host?: string;
+	/** Where a bind or socket failure is reported. Defaults to a stderr JSON line. */
+	readonly onError?: (error: Error) => void;
+}
+
+// This module is imported by the logger, so it cannot import the logger back.
+// One JSON object on stderr matches the shape every service writes anyway.
+const reportToStderr = (error: Error): void => {
+	process.stderr.write(
+		`${JSON.stringify({
+			level: "error",
+			time: new Date().toISOString(),
+			msg: "metrics server unavailable; this service is serving no metrics",
+			error: error.message,
+		})}\n`,
+	);
+};
+
+/**
+ * The listener D2 adds to each worker image, for `/metrics` alone. Bound to the
+ * compose network and never published to the host: the only host ports in the
+ * deployment stay caddy's 80 and 443.
+ *
+ * A port it cannot bind is reported and then dropped. `listen` resolves on a
+ * later tick, by which time the poll loop is already running, so an unhandled
+ * `'error'` event here would kill a worker in the middle of syncing mail —
+ * 9464 is the OpenTelemetry Prometheus exporter's default, so a collector on
+ * the same host is a real trigger. An observability endpoint must never be able
+ * to stop mail from arriving; absent metrics are the correct failure.
+ *
+ * Unreferenced from the event loop, so it never keeps a worker alive past the
+ * end of its poll loop.
+ */
+export const startMetricsServer = (
+	options: MetricsServerOptions = {},
+): Server => {
+	const port =
+		options.port ??
+		Number(process.env.METRICS_PORT ?? String(DEFAULT_METRICS_PORT));
+	const host = options.host ?? process.env.METRICS_HOST ?? "0.0.0.0";
+	const onError = options.onError ?? reportToStderr;
+	const server = createServer(createMetricsRequestListener());
+	server.unref();
+	server.on("error", onError);
+	server.listen(port, host);
+	return server;
+};