@remit/sqs-client 0.0.8 → 0.0.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/sqs-client",
-  "version": "0.0.8",
+  "version": "0.0.9",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/heartbeat.test.ts

@@ -0,0 +1,102 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { afterEach, describe, it } from "node:test";
+import { clearHeartbeats, createHeartbeat } from "./heartbeat.js";
+
+const heartbeatPrefix = async (): Promise<string> => {
+	const dir = await mkdtemp(join(tmpdir(), "remit-heartbeat-"));
+	return join(dir, "imap-worker");
+};
+
+describe("createHeartbeat", () => {
+	afterEach(() => {
+		delete process.env.WORKER_HEARTBEAT_PREFIX;
+	});
+
+	it("writes one file per queue and advances its mtime on the next beat", async () => {
+		const prefix = await heartbeatPrefix();
+		const body = createHeartbeat("remit-body", prefix);
+		const flags = createHeartbeat("remit-flags.fifo", prefix);
+
+		await body();
+		await flags();
+		assert.deepEqual((await readdir(dirname(prefix))).sort(), [
+			"imap-worker.remit-body",
+			"imap-worker.remit-flags.fifo",
+		]);
+
+		const contents = await readFile(`${prefix}.remit-body`, "utf8");
+		assert.ok(
+			!Number.isNaN(Date.parse(contents.trim())),
+			`heartbeat contents should be a timestamp, got ${contents}`,
+		);
+
+		// The healthcheck reads mtime and nothing else, so a rewrite that leaves
+		// the file byte-identical would have to advance it anyway.
+		const first = await stat(`${prefix}.remit-body`);
+		await new Promise((resolve) => setTimeout(resolve, 20));
+		await body();
+		const second = await stat(`${prefix}.remit-body`);
+		assert.ok(
+			second.mtimeMs > first.mtimeMs,
+			`the second beat did not advance mtime: ${second.mtimeMs} <= ${first.mtimeMs}`,
+		);
+	});
+
+	it("writes under WORKER_HEARTBEAT_PREFIX when no prefix is given", async () => {
+		const prefix = await heartbeatPrefix();
+		process.env.WORKER_HEARTBEAT_PREFIX = prefix;
+
+		await createHeartbeat("remit-smtp")();
+
+		assert.equal((await stat(`${prefix}.remit-smtp`)).isFile(), true);
+	});
+
+	it("writes nothing when WORKER_HEARTBEAT_PREFIX is unset", async () => {
+		const prefix = await heartbeatPrefix();
+
+		await createHeartbeat("remit-smtp")();
+
+		assert.deepEqual(await readdir(dirname(prefix)), []);
+	});
+
+	it("rejects when the heartbeat directory does not exist", async () => {
+		const heartbeat = createHeartbeat(
+			"remit-smtp",
+			"/nonexistent/remit/worker",
+		);
+
+		await assert.rejects(() => heartbeat(), /ENOENT/);
+	});
+});
+
+describe("clearHeartbeats", () => {
+	afterEach(() => {
+		delete process.env.WORKER_HEARTBEAT_PREFIX;
+	});
+
+	it("removes this service's files and leaves every other service's alone", async () => {
+		const prefix = await heartbeatPrefix();
+		const directory = dirname(prefix);
+		await writeFile(`${prefix}.remit-body`, "stale\n");
+		await writeFile(`${prefix}.remit-retired-queue`, "stale\n");
+		await writeFile(join(directory, "smtp-worker.remit-smtp"), "fresh\n");
+
+		await clearHeartbeats(prefix);
+
+		assert.deepEqual(await readdir(directory), ["smtp-worker.remit-smtp"]);
+	});
+
+	it("does nothing when WORKER_HEARTBEAT_PREFIX is unset", async () => {
+		await clearHeartbeats();
+	});
+
+	it("rejects when the heartbeat directory does not exist", async () => {
+		await assert.rejects(
+			() => clearHeartbeats("/nonexistent/remit/worker"),
+			/ENOENT/,
+		);
+	});
+});

package/src/heartbeat.ts

@@ -0,0 +1,68 @@
+import { readdir, rm, writeFile } from "node:fs/promises";
+import { basename, dirname, join } from "node:path";
+
+/**
+ * Poll-loop liveness as a file (docs/design/standalone-observability.md, D1).
+ *
+ * A worker whose poll loop wedges inside a socket read never exits, so
+ * `restart: unless-stopped` never fires and `docker compose ps` reports it
+ * running while mail stops moving. Each loop rewrites its own file once per
+ * receive attempt; a container healthcheck reads the age of the oldest of them
+ * and fails past a threshold above the longest legitimate handler run.
+ *
+ * One file per loop, not one per container: a worker runs a loop per queue, and
+ * a receive loop only ever talks to the queue sidecar. One handler wedged in an
+ * IMAP read leaves imap-worker's other five loops long-polling, so a
+ * container-wide file stays fresh through exactly the failure this exists to
+ * catch. One writer per file also means a reader cannot catch a torn write.
+ *
+ * A file rather than an HTTP route because an HTTP handler answers from a
+ * process whose poll loop is wedged — any honest health endpoint would have to
+ * consult this same timestamp.
+ *
+ * The heartbeat proves the loop is turning, not that the last unit of work
+ * succeeded. A loop that receives, fails every handler and rewrites its file
+ * each cycle is healthy here.
+ */
+export type Heartbeat = () => Promise<void>;
+
+const noHeartbeat: Heartbeat = () => Promise.resolve();
+
+/**
+ * `WORKER_HEARTBEAT_PREFIX` is set per worker service by the self-host compose
+ * stack, which owns both the path and the volume it lands on; each loop appends
+ * its own queue name. Unset — the Lambda deployment, where the poller does not
+ * run at all, and any local process — writes nothing.
+ *
+ * The written value is the timestamp itself, so the file is readable as well as
+ * stat-able: an empty rewrite is not guaranteed to advance mtime when the file
+ * is already zero-length, and mtime is the whole signal.
+ */
+export const createHeartbeat = (
+	name: string,
+	prefix: string | undefined = process.env.WORKER_HEARTBEAT_PREFIX,
+): Heartbeat => {
+	if (!prefix) return noHeartbeat;
+	const filePath = `${prefix}.${name}`;
+	return () => writeFile(filePath, `${new Date().toISOString()}\n`);
+};
+
+/**
+ * Drops the files this service left on the volume, before its loops start
+ * writing again. The healthcheck fails on the oldest file it finds, so a queue
+ * a later version stops polling would otherwise leave a file nobody rewrites,
+ * and a worker that can never report healthy again.
+ */
+export const clearHeartbeats = async (
+	prefix: string | undefined = process.env.WORKER_HEARTBEAT_PREFIX,
+): Promise<void> => {
+	if (!prefix) return;
+	const directory = dirname(prefix);
+	const owned = `${basename(prefix)}.`;
+	const names = await readdir(directory);
+	await Promise.all(
+		names
+			.filter((name) => name.startsWith(owned))
+			.map((name) => rm(join(directory, name))),
+	);
+};

package/src/poller.test.ts

@@ -7,8 +7,8 @@ import { runQueuePoller } from "./poller.js";
 const QUEUE_URL = "http://localhost:9324/000000000000/remit-test";
 
 const buildLog = () => ({
-	info: mock.fn(() => {}),
-	error: mock.fn(() => {}),
+	info: mock.fn((_fields: Record<string, unknown>, _message: string) => {}),
+	error: mock.fn((_fields: Record<string, unknown>, _message: string) => {}),
 });
 
 const buildMessage = (id: string) => ({
@@ -81,6 +81,102 @@ describe("runQueuePoller", () => {
 		assert.deepEqual(deletedReceiptHandles, ["receipt-m1"]);
 	});
 
+	it("gives every target its own heartbeat, beaten at the top of each receive attempt", async () => {
+		const beatsByQueue = new Map<string, number>();
+		let receiveCalls = 0;
+
+		const sendMock = mock.method(
+			SQSClient.prototype,
+			"send",
+			// biome-ignore lint/suspicious/noExplicitAny: minimal SDK command shape, not worth typing per-command here
+			async function (this: SQSClient, command: any) {
+				if (command.constructor.name !== "ReceiveMessageCommand") {
+					throw new Error(`unexpected command: ${command.constructor.name}`);
+				}
+				// Two empty receives per target, then shut down: the heartbeat has
+				// to be written before the receive, and on an empty poll too — an
+				// idle queue is the normal state of a healthy worker.
+				receiveCalls++;
+				if (receiveCalls >= 4) process.emit("SIGWINCH", "SIGWINCH");
+				return { Messages: [] };
+			},
+		);
+
+		try {
+			await runQueuePoller({
+				targets: [
+					{ queueUrl: QUEUE_URL, handler: async () => {}, functionName: "a" },
+					{
+						queueUrl: `${QUEUE_URL}-2`,
+						handler: async () => {},
+						functionName: "b",
+					},
+				],
+				log: buildLog(),
+				signals: ["SIGWINCH"],
+				createHeartbeat: (name) => async () => {
+					beatsByQueue.set(name, (beatsByQueue.get(name) ?? 0) + 1);
+				},
+			});
+		} finally {
+			sendMock.mock.restore();
+		}
+
+		assert.ok(receiveCalls >= 4, `only ${receiveCalls} receives ran`);
+		// One heartbeat per queue, named after it: a wedged loop has to go stale
+		// on its own rather than ride on a sibling that is still turning.
+		assert.deepEqual([...beatsByQueue.keys()].sort(), [
+			"remit-test",
+			"remit-test-2",
+		]);
+		assert.equal(
+			[...beatsByQueue.values()].reduce((sum, count) => sum + count, 0),
+			receiveCalls,
+		);
+	});
+
+	it("keeps polling and logs when a heartbeat write fails", async () => {
+		let receiveCalls = 0;
+		const log = buildLog();
+
+		const sendMock = mock.method(
+			SQSClient.prototype,
+			"send",
+			// biome-ignore lint/suspicious/noExplicitAny: minimal SDK command shape, not worth typing per-command here
+			async function (this: SQSClient, command: any) {
+				if (command.constructor.name !== "ReceiveMessageCommand") {
+					throw new Error(`unexpected command: ${command.constructor.name}`);
+				}
+				receiveCalls++;
+				if (receiveCalls >= 2) process.emit("SIGWINCH", "SIGWINCH");
+				return { Messages: [] };
+			},
+		);
+
+		try {
+			// A full disk is the likeliest cause, and the moment the loop must stay
+			// up: the healthcheck reports the missed beat by itself.
+			await runQueuePoller({
+				targets: [
+					{ queueUrl: QUEUE_URL, handler: async () => {}, functionName: "a" },
+				],
+				log,
+				signals: ["SIGWINCH"],
+				createHeartbeat: () => () =>
+					Promise.reject(new Error("ENOSPC: no space left on device")),
+			});
+		} finally {
+			sendMock.mock.restore();
+		}
+
+		assert.ok(receiveCalls >= 2, `only ${receiveCalls} receives ran`);
+		const failures = log.error.mock.calls.filter(
+			(call) => call.arguments[1] === "poller: heartbeat write failed",
+		);
+		assert.equal(failures.length, receiveCalls);
+		assert.match(String(failures[0]?.arguments[0]?.error), /ENOSPC/);
+	});
+
 	it("throws when constructed with no targets", async () => {
 		await assert.rejects(
 			() => runQueuePoller({ targets: [], log: buildLog() }),

package/src/poller.ts

@@ -1,3 +1,4 @@
+import { inspect } from "node:util";
 import {
 	DeleteMessageCommand,
 	ReceiveMessageCommand,
@@ -8,6 +9,11 @@ import type {
 	SQSEvent,
 	SQSHandler,
 } from "aws-lambda";
+import {
+	clearHeartbeats,
+	createHeartbeat,
+	type Heartbeat,
+} from "./heartbeat.js";
 import { createQueueProducer } from "./producer.js";
 
 /**
@@ -41,17 +47,28 @@ export interface RunQueuePollerOptions {
 	readonly targets: readonly QueuePollerTarget[];
 	readonly log: QueuePollerLog;
 	readonly signals?: readonly NodeJS.Signals[];
+	/** One per target. Defaults to a file under `WORKER_HEARTBEAT_PREFIX`. */
+	readonly createHeartbeat?: (name: string) => Heartbeat;
 }
 
-const DEFAULT_MAX_MESSAGES = 10; // SQS API limit
+// One message per receive, not the SQS maximum of 10. Handlers process a batch
+// sequentially, so a batch of ten is ten handler runs between two heartbeats:
+// ten stalled SMTP sends at nodemailer's 300 s socket timeout is 50 minutes of
+// legitimate silence, and a threshold that tolerates it detects nothing. At one
+// message the quiet stretch a healthy loop can produce is one handler run, and
+// the extra receive per message is a millisecond against the local queue
+// sidecar — the same total work either way.
+const DEFAULT_MAX_MESSAGES = 1;
 const DEFAULT_VISIBILITY_TIMEOUT_SECONDS = 300;
 
 const pollTarget = async (
 	target: QueuePollerTarget,
 	log: QueuePollerLog,
 	isShuttingDown: () => boolean,
+	buildHeartbeat: (name: string) => Heartbeat,
 ): Promise<void> => {
 	const queueName = new URL(target.queueUrl).pathname.split("/").pop();
+	const heartbeat = buildHeartbeat(queueName ?? target.functionName);
 	const sqs = createQueueProducer({ queueUrl: target.queueUrl });
 	const maxMessages = target.maxMessages ?? DEFAULT_MAX_MESSAGES;
 	const visibilityTimeout =
@@ -68,6 +85,22 @@ const pollTarget = async (
 	const LONG_POLL_WAIT_SECONDS = 20;
 
 	while (!isShuttingDown()) {
+		// Top of the receive attempt, before the long poll: this loop's own file,
+		// so a loop that stops turning — wedged in a socket read, or in a handler
+		// that never returns — goes stale on its own however busy its siblings are.
+		//
+		// A write that fails must not take the worker down with it. A full disk is
+		// the likeliest cause and the moment the loop should stay up, failing
+		// records into the dead-letter queues with logs rather than crash-looping
+		// on the monitoring. The missed beat is itself the signal: the healthcheck
+		// goes stale, which is what this file exists to report.
+		await heartbeat().catch((error) => {
+			log.error(
+				{ queue: queueName, error: inspect(error) },
+				"poller: heartbeat write failed",
+			);
+		});
+
 		const response = await sqs.send(
 			new ReceiveMessageCommand({
 				QueueUrl: target.queueUrl,
@@ -129,9 +162,9 @@ const pollTarget = async (
 		);
 		const succeeded = messages.filter((m) => !failedIds.has(m.messageId));
 
-		// Batch is capped at maxMessages (SQS API limit: 10), so a handful of
-		// concurrent DeleteMessage calls is safe and avoids paying N sequential
-		// round trips between every receive/handler cycle.
+		// Batch is capped at maxMessages (one by default, 10 is the SQS ceiling),
+		// so a handful of concurrent DeleteMessage calls is safe and avoids paying
+		// N sequential round trips between every receive/handler cycle.
 		await Promise.all(
 			succeeded.map((message) =>
 				sqs.send(
@@ -161,17 +194,29 @@ const pollTarget = async (
  * received (default: SIGINT, SIGTERM), then lets each loop finish its
  * current iteration and returns. Rejects (crashes the process) if any loop
  * throws — a stuck poller should exit loudly, not degrade silently.
+ *
+ * Every loop writes its own heartbeat file (see heartbeat.ts) at the top of its
+ * receive attempt, which is how a loop that hangs instead of throwing — the
+ * case this cannot make loud by itself — becomes visible.
  */
 export const runQueuePoller = async (
 	options: RunQueuePollerOptions,
 ): Promise<void> => {
 	const { targets, log } = options;
 	const signals = options.signals ?? ["SIGINT", "SIGTERM"];
+	const buildHeartbeat = options.createHeartbeat ?? createHeartbeat;
 
 	if (targets.length === 0) {
 		throw new Error("runQueuePoller: no targets configured");
 	}
 
+	// Files from the previous run of this service, before its loops write again:
+	// the healthcheck reads the oldest file it finds, so one left behind by a
+	// queue this version no longer polls would fail the check forever.
+	await clearHeartbeats().catch((error) => {
+		log.error({ error: inspect(error) }, "poller: heartbeat cleanup failed");
+	});
+
 	let shuttingDown = false;
 	const isShuttingDown = () => shuttingDown;
 	const onSignal = (signal: NodeJS.Signals) => {
@@ -185,6 +230,8 @@ export const runQueuePoller = async (
 	log.info({ queues: targets.map((t) => t.queueUrl) }, "poller: starting");
 
 	await Promise.all(
-		targets.map((target) => pollTarget(target, log, isShuttingDown)),
+		targets.map((target) =>
+			pollTarget(target, log, isShuttingDown, buildHeartbeat),
+		),
 	);
 };