@remit/queue-sidecar 0.0.11 → 0.0.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/queue-sidecar",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/index.ts

@@ -4,6 +4,7 @@ export {
 	parseQueuesConfig,
 	type QueuesConfig,
 } from "./queues-config.js";
+export { redriveOnRelease } from "./redrive.js";
 export {
 	type CreateSidecarOptions,
 	createSidecarServer,

package/src/main.ts

@@ -1,6 +1,7 @@
 import { mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 import { bootstrapQueues, loadQueuesConfig } from "./queues-config.js";
+import { redriveOnRelease } from "./redrive.js";
 import { createSidecarServer, type SidecarLog } from "./server.js";
 import { QueueStore } from "./store.js";
 
@@ -50,6 +51,9 @@ if (queuesConfigPath) {
 	log.info({}, "sidecar: no queues config; queues created on demand");
 }
 
+const release = process.env.QUEUE_SIDECAR_RELEASE;
+if (release) redriveOnRelease(store, release);
+
 const server = createSidecarServer({ store, accountId, log });
 
 server.listen(port, host, () => {

package/src/metrics.test.ts

@@ -158,6 +158,19 @@ describe("the sidecar /metrics endpoint", () => {
 		store.purgeQueue("work");
 	});
 
+	it("exports a received-but-undeleted message as in flight", async () => {
+		store.purgeQueue("work-dlq");
+		store.sendMessage({ queueName: "work-dlq", body: "peeked" });
+		store.receiveMessages({ queueName: "work-dlq", maxMessages: 1 });
+		const { body } = await get("/metrics");
+		assert.match(
+			body,
+			/^remit_queue_messages_in_flight\{queue="work-dlq",role="dead_letter"\} 1$/m,
+		);
+		assert.equal(sample(body, ['queue="work-dlq"']), 0);
+		store.purgeQueue("work-dlq");
+	});
+
 	it("counts only visible messages as depth", () => {
 		store.sendMessage({ queueName: "work", body: "c" });
 		store.receiveMessages({ queueName: "work", maxMessages: 1 });

package/src/metrics.ts

@@ -17,6 +17,13 @@ const queueMessages = new Gauge({
 	registers: [registry],
 });
 
+const queueMessagesInFlight = new Gauge({
+	name: "remit_queue_messages_in_flight",
+	help: "Messages received from a queue and not yet deleted or visible again, by queue and role.",
+	labelNames: ["queue", "role"],
+	registers: [registry],
+});
+
 const queueLastReceiveAge = new Gauge({
 	name: "remit_queue_last_receive_age_seconds",
 	help: "Seconds since a queue last handed a message to a consumer, or since the sidecar started if it has not.",
@@ -66,11 +73,16 @@ export const collectQueueDepths = (store: QueueDepthSource): void => {
 	}
 	const roles = queueRoles(queues);
 	queueMessages.reset();
+	queueMessagesInFlight.reset();
 	queueLastReceiveAge.reset();
 	for (const queue of queues) {
 		const attributes = store.getQueueAttributes(queue.name);
 		const labels = { queue: queue.name, role: roles.get(queue.name) ?? "work" };
 		queueMessages.set(labels, attributes.approximateNumberOfMessages);
+		queueMessagesInFlight.set(
+			labels,
+			attributes.approximateNumberOfMessagesNotVisible,
+		);
 		if (labels.role === "work") {
 			queueLastReceiveAge.set(labels, store.lastReceiveAgeSeconds(queue.name));
 		}

package/src/protocol.ts

@@ -140,6 +140,9 @@ export const receiveMessageResult = (messages: ReceiveMessageXml[]): string =>
 export const queueUrlResult = (queueUrl: string): string =>
 	tag("QueueUrl", queueUrl);
 
+export const taskHandleResult = (taskHandle: string): string =>
+	tag("TaskHandle", taskHandle);
+
 export const queueAttributesResult = (
 	attributes: Record<string, string>,
 ): string =>

package/src/redrive-cli.ts

@@ -0,0 +1,9 @@
+import { runRedriveCli } from "./redrive.js";
+
+process.exit(
+	await runRedriveCli(process.argv.slice(2), {
+		env: process.env,
+		out: (text) => process.stdout.write(text),
+		err: (text) => process.stderr.write(text),
+	}),
+);

package/src/redrive.test.ts

@@ -0,0 +1,104 @@
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import type { Server } from "node:http";
+import type { AddressInfo } from "node:net";
+import { join } from "node:path";
+import { after, before, describe, it } from "node:test";
+import { fileURLToPath } from "node:url";
+import { bootstrapQueues, loadQueuesConfig } from "./queues-config.js";
+import { runRedriveCli } from "./redrive.js";
+import { createSidecarServer } from "./server.js";
+import { QueueStore } from "./store.js";
+
+const here = fileURLToPath(new URL(".", import.meta.url));
+const tmpRoot = join(here, "..", ".tmp", "redrive");
+
+describe("remit redrive, against the shipped queue definitions", () => {
+	let store: QueueStore;
+	let server: Server;
+	let port: number;
+
+	before(async () => {
+		mkdirSync(tmpRoot, { recursive: true });
+		store = new QueueStore(join(mkdtempSync(join(tmpRoot, "q-")), "q.db"));
+		bootstrapQueues(
+			store,
+			loadQueuesConfig(join(here, "..", "..", "..", "deploy/vps/queues.json")),
+		);
+		server = createSidecarServer({ store });
+		port = await new Promise((resolve) =>
+			server.listen(0, "127.0.0.1", () =>
+				resolve((server.address() as AddressInfo).port),
+			),
+		);
+	});
+
+	after(async () => {
+		await new Promise((resolve) => server.close(resolve));
+		store.close();
+		rmSync(tmpRoot, { recursive: true, force: true });
+	});
+
+	const cli = async (args: string[]) => {
+		const out: string[] = [];
+		const err: string[] = [];
+		const code = await runRedriveCli(args, {
+			env: { QUEUE_SIDECAR_PORT: String(port) },
+			out: (text) => out.push(text),
+			err: (text) => err.push(text),
+		});
+		return { code, out: out.join(""), err: err.join("") };
+	};
+
+	it("returns a body dead-letter queue to the body queue", async () => {
+		store.sendMessage({ queueName: "remit-body-dlq", body: "batch-1" });
+		store.sendMessage({ queueName: "remit-body-dlq", body: "batch-2" });
+
+		const result = await cli(["remit-body-dlq"]);
+
+		assert.equal(result.code, 0);
+		assert.match(result.out, /Moved 2 message\(s\) from remit-body-dlq/);
+		assert.deepEqual(
+			store
+				.receiveMessages({ queueName: "remit-body", maxMessages: 10 })
+				.map((message) => message.body),
+			["batch-1", "batch-2"],
+		);
+	});
+
+	it("refuses the send dead-letter queue, leaving it where it is", async () => {
+		store.sendMessage({ queueName: "remit-smtp-dlq", body: "send" });
+
+		const result = await cli(["remit-smtp-dlq"]);
+
+		assert.equal(result.code, 2);
+		assert.match(result.err, /resend|deliver mail/);
+		assert.equal(
+			store.receiveMessages({ queueName: "remit-smtp", maxMessages: 10 })
+				.length,
+			0,
+		);
+	});
+
+	it("points a sync dead-letter queue at a purge instead", async () => {
+		const result = await cli(["remit-mailboxes-dlq.fifo"]);
+
+		assert.equal(result.code, 2);
+		assert.match(result.err, /Purge it instead/);
+	});
+
+	it("names the queues it redrives for any other name", async () => {
+		const result = await cli(["remit-body"]);
+
+		assert.equal(result.code, 2);
+		assert.match(result.err, /it redrives remit-body-dlq/);
+	});
+
+	it("takes exactly one queue", async () => {
+		assert.equal((await cli([])).code, 2);
+		assert.equal(
+			(await cli(["remit-body-dlq", "remit-flags-dlq.fifo"])).code,
+			2,
+		);
+	});
+});

package/src/redrive.ts

@@ -0,0 +1,139 @@
+import type { QueueStore } from "./store.js";
+
+export type RedriveVerdict =
+	| { kind: "redrive" }
+	| { kind: "refuse"; reason: string };
+
+const REPLAYABLE = new Set([
+	"remit-body-dlq",
+	"remit-flags-dlq.fifo",
+	"remit-message-mgmt-dlq",
+]);
+
+const RESYNCED = new Set([
+	"remit-mailboxes-dlq.fifo",
+	"remit-messages-dlq.fifo",
+]);
+
+/**
+ * Which dead-letter queues are safe to replay. Body syncs, flag pushes and
+ * filings re-read the row before they act, so a replay months later does the
+ * right thing or nothing. A send does not: its outbox row may still read
+ * `sending`, and the replay would deliver mail the user gave up on.
+ */
+export const redriveVerdict = (deadLetterQueue: string): RedriveVerdict => {
+	if (REPLAYABLE.has(deadLetterQueue)) return { kind: "redrive" };
+	if (deadLetterQueue === "remit-smtp-dlq") {
+		return {
+			kind: "refuse",
+			reason:
+				"remit-smtp-dlq holds sends. Replaying one can deliver mail the user stopped expecting, possibly months old; read each message and delete it instead.",
+		};
+	}
+	if (RESYNCED.has(deadLetterQueue)) {
+		return {
+			kind: "refuse",
+			reason: `${deadLetterQueue} holds folder syncs, and the scheduler queues a fresh one for every folder on its own. Purge it instead.`,
+		};
+	}
+	return {
+		kind: "refuse",
+		reason: `${deadLetterQueue} is not a dead-letter queue remit redrives; it redrives remit-body-dlq, remit-flags-dlq.fifo and remit-message-mgmt-dlq.`,
+	};
+};
+
+export const redriveOnRelease = (store: QueueStore, release: string): void =>
+	store.redriveForRelease(
+		release,
+		store
+			.listQueues()
+			.map((queue) => queue.name)
+			.filter((name) => redriveVerdict(name).kind === "redrive"),
+	);
+
+export interface RedriveOptions {
+	deadLetterQueue: string;
+	baseUrl: string;
+	accountId?: string;
+}
+
+export type RedriveOutcome =
+	| { kind: "moved"; count: number }
+	| { kind: "refused"; reason: string }
+	| { kind: "failed"; reason: string };
+
+const call = async (
+	baseUrl: string,
+	params: Record<string, string>,
+): Promise<{ ok: true; body: string } | { ok: false; reason: string }> => {
+	const response = await fetch(`${baseUrl}/`, {
+		method: "POST",
+		headers: { "content-type": "application/x-www-form-urlencoded" },
+		body: new URLSearchParams({ ...params, Version: "2012-11-05" }),
+	});
+	const body = await response.text();
+	if (response.ok) return { ok: true, body };
+	return {
+		ok: false,
+		reason: body.match(/<Message>(.*)<\/Message>/)?.[1] ?? body,
+	};
+};
+
+export const redrive = async (
+	options: RedriveOptions,
+): Promise<RedriveOutcome> => {
+	const { deadLetterQueue, baseUrl } = options;
+	const accountId = options.accountId ?? "000000000000";
+	const verdict = redriveVerdict(deadLetterQueue);
+	if (verdict.kind === "refuse") {
+		return { kind: "refused", reason: verdict.reason };
+	}
+
+	const depth = await call(baseUrl, {
+		Action: "GetQueueAttributes",
+		QueueUrl: `${baseUrl}/${accountId}/${deadLetterQueue}`,
+		"AttributeName.1": "ApproximateNumberOfMessages",
+	});
+	if (!depth.ok) return { kind: "failed", reason: depth.reason };
+
+	const moved = await call(baseUrl, {
+		Action: "StartMessageMoveTask",
+		SourceArn: `arn:aws:sqs:local:${accountId}:${deadLetterQueue}`,
+	});
+	if (!moved.ok) return { kind: "failed", reason: moved.reason };
+
+	return {
+		kind: "moved",
+		count: Number(depth.body.match(/<Value>(\d+)<\/Value>/)?.[1] ?? "0"),
+	};
+};
+
+export interface RedriveCliIo {
+	env: NodeJS.ProcessEnv;
+	out: (text: string) => void;
+	err: (text: string) => void;
+}
+
+export const runRedriveCli = async (
+	args: string[],
+	io: RedriveCliIo,
+): Promise<number> => {
+	const [deadLetterQueue, ...rest] = args;
+	if (!deadLetterQueue || rest.length > 0) {
+		io.err("usage: redrive <dead-letter-queue>\n");
+		return 2;
+	}
+	const outcome = await redrive({
+		deadLetterQueue,
+		baseUrl: `http://127.0.0.1:${io.env.QUEUE_SIDECAR_PORT ?? "9324"}`,
+		accountId: io.env.QUEUE_SIDECAR_ACCOUNT_ID,
+	});
+	if (outcome.kind === "moved") {
+		io.out(
+			`Moved ${outcome.count} message(s) from ${deadLetterQueue} back to the queue it came from.\n`,
+		);
+		return 0;
+	}
+	io.err(`redrive: ${outcome.reason}\n`);
+	return outcome.kind === "refused" ? 2 : 1;
+};

package/src/release-redrive.test.ts

@@ -0,0 +1,86 @@
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { after, beforeEach, describe, it } from "node:test";
+import { fileURLToPath } from "node:url";
+import { bootstrapQueues, loadQueuesConfig } from "./queues-config.js";
+import { redriveOnRelease } from "./redrive.js";
+import { QueueStore } from "./store.js";
+
+const here = fileURLToPath(new URL(".", import.meta.url));
+const tmpRoot = join(here, "..", ".tmp", "release-redrive");
+const shippedQueues = loadQueuesConfig(
+	join(here, "..", "..", "..", "deploy/vps/queues.json"),
+);
+
+const bodies = (store: QueueStore, queueName: string): string[] =>
+	store
+		.receiveMessages({
+			queueName,
+			maxMessages: 10,
+			visibilityTimeoutSeconds: 0,
+		})
+		.map((message) => message.body);
+
+const failUntilDeadLettered = (store: QueueStore, queueName: string): void => {
+	for (let attempt = 0; attempt <= 3; attempt++) {
+		store.receiveMessages({
+			queueName,
+			maxMessages: 10,
+			visibilityTimeoutSeconds: 0,
+		});
+	}
+};
+
+describe("the dead-letter redrive a release runs on start", () => {
+	let dbPath: string;
+
+	beforeEach(() => {
+		mkdirSync(tmpRoot, { recursive: true });
+		dbPath = join(mkdtempSync(join(tmpRoot, "q-")), "q.db");
+		const store = new QueueStore(dbPath);
+		bootstrapQueues(store, shippedQueues);
+		store.close();
+	});
+
+	after(() => rmSync(tmpRoot, { recursive: true, force: true }));
+
+	const boot = <T>(run: (store: QueueStore) => T): T => {
+		const store = new QueueStore(dbPath);
+		try {
+			return run(store);
+		} finally {
+			store.close();
+		}
+	};
+
+	it("returns a dead-lettered body batch to the body queue once per release", () => {
+		boot((store) =>
+			store.sendMessage({ queueName: "remit-body-dlq", body: "batch-1" }),
+		);
+
+		boot((store) => {
+			redriveOnRelease(store, "v1.4.0");
+			assert.deepEqual(bodies(store, "remit-body"), ["batch-1"]);
+			failUntilDeadLettered(store, "remit-body");
+			assert.equal(
+				store.getQueueAttributes("remit-body-dlq").approximateNumberOfMessages,
+				1,
+			);
+		});
+
+		boot((store) => {
+			redriveOnRelease(store, "v1.4.0");
+			assert.deepEqual(bodies(store, "remit-body"), []);
+			assert.equal(
+				store.getQueueAttributes("remit-body-dlq").approximateNumberOfMessages,
+				1,
+			);
+		});
+
+		boot((store) => {
+			redriveOnRelease(store, "v1.5.0");
+			assert.deepEqual(bodies(store, "remit-body"), ["batch-1"]);
+		});
+	});
+});

package/src/server.ts

@@ -17,6 +17,7 @@ import {
 	receiveMessageResult,
 	sendMessageBatchResult,
 	sendMessageResult,
+	taskHandleResult,
 } from "./protocol.js";
 import {
 	QueueDoesNotExistError,
@@ -73,6 +74,12 @@ class InvalidRequestError extends SqsError {
 	}
 }
 
+const queueNameFromArn = (arn: string): string => {
+	const name = arn.split(":").at(-1);
+	if (!name) throw new InvalidRequestError(`malformed queue ARN: ${arn}`);
+	return name;
+};
+
 const collectIndexed = (params: URLSearchParams, prefix: string): string[] => {
 	const values: string[] = [];
 	for (let index = 1; ; index += 1) {
@@ -208,6 +215,21 @@ export const createSidecarServer = (options: CreateSidecarOptions): Server => {
 			};
 		}
 
+		if (action === "StartMessageMoveTask") {
+			const sourceArn = params.get("SourceArn");
+			if (!sourceArn) {
+				throw new InvalidRequestError(
+					"StartMessageMoveTask requires SourceArn",
+				);
+			}
+			const moved = store.redriveDeadLetters(queueNameFromArn(sourceArn));
+			log.info({ sourceArn, moved }, "sidecar: dead letters redriven");
+			return {
+				status: 200,
+				body: queryResponse(action, taskHandleResult(randomUUID()), requestId),
+			};
+		}
+
 		const queueName = requireQueueName(params);
 
 		if (action === "SendMessage") {

package/src/sqs.test.ts

@@ -14,6 +14,7 @@ import {
 	SendMessageBatchCommand,
 	SendMessageCommand,
 	SQSClient,
+	StartMessageMoveTaskCommand,
 } from "@aws-sdk/client-sqs";
 import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
 import { bootstrapQueues, parseQueuesConfig } from "./queues-config.js";
@@ -335,6 +336,63 @@ describe("SQS wire protocol via the AWS SDK", () => {
 		assert.equal(dlq.Messages?.[0].Body, "poison");
 	});
 
+	it("moves a dead-lettered message back to its source with a fresh budget", async () => {
+		const url = h.queueUrl("work");
+		const dlqUrl = h.queueUrl("work-dlq");
+		await h.client.send(new PurgeQueueCommand({ QueueUrl: url }));
+		await h.client.send(new PurgeQueueCommand({ QueueUrl: dlqUrl }));
+		await h.client.send(
+			new SendMessageCommand({ QueueUrl: url, MessageBody: "stranded" }),
+		);
+		for (let attempt = 1; attempt <= 2; attempt += 1) {
+			await h.client.send(
+				new ReceiveMessageCommand({ QueueUrl: url, WaitTimeSeconds: 2 }),
+			);
+			await delay(1200);
+		}
+		await h.client.send(
+			new ReceiveMessageCommand({ QueueUrl: url, WaitTimeSeconds: 1 }),
+		);
+
+		const task = await h.client.send(
+			new StartMessageMoveTaskCommand({
+				SourceArn: "arn:aws:sqs:local:000000000000:work-dlq",
+			}),
+		);
+
+		assert.ok(task.TaskHandle);
+		const dlqDepth = await h.client.send(
+			new GetQueueAttributesCommand({
+				QueueUrl: dlqUrl,
+				AttributeNames: ["ApproximateNumberOfMessages"],
+			}),
+		);
+		assert.equal(dlqDepth.Attributes?.ApproximateNumberOfMessages, "0");
+		const redelivered = await h.client.send(
+			new ReceiveMessageCommand({
+				QueueUrl: url,
+				WaitTimeSeconds: 2,
+				MessageSystemAttributeNames: ["ApproximateReceiveCount"],
+			}),
+		);
+		assert.equal(redelivered.Messages?.[0]?.Body, "stranded");
+		assert.equal(
+			redelivered.Messages?.[0]?.Attributes?.ApproximateReceiveCount,
+			"1",
+		);
+	});
+
+	it("refuses to move messages out of a queue that does not exist", async () => {
+		await assert.rejects(
+			h.client.send(
+				new StartMessageMoveTaskCommand({
+					SourceArn: "arn:aws:sqs:local:000000000000:nowhere-dlq",
+				}),
+			),
+			/does not exist/,
+		);
+	});
+
 	it("reports queue attributes for DLQ depth monitoring", async () => {
 		const url = h.queueUrl("sync.fifo");
 		await h.client.send(new PurgeQueueCommand({ QueueUrl: url }));

package/src/store.test.ts

@@ -284,7 +284,7 @@ describe("QueueStore", () => {
 		});
 		assert.equal(dlq.length, 1);
 		assert.equal(dlq[0].body, "poison");
-		assert.equal(dlq[0].receiveCount, 1, "receive count resets in the DLQ");
+		assert.equal(dlq[0].receiveCount, 3, "the DLQ keeps the receive count");
 	});
 
 	it("persists enqueued messages across a store restart", () => {

package/src/store.ts

@@ -150,6 +150,10 @@ export class QueueStore {
 				created_at INTEGER NOT NULL,
 				PRIMARY KEY (queue_name, dedup_id)
 			);
+			CREATE TABLE IF NOT EXISTS release_redrive (
+				id INTEGER PRIMARY KEY CHECK (id = 1),
+				release TEXT NOT NULL
+			);
 		`);
 	}
 
@@ -321,7 +325,9 @@ export class QueueStore {
 					queue.deadLetterTargetName !== null &&
 					candidate.receive_count >= queue.maxReceiveCount
 				) {
-					this.moveToDeadLetter(candidate, queue.deadLetterTargetName, now);
+					this.db
+						.prepare("UPDATE messages SET queue_name = ? WHERE id = ?")
+						.run(queue.deadLetterTargetName, candidate.id);
 					continue;
 				}
 
@@ -449,9 +455,52 @@ export class QueueStore {
 			.all({ queue: queue.name, now, limit }) as MessageRow[];
 	}
 
-	private moveToDeadLetter(
+	/**
+	 * Return everything a dead-letter queue holds to the queue it came from, as
+	 * SQS StartMessageMoveTask does. Each message goes to the back of that queue
+	 * with a fresh receive count, so it gets the full redelivery budget again and
+	 * a FIFO group still sees it after anything sent since.
+	 */
+	redriveDeadLetters(deadLetterName: string, now = Date.now()): number {
+		this.requireQueue(deadLetterName);
+		const source = this.listQueues().find(
+			(queue) => queue.deadLetterTargetName === deadLetterName,
+		);
+		if (!source) {
+			throw new InvalidParameterValueError(
+				`${deadLetterName} is not the dead-letter queue of any queue`,
+			);
+		}
+		const destination = source.name;
+
+		const redrive = this.db.transaction((): number => {
+			const rows = this.db
+				.prepare("SELECT * FROM messages WHERE queue_name = ? ORDER BY id ASC")
+				.all(deadLetterName) as MessageRow[];
+			for (const row of rows) this.moveToQueue(row, destination, now);
+			return rows.length;
+		});
+		return redrive();
+	}
+
+	redriveForRelease(release: string, deadLetterNames: string[]): void {
+		this.db.transaction((): void => {
+			const marker = this.db
+				.prepare("SELECT release FROM release_redrive WHERE id = 1")
+				.get() as { release: string } | undefined;
+			if (marker?.release === release) return;
+			for (const name of deadLetterNames) this.redriveDeadLetters(name);
+			this.db
+				.prepare(
+					"INSERT INTO release_redrive (id, release) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET release = excluded.release",
+				)
+				.run(release);
+		})();
+	}
+
+	private moveToQueue(
 		message: MessageRow,
-		deadLetterName: string,
+		targetName: string,
 		now: number,
 	): void {
 		this.db.prepare("DELETE FROM messages WHERE id = ?").run(message.id);
@@ -465,7 +514,7 @@ export class QueueStore {
 			)
 			.run(
 				message.message_id,
-				deadLetterName,
+				targetName,
 				message.body,
 				message.md5_body,
 				message.group_id,