@remit/queue-sidecar 0.0.1

package/src/sqs.test.ts

@@ -0,0 +1,386 @@
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import type { Server } from "node:http";
+import type { AddressInfo } from "node:net";
+import { join } from "node:path";
+import { after, before, describe, it } from "node:test";
+import { fileURLToPath } from "node:url";
+import {
+	CreateQueueCommand,
+	DeleteMessageCommand,
+	GetQueueAttributesCommand,
+	PurgeQueueCommand,
+	ReceiveMessageCommand,
+	SendMessageBatchCommand,
+	SendMessageCommand,
+	SQSClient,
+} from "@aws-sdk/client-sqs";
+import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
+import { bootstrapQueues, parseQueuesConfig } from "./queues-config.js";
+import { createSidecarServer } from "./server.js";
+import { QueueStore } from "./store.js";
+
+const tmpRoot = join(
+	fileURLToPath(new URL(".", import.meta.url)),
+	"..",
+	".tmp",
+	"sqs",
+);
+
+const config = parseQueuesConfig({
+	queues: [
+		{
+			name: "sync.fifo",
+			fifo: true,
+			visibilityTimeoutSeconds: 30,
+			deadLetter: { name: "sync-dlq.fifo", maxReceiveCount: 3 },
+		},
+		{ name: "sync-dlq.fifo", fifo: true, visibilityTimeoutSeconds: 30 },
+		{
+			name: "work",
+			visibilityTimeoutSeconds: 1,
+			deadLetter: { name: "work-dlq", maxReceiveCount: 2 },
+		},
+		{ name: "work-dlq", visibilityTimeoutSeconds: 30 },
+	],
+});
+
+interface Harness {
+	client: SQSClient;
+	baseUrl: string;
+	queueUrl: (name: string) => string;
+	restart: () => Promise<void>;
+	stop: () => Promise<void>;
+	dbPath: string;
+}
+
+const listen = (server: Server, port = 0): Promise<number> =>
+	new Promise((resolve) => {
+		server.listen(port, "127.0.0.1", () => {
+			resolve((server.address() as AddressInfo).port);
+		});
+	});
+
+const startHarness = async (): Promise<Harness> => {
+	mkdirSync(tmpRoot, { recursive: true });
+	const dir = mkdtempSync(join(tmpRoot, "sqs-"));
+	const dbPath = join(dir, "queue.db");
+
+	let store = new QueueStore(dbPath);
+	bootstrapQueues(store, config);
+	let server = createSidecarServer({ store, longPollIntervalMs: 25 });
+	let port = await listen(server);
+
+	const baseUrl = () => `http://127.0.0.1:${port}`;
+	const client = new SQSClient({
+		endpoint: baseUrl(),
+		protocol: AwsQueryProtocol,
+		region: "local",
+		credentials: { accessKeyId: "local", secretAccessKey: "local" },
+		maxAttempts: 3,
+	});
+
+	const closeServer = (): Promise<void> =>
+		new Promise((resolve) => server.close(() => resolve()));
+
+	return {
+		client,
+		get baseUrl() {
+			return baseUrl();
+		},
+		queueUrl: (name) => `${baseUrl()}/000000000000/${name}`,
+		restart: async () => {
+			await closeServer();
+			store.close();
+			store = new QueueStore(dbPath);
+			server = createSidecarServer({ store, longPollIntervalMs: 25 });
+			port = await listen(server, port);
+		},
+		stop: async () => {
+			await closeServer();
+			store.close();
+		},
+		dbPath,
+	};
+};
+
+describe("SQS wire protocol via the AWS SDK", () => {
+	let h: Harness;
+
+	before(async () => {
+		h = await startHarness();
+	});
+
+	after(async () => {
+		await h.stop();
+		rmSync(tmpRoot, { recursive: true, force: true });
+	});
+
+	it("creates a queue and returns its URL", async () => {
+		const result = await h.client.send(
+			new CreateQueueCommand({
+				QueueName: "sync.fifo",
+				Attributes: { FifoQueue: "true" },
+			}),
+		);
+		assert.equal(result.QueueUrl, h.queueUrl("sync.fifo"));
+	});
+
+	it("sends and receives a message through the SDK", async () => {
+		const url = h.queueUrl("sync.fifo");
+		const sent = await h.client.send(
+			new SendMessageCommand({
+				QueueUrl: url,
+				MessageBody: "hello world",
+				MessageGroupId: "g-send",
+				MessageDeduplicationId: "d-send",
+			}),
+		);
+		assert.ok(sent.MessageId);
+		assert.equal(sent.MD5OfMessageBody, "5eb63bbbe01eeed093cb22bb8f5acdc3");
+
+		const received = await h.client.send(
+			new ReceiveMessageCommand({
+				QueueUrl: url,
+				MaxNumberOfMessages: 10,
+				WaitTimeSeconds: 2,
+			}),
+		);
+		assert.equal(received.Messages?.length, 1);
+		assert.equal(received.Messages?.[0].Body, "hello world");
+
+		await h.client.send(
+			new DeleteMessageCommand({
+				QueueUrl: url,
+				ReceiptHandle: received.Messages?.[0].ReceiptHandle,
+			}),
+		);
+	});
+
+	it("preserves FIFO order within a message group", async () => {
+		const url = h.queueUrl("sync.fifo");
+		await h.client.send(new PurgeQueueCommand({ QueueUrl: url }));
+		for (const body of ["one", "two", "three"]) {
+			await h.client.send(
+				new SendMessageCommand({
+					QueueUrl: url,
+					MessageBody: body,
+					MessageGroupId: "ordered",
+					MessageDeduplicationId: `order-${body}`,
+				}),
+			);
+		}
+
+		const bodies: string[] = [];
+		for (let i = 0; i < 3; i += 1) {
+			const received = await h.client.send(
+				new ReceiveMessageCommand({
+					QueueUrl: url,
+					MaxNumberOfMessages: 10,
+					WaitTimeSeconds: 2,
+				}),
+			);
+			assert.equal(received.Messages?.length, 1);
+			const message = received.Messages?.[0];
+			if (!message?.Body || !message.ReceiptHandle)
+				throw new Error("no message");
+			bodies.push(message.Body);
+			await h.client.send(
+				new DeleteMessageCommand({
+					QueueUrl: url,
+					ReceiptHandle: message.ReceiptHandle,
+				}),
+			);
+		}
+		assert.deepEqual(bodies, ["one", "two", "three"]);
+	});
+
+	it("sends a batch and reports per-entry ids", async () => {
+		const url = h.queueUrl("sync.fifo");
+		await h.client.send(new PurgeQueueCommand({ QueueUrl: url }));
+		const result = await h.client.send(
+			new SendMessageBatchCommand({
+				QueueUrl: url,
+				Entries: [
+					{
+						Id: "a",
+						MessageBody: "batch-a",
+						MessageGroupId: "batch",
+						MessageDeduplicationId: "batch-a",
+					},
+					{
+						Id: "b",
+						MessageBody: "batch-b",
+						MessageGroupId: "batch",
+						MessageDeduplicationId: "batch-b",
+					},
+				],
+			}),
+		);
+		assert.equal(result.Successful?.length, 2);
+		assert.deepEqual(result.Successful?.map((e) => e.Id).sort(), ["a", "b"]);
+	});
+
+	it("returns per-entry failures for a partially invalid batch", async () => {
+		const url = h.queueUrl("sync.fifo");
+		await h.client.send(new PurgeQueueCommand({ QueueUrl: url }));
+		const result = await h.client.send(
+			new SendMessageBatchCommand({
+				QueueUrl: url,
+				Entries: [
+					{
+						Id: "good",
+						MessageBody: "ok",
+						MessageGroupId: "g",
+						MessageDeduplicationId: "good-1",
+					},
+					// No MessageGroupId on a FIFO queue — SQS rejects this entry,
+					// the valid one still lands.
+					{ Id: "bad", MessageBody: "no-group" },
+				],
+			}),
+		);
+		assert.equal(result.Successful?.length, 1);
+		assert.equal(result.Successful?.[0].Id, "good");
+		assert.equal(result.Failed?.length, 1);
+		assert.equal(result.Failed?.[0].Id, "bad");
+		assert.equal(result.Failed?.[0].Code, "MissingParameter");
+		assert.equal(result.Failed?.[0].SenderFault, true);
+	});
+
+	it("rejects a FIFO SendMessage without a MessageGroupId", async () => {
+		const url = h.queueUrl("sync.fifo");
+		await assert.rejects(
+			h.client.send(
+				new SendMessageCommand({ QueueUrl: url, MessageBody: "no-group" }),
+			),
+			(error: Error) => {
+				assert.match(`${error.name} ${error.message}`, /MissingParameter/);
+				return true;
+			},
+		);
+	});
+
+	it("redelivers after the visibility timeout and reports the receive count", async () => {
+		const url = h.queueUrl("work");
+		await h.client.send(new PurgeQueueCommand({ QueueUrl: url }));
+		await h.client.send(
+			new SendMessageCommand({ QueueUrl: url, MessageBody: "retry-me" }),
+		);
+
+		const first = await h.client.send(
+			new ReceiveMessageCommand({
+				QueueUrl: url,
+				MaxNumberOfMessages: 1,
+				WaitTimeSeconds: 2,
+				MessageSystemAttributeNames: ["ApproximateReceiveCount"],
+			}),
+		);
+		assert.equal(first.Messages?.length, 1);
+		assert.equal(first.Messages?.[0].Attributes?.ApproximateReceiveCount, "1");
+
+		await delay(1500);
+
+		const second = await h.client.send(
+			new ReceiveMessageCommand({
+				QueueUrl: url,
+				MaxNumberOfMessages: 1,
+				WaitTimeSeconds: 2,
+				MessageSystemAttributeNames: ["ApproximateReceiveCount"],
+			}),
+		);
+		assert.equal(second.Messages?.length, 1);
+		assert.equal(second.Messages?.[0].Attributes?.ApproximateReceiveCount, "2");
+		assert.equal(second.Messages?.[0].Body, "retry-me");
+	});
+
+	it("redrives a poison message to the DLQ after maxReceiveCount", async () => {
+		const url = h.queueUrl("work");
+		const dlqUrl = h.queueUrl("work-dlq");
+		await h.client.send(new PurgeQueueCommand({ QueueUrl: url }));
+		await h.client.send(new PurgeQueueCommand({ QueueUrl: dlqUrl }));
+		await h.client.send(
+			new SendMessageCommand({ QueueUrl: url, MessageBody: "poison" }),
+		);
+
+		for (let attempt = 1; attempt <= 2; attempt += 1) {
+			const received = await h.client.send(
+				new ReceiveMessageCommand({
+					QueueUrl: url,
+					MaxNumberOfMessages: 1,
+					WaitTimeSeconds: 2,
+				}),
+			);
+			assert.equal(received.Messages?.length, 1, `delivery ${attempt}`);
+			await delay(1200);
+		}
+
+		const drained = await h.client.send(
+			new ReceiveMessageCommand({
+				QueueUrl: url,
+				MaxNumberOfMessages: 1,
+				WaitTimeSeconds: 1,
+			}),
+		);
+		assert.equal(drained.Messages?.length ?? 0, 0, "source queue is empty");
+
+		const dlq = await h.client.send(
+			new ReceiveMessageCommand({
+				QueueUrl: dlqUrl,
+				MaxNumberOfMessages: 1,
+				WaitTimeSeconds: 2,
+			}),
+		);
+		assert.equal(dlq.Messages?.length, 1);
+		assert.equal(dlq.Messages?.[0].Body, "poison");
+	});
+
+	it("reports queue attributes for DLQ depth monitoring", async () => {
+		const url = h.queueUrl("sync.fifo");
+		await h.client.send(new PurgeQueueCommand({ QueueUrl: url }));
+		await h.client.send(
+			new SendMessageCommand({
+				QueueUrl: url,
+				MessageBody: "depth",
+				MessageGroupId: "g",
+				MessageDeduplicationId: "depth-1",
+			}),
+		);
+		const attributes = await h.client.send(
+			new GetQueueAttributesCommand({
+				QueueUrl: url,
+				AttributeNames: ["All"],
+			}),
+		);
+		assert.equal(attributes.Attributes?.ApproximateNumberOfMessages, "1");
+		assert.equal(attributes.Attributes?.FifoQueue, "true");
+	});
+
+	it("keeps enqueued messages across a sidecar restart", async () => {
+		const url = h.queueUrl("sync.fifo");
+		await h.client.send(new PurgeQueueCommand({ QueueUrl: url }));
+		await h.client.send(
+			new SendMessageCommand({
+				QueueUrl: url,
+				MessageBody: "survive-restart",
+				MessageGroupId: "durable",
+				MessageDeduplicationId: "durable-1",
+			}),
+		);
+
+		await h.restart();
+
+		const received = await h.client.send(
+			new ReceiveMessageCommand({
+				QueueUrl: url,
+				MaxNumberOfMessages: 10,
+				WaitTimeSeconds: 2,
+			}),
+		);
+		assert.equal(received.Messages?.length, 1);
+		assert.equal(received.Messages?.[0].Body, "survive-restart");
+	});
+});
+
+const delay = (ms: number): Promise<void> =>
+	new Promise((resolve) => setTimeout(resolve, ms));

package/src/store.test.ts

@@ -0,0 +1,320 @@
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { after, beforeEach, describe, it } from "node:test";
+import { fileURLToPath } from "node:url";
+import { bootstrapQueues, parseQueuesConfig } from "./queues-config.js";
+import {
+	InvalidParameterValueError,
+	MissingParameterError,
+	QueueStore,
+} from "./store.js";
+
+const tmpRoot = join(
+	fileURLToPath(new URL(".", import.meta.url)),
+	"..",
+	".tmp",
+	"store",
+);
+
+const makeTempDir = (): string => {
+	mkdirSync(tmpRoot, { recursive: true });
+	return mkdtempSync(join(tmpRoot, "store-"));
+};
+
+const config = parseQueuesConfig({
+	queues: [
+		{
+			name: "orders.fifo",
+			fifo: true,
+			visibilityTimeoutSeconds: 30,
+			deadLetter: { name: "orders-dlq.fifo", maxReceiveCount: 3 },
+		},
+		{ name: "orders-dlq.fifo", fifo: true, visibilityTimeoutSeconds: 30 },
+		{
+			name: "jobs",
+			visibilityTimeoutSeconds: 30,
+			deadLetter: { name: "jobs-dlq", maxReceiveCount: 2 },
+		},
+		{ name: "jobs-dlq", visibilityTimeoutSeconds: 30 },
+	],
+});
+
+describe("QueueStore", () => {
+	let dir: string;
+	let store: QueueStore;
+
+	beforeEach(() => {
+		dir = makeTempDir();
+		store = new QueueStore(join(dir, "queue.db"));
+		bootstrapQueues(store, config);
+	});
+
+	after(() => {
+		rmSync(tmpRoot, { recursive: true, force: true });
+	});
+
+	it("round-trips a standard message and computes its md5", () => {
+		const sent = store.sendMessage({ queueName: "jobs", body: "hello" });
+		assert.equal(sent.md5OfBody, "5d41402abc4b2a76b9719d911017c592");
+
+		const received = store.receiveMessages({
+			queueName: "jobs",
+			maxMessages: 10,
+		});
+		assert.equal(received.length, 1);
+		assert.equal(received[0].body, "hello");
+		assert.equal(received[0].receiveCount, 1);
+
+		store.deleteMessage("jobs", received[0].receiptHandle);
+		assert.equal(
+			store.receiveMessages({ queueName: "jobs", maxMessages: 10 }).length,
+			0,
+		);
+	});
+
+	it("rejects a FIFO send with no MessageGroupId", () => {
+		assert.throws(
+			() =>
+				store.sendMessage({
+					queueName: "orders.fifo",
+					body: "x",
+					deduplicationId: "d",
+				}),
+			MissingParameterError,
+		);
+	});
+
+	it("rejects a FIFO send with neither a dedup id nor content-based dedup", () => {
+		assert.throws(
+			() =>
+				store.sendMessage({
+					queueName: "orders.fifo",
+					body: "x",
+					groupId: "g1",
+				}),
+			InvalidParameterValueError,
+		);
+	});
+
+	it("delivers FIFO messages in send order within a group", () => {
+		const now = 1_000;
+		for (const body of ["a", "b", "c"]) {
+			store.sendMessage({
+				queueName: "orders.fifo",
+				body,
+				groupId: "g1",
+				deduplicationId: body,
+				now,
+			});
+		}
+
+		const bodies: string[] = [];
+		for (let i = 0; i < 3; i += 1) {
+			const [message] = store.receiveMessages({
+				queueName: "orders.fifo",
+				maxMessages: 10,
+				visibilityTimeoutSeconds: 30,
+				now,
+			});
+			assert.ok(message, "a message is available");
+			bodies.push(message.body);
+			store.deleteMessage("orders.fifo", message.receiptHandle);
+		}
+		assert.deepEqual(bodies, ["a", "b", "c"]);
+	});
+
+	it("locks a FIFO group while a message is in flight", () => {
+		const now = 1_000;
+		store.sendMessage({
+			queueName: "orders.fifo",
+			body: "a",
+			groupId: "g1",
+			deduplicationId: "a",
+			now,
+		});
+		store.sendMessage({
+			queueName: "orders.fifo",
+			body: "b",
+			groupId: "g1",
+			deduplicationId: "b",
+			now,
+		});
+
+		const first = store.receiveMessages({
+			queueName: "orders.fifo",
+			maxMessages: 10,
+			visibilityTimeoutSeconds: 30,
+			now,
+		});
+		assert.equal(first.length, 1);
+		assert.equal(first[0].body, "a");
+
+		const blocked = store.receiveMessages({
+			queueName: "orders.fifo",
+			maxMessages: 10,
+			visibilityTimeoutSeconds: 30,
+			now,
+		});
+		assert.equal(
+			blocked.length,
+			0,
+			"group is locked until the first is deleted",
+		);
+
+		store.deleteMessage("orders.fifo", first[0].receiptHandle);
+		const next = store.receiveMessages({
+			queueName: "orders.fifo",
+			maxMessages: 10,
+			visibilityTimeoutSeconds: 30,
+			now,
+		});
+		assert.equal(next[0].body, "b");
+	});
+
+	it("parallelises distinct FIFO groups", () => {
+		const now = 1_000;
+		store.sendMessage({
+			queueName: "orders.fifo",
+			body: "g1-a",
+			groupId: "g1",
+			deduplicationId: "g1-a",
+			now,
+		});
+		store.sendMessage({
+			queueName: "orders.fifo",
+			body: "g2-a",
+			groupId: "g2",
+			deduplicationId: "g2-a",
+			now,
+		});
+		const received = store.receiveMessages({
+			queueName: "orders.fifo",
+			maxMessages: 10,
+			visibilityTimeoutSeconds: 30,
+			now,
+		});
+		assert.deepEqual(received.map((m) => m.body).sort(), ["g1-a", "g2-a"]);
+	});
+
+	it("deduplicates FIFO sends inside the dedup window", () => {
+		const now = 1_000;
+		const first = store.sendMessage({
+			queueName: "orders.fifo",
+			body: "a",
+			groupId: "g1",
+			deduplicationId: "dup",
+			now,
+		});
+		const second = store.sendMessage({
+			queueName: "orders.fifo",
+			body: "a-again",
+			groupId: "g1",
+			deduplicationId: "dup",
+			now: now + 1000,
+		});
+		assert.equal(second.deduplicated, true);
+		assert.equal(second.messageId, first.messageId);
+
+		const received = store.receiveMessages({
+			queueName: "orders.fifo",
+			maxMessages: 10,
+			now,
+		});
+		assert.equal(received.length, 1);
+		assert.equal(received[0].body, "a");
+	});
+
+	it("redelivers a message after its visibility timeout expires", () => {
+		const now = 1_000;
+		store.sendMessage({ queueName: "jobs", body: "x", now });
+
+		const first = store.receiveMessages({
+			queueName: "jobs",
+			maxMessages: 10,
+			visibilityTimeoutSeconds: 5,
+			now,
+		});
+		assert.equal(first.length, 1);
+
+		const stillHidden = store.receiveMessages({
+			queueName: "jobs",
+			maxMessages: 10,
+			now: now + 4_000,
+		});
+		assert.equal(stillHidden.length, 0);
+
+		const redelivered = store.receiveMessages({
+			queueName: "jobs",
+			maxMessages: 10,
+			now: now + 6_000,
+		});
+		assert.equal(redelivered.length, 1);
+		assert.equal(redelivered[0].receiveCount, 2);
+	});
+
+	it("moves a message to the DLQ after maxReceiveCount deliveries", () => {
+		let now = 1_000;
+		store.sendMessage({ queueName: "jobs", body: "poison", now });
+
+		for (let attempt = 1; attempt <= 2; attempt += 1) {
+			const received = store.receiveMessages({
+				queueName: "jobs",
+				maxMessages: 10,
+				visibilityTimeoutSeconds: 1,
+				now,
+			});
+			assert.equal(received.length, 1, `delivery ${attempt}`);
+			assert.equal(received[0].receiveCount, attempt);
+			now += 2_000;
+		}
+
+		const afterMax = store.receiveMessages({
+			queueName: "jobs",
+			maxMessages: 10,
+			now,
+		});
+		assert.equal(afterMax.length, 0, "source queue is drained");
+
+		const dlq = store.receiveMessages({
+			queueName: "jobs-dlq",
+			maxMessages: 10,
+			now,
+		});
+		assert.equal(dlq.length, 1);
+		assert.equal(dlq[0].body, "poison");
+		assert.equal(dlq[0].receiveCount, 1, "receive count resets in the DLQ");
+	});
+
+	it("persists enqueued messages across a store restart", () => {
+		const path = join(dir, "durable.db");
+		const first = new QueueStore(path);
+		bootstrapQueues(first, config);
+		first.sendMessage({
+			queueName: "orders.fifo",
+			body: "durable",
+			groupId: "g1",
+			deduplicationId: "d1",
+		});
+		first.close();
+
+		const reopened = new QueueStore(path);
+		const received = reopened.receiveMessages({
+			queueName: "orders.fifo",
+			maxMessages: 10,
+		});
+		assert.equal(received.length, 1);
+		assert.equal(received[0].body, "durable");
+		reopened.close();
+	});
+
+	it("purges all messages from a queue", () => {
+		store.sendMessage({ queueName: "jobs", body: "1" });
+		store.sendMessage({ queueName: "jobs", body: "2" });
+		store.purgeQueue("jobs");
+		assert.equal(
+			store.receiveMessages({ queueName: "jobs", maxMessages: 10 }).length,
+			0,
+		);
+	});
+});