@remit/queue-sidecar 0.0.1

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@remit/queue-sidecar",
+  "version": "0.0.1",
+  "type": "module",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "default": "./src/index.ts"
+    }
+  },
+  "bin": {
+    "queue-sidecar": "src/main.ts"
+  },
+  "scripts": {
+    "start": "node --import tsx src/main.ts",
+    "test:typecheck": "tsgo --noEmit",
+    "test:run": "node --import tsx --test 'src/**/*.test.ts'",
+    "test": "npm run test:typecheck && npm run test:run"
+  },
+  "dependencies": {
+    "better-sqlite3": "^12.11.1"
+  },
+  "devDependencies": {
+    "@aws-sdk/client-sqs": "*",
+    "@aws-sdk/core": "*",
+    "@types/better-sqlite3": "^7.6.13"
+  },
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/remit-mail/reader.git",
+    "directory": "packages/queue-sidecar"
+  }
+}

package/src/crash-recovery.test.ts

@@ -0,0 +1,128 @@
+import assert from "node:assert/strict";
+import { type ChildProcess, spawn } from "node:child_process";
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { createServer, get } from "node:http";
+import type { AddressInfo } from "node:net";
+import { join } from "node:path";
+import { after, describe, it } from "node:test";
+import { fileURLToPath } from "node:url";
+import {
+	ReceiveMessageCommand,
+	SendMessageCommand,
+	SQSClient,
+} from "@aws-sdk/client-sqs";
+import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
+
+const here = fileURLToPath(new URL(".", import.meta.url));
+const tmpRoot = join(here, "..", ".tmp", "crash");
+const mainEntry = join(here, "main.ts");
+
+const delay = (ms: number): Promise<void> =>
+	new Promise((resolve) => setTimeout(resolve, ms));
+
+const freePort = (): Promise<number> =>
+	new Promise((resolve) => {
+		const probe = createServer();
+		probe.listen(0, "127.0.0.1", () => {
+			const { port } = probe.address() as AddressInfo;
+			probe.close(() => resolve(port));
+		});
+	});
+
+const httpOk = (url: string): Promise<boolean> =>
+	new Promise((resolve) => {
+		const req = get(url, (res) => {
+			res.resume();
+			resolve(res.statusCode === 200);
+		});
+		req.on("error", () => resolve(false));
+	});
+
+const waitForHealth = async (port: number): Promise<void> => {
+	const deadline = Date.now() + 15_000;
+	while (Date.now() < deadline) {
+		if (await httpOk(`http://127.0.0.1:${port}/health`)) return;
+		await delay(200);
+	}
+	throw new Error("sidecar child did not become healthy in time");
+};
+
+const spawnSidecar = async (
+	port: number,
+	dbPath: string,
+	configPath: string,
+): Promise<ChildProcess> => {
+	const child = spawn(process.execPath, ["--import", "tsx", mainEntry], {
+		env: {
+			...process.env,
+			QUEUE_SIDECAR_HOST: "127.0.0.1",
+			QUEUE_SIDECAR_PORT: String(port),
+			QUEUE_SIDECAR_DB: dbPath,
+			QUEUE_SIDECAR_QUEUES_CONFIG: configPath,
+		},
+		stdio: "ignore",
+	});
+	await waitForHealth(port);
+	return child;
+};
+
+const waitForExit = (child: ChildProcess): Promise<void> =>
+	new Promise((resolve) => child.on("exit", () => resolve()));
+
+describe("crash recovery (SIGKILL, no graceful close)", () => {
+	after(() => {
+		rmSync(tmpRoot, { recursive: true, force: true });
+	});
+
+	it("recovers a committed send after the process is killed uncleanly", async () => {
+		mkdirSync(tmpRoot, { recursive: true });
+		const dbPath = join(tmpRoot, "queue.db");
+		const configPath = join(tmpRoot, "queues.json");
+		writeFileSync(
+			configPath,
+			JSON.stringify({
+				queues: [{ name: "outbox", visibilityTimeoutSeconds: 30 }],
+			}),
+		);
+
+		const port = await freePort();
+		const client = new SQSClient({
+			endpoint: `http://127.0.0.1:${port}`,
+			protocol: AwsQueryProtocol,
+			region: "local",
+			credentials: { accessKeyId: "local", secretAccessKey: "local" },
+			maxAttempts: 1,
+		});
+		const queueUrl = `http://127.0.0.1:${port}/000000000000/outbox`;
+
+		const first = await spawnSidecar(port, dbPath, configPath);
+		// The send resolves only after the sidecar has committed it — under
+		// synchronous=FULL that commit is fsync'd to the WAL before the response.
+		await client.send(
+			new SendMessageCommand({
+				QueueUrl: queueUrl,
+				MessageBody: "unsent-mail",
+			}),
+		);
+
+		// Kill -9: no SIGTERM handler runs, no graceful close, no WAL
+		// checkpoint. This is the host-crash / power-loss case, not a clean
+		// restart — the recovery path the outbox durability requirement is about.
+		first.kill("SIGKILL");
+		await waitForExit(first);
+
+		const second = await spawnSidecar(port, dbPath, configPath);
+		const received = await client.send(
+			new ReceiveMessageCommand({
+				QueueUrl: queueUrl,
+				MaxNumberOfMessages: 10,
+				WaitTimeSeconds: 2,
+			}),
+		);
+		second.kill("SIGKILL");
+		await waitForExit(second);
+
+		assert.equal(received.Messages?.length, 1);
+		assert.equal(received.Messages?.[0].Body, "unsent-mail");
+	});
+});

package/src/index.ts

@@ -0,0 +1,25 @@
+export {
+	bootstrapQueues,
+	loadQueuesConfig,
+	parseQueuesConfig,
+	type QueuesConfig,
+} from "./queues-config.js";
+export {
+	type CreateSidecarOptions,
+	createSidecarServer,
+	type SidecarLog,
+} from "./server.js";
+export {
+	type FifoSendInput,
+	InvalidParameterValueError,
+	MissingParameterError,
+	type QueueAttributes,
+	type QueueDefinition,
+	QueueDoesNotExistError,
+	type QueueRecord,
+	QueueStore,
+	type ReceivedMessage,
+	type SendResult,
+	SqsError,
+	validateFifoSend,
+} from "./store.js";

package/src/main.ts

@@ -0,0 +1,54 @@
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { bootstrapQueues, loadQueuesConfig } from "./queues-config.js";
+import { createSidecarServer, type SidecarLog } from "./server.js";
+import { QueueStore } from "./store.js";
+
+const log: SidecarLog = {
+	info: (fields, message) =>
+		process.stdout.write(
+			`${JSON.stringify({ level: "info", message, ...fields })}\n`,
+		),
+	error: (fields, message) =>
+		process.stderr.write(
+			`${JSON.stringify({ level: "error", message, ...fields })}\n`,
+		),
+};
+
+const port = Number(process.env.QUEUE_SIDECAR_PORT ?? "9324");
+const host = process.env.QUEUE_SIDECAR_HOST ?? "0.0.0.0";
+const dbPath = process.env.QUEUE_SIDECAR_DB ?? "/data/queue/queue.db";
+const accountId = process.env.QUEUE_SIDECAR_ACCOUNT_ID ?? "000000000000";
+const queuesConfigPath = process.env.QUEUE_SIDECAR_QUEUES_CONFIG;
+
+mkdirSync(dirname(dbPath), { recursive: true });
+const store = new QueueStore(dbPath);
+
+if (queuesConfigPath) {
+	const config = loadQueuesConfig(queuesConfigPath);
+	bootstrapQueues(store, config);
+	log.info(
+		{ path: queuesConfigPath, queues: config.queues.length },
+		"sidecar: queues bootstrapped",
+	);
+} else {
+	log.info({}, "sidecar: no queues config; queues created on demand");
+}
+
+const server = createSidecarServer({ store, accountId, log });
+
+server.listen(port, host, () => {
+	log.info({ host, port, dbPath }, "sidecar: listening");
+});
+
+const shutdown = (signal: NodeJS.Signals): void => {
+	log.info({ signal }, "sidecar: shutting down");
+	server.close(() => {
+		store.close();
+		process.exit(0);
+	});
+};
+
+for (const signal of ["SIGINT", "SIGTERM"] as const) {
+	process.on(signal, shutdown);
+}

package/src/protocol.ts

@@ -0,0 +1,151 @@
+const XMLNS = "http://queue.amazonaws.com/doc/2012-11-05/";
+
+export const escapeXml = (value: string): string =>
+	value
+		.replace(/&/g, "&")
+		.replace(/</g, "&lt;")
+		.replace(/>/g, "&gt;")
+		.replace(/"/g, "&quot;")
+		.replace(/'/g, "&apos;");
+
+const tag = (name: string, value: string): string =>
+	`<${name}>${escapeXml(value)}</${name}>`;
+
+const metadata = (requestId: string): string =>
+	`<ResponseMetadata><RequestId>${requestId}</RequestId></ResponseMetadata>`;
+
+/**
+ * Wraps an action's result body in the AWS Query envelope the SDK's
+ * `AwsQueryProtocol` deserializer expects: `<ActionResponse>` containing
+ * `<ActionResult>` and `<ResponseMetadata>`.
+ */
+export const queryResponse = (
+	action: string,
+	resultBody: string,
+	requestId: string,
+): string =>
+	`<?xml version="1.0"?>` +
+	`<${action}Response xmlns="${XMLNS}">` +
+	`<${action}Result>${resultBody}</${action}Result>` +
+	metadata(requestId) +
+	`</${action}Response>`;
+
+/** For actions with no result payload (DeleteMessage, PurgeQueue). */
+export const queryResponseNoResult = (
+	action: string,
+	requestId: string,
+): string =>
+	`<?xml version="1.0"?>` +
+	`<${action}Response xmlns="${XMLNS}">` +
+	metadata(requestId) +
+	`</${action}Response>`;
+
+export const errorResponse = (
+	code: string,
+	message: string,
+	requestId: string,
+	senderFault = true,
+): string =>
+	`<?xml version="1.0"?>` +
+	`<ErrorResponse xmlns="${XMLNS}">` +
+	`<Error>` +
+	tag("Type", senderFault ? "Sender" : "Receiver") +
+	tag("Code", code) +
+	tag("Message", message) +
+	`</Error>` +
+	`<RequestId>${requestId}</RequestId>` +
+	`</ErrorResponse>`;
+
+export const sendMessageResult = (result: {
+	messageId: string;
+	md5OfBody: string;
+	sequenceNumber: string | null;
+}): string =>
+	tag("MessageId", result.messageId) +
+	tag("MD5OfMessageBody", result.md5OfBody) +
+	(result.sequenceNumber ? tag("SequenceNumber", result.sequenceNumber) : "");
+
+export interface BatchSuccessEntry {
+	id: string;
+	messageId: string;
+	md5OfBody: string;
+	sequenceNumber: string | null;
+}
+
+export interface BatchErrorEntry {
+	id: string;
+	code: string;
+	message: string;
+	senderFault: boolean;
+}
+
+export const sendMessageBatchResult = (
+	successful: BatchSuccessEntry[],
+	failed: BatchErrorEntry[],
+): string => {
+	const successXml = successful
+		.map(
+			(e) =>
+				`<SendMessageBatchResultEntry>` +
+				tag("Id", e.id) +
+				tag("MessageId", e.messageId) +
+				tag("MD5OfMessageBody", e.md5OfBody) +
+				(e.sequenceNumber ? tag("SequenceNumber", e.sequenceNumber) : "") +
+				`</SendMessageBatchResultEntry>`,
+		)
+		.join("");
+	const errorXml = failed
+		.map(
+			(e) =>
+				`<BatchResultErrorEntry>` +
+				tag("Id", e.id) +
+				tag("Code", e.code) +
+				tag("Message", e.message) +
+				tag("SenderFault", e.senderFault ? "true" : "false") +
+				`</BatchResultErrorEntry>`,
+		)
+		.join("");
+	return successXml + errorXml;
+};
+
+export interface ReceiveMessageXml {
+	messageId: string;
+	receiptHandle: string;
+	md5OfBody: string;
+	body: string;
+	attributes: Record<string, string>;
+}
+
+export const receiveMessageResult = (messages: ReceiveMessageXml[]): string =>
+	messages
+		.map((m) => {
+			const attrs = Object.entries(m.attributes)
+				.map(
+					([name, value]) =>
+						`<Attribute>${tag("Name", name)}${tag("Value", value)}</Attribute>`,
+				)
+				.join("");
+			return (
+				`<Message>` +
+				tag("MessageId", m.messageId) +
+				tag("ReceiptHandle", m.receiptHandle) +
+				tag("MD5OfBody", m.md5OfBody) +
+				tag("Body", m.body) +
+				attrs +
+				`</Message>`
+			);
+		})
+		.join("");
+
+export const queueUrlResult = (queueUrl: string): string =>
+	tag("QueueUrl", queueUrl);
+
+export const queueAttributesResult = (
+	attributes: Record<string, string>,
+): string =>
+	Object.entries(attributes)
+		.map(
+			([name, value]) =>
+				`<Attribute>${tag("Name", name)}${tag("Value", value)}</Attribute>`,
+		)
+		.join("");

package/src/queues-config.ts

@@ -0,0 +1,104 @@
+import { readFileSync } from "node:fs";
+import type { QueueDefinition, QueueStore } from "./store.js";
+
+export interface QueuesConfig {
+	readonly accountId: string;
+	readonly queues: readonly QueueDefinition[];
+}
+
+interface RawDeadLetter {
+	name: unknown;
+	maxReceiveCount: unknown;
+}
+
+interface RawQueue {
+	name?: unknown;
+	fifo?: unknown;
+	visibilityTimeoutSeconds?: unknown;
+	contentBasedDeduplication?: unknown;
+	deadLetter?: RawDeadLetter;
+}
+
+interface RawConfig {
+	accountId?: unknown;
+	queues?: unknown;
+}
+
+const DEFAULT_VISIBILITY_TIMEOUT_SECONDS = 30;
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+	typeof value === "object" && value !== null;
+
+const parseQueue = (raw: unknown): QueueDefinition => {
+	if (!isRecord(raw)) throw new Error("queue entry must be an object");
+	const { name, fifo, visibilityTimeoutSeconds, contentBasedDeduplication } =
+		raw as RawQueue;
+	if (typeof name !== "string" || name.length === 0) {
+		throw new Error("queue entry requires a non-empty string name");
+	}
+	const isFifo = typeof fifo === "boolean" ? fifo : name.endsWith(".fifo");
+
+	const definition: QueueDefinition = {
+		name,
+		fifo: isFifo,
+		visibilityTimeoutSeconds:
+			typeof visibilityTimeoutSeconds === "number"
+				? visibilityTimeoutSeconds
+				: DEFAULT_VISIBILITY_TIMEOUT_SECONDS,
+		contentBasedDeduplication: contentBasedDeduplication === true,
+	};
+
+	const deadLetter = (raw as RawQueue).deadLetter;
+	if (deadLetter === undefined) return definition;
+	if (
+		typeof deadLetter.name !== "string" ||
+		typeof deadLetter.maxReceiveCount !== "number"
+	) {
+		throw new Error(
+			`queue ${name}: deadLetter requires a string name and numeric maxReceiveCount`,
+		);
+	}
+	return {
+		...definition,
+		deadLetterTargetName: deadLetter.name,
+		maxReceiveCount: deadLetter.maxReceiveCount,
+	};
+};
+
+export const parseQueuesConfig = (raw: unknown): QueuesConfig => {
+	if (!isRecord(raw)) throw new Error("queues config must be an object");
+	const { accountId, queues } = raw as RawConfig;
+	if (!Array.isArray(queues)) {
+		throw new Error("queues config requires a `queues` array");
+	}
+	return {
+		accountId: typeof accountId === "string" ? accountId : "000000000000",
+		queues: queues.map(parseQueue),
+	};
+};
+
+export const loadQueuesConfig = (path: string): QueuesConfig =>
+	parseQueuesConfig(JSON.parse(readFileSync(path, "utf8")));
+
+/**
+ * Declares every queue in the config on the store. Dead-letter targets are
+ * declared before the source queues that reference them so a redrive never
+ * points at a queue the store has not seen.
+ */
+export const bootstrapQueues = (
+	store: QueueStore,
+	config: QueuesConfig,
+): void => {
+	const deadLetterNames = new Set(
+		config.queues
+			.map((q) => q.deadLetterTargetName)
+			.filter((name): name is string => name !== undefined),
+	);
+	const ordered = [
+		...config.queues.filter((q) => deadLetterNames.has(q.name)),
+		...config.queues.filter((q) => !deadLetterNames.has(q.name)),
+	];
+	for (const queue of ordered) {
+		store.upsertQueue(queue);
+	}
+};