@remit/sqs-client 0.0.1

package/package.json

@@ -0,0 +1,40 @@
+{
+	"name": "@remit/sqs-client",
+	"version": "0.0.1",
+	"type": "module",
+	"main": "src/index.ts",
+	"types": "src/index.ts",
+	"exports": {
+		".": {
+			"types": "./src/index.ts",
+			"default": "./src/index.ts"
+		},
+		"./poller": {
+			"types": "./src/poller.ts",
+			"default": "./src/poller.ts"
+		},
+		"./producer": {
+			"types": "./src/producer.ts",
+			"default": "./src/producer.ts"
+		}
+	},
+	"scripts": {
+		"test:typecheck": "tsgo --noEmit",
+		"test:run": "node --env-file=../../localhost-test-unit.env --test 'src/**/*.test.ts'",
+		"test": "npm run test:typecheck && npm run test:run"
+	},
+	"devDependencies": {
+		"@aws-sdk/client-sqs": "*",
+		"@aws-sdk/core": "*",
+		"@types/aws-lambda": "*"
+	},
+	"license": "MIT",
+	"publishConfig": {
+		"access": "public"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/remit-mail/remit.git",
+		"directory": "packages/sqs-client"
+	}
+}

package/src/index.test.ts

@@ -0,0 +1,39 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { resolveSqsCredentials } from "./index.js";
+
+test("returns the SQS pair when both are set", () => {
+	const creds = resolveSqsCredentials({
+		SQS_ACCESS_KEY_ID: "SCWXXX",
+		SQS_SECRET_ACCESS_KEY: "secret",
+	});
+	assert.deepEqual(creds, { accessKeyId: "SCWXXX", secretAccessKey: "secret" });
+});
+
+test("falls back to the default chain when the access key is missing", () => {
+	assert.equal(
+		resolveSqsCredentials({ SQS_SECRET_ACCESS_KEY: "secret" }),
+		undefined,
+	);
+});
+
+test("falls back to the default chain when the secret is missing", () => {
+	assert.equal(
+		resolveSqsCredentials({ SQS_ACCESS_KEY_ID: "SCWXXX" }),
+		undefined,
+	);
+});
+
+test("falls back to the default chain when neither is set", () => {
+	assert.equal(resolveSqsCredentials({}), undefined);
+});
+
+test("ignores the AWS S3 credentials — reads only the SQS pair", () => {
+	assert.equal(
+		resolveSqsCredentials({
+			AWS_ACCESS_KEY_ID: "AKIA",
+			AWS_SECRET_ACCESS_KEY: "aws-secret",
+		}),
+		undefined,
+	);
+});

package/src/index.ts

@@ -0,0 +1,23 @@
+export interface SqsStaticCredentials {
+	accessKeyId: string;
+	secretAccessKey: string;
+}
+
+// On AWS, SQS shares the process-wide credential chain (AWS_ACCESS_KEY_ID / IAM
+// role). On Scaleway, Messaging & Queuing has its own credential pair, distinct
+// from the Object Storage (S3) keys, so it can't ride the default chain — the app
+// is handed SQS_ACCESS_KEY_ID / SQS_SECRET_ACCESS_KEY and must feed them to the
+// SQS client explicitly. When they are absent (AWS, or local ElasticMQ), returning
+// undefined lets the SDK fall back to its default credential provider chain.
+export function resolveSqsCredentials(
+	env: NodeJS.ProcessEnv = process.env,
+): SqsStaticCredentials | undefined {
+	const accessKeyId = env.SQS_ACCESS_KEY_ID;
+	const secretAccessKey = env.SQS_SECRET_ACCESS_KEY;
+
+	if (accessKeyId && secretAccessKey) {
+		return { accessKeyId, secretAccessKey };
+	}
+
+	return undefined;
+}

package/src/poller.test.ts

@@ -0,0 +1,90 @@
+import assert from "node:assert/strict";
+import { describe, it, mock } from "node:test";
+import { SQSClient } from "@aws-sdk/client-sqs";
+import type { SQSBatchResponse, SQSEvent } from "aws-lambda";
+import { runQueuePoller } from "./poller.js";
+
+const QUEUE_URL = "http://localhost:9324/000000000000/remit-test";
+
+const buildLog = () => ({
+	info: mock.fn(() => {}),
+	error: mock.fn(() => {}),
+});
+
+const buildMessage = (id: string) => ({
+	MessageId: id,
+	ReceiptHandle: `receipt-${id}`,
+	Body: "{}",
+	Attributes: { ApproximateReceiveCount: "1" },
+});
+
+describe("runQueuePoller", () => {
+	it("deletes succeeded messages and leaves batchItemFailures for redelivery, then stops on signal", async () => {
+		const deletedReceiptHandles: string[] = [];
+		let receiveCalls = 0;
+
+		// SQSClient.prototype.send is mocked globally (poller.ts constructs its
+		// own client internally — there is no injection seam), matching the
+		// same shape every AWS SDK v3 command carries: `{ input, constructor }`.
+		const sendMock = mock.method(
+			SQSClient.prototype,
+			"send",
+			// biome-ignore lint/suspicious/noExplicitAny: minimal SDK command shape, not worth typing per-command here
+			async function (this: SQSClient, command: any) {
+				if (command.constructor.name === "ReceiveMessageCommand") {
+					receiveCalls++;
+					if (receiveCalls === 1) {
+						// Simulate the shutdown signal landing while this batch is
+						// in flight — the loop is expected to finish processing it
+						// (delete succeeded / leave failures) and then exit before
+						// issuing a second ReceiveMessage.
+						process.emit("SIGWINCH", "SIGWINCH");
+						return { Messages: [buildMessage("m1"), buildMessage("m2")] };
+					}
+					throw new Error(
+						"unexpected second ReceiveMessage — poller did not stop on signal",
+					);
+				}
+				if (command.constructor.name === "DeleteMessageCommand") {
+					deletedReceiptHandles.push(command.input.ReceiptHandle);
+					return {};
+				}
+				throw new Error(`unexpected command: ${command.constructor.name}`);
+			},
+		);
+
+		const handler = mock.fn(
+			async (_event: SQSEvent): Promise<SQSBatchResponse> => ({
+				batchItemFailures: [{ itemIdentifier: "m2" }],
+			}),
+		);
+
+		try {
+			await runQueuePoller({
+				targets: [{ queueUrl: QUEUE_URL, handler, functionName: "test-fn" }],
+				log: buildLog(),
+				signals: ["SIGWINCH"],
+			});
+		} finally {
+			sendMock.mock.restore();
+		}
+
+		assert.equal(handler.mock.calls.length, 1);
+		const event = handler.mock.calls[0]?.arguments[0] as SQSEvent;
+		assert.deepEqual(
+			event.Records.map((r) => r.messageId),
+			["m1", "m2"],
+		);
+
+		// m1 succeeded (not in batchItemFailures) -> deleted. m2 failed -> left
+		// for redelivery, matching the SQS batchItemFailures contract.
+		assert.deepEqual(deletedReceiptHandles, ["receipt-m1"]);
+	});
+
+	it("throws when constructed with no targets", async () => {
+		await assert.rejects(
+			() => runQueuePoller({ targets: [], log: buildLog() }),
+			/no targets configured/,
+		);
+	});
+});

package/src/poller.ts

@@ -0,0 +1,190 @@
+import {
+	DeleteMessageCommand,
+	ReceiveMessageCommand,
+} from "@aws-sdk/client-sqs";
+import type {
+	Context,
+	SQSBatchResponse,
+	SQSEvent,
+	SQSHandler,
+} from "aws-lambda";
+import { createQueueProducer } from "./producer.js";
+
+/**
+ * Production queue poller — the non-e2e counterpart of the per-worker
+ * `e2e-processor-shim.ts` scripts. It receives from an SQS-compatible
+ * endpoint (real SQS or ElasticMQ), wraps each batch in an `SQSEvent`, and
+ * invokes the exact production Lambda `handler`, honouring
+ * `batchItemFailures` the way the SQS service does: delete what succeeded,
+ * leave failures for redelivery.
+ *
+ * Unlike the e2e shim (one child process per queue, via `node:cluster`),
+ * this runs every target queue as a concurrent poll loop inside a single
+ * process — the right shape for a container whose only job is "poll these
+ * queues, invoke this handler, exit on SIGTERM."
+ */
+export interface QueuePollerLog {
+	info: (fields: Record<string, unknown>, message: string) => void;
+	error: (fields: Record<string, unknown>, message: string) => void;
+}
+
+export interface QueuePollerTarget {
+	readonly queueUrl: string;
+	readonly handler: SQSHandler;
+	/** Reported as the Lambda `Context.functionName` the handler observes. */
+	readonly functionName: string;
+	readonly maxMessages?: number;
+	readonly visibilityTimeoutSeconds?: number;
+}
+
+export interface RunQueuePollerOptions {
+	readonly targets: readonly QueuePollerTarget[];
+	readonly log: QueuePollerLog;
+	readonly signals?: readonly NodeJS.Signals[];
+}
+
+const DEFAULT_MAX_MESSAGES = 10; // SQS API limit
+const DEFAULT_VISIBILITY_TIMEOUT_SECONDS = 300;
+
+const pollTarget = async (
+	target: QueuePollerTarget,
+	log: QueuePollerLog,
+	isShuttingDown: () => boolean,
+): Promise<void> => {
+	const queueName = new URL(target.queueUrl).pathname.split("/").pop();
+	const sqs = createQueueProducer({ queueUrl: target.queueUrl });
+	const maxMessages = target.maxMessages ?? DEFAULT_MAX_MESSAGES;
+	const visibilityTimeout =
+		target.visibilityTimeoutSeconds ?? DEFAULT_VISIBILITY_TIMEOUT_SECONDS;
+	const lambdaContext = { functionName: target.functionName } as Context;
+
+	log.info({ queue: queueName, maxMessages }, "poller: started");
+
+	// Always long-poll (SQS max WaitTimeSeconds). An empty long poll is free
+	// and returns as soon as a message arrives, so this is never slower than
+	// short-polling under load — it just stops burning a ReceiveMessage call
+	// (and, against real SQS, its cost) every loop iteration while traffic is
+	// continuous.
+	const LONG_POLL_WAIT_SECONDS = 20;
+
+	while (!isShuttingDown()) {
+		const response = await sqs.send(
+			new ReceiveMessageCommand({
+				QueueUrl: target.queueUrl,
+				MaxNumberOfMessages: maxMessages,
+				WaitTimeSeconds: LONG_POLL_WAIT_SECONDS,
+				VisibilityTimeout: visibilityTimeout,
+				MessageSystemAttributeNames: ["ApproximateReceiveCount"],
+			}),
+		);
+
+		if (!response.Messages || response.Messages.length === 0) {
+			continue;
+		}
+
+		const messages = response.Messages.flatMap((m) =>
+			m.Body && m.ReceiptHandle && m.MessageId
+				? [
+						{
+							messageId: m.MessageId,
+							receiptHandle: m.ReceiptHandle,
+							body: m.Body,
+							receiveCount: m.Attributes?.ApproximateReceiveCount ?? "1",
+						},
+					]
+				: [],
+		);
+		if (messages.length === 0) continue;
+
+		log.info(
+			{ queue: queueName, count: messages.length },
+			"poller: invoking handler",
+		);
+
+		const event: SQSEvent = {
+			Records: messages.map((m) => ({
+				messageId: m.messageId,
+				receiptHandle: m.receiptHandle,
+				body: m.body,
+				attributes: {
+					ApproximateReceiveCount: m.receiveCount,
+					SentTimestamp: "0",
+					SenderId: "poller",
+					ApproximateFirstReceiveTimestamp: "0",
+				},
+				messageAttributes: {},
+				md5OfBody: "",
+				eventSource: "aws:sqs",
+				eventSourceARN: target.queueUrl,
+				awsRegion: "local",
+			})),
+		};
+
+		const result = (await target.handler(event, lambdaContext, () => {})) as
+			| SQSBatchResponse
+			| undefined;
+
+		const failedIds = new Set(
+			(result?.batchItemFailures ?? []).map((f) => f.itemIdentifier),
+		);
+		const succeeded = messages.filter((m) => !failedIds.has(m.messageId));
+
+		// Batch is capped at maxMessages (SQS API limit: 10), so a handful of
+		// concurrent DeleteMessage calls is safe and avoids paying N sequential
+		// round trips between every receive/handler cycle.
+		await Promise.all(
+			succeeded.map((message) =>
+				sqs.send(
+					new DeleteMessageCommand({
+						QueueUrl: target.queueUrl,
+						ReceiptHandle: message.receiptHandle,
+					}),
+				),
+			),
+		);
+
+		log.info(
+			{
+				queue: queueName,
+				deleted: succeeded.length,
+				leftForRedelivery: failedIds.size,
+			},
+			"poller: batch processed",
+		);
+	}
+
+	log.info({ queue: queueName }, "poller: stopped");
+};
+
+/**
+ * Runs every target's poll loop concurrently until a shutdown signal is
+ * received (default: SIGINT, SIGTERM), then lets each loop finish its
+ * current iteration and returns. Rejects (crashes the process) if any loop
+ * throws — a stuck poller should exit loudly, not degrade silently.
+ */
+export const runQueuePoller = async (
+	options: RunQueuePollerOptions,
+): Promise<void> => {
+	const { targets, log } = options;
+	const signals = options.signals ?? ["SIGINT", "SIGTERM"];
+
+	if (targets.length === 0) {
+		throw new Error("runQueuePoller: no targets configured");
+	}
+
+	let shuttingDown = false;
+	const isShuttingDown = () => shuttingDown;
+	const onSignal = (signal: NodeJS.Signals) => {
+		log.info({ signal }, "poller: shutdown signal received");
+		shuttingDown = true;
+	};
+	for (const signal of signals) {
+		process.on(signal, onSignal);
+	}
+
+	log.info({ queues: targets.map((t) => t.queueUrl) }, "poller: starting");
+
+	await Promise.all(
+		targets.map((target) => pollTarget(target, log, isShuttingDown)),
+	);
+};

package/src/producer.test.ts

@@ -0,0 +1,28 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { createQueueProducer } from "./producer.js";
+
+describe("createQueueProducer", () => {
+	it("returns a real SQSClient for a local queue URL", () => {
+		const client = createQueueProducer({
+			queueUrl: "http://localhost:4566/000000000000/test-queue",
+		});
+		assert.equal(typeof client.send, "function");
+	});
+
+	it("returns a real SQSClient for a real AWS queue URL", () => {
+		const client = createQueueProducer({
+			queueUrl:
+				"https://sqs.eu-west-1.amazonaws.com/123456789012/remit-dev-mailboxes.fifo",
+		});
+		assert.equal(typeof client.send, "function");
+	});
+
+	it("accepts an explicit endpoint override", () => {
+		const client = createQueueProducer({
+			queueUrl: "http://localhost:4566/000000000000/test-queue",
+			endpoint: "http://custom-endpoint:1234",
+		});
+		assert.equal(typeof client.send, "function");
+	});
+});

package/src/producer.ts

@@ -0,0 +1,32 @@
+import { SQSClient } from "@aws-sdk/client-sqs";
+import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
+import { resolveSqsCredentials } from "./index.js";
+
+export interface CreateQueueProducerOptions {
+	readonly queueUrl: string;
+	readonly endpoint?: string;
+	readonly env?: NodeJS.ProcessEnv;
+}
+
+const isLocalEndpoint = (queueUrl: string): boolean =>
+	queueUrl.startsWith("http://") || queueUrl.startsWith("https://localhost");
+
+/**
+ * Producer-side counterpart of `runQueuePoller`: the one place an
+ * SQS-compatible client is constructed for sending. A queue URL that points at
+ * a local endpoint (ElasticMQ) gets its origin as the SDK endpoint and the
+ * query protocol ElasticMQ speaks; real SQS and Scaleway M&Q resolve their own
+ * endpoint and ride the default credential chain unless
+ * `SQS_ACCESS_KEY_ID`/`SQS_SECRET_ACCESS_KEY` are present.
+ */
+export const createQueueProducer = (
+	options: CreateQueueProducerOptions,
+): SQSClient => {
+	const { queueUrl, endpoint, env } = options;
+	const local = isLocalEndpoint(queueUrl);
+	return new SQSClient({
+		endpoint: endpoint ?? (local ? new URL(queueUrl).origin : undefined),
+		...(local && { protocol: AwsQueryProtocol }),
+		credentials: resolveSqsCredentials(env),
+	});
+};

package/tsconfig.json

@@ -0,0 +1,8 @@
+{
+	"extends": "../../tsconfig.json",
+	"compilerOptions": {
+		"outDir": "dist",
+		"rootDir": "src"
+	},
+	"include": ["src/**/*.ts"]
+}