@remit/account-worker 0.0.3 → 0.0.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/account-worker",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/cascade.test.ts

@@ -12,69 +12,66 @@ const modelsDir = resolve(
 );
 
 describe("cascade entity coverage snapshot", () => {
-	it(
-		"covers every entity model in remit-electrodb-service",
-		{ skip: !existsSync(modelsDir) },
-		() => {
-			const modelFiles = readdirSync(modelsDir)
-				.filter(
-					(f) =>
-						f.endsWith(".ts") && !f.endsWith(".test.ts") && f !== "index.ts",
-				)
-				.map((f) => f.replace(".ts", ""));
+	it("covers every entity model in remit-electrodb-service", {
+		skip: !existsSync(modelsDir),
+	}, () => {
+		const modelFiles = readdirSync(modelsDir)
+			.filter(
+				(f) => f.endsWith(".ts") && !f.endsWith(".test.ts") && f !== "index.ts",
+			)
+			.map((f) => f.replace(".ts", ""));
 
-			// Map model file names to the entity types they contain
-			const entityTypesByFile: Record<string, string[]> = {
-				"account-config": ["AccountConfig"],
-				"account-export-request": [],
-				"account-setting": ["AccountSetting"],
-				"account-setting-registry": [],
-				account: ["Account"],
-				address: ["Address", "EnvelopeAddress"],
-				envelope: [
-					"Envelope",
-					"MessageReference",
-					"BodyPart",
-					"BodyPartParameter",
-					"RawMessageStorage",
-					"BodyPartStorage",
-					"BodyPartContent",
-				],
-				filter: ["Filter"],
-				"filter-anchor": ["FilterAnchor"],
-				label: ["Label"],
-				mailbox: ["Mailbox"],
-				"mailbox-lock": ["MailboxLock"],
-				"mailbox-special-use": [],
-				message: ["Message"],
-				"message-flag": ["MessageFlag"],
-				"message-flag-push": ["MessageFlagPush"],
-				"message-label": ["MessageLabel"],
-				"message-placement-move": ["MessagePlacementMove"],
-				"organize-job-request": [],
-				"outbox-message": ["OutboxMessage"],
-				"thread-message": ["ThreadMessage"],
-				"wellknown-rule": [],
-			};
+		// Map model file names to the entity types they contain
+		const entityTypesByFile: Record<string, string[]> = {
+			"account-config": ["AccountConfig"],
+			"account-export-request": [],
+			"account-setting": ["AccountSetting"],
+			"account-setting-registry": [],
+			account: ["Account"],
+			address: ["Address", "EnvelopeAddress"],
+			envelope: [
+				"Envelope",
+				"MessageReference",
+				"BodyPart",
+				"BodyPartParameter",
+				"RawMessageStorage",
+				"BodyPartStorage",
+				"BodyPartContent",
+			],
+			filter: ["Filter"],
+			"filter-anchor": ["FilterAnchor"],
+			label: ["Label"],
+			mailbox: ["Mailbox"],
+			"mailbox-lock": ["MailboxLock"],
+			"mailbox-special-use": [],
+			message: ["Message"],
+			"message-flag": ["MessageFlag"],
+			"message-flag-push": ["MessageFlagPush"],
+			"message-label": ["MessageLabel"],
+			"message-placement-move": ["MessagePlacementMove"],
+			"organize-job-request": [],
+			"outbox-message": ["OutboxMessage"],
+			"thread-message": ["ThreadMessage"],
+			"wellknown-rule": [],
+		};
 
-			const allEntityTypes = Object.values(entityTypesByFile).flat().sort();
-			const coveredSorted = [...COVERED_ENTITY_TYPES].sort();
+		const allEntityTypes = Object.values(entityTypesByFile).flat().sort();
+		const coveredSorted = [...COVERED_ENTITY_TYPES].sort();
 
-			assert.deepStrictEqual(
-				coveredSorted,
-				allEntityTypes,
-				`Cascade does not cover all entity types.\n` +
-					`Missing: ${allEntityTypes.filter((t) => !(coveredSorted as readonly string[]).includes(t)).join(", ")}\n` +
-					`Extra: ${(coveredSorted as readonly string[]).filter((t) => !allEntityTypes.includes(t)).join(", ")}`,
-			);
+		assert.deepStrictEqual(
+			coveredSorted,
+			allEntityTypes,
+			`Cascade does not cover all entity types.\n` +
+				`Missing: ${allEntityTypes.filter((t) => !(coveredSorted as readonly string[]).includes(t)).join(", ")}\n` +
+				`Extra: ${(coveredSorted as readonly string[]).filter((t) => !allEntityTypes.includes(t)).join(", ")}`,
+		);
 
-			// Verify all model files are accounted for
-			const unmappedFiles = modelFiles.filter((f) => !(f in entityTypesByFile));
-			assert.deepStrictEqual(
-				unmappedFiles,
-				[],
-				`Model files not mapped to entity types: ${unmappedFiles.join(", ")}`,
-			);
-		},
-	);
+		// Verify all model files are accounted for
+		const unmappedFiles = modelFiles.filter((f) => !(f in entityTypesByFile));
+		assert.deepStrictEqual(
+			unmappedFiles,
+			[],
+			`Model files not mapped to entity types: ${unmappedFiles.join(", ")}`,
+		);
+	});
 });

package/src/config.test.ts

@@ -0,0 +1,37 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
+import { getSqsClient } from "./config.js";
+
+// The queue URL a self-hosted compose stack hands the account worker
+// (deploy/vps/remit.env.template). It is a plain http:// URL that is not
+// localhost, which is exactly the case the endpoint heuristic used to miss.
+const composeQueueUrl =
+	"http://queue:9324/000000000000/remit-account-purge-delete.fifo";
+const awsQueueUrl =
+	"https://sqs.eu-west-1.amazonaws.com/123456789012/remit-account-purge-delete.fifo";
+
+describe("getSqsClient", () => {
+	// Regression: the account-deletion cascade sends through this client. It
+	// used to be built once at import with no endpoint and no protocol, so on a
+	// self-hosted stack every cascade send went to real SQS instead of the local
+	// queue server and account deletion never completed.
+	it("selects the query protocol and the queue origin for a compose-stack queue URL", async () => {
+		const client = getSqsClient(composeQueueUrl);
+		assert.equal(client.config.protocol instanceof AwsQueryProtocol, true);
+		const endpoint = await client.config.endpoint?.();
+		assert.equal(endpoint?.hostname, "queue");
+		assert.equal(endpoint?.port, 9324);
+	});
+
+	it("leaves the default protocol and endpoint for a real SQS queue URL", async () => {
+		const client = getSqsClient(awsQueueUrl);
+		assert.equal(client.config.protocol instanceof AwsQueryProtocol, false);
+		assert.equal(client.config.endpoint, undefined);
+	});
+
+	it("reuses one client per queue URL and keeps distinct queues apart", () => {
+		assert.equal(getSqsClient(composeQueueUrl), getSqsClient(composeQueueUrl));
+		assert.notEqual(getSqsClient(composeQueueUrl), getSqsClient(awsQueueUrl));
+	});
+});

package/src/config.ts

@@ -1,14 +1,25 @@
-import { SQSClient } from "@aws-sdk/client-sqs";
+import type { SQSClient } from "@aws-sdk/client-sqs";
 import { getClient } from "@remit/backend/client";
-import { resolveSqsCredentials } from "@remit/sqs-client";
+import { createQueueProducer } from "@remit/sqs-client/producer";
 import type { StorageService } from "@remit/storage-service";
 import { createStorageService } from "@remit/storage-service/s3";
 import { env } from "expect-env";
 import type { CascadeServices } from "./cascade.js";
 
-export const sqsClient = new SQSClient({
-	credentials: resolveSqsCredentials(),
-});
+// A client per queue URL, resolved at send time rather than at import. The
+// endpoint and wire protocol are derived from the URL, so a self-hosted stack
+// (http://queue:9324/...) gets the query protocol its queue server speaks while
+// real SQS keeps the default. Building at import is not an option: the queue
+// URLs are read lazily for the reason given below.
+const sqsClients = new Map<string, SQSClient>();
+
+export const getSqsClient = (queueUrl: string): SQSClient => {
+	const existing = sqsClients.get(queueUrl);
+	if (existing) return existing;
+	const client = createQueueProducer({ queueUrl });
+	sqsClients.set(queueUrl, client);
+	return client;
+};
 
 // SQS env vars are lazy-evaluated. The fanout worker needs them at handler time;
 // the finalize worker imports `getCascadeServices` from this module but talks to

package/src/handlers/account-fanout.ts

@@ -1,9 +1,5 @@
 import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
-import {
-	createLogger,
-	type Logger,
-	withTelemetry,
-} from "@remit/logger-lambda";
+import { createLogger, type Logger, withTelemetry } from "@remit/logger-lambda";
 import type { SQSBatchResponse, SQSEvent, SQSHandler } from "aws-lambda";
 import type { CascadeServices } from "../cascade.js";
 import { enumerateCascadeEntities } from "../cascade.js";
@@ -12,7 +8,7 @@ import {
 	getAccountPurgeDeleteQueueUrl,
 	getCascadeServices,
 	getImapWorkerQueueUrl,
-	sqsClient,
+	getSqsClient,
 } from "../config.js";
 import {
 	type DeletionCapabilities,
@@ -49,12 +45,13 @@ export const processAccountFanout = async (
 	deps: ProcessAccountFanoutDeps = {},
 ): Promise<void> => {
 	const services = deps.services ?? (await getCascadeServices());
-	const sqs = deps.sqs ?? sqsClient;
+	const sqsFor = (queueUrl: string): SQSClient =>
+		deps.sqs ?? getSqsClient(queueUrl);
 
 	if (event.type === "AccountDataPurge") {
 		await processAccountDataPurge(event, log, {
 			services,
-			sqs,
+			sqs: deps.sqs,
 			accountPurgeDeleteQueueUrl:
 				deps.accountPurgeDeleteQueueUrl ?? getAccountPurgeDeleteQueueUrl(),
 		});
@@ -71,14 +68,14 @@ export const processAccountFanout = async (
 		return;
 	}
 
-	await processAccountDelete(event, log, services, sqs, deps);
+	await processAccountDelete(event, log, services, sqsFor, deps);
 };
 
 const processAccountDelete = async (
 	event: AccountDeleteEvent,
 	log: Logger,
 	services: CascadeServices,
-	sqs: SQSClient,
+	sqsFor: (queueUrl: string) => SQSClient,
 	deps: ProcessAccountFanoutDeps,
 ): Promise<void> => {
 	const { accountConfigId } = event;
@@ -102,7 +99,7 @@ const processAccountDelete = async (
 		.map((e) => e.key.accountId);
 
 	for (const accountId of accountIds) {
-		await sqs.send(
+		await sqsFor(imapWorkerQueueUrl).send(
 			new SendMessageCommand({
 				QueueUrl: imapWorkerQueueUrl,
 				MessageBody: JSON.stringify({
@@ -124,7 +121,7 @@ const processAccountDelete = async (
 		type: "FinalizeAccountDelete",
 		accountConfigId,
 	};
-	await sqs.send(
+	await sqsFor(accountFinalizeQueueUrl).send(
 		new SendMessageCommand({
 			QueueUrl: accountFinalizeQueueUrl,
 			MessageBody: JSON.stringify(finalizeEvent),

package/src/handlers/account-finalize.ts

@@ -1,9 +1,5 @@
 import { inspect } from "node:util";
-import {
-	createLogger,
-	type Logger,
-	withTelemetry,
-} from "@remit/logger-lambda";
+import { createLogger, type Logger, withTelemetry } from "@remit/logger-lambda";
 import type { SQSEvent, SQSHandler } from "aws-lambda";
 import {
 	type CascadeEntity,

package/src/handlers/account-purge.ts

@@ -11,10 +11,10 @@ import { Entity } from "electrodb";
 import type { CascadeServices } from "../cascade.js";
 import {
 	dataBackend as defaultDataBackend,
-	sqsClient as defaultSqsClient,
 	getAccountPurgeDeleteQueueUrl,
 	getCascadeServices,
 	getSearchIndexQueueUrl,
+	getSqsClient,
 } from "../config.js";
 import type {
 	AccountDataPurgeEvent,
@@ -162,7 +162,8 @@ export const processAccountDataPurge = async (
 ): Promise<void> => {
 	const { accountId, accountConfigId } = event;
 	const services = deps.services ?? (await getCascadeServices());
-	const sqs = deps.sqs ?? defaultSqsClient;
+	const sqsFor = (queueUrl: string): SQSClient =>
+		deps.sqs ?? getSqsClient(queueUrl);
 	const accountPurgeDeleteQueueUrl =
 		deps.accountPurgeDeleteQueueUrl ?? getAccountPurgeDeleteQueueUrl();
 	const searchIndexQueueUrl =
@@ -200,7 +201,12 @@ export const processAccountDataPurge = async (
 	// here too would double the removal. On DynamoDB this fanout is the sole
 	// producer of the vector deletes (#457).
 	if (dataBackend !== "postgres") {
-		await enqueueVectorDeletes(sqs, searchIndexQueueUrl, accountId, messageIds);
+		await enqueueVectorDeletes(
+			sqsFor(searchIndexQueueUrl),
+			searchIndexQueueUrl,
+			accountId,
+			messageIds,
+		);
 		log.info(
 			{ accountConfigId, accountId, count: messageIds.length },
 			"Enqueued search-index deletes for purged account",
@@ -208,7 +214,7 @@ export const processAccountDataPurge = async (
 	}
 
 	await enqueuePurgeFinalize(
-		sqs,
+		sqsFor(accountPurgeDeleteQueueUrl),
 		accountPurgeDeleteQueueUrl,
 		accountId,
 		accountConfigId,