@remit/imap-worker 0.0.88 → 0.0.89

package/src/index.ts

@@ -9,8 +9,14 @@ import type { SQSBatchResponse, SQSEvent, SQSHandler } from "aws-lambda";
 import type { WorkerEvent } from "./events.js";
 import { imapFailureKind } from "./failure-kind.js";
 import { processEvent } from "./processor.js";
+import {
+	setWorkerQueues,
+	sqsWorkerQueues,
+	workerQueueUrlsFromEnv,
+} from "./queues.js";
 
 installNodeLogger();
+setWorkerQueues(sqsWorkerQueues(workerQueueUrlsFromEnv()));
 
 const log = createLogger();
 

package/src/mail-sync-selection.sqlite.test.ts

@@ -1,13 +1,13 @@
 import assert from "node:assert/strict";
 import { randomUUID } from "node:crypto";
 import { afterEach, beforeEach, describe, it } from "node:test";
-import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
 import {
 	_resetForTest,
 	type RemitClient,
 	setClient,
 } from "@remit/backend/client";
 import type { AccountItem } from "@remit/data-ports";
+import { recordingQueue } from "@remit/data-ports/recording-queue";
 import {
 	AccountAuthType,
 	AccountService,
@@ -25,6 +25,7 @@ import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
 import type { Logger } from "@remit/logger";
 import { syncMailboxes } from "./handlers/sync-mailboxes.js";
 import { syncMessages } from "./handlers/sync-messages.js";
+import { setWorkerQueues } from "./queues.js";
 import { runSchedulerTick } from "./scheduler/run-tick.js";
 
 const NOW = 1_700_000_000_000;
@@ -69,6 +70,15 @@ beforeEach(() => {
 		mailboxLock: locks,
 	} as unknown as RemitClient;
 	setClient(client);
+	const queue = recordingQueue();
+	setWorkerQueues({
+		mailboxes: queue,
+		messages: queue,
+		flags: queue,
+		body: queue,
+		mailboxManagement: queue,
+		messageManagement: queue,
+	});
 });
 
 afterEach(() => {
@@ -130,18 +140,11 @@ const seedSyncedAccount = async (): Promise<{
 };
 
 const tick = async (): Promise<{ enqueued: number; accountIds: string[] }> => {
-	const sent: SendMessageCommand[] = [];
-	const sqsClient = {
-		send: async (command: SendMessageCommand) => {
-			sent.push(command);
-			return {};
-		},
-	} as unknown as SQSClient;
+	const queue = recordingQueue();
 	const result = await runSchedulerTick({
 		redriveStrandedPlacements: async () => 0,
 		accountService: accounts,
-		sqsClient,
-		queueUrl: "https://queue.test/mailboxes",
+		queue,
 		log: silentLogger,
 		tickIntervalMs: 60_000,
 		offlineIntervalMs: 0,
@@ -149,9 +152,7 @@ const tick = async (): Promise<{ enqueued: number; accountIds: string[] }> => {
 	});
 	return {
 		enqueued: result.enqueued,
-		accountIds: sent.map(
-			(command) => JSON.parse(command.input.MessageBody ?? "{}").accountId,
-		),
+		accountIds: queue.sent.map((message) => String(message.body.accountId)),
 	};
 };
 

package/src/pending-sync.sqlite.test.ts

@@ -1,18 +1,21 @@
 import assert from "node:assert/strict";
-import { afterEach, before, beforeEach, describe, it } from "node:test";
-import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
+import { afterEach, beforeEach, describe, it } from "node:test";
 import {
 	_resetForTest,
 	type RemitClient,
 	setClient,
 } from "@remit/backend/client";
+import {
+	type RecordingQueue,
+	recordingQueue,
+} from "@remit/data-ports/recording-queue";
 import { MailboxLockRepo } from "@remit/drizzle-service";
 import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
-import { mockClient } from "aws-sdk-client-mock";
-import type * as Emit from "./emit.js";
+import { emitEvent, SYNC_MESSAGES_PENDING } from "./emit.js";
 import type { SyncMessagesEvent } from "./events.js";
 import { emitMailboxResync, emitMoveResync } from "./handlers/message-move.js";
 import { emitSyncMessagesEvents } from "./handlers/sync-mailboxes.js";
+import { setWorkerQueues } from "./queues.js";
 
 const ACCOUNT = "account-1366";
 const FOLDERS = [
@@ -21,20 +24,17 @@ const FOLDERS = [
 	{ mailboxId: "mbx-archive", fullPath: "Archive" },
 ];
 
-const sqsMock = mockClient(SQSClient);
-
-let emitEvent: typeof Emit.emitEvent;
-let SYNC_MESSAGES_PENDING: typeof Emit.SYNC_MESSAGES_PENDING;
+let queue: RecordingQueue;
 let locks: MailboxLockRepo;
 let closeStore: () => void;
 
 type Queued = { mailboxId: string; eventId: string };
 
 const queued = (): Queued[] =>
-	sqsMock.commandCalls(SendMessageCommand).map((call) => {
-		const body = JSON.parse(call.args[0].input.MessageBody ?? "{}") as Queued;
-		return { mailboxId: body.mailboxId, eventId: body.eventId };
-	});
+	queue.sent.map(({ body }) => ({
+		mailboxId: String(body.mailboxId),
+		eventId: String(body.eventId),
+	}));
 
 const inboxSync = (): Omit<SyncMessagesEvent, "eventId" | "timestamp"> => ({
 	type: "SYNC_MESSAGES",
@@ -45,18 +45,16 @@ const inboxSync = (): Omit<SyncMessagesEvent, "eventId" | "timestamp"> => ({
 const fanOut = (): Promise<void> =>
 	emitSyncMessagesEvents(ACCOUNT, FOLDERS, emitEvent);
 
-before(async () => {
-	const fifo = (name: string) =>
-		`https://sqs.eu-west-1.amazonaws.com/0/test-${name}.fifo`;
-	process.env.SQS_QUEUE_URL_MAILBOXES = fifo("mailboxes");
-	process.env.SQS_QUEUE_URL_MESSAGES = fifo("messages");
-	process.env.SQS_QUEUE_URL_FLAGS = fifo("flags");
-	({ emitEvent, SYNC_MESSAGES_PENDING } = await import("./emit.js"));
-});
-
 beforeEach(() => {
-	sqsMock.reset();
-	sqsMock.on(SendMessageCommand).resolves({});
+	queue = recordingQueue();
+	setWorkerQueues({
+		mailboxes: queue,
+		messages: queue,
+		flags: queue,
+		body: queue,
+		mailboxManagement: queue,
+		messageManagement: queue,
+	});
 	const store = createShippedSqliteDb();
 	closeStore = store.close;
 	locks = new MailboxLockRepo(store.db);
@@ -140,10 +138,11 @@ describe("SYNC_MESSAGES under explicit refreshes (#1366)", () => {
 	});
 
 	it("releases the marker when the send fails, so the next round is not swallowed", async () => {
-		sqsMock
-			.on(SendMessageCommand)
-			.rejectsOnce(new Error("queue unreachable"))
-			.resolves({});
+		let sends = 0;
+		queue.answer(async () => {
+			sends += 1;
+			if (sends === 1) throw new Error("queue unreachable");
+		});
 
 		await assert.rejects(emitEvent(inboxSync()), /queue unreachable/);
 		await emitEvent(inboxSync());
@@ -162,6 +161,6 @@ describe("SYNC_MESSAGES under explicit refreshes (#1366)", () => {
 		} as unknown as RemitClient);
 
 		await assert.rejects(emitEvent(inboxSync()), /marker store down/);
-		assert.equal(sqsMock.commandCalls(SendMessageCommand).length, 0);
+		assert.equal(queue.sent.length, 0);
 	});
 });

package/src/processor.ts

@@ -54,14 +54,6 @@ export const processEvent = async (
 			return handleAppendSentMessage(event, log, receiveCount);
 		case "DELETE_ACCOUNT_OBJECTS":
 			return handleDeleteAccountObjects(event, log);
-		case "IMAP_WORKER_STOP":
-			// Tombstone fence on the account row already stops processing;
-			// this event acks the cascade contract and is a no-op today.
-			log.info(
-				{ accountConfigId: event.accountConfigId, accountId: event.accountId },
-				"Imap worker stop signal received",
-			);
-			return;
 		default:
 			throw new Error(`Unknown event type: ${(event as WorkerEvent).type}`);
 	}

package/src/queues.ts

@@ -0,0 +1,49 @@
+import type { FifoQueueProducer, QueueProducer } from "@remit/data-ports";
+import { createQueueProducer } from "@remit/sqs-client/producer";
+import { env } from "expect-env";
+import type { ImapEvent, WorkerEvent } from "./events.js";
+
+export interface WorkerQueues {
+	mailboxes: FifoQueueProducer<ImapEvent>;
+	messages: FifoQueueProducer<ImapEvent>;
+	flags: FifoQueueProducer<ImapEvent>;
+	body: QueueProducer<ImapEvent>;
+	mailboxManagement: QueueProducer<ImapEvent>;
+	messageManagement: QueueProducer<WorkerEvent>;
+}
+
+export type WorkerQueueUrls = Record<keyof WorkerQueues, string>;
+
+export const workerQueueUrlsFromEnv = (): WorkerQueueUrls => ({
+	mailboxes: env.SQS_QUEUE_URL_MAILBOXES,
+	messages: env.SQS_QUEUE_URL_MESSAGES,
+	flags: env.SQS_QUEUE_URL_FLAGS,
+	body: env.SQS_QUEUE_URL_BODY,
+	mailboxManagement: env.SQS_QUEUE_URL_MAILBOX_MGMT,
+	messageManagement: env.SQS_QUEUE_URL_MESSAGE_MGMT,
+});
+
+export const sqsWorkerQueues = (urls: WorkerQueueUrls): WorkerQueues => ({
+	mailboxes: createQueueProducer({ queueUrl: urls.mailboxes }),
+	messages: createQueueProducer({ queueUrl: urls.messages }),
+	flags: createQueueProducer({ queueUrl: urls.flags }),
+	body: createQueueProducer({ queueUrl: urls.body }),
+	mailboxManagement: createQueueProducer({ queueUrl: urls.mailboxManagement }),
+	messageManagement: createQueueProducer({ queueUrl: urls.messageManagement }),
+});
+
+let composed: WorkerQueues | null = null;
+
+/** Register the worker's queues from its entry, before the first event. */
+export const setWorkerQueues = (queues: WorkerQueues): void => {
+	composed = queues;
+};
+
+export const workerQueues = (): WorkerQueues => {
+	if (!composed) {
+		throw new Error(
+			"no worker queues registered — register them with setWorkerQueues() from the entry",
+		);
+	}
+	return composed;
+};

package/src/scheduler/attachment-sweep.sqlite.test.ts

@@ -58,6 +58,7 @@ describe("the outbox attachment sweep", () => {
 		const secondDraft = await arrange(second);
 
 		const { handler } = await import("./handler.js");
+		store.bindWorkerQueues();
 		await handler(
 			{ time: new Date().toISOString() } as never,
 			{} as never,

package/src/scheduler/half-deleted-accounts.sqlite.test.ts

@@ -1,17 +1,19 @@
 import assert from "node:assert/strict";
 import { randomBytes } from "node:crypto";
 import { after, before, beforeEach, describe, it } from "node:test";
-import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
 import { finishHalfDeletedAccounts } from "@remit/backend/account-deletion";
 import type { RemitClient } from "@remit/backend/client";
 import { importConfig } from "@remit/config-transfer";
 import type { AccountItem } from "@remit/data-ports";
+import {
+	failingQueue,
+	type RecordingQueue,
+	recordingQueue,
+} from "@remit/data-ports/recording-queue";
 import { AccountAuthType, ConnectionState } from "@remit/domain-enums";
 import { noopLogger } from "@remit/logger";
 import { openStore } from "../test-helpers/test-world.js";
 
-const FANOUT_URL = "http://queue:9324/000000000000/remit-account-fanout";
-
 let store: Awaited<ReturnType<typeof openStore>>;
 let client: RemitClient;
 
@@ -105,21 +107,11 @@ const importAccount = async (email: string): Promise<AccountItem> => {
 	return client.account.get(accountId);
 };
 
-const recordingSqs = (sent: SendMessageCommand[], fail = false): SQSClient =>
-	({
-		send: async (command: SendMessageCommand) => {
-			if (fail) throw new Error("queue unreachable");
-			sent.push(command);
-			return {};
-		},
-	}) as unknown as SQSClient;
-
-const heal = (sqsClient: SQSClient) =>
+const heal = (queue: RecordingQueue) =>
 	finishHalfDeletedAccounts({
 		account: client.account,
 		configImport: client.configImport,
-		sqsClient,
-		accountFanoutQueueUrl: FANOUT_URL,
+		queue,
 		log: noopLogger,
 	});
 
@@ -150,13 +142,11 @@ describe("finishing an account deleted before #1384 on scheduler start", () => {
 		]);
 
 	it("tombstones it and asks for the same purge a delete asks for", async () => {
-		const sent: SendMessageCommand[] = [];
+		const queue = recordingQueue();
 
-		await heal(recordingSqs(sent));
+		await heal(queue);
 
-		const purges = sent
-			.filter((command) => command.input.QueueUrl === FANOUT_URL)
-			.map((command) => JSON.parse(command.input.MessageBody ?? "{}"));
+		const purges = queue.sent.map((message) => message.body);
 		assert.ok(
 			purges.some(
 				(body) =>
@@ -175,27 +165,30 @@ describe("finishing an account deleted before #1384 on scheduler start", () => {
 		assert.ok(
 			!purges.some((body) =>
 				[live.accountId, imported.accountId, deleted.accountId].includes(
-					body.accountId,
+					String(body.accountId),
 				),
 			),
 		);
 	});
 
 	it("does nothing for it on the next start", async () => {
-		await heal(recordingSqs([]));
-		const sent: SendMessageCommand[] = [];
+		await heal(recordingQueue());
+		const queue = recordingQueue();
 
-		await heal(recordingSqs(sent));
+		await heal(queue);
 
 		assert.ok(
-			!sent.some((command) =>
-				(command.input.MessageBody ?? "").includes(halfDeleted.accountId),
+			!queue.sent.some(
+				(message) => message.body.accountId === halfDeleted.accountId,
 			),
 		);
 	});
 
 	it("fails the start and leaves it to be found again when the purge cannot be asked for", async () => {
-		await assert.rejects(heal(recordingSqs([], true)), /queue unreachable/);
+		await assert.rejects(
+			heal(failingQueue(new Error("queue unreachable"))),
+			/queue unreachable/,
+		);
 
 		const after = await client.account.get(halfDeleted.accountId);
 		assert.equal(after.deletedAt, undefined);

package/src/scheduler/handler.ts

@@ -1,10 +1,14 @@
 import { getClient } from "@remit/backend/client";
 import { createLogger } from "@remit/logger";
 import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
-import { createQueueProducer } from "@remit/sqs-client/producer";
 import type { ScheduledHandler } from "aws-lambda";
-import { env } from "expect-env";
 import { emitEvent } from "../emit.js";
+import {
+	setWorkerQueues,
+	sqsWorkerQueues,
+	workerQueues,
+	workerQueueUrlsFromEnv,
+} from "../queues.js";
 import { buildAttachmentSweep } from "./attachment-sweep.js";
 import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
 import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
@@ -15,8 +19,7 @@ installNodeLogger();
 
 const log = createLogger();
 
-const mailboxesQueueUrl = env.SQS_QUEUE_URL_MAILBOXES;
-const sqsClient = createQueueProducer({ queueUrl: mailboxesQueueUrl });
+setWorkerQueues(sqsWorkerQueues(workerQueueUrlsFromEnv()));
 
 /**
  * EventBridge-scheduled entry point for the periodic mailbox-sync tick
@@ -41,8 +44,7 @@ export const handler: ScheduledHandler = withTelemetry(async (event) => {
 
 	await runSchedulerTick({
 		accountService: client.account,
-		sqsClient,
-		queueUrl: mailboxesQueueUrl,
+		queue: workerQueues().mailboxes,
 		log,
 		tickIntervalMs: getTickIntervalMs(),
 		offlineIntervalMs: getOfflineIntervalMs(),

package/src/scheduler/run-tick.sqlite.test.ts

@@ -1,7 +1,7 @@
 import assert from "node:assert/strict";
 import { afterEach, beforeEach, describe, it } from "node:test";
-import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
 import type { AccountItem } from "@remit/data-ports";
+import { recordingQueue } from "@remit/data-ports/recording-queue";
 import {
 	AccountAuthType,
 	AccountService,
@@ -60,18 +60,12 @@ const account = async (
 const tick = async (
 	options: Partial<Parameters<typeof runSchedulerTick>[0]> = {},
 ) => {
-	const sent: SendMessageCommand[] = [];
+	const queue = recordingQueue();
+	const { sent } = queue;
 	const logs: LogLine[] = [];
 	const result = await runSchedulerTick({
 		accountService: accounts,
-		sqsClient: {
-			send: async (command: SendMessageCommand) => {
-				sent.push(command);
-				return {};
-			},
-		} as unknown as SQSClient,
-		queueUrl:
-			"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
+		queue,
 		log: capturingLogger(logs),
 		tickIntervalMs: TICK_INTERVAL_MS,
 		offlineIntervalMs: OFFLINE_INTERVAL_MS,
@@ -79,9 +73,7 @@ const tick = async (
 		redriveStrandedPlacements: async () => 0,
 		...options,
 	});
-	const enqueued = sent.map(
-		(command) => JSON.parse(command.input.MessageBody ?? "{}").accountId,
-	);
+	const enqueued = sent.map((message) => String(message.body.accountId));
 	return { result, sent, enqueued, logs };
 };
 
@@ -129,7 +121,7 @@ describe("a scheduler tick", () => {
 
 		const { sent } = await tick();
 
-		const dedupId = sent[0]?.input.MessageDeduplicationId;
+		const dedupId = sent[0]?.deduplicationId;
 		assert.ok(
 			dedupId?.startsWith(`SYNC_MAILBOXES:scheduled:${live.accountId}:`),
 		);

package/src/scheduler/run-tick.ts

@@ -1,9 +1,13 @@
-import type { SQSClient } from "@aws-sdk/client-sqs";
 import {
 	buildScheduledSyncDedupId,
+	type SyncMailboxesEvent,
 	triggerAccountSync,
 } from "@remit/backend/trigger-sync";
-import type { AccountItem, IAccountRepository } from "@remit/data-ports";
+import type {
+	AccountItem,
+	FifoQueueProducer,
+	IAccountRepository,
+} from "@remit/data-ports";
 import type { Logger } from "@remit/logger";
 import { hasStoredCredential } from "@remit/mailbox-service/account-credentials";
 import pMap from "p-map";
@@ -22,8 +26,7 @@ import { isSyncDue } from "./decide-due.js";
 
 export interface RunSchedulerTickDeps {
 	accountService: Pick<IAccountRepository, "listAllAccountsPage">;
-	sqsClient: SQSClient;
-	queueUrl: string;
+	queue: FifoQueueProducer<SyncMailboxesEvent>;
 	log: Logger;
 	offlineIntervalMs: number;
 	/**
@@ -125,14 +128,8 @@ const isEligible = (account: AccountItem): boolean => {
 export const runSchedulerTick = async (
 	deps: RunSchedulerTickDeps,
 ): Promise<SchedulerTickResult> => {
-	const {
-		accountService,
-		sqsClient,
-		queueUrl,
-		log,
-		offlineIntervalMs,
-		tickIntervalMs,
-	} = deps;
+	const { accountService, queue, log, offlineIntervalMs, tickIntervalMs } =
+		deps;
 	const now = deps.now ?? Date.now();
 
 	let cursor: string | undefined;
@@ -161,8 +158,7 @@ export const runSchedulerTick = async (
 			due,
 			(account) =>
 				triggerAccountSync({
-					sqsClient,
-					queueUrl,
+					queue,
 					accountId: account.accountId,
 					dedupId: buildScheduledSyncDedupId(
 						account.accountId,

package/src/scheduler/runner.ts

@@ -4,11 +4,16 @@ import { finishHalfDeletedAccounts } from "@remit/backend/account-deletion";
 import { getClient } from "@remit/backend/client";
 import { createLogger } from "@remit/logger";
 import { installNodeLogger } from "@remit/logger-lambda/node";
+import { backfillSearchText } from "@remit/mailbox-service";
 import { clearHeartbeats, createHeartbeat } from "@remit/sqs-client/heartbeat";
-import { createQueueProducer } from "@remit/sqs-client/producer";
 
-import { env } from "expect-env";
 import { emitEvent } from "../emit.js";
+import {
+	setWorkerQueues,
+	sqsWorkerQueues,
+	workerQueues,
+	workerQueueUrlsFromEnv,
+} from "../queues.js";
 import { buildAttachmentSweep } from "./attachment-sweep.js";
 import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
 import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
@@ -40,9 +45,7 @@ installNodeLogger();
 
 const log = createLogger();
 
-const mailboxesQueueUrl = env.SQS_QUEUE_URL_MAILBOXES;
-const sqsClient = createQueueProducer({ queueUrl: mailboxesQueueUrl });
-const accountFanoutQueueUrl = env.SQS_QUEUE_URL_ACCOUNT_FANOUT;
+setWorkerQueues(sqsWorkerQueues(workerQueueUrlsFromEnv()));
 
 const tickIntervalMs = getTickIntervalMs();
 const offlineIntervalMs = getOfflineIntervalMs();
@@ -81,16 +84,20 @@ const runLoop = async (): Promise<void> => {
 	await finishHalfDeletedAccounts({
 		account,
 		configImport: client.configImport,
-		sqsClient: createQueueProducer({ queueUrl: accountFanoutQueueUrl }),
-		accountFanoutQueueUrl,
+		queue: client.queues.accountFanout,
 		log,
 	});
-	await runSchedulerLoop({
+	const searchTextBackfill = backfillSearchText(
+		{ messageService: client.message, storageService: client.storage },
+		log,
+	).then((result) => {
+		log.debug({ ...result }, "Search text backfill finished");
+	});
+	const schedulerLoop = runSchedulerLoop({
 		tick: runSchedulerTick,
 		tickDeps: {
 			accountService: account,
-			sqsClient,
-			queueUrl: mailboxesQueueUrl,
+			queue: workerQueues().mailboxes,
 			log,
 			tickIntervalMs,
 			offlineIntervalMs,
@@ -110,6 +117,7 @@ const runLoop = async (): Promise<void> => {
 		onHeartbeatError: onBeatError,
 		tickIntervalMs,
 	});
+	await Promise.all([searchTextBackfill, schedulerLoop]);
 };
 
 runLoop()

package/src/stranded-placement.sqlite.test.ts

@@ -84,7 +84,7 @@ const placementMover = (w: World): PlacementMoveService =>
 		addressService: w.client.address,
 		mailboxSpecialUseService: w.client.mailboxSpecialUse,
 		mailboxService: w.client.mailbox,
-		sqsQueueUrl: process.env.SQS_QUEUE_URL_MESSAGE_MGMT ?? "",
+		queue: w.client.queues.messageManagement,
 	});
 
 const placementOf = async (a: Arranged) => {
@@ -131,7 +131,8 @@ const queueRefuses = (w: World): void => {
 };
 
 const queueTakes = (w: World): void => {
-	w.queue.answer(async () => ({ Failed: [] }));
+	w.queue.answer(async () => ({}));
+	w.queue.refuse([]);
 };
 
 const redriveLater = async (
@@ -255,9 +256,7 @@ describe("a move whose event dies leaves no row moving", () => {
 
 	it("hands back a move to Trash whose batch entry the queue refused", async () => {
 		const a = await arrange();
-		a.w.queue.answer(async () => ({
-			Failed: [{ Id: "0", Code: "InvalidParameterValue", SenderFault: true }],
-		}));
+		a.w.queue.refuse([0]);
 
 		await assert.rejects(
 			a.w.client.messageMove.deleteMessages(