@remit/imap-worker 0.0.57 → 0.0.59

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.57",
+  "version": "0.0.59",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -31,6 +31,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1055.0",
+    "@remit/config-transfer": "*",
     "@remit/data-ports": "*",
     "@remit/smtp-service": "*",
     "@remit/logger-lambda": "*",

package/src/handlers/sync-mailboxes.test.ts

@@ -2,9 +2,11 @@ import assert from "node:assert/strict";
 import { describe, it } from "node:test";
 import {
 	DEFAULT_MAILBOX_FRESHNESS_MS,
+	emitSyncMessagesEvents,
 	MAILBOX_FRESHNESS_MS,
 	mailboxNeedsSync,
 	resolveMailboxFreshnessMs,
+	splitInboxFirst,
 } from "./sync-mailboxes.js";
 
 const NOW = 1_700_000_000_000;
@@ -84,3 +86,99 @@ describe("resolveMailboxFreshnessMs", () => {
 		);
 	});
 });
+
+describe("splitInboxFirst", () => {
+	it("pulls INBOX out of the list whatever case it is spelled in", () => {
+		const { inbox, rest } = splitInboxFirst([
+			{ mailboxId: "mb-sent", fullPath: "Sent" },
+			{ mailboxId: "mb-inbox", fullPath: "Inbox" },
+			{ mailboxId: "mb-junk", fullPath: "Junk" },
+		]);
+
+		assert.equal(inbox?.mailboxId, "mb-inbox");
+		assert.deepEqual(
+			rest.map((mailbox) => mailbox.mailboxId),
+			["mb-sent", "mb-junk"],
+		);
+	});
+
+	it("leaves an account without an INBOX intact", () => {
+		const { inbox, rest } = splitInboxFirst([
+			{ mailboxId: "mb-sent", fullPath: "Sent" },
+		]);
+
+		assert.equal(inbox, undefined);
+		assert.deepEqual(
+			rest.map((mailbox) => mailbox.mailboxId),
+			["mb-sent"],
+		);
+	});
+});
+
+describe("emitSyncMessagesEvents", () => {
+	// Every event of an account shares one FIFO group, so arrival order is
+	// service order. pMap only bounds concurrency: it used to race all the
+	// emits, letting INBOX queue behind up to nineteen other folders while a
+	// person waited for their new mail.
+	it("emits INBOX before any other folder", async () => {
+		const emitted: string[] = [];
+		const mailboxes = Array.from({ length: 25 }, (_, index) => ({
+			mailboxId: `mb-${index}`,
+			fullPath: `Folder ${index}`,
+		}));
+		mailboxes.push({ mailboxId: "mb-inbox", fullPath: "INBOX" });
+
+		await emitSyncMessagesEvents("acc-1", mailboxes, async (event) => {
+			emitted.push(event.mailboxId);
+		});
+
+		assert.equal(emitted[0], "mb-inbox");
+		assert.equal(emitted.length, 26);
+	});
+
+	// Emitting INBOX and awaiting it is the whole fix: a fire-and-forget emit
+	// would leave it racing the fan-out again.
+	it("waits for the INBOX emit to resolve before emitting the rest", async () => {
+		const emitted: string[] = [];
+		let releaseInbox: () => void = () => {};
+		const inboxEmitted = new Promise<void>((resolve) => {
+			releaseInbox = resolve;
+		});
+
+		const done = emitSyncMessagesEvents(
+			"acc-1",
+			[
+				{ mailboxId: "mb-inbox", fullPath: "INBOX" },
+				{ mailboxId: "mb-sent", fullPath: "Sent" },
+			],
+			async (event) => {
+				emitted.push(event.mailboxId);
+				if (event.mailboxId === "mb-inbox") await inboxEmitted;
+			},
+		);
+
+		await new Promise((resolve) => setImmediate(resolve));
+		assert.deepEqual(emitted, ["mb-inbox"]);
+
+		releaseInbox();
+		await done;
+		assert.deepEqual(emitted, ["mb-inbox", "mb-sent"]);
+	});
+
+	it("emits every folder for an account with no INBOX", async () => {
+		const emitted: string[] = [];
+
+		await emitSyncMessagesEvents(
+			"acc-1",
+			[
+				{ mailboxId: "mb-sent", fullPath: "Sent" },
+				{ mailboxId: "mb-junk", fullPath: "Junk" },
+			],
+			async (event) => {
+				emitted.push(event.mailboxId);
+			},
+		);
+
+		assert.deepEqual(emitted.sort(), ["mb-junk", "mb-sent"]);
+	});
+});

package/src/handlers/sync-mailboxes.ts

@@ -1,10 +1,16 @@
 import { getClient } from "@remit/backend/client";
+import { writeFolderRoleAppointment } from "@remit/backend/folder-role-appointments";
+import {
+	bindImportedFolders,
+	type ConfigBinderDeps,
+} from "@remit/config-transfer";
 import type {
 	AccountItem,
 	IAccountRepository,
 	IMailboxRepository,
 	IMailboxSpecialUseRepository,
 } from "@remit/data-ports";
+import type { CanonicalMailboxRoleValue } from "@remit/data-ports/folder-role";
 import { SyncPhase } from "@remit/domain-enums";
 import type { Logger } from "@remit/logger-lambda";
 import { RefreshTokenError } from "@remit/mail-oauth-service";
@@ -31,9 +37,9 @@ const EVENT_EMIT_CONCURRENCY = 20;
  * client produces when it loads (`GET /config` triggers a sync per account)
  * into one round of IMAP work.
  *
- * The web client floors its automatic poll at this same window
- * (`MIN_POLL_INTERVAL_MS` in useStaleAccountSync), so the one caller that
- * skips this gate on a timer still cannot drive a fan-out faster than it.
+ * This gates side-effect triggers only. The web client's automatic poll asks
+ * by name and so skips it; what bounds that one is its own floor
+ * (`MIN_POLL_INTERVAL_MS` in useStaleAccountSync, 30s).
  */
 export const DEFAULT_MAILBOX_FRESHNESS_MS = 60_000;
 
@@ -82,12 +88,25 @@ export const syncMailboxes = async (
 	event: SyncMailboxesEvent,
 	log: Logger,
 ): Promise<void> => {
+	const client = await getClient();
 	const {
 		account: accountService,
 		mailbox: mailboxService,
 		mailboxSpecialUse: mailboxSpecialUseService,
 		secrets,
-	} = await getClient();
+	} = client;
+	const binder: ConfigBinderDeps = {
+		repositories: client,
+		appointFolderRole: (configId, accountId, role, mailboxId, lastKnownPath) =>
+			writeFolderRoleAppointment(
+				client.accountSetting,
+				configId,
+				accountId,
+				role as CanonicalMailboxRoleValue,
+				mailboxId,
+				lastKnownPath,
+			),
+	};
 
 	const { accountId } = event;
 	log.info({ event: event.type, accountId }, "Handling event");
@@ -125,6 +144,7 @@ export const syncMailboxes = async (
 					mailboxService,
 					mailboxSpecialUseService,
 					accountService,
+					binder,
 					log,
 				);
 			} catch (err) {
@@ -157,6 +177,7 @@ const syncMailboxesForAccount = async (
 	mailboxService: IMailboxRepository,
 	mailboxSpecialUseService: IMailboxSpecialUseRepository,
 	accountService: IAccountRepository,
+	binder: ConfigBinderDeps,
 	log: Logger,
 ): Promise<void> => {
 	const { accountId } = account;
@@ -194,6 +215,20 @@ const syncMailboxesForAccount = async (
 
 	log.info({ result }, "Mailbox sync complete");
 
+	// The folder list this account holds is now known, which is the one thing a
+	// configuration import could not know when it ran: every folder in a file is
+	// named by IMAP path, and the ids are minted here. Binding what is bindable
+	// belongs at exactly this point, and is idempotent, so a later discovery
+	// simply finds nothing left to do.
+	const bound = await bindImportedFolders(
+		binder,
+		account.accountConfigId,
+		accountId,
+	);
+	if (bound.bound > 0 || bound.stillPending > 0) {
+		log.info({ accountId, ...bound }, "Bound imported folder references");
+	}
+
 	const allMailboxes = await collectAllMailboxes(accountId, mailboxService);
 
 	// This fan-out is where an account's IMAP work is decided: one SEARCH per
@@ -264,20 +299,54 @@ const syncMailboxesForAccount = async (
 		mailboxCountSynced: skipped,
 	});
 
-	// Emit events in parallel with concurrency limit
-	// INBOX is first in the sorted list, so it gets priority
-	await pMap(
-		mailboxes,
-		({ mailboxId }) => {
-			const syncEvent: Omit<SyncMessagesEvent, "eventId" | "timestamp"> = {
-				type: "SYNC_MESSAGES",
-				accountId,
-				mailboxId,
-			};
-			return emitEvent(syncEvent);
-		},
-		{ concurrency: EVENT_EMIT_CONCURRENCY },
+	await emitSyncMessagesEvents(accountId, mailboxes, emitEvent);
+};
+
+type SyncMessagesInput = Omit<SyncMessagesEvent, "eventId" | "timestamp">;
+
+/**
+ * Separate INBOX from the rest of the fan-out. `collectAllMailboxes` sorts it
+ * first, but sort order alone decides nothing once the emits go out together.
+ */
+export const splitInboxFirst = <T extends { fullPath: string }>(
+	mailboxes: readonly T[],
+): { inbox: T | undefined; rest: T[] } => {
+	const at = mailboxes.findIndex(
+		(mailbox) => mailbox.fullPath.toUpperCase() === "INBOX",
 	);
+	if (at === -1) return { inbox: undefined, rest: [...mailboxes] };
+	return {
+		inbox: mailboxes[at],
+		rest: mailboxes.filter((_, index) => index !== at),
+	};
+};
+
+/**
+ * Emit INBOX's SYNC_MESSAGES event and wait for it before emitting the rest.
+ *
+ * Every event of an account shares one FIFO group (`MessageGroupId =
+ * accountId`), so the queue serves them strictly in arrival order. Emitting
+ * the whole list through `pMap` only bounds concurrency — it races the writes,
+ * and INBOX could land behind up to `EVENT_EMIT_CONCURRENCY - 1` other
+ * folders. The mail a person pressed refresh for then arrived only after every
+ * one of those folders had finished syncing.
+ */
+export const emitSyncMessagesEvents = async (
+	accountId: string,
+	mailboxes: readonly { mailboxId: string; fullPath: string }[],
+	emit: (event: SyncMessagesInput) => Promise<unknown>,
+): Promise<void> => {
+	const eventFor = (mailboxId: string): SyncMessagesInput => ({
+		type: "SYNC_MESSAGES",
+		accountId,
+		mailboxId,
+	});
+
+	const { inbox, rest } = splitInboxFirst(mailboxes);
+	if (inbox) await emit(eventFor(inbox.mailboxId));
+	await pMap(rest, ({ mailboxId }) => emit(eventFor(mailboxId)), {
+		concurrency: EVENT_EMIT_CONCURRENCY,
+	});
 };
 
 type MailboxSortEntry = {