@remit/imap-worker 0.0.84 → 0.0.86

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.84",
+  "version": "0.0.86",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/emit.test.ts

@@ -1,8 +1,15 @@
 import assert from "node:assert/strict";
-import { before, beforeEach, describe, it } from "node:test";
+import { afterEach, before, beforeEach, describe, it } from "node:test";
 import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
+import {
+	_resetForTest,
+	type RemitClient,
+	setClient,
+} from "@remit/backend/client";
+import { MailboxLockRepo } from "@remit/drizzle-service";
+import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
 import { mockClient } from "aws-sdk-client-mock";
-import type { emitEvent as EmitEvent } from "./emit.js";
+import type * as Emit from "./emit.js";
 import type {
 	FlagPushEvent,
 	SyncMailboxesEvent,
@@ -20,7 +27,10 @@ const fifo = (name: string) =>
 
 const sqsMock = mockClient(SQSClient);
 
-let emitEvent: typeof EmitEvent;
+let emitEvent: typeof Emit.emitEvent;
+let SYNC_MESSAGES_PENDING: typeof Emit.SYNC_MESSAGES_PENDING;
+let locks: MailboxLockRepo;
+let closeStore: () => void;
 
 const sentCommands = (): SendMessageCommand["input"][] =>
 	sqsMock.commandCalls(SendMessageCommand).map((call) => call.args[0].input);
@@ -32,11 +42,21 @@ before(async () => {
 	process.env.SQS_QUEUE_URL_MAILBOXES = fifo("mailboxes");
 	process.env.SQS_QUEUE_URL_MESSAGES = fifo("messages");
 	process.env.SQS_QUEUE_URL_FLAGS = fifo("flags");
-	({ emitEvent } = await import("./emit.js"));
+	({ emitEvent, SYNC_MESSAGES_PENDING } = await import("./emit.js"));
 });
 
 beforeEach(() => {
 	sqsMock.reset();
+	sqsMock.on(SendMessageCommand).resolves({});
+	const store = createShippedSqliteDb();
+	closeStore = store.close;
+	locks = new MailboxLockRepo(store.db);
+	setClient({ mailboxLock: locks } as unknown as RemitClient);
+});
+
+afterEach(() => {
+	_resetForTest();
+	closeStore();
 });
 
 describe("emitEvent on a FIFO queue", () => {
@@ -68,6 +88,14 @@ describe("emitEvent on a FIFO queue", () => {
 		};
 
 		await emitEvent(event);
+		const [first] = sentCommands();
+		if (!first) throw new Error("expected a send");
+		await locks.releaseLock(
+			"acc-1",
+			"mbx-1",
+			SYNC_MESSAGES_PENDING,
+			String(bodyOf(first).eventId),
+		);
 		await emitEvent(event);
 
 		const sent = sentCommands();

package/src/emit.ts

@@ -1,5 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { SendMessageCommand } from "@aws-sdk/client-sqs";
+import { getClient } from "@remit/backend/client";
+import { recordImapFailure } from "@remit/logger-lambda";
 import {
 	createQueueProducer,
 	isLocalEndpoint,
@@ -9,6 +11,10 @@ import type { ImapEvent } from "./events.js";
 
 type EventInput = Omit<ImapEvent, "eventId" | "timestamp">;
 
+export const SYNC_MESSAGES_PENDING = "SYNC_MESSAGES_PENDING";
+
+const SYNC_MESSAGES_PENDING_TTL_SECONDS = 300;
+
 const mailboxesQueueUrl = env.SQS_QUEUE_URL_MAILBOXES;
 const messagesQueueUrl = env.SQS_QUEUE_URL_MESSAGES;
 // SYNC_MESSAGE_BODY events route to the single standard body queue (#612). It is
@@ -77,29 +83,58 @@ export const emitEvent = async (
 	// ElasticMQ FIFO queues don't support per-message DelaySeconds
 	const useDelay = options?.delaySeconds && !isLocal;
 
-	await sqs.send(
-		new SendMessageCommand({
-			QueueUrl: queueUrl,
-			MessageBody: JSON.stringify(fullEvent),
-			// Delay delivery for retry backoff (skip for local ElasticMQ)
-			...(useDelay && { DelaySeconds: options.delaySeconds }),
-			// FIFO queue parameters — only set if the queue is FIFO. The
-			// deduplication id is the event's own id, so the queue suppresses a
-			// re-send of one event (the retry SQS's own idempotency guard is for)
-			// and nothing else. A shared id per account or per mailbox instead made
-			// the 5-minute window a rate limiter: the second sync of a mailbox
-			// within five minutes was discarded before any worker saw it, so mail
-			// that arrived after a sync could not be fetched until the window
-			// elapsed (issue #37).
-			//
-			// Dropping events is not how repeated work is bounded — that is the
-			// freshness gate in the sync-mailboxes fan-out, which decides whether
-			// a mailbox is worth enumerating at all. MessageGroupId only orders an
-			// account's events; it makes duplicates serial, not cheap.
-			...(useFifo && {
-				MessageGroupId: event.accountId,
-				MessageDeduplicationId: fullEvent.eventId,
+	const send = () =>
+		sqs.send(
+			new SendMessageCommand({
+				QueueUrl: queueUrl,
+				MessageBody: JSON.stringify(fullEvent),
+				// Delay delivery for retry backoff (skip for local ElasticMQ)
+				...(useDelay && { DelaySeconds: options.delaySeconds }),
+				// FIFO queue parameters — only set if the queue is FIFO. The
+				// deduplication id is the event's own id, so the queue suppresses a
+				// re-send of one event (the retry SQS's own idempotency guard is for)
+				// and nothing else. A shared id per account or per mailbox instead made
+				// the 5-minute window a rate limiter: the second sync of a mailbox
+				// within five minutes was discarded before any worker saw it, so mail
+				// that arrived after a sync could not be fetched until the window
+				// elapsed (issue #37).
+				...(useFifo && {
+					MessageGroupId: event.accountId,
+					MessageDeduplicationId: fullEvent.eventId,
+				}),
 			}),
-		}),
+		);
+	if (fullEvent.type !== "SYNC_MESSAGES") {
+		await send();
+		return;
+	}
+
+	const { mailboxLock } = await getClient();
+	const { accountId, mailboxId, eventId } = fullEvent;
+	const previous = await mailboxLock.get(
+		accountId,
+		mailboxId,
+		SYNC_MESSAGES_PENDING,
 	);
+	const acquired = await mailboxLock.tryAcquireLock(
+		mailboxId,
+		SYNC_MESSAGES_PENDING,
+		accountId,
+		eventId,
+		SYNC_MESSAGES_PENDING_TTL_SECONDS,
+	);
+	if (!acquired) return;
+	if (previous && previous.ttl < Math.floor(Date.now() / 1000)) {
+		recordImapFailure("SYNC_PENDING_EXPIRED", "other");
+	}
+
+	await send().catch(async (error: unknown) => {
+		await mailboxLock.releaseLock(
+			accountId,
+			mailboxId,
+			SYNC_MESSAGES_PENDING,
+			eventId,
+		);
+		throw error;
+	});
 };

package/src/handlers/imported-folder.test.ts

@@ -0,0 +1,63 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import type { MailboxItem } from "@remit/data-ports";
+import type { CreateMailboxQueueInput } from "@remit/mailbox-service";
+import { createImportedFolder } from "./imported-folder.js";
+
+const inbox = {
+	mailboxId: "mbx-inbox",
+	accountId: "acc-1",
+	fullPath: "INBOX",
+	namespaceType: "personal",
+	namespacePrefix: "INBOX.",
+	hierarchyDelimiter: ".",
+} as MailboxItem;
+
+describe("createImportedFolder (#1103)", () => {
+	it("creates the folder through the pending folder-create path, in the INBOX namespace", async () => {
+		const calls: [CreateMailboxQueueInput, string, boolean | undefined][] = [];
+		const create = createImportedFolder(
+			{ findByPath: async () => inbox },
+			{
+				createMailbox: async (input, accountId, subscribe) => {
+					calls.push([input, accountId, subscribe]);
+					return {
+						outcome: "Created" as const,
+						mailbox: {
+							...inbox,
+							mailboxId: "mbx-new",
+							fullPath: input.fullPath,
+						},
+					};
+				},
+			},
+		);
+
+		const created = await create("acc-1", "INBOX.Facturen");
+
+		assert.equal(
+			created.outcome === "Created" && created.mailbox.mailboxId,
+			"mbx-new",
+		);
+		assert.equal(calls.length, 1);
+		const [input, accountId, subscribe] = calls[0];
+		assert.equal(accountId, "acc-1");
+		assert.equal(subscribe, true);
+		assert.equal(input.fullPath, "INBOX.Facturen");
+		assert.equal(input.namespacePrefix, "INBOX.");
+		assert.equal(input.hierarchyDelimiter, ".");
+	});
+
+	it("refuses to guess a namespace for an account with no INBOX", async () => {
+		const create = createImportedFolder(
+			{ findByPath: async () => null },
+			{
+				createMailbox: async () => {
+					throw new Error("must not be called");
+				},
+			},
+		);
+
+		await assert.rejects(create("acc-1", "Facturen"), /no INBOX/);
+	});
+});

package/src/handlers/imported-folder.ts

@@ -0,0 +1,38 @@
+import type { CreateImportedFolder } from "@remit/config-transfer";
+import type { IMailboxRepository } from "@remit/data-ports";
+import type { MailboxQueueService } from "@remit/mailbox-service";
+
+export const createImportedFolder =
+	(
+		mailbox: Pick<IMailboxRepository, "findByPath">,
+		queue: Pick<MailboxQueueService, "createMailbox">,
+	): CreateImportedFolder =>
+	async (accountId, folderPath) => {
+		const inbox = await mailbox.findByPath(accountId, "INBOX");
+		if (!inbox) {
+			throw new Error(
+				`Account ${accountId} has no INBOX to take a namespace from, so "${folderPath}" cannot be created`,
+			);
+		}
+		return queue.createMailbox(
+			{
+				accountId,
+				namespaceType: inbox.namespaceType,
+				namespacePrefix: inbox.namespacePrefix,
+				hierarchyDelimiter: inbox.hierarchyDelimiter,
+				fullPath: folderPath,
+				uidValidity: 0,
+				uidNext: 1,
+				highestModseq: "0",
+				messageCount: 0,
+				unseenCount: 0,
+				deletedCount: 0,
+				totalSize: 0,
+				lastSyncUid: 0,
+				highWaterMarkUid: 0,
+				lastMessageSyncAt: 0,
+			},
+			accountId,
+			true,
+		);
+	};

package/src/handlers/sync-mailboxes.ts

@@ -3,6 +3,7 @@ import { writeFolderRoleAppointment } from "@remit/backend/folder-role-appointme
 import {
 	bindImportedFolders,
 	type ConfigBinderDeps,
+	disableFiltersMissingFolders,
 } from "@remit/config-transfer";
 import type {
 	AccountItem,
@@ -16,10 +17,12 @@ import type { Logger } from "@remit/logger-lambda";
 import { RefreshTokenError } from "@remit/mail-oauth-service";
 import {
 	createConnectionWithCredentials,
+	MailboxQueueService,
 	MailboxSyncService,
 	MailConnectionError,
 	type MailCredentials,
 } from "@remit/mailbox-service";
+import { env } from "expect-env";
 import pMap from "p-map";
 import {
 	isAccountDeleted,
@@ -30,6 +33,7 @@ import { emitEvent } from "../emit.js";
 import type { SyncMailboxesEvent, SyncMessagesEvent } from "../events.js";
 import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
 import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
+import { createImportedFolder } from "./imported-folder.js";
 import { orderMailboxesForSync } from "./mailbox-sync-order.js";
 
 const EVENT_EMIT_CONCURRENCY = 20;
@@ -110,6 +114,14 @@ export const syncMailboxes = async (
 				mailboxId,
 				lastKnownPath,
 			),
+		createFolder: createImportedFolder(
+			mailboxService,
+			new MailboxQueueService({
+				mailboxService,
+				sqsQueueUrl: env.SQS_QUEUE_URL_MAILBOX_MGMT,
+				logger: log,
+			}),
+		),
 	};
 
 	const { accountId } = event;
@@ -233,10 +245,21 @@ const syncMailboxesForAccount = async (
 		account.accountConfigId,
 		accountId,
 	);
-	if (bound.bound > 0 || bound.dropped > 0 || bound.stillPending > 0) {
+	if (Object.values(bound).some((count) => count > 0)) {
 		log.info({ accountId, ...bound }, "Bound imported folder references");
 	}
 
+	const disabled = await disableFiltersMissingFolders(
+		binder.repositories,
+		account.accountConfigId,
+	);
+	if (disabled > 0) {
+		log.warn(
+			{ accountId, disabled },
+			"Disabled filters whose move folder no longer exists",
+		);
+	}
+
 	const allMailboxes = await collectAllMailboxes(accountId, mailboxService);
 
 	// This fan-out is where an account's IMAP work is decided: one SEARCH per

package/src/handlers/sync-messages.test.ts

@@ -287,6 +287,7 @@ const buildSyncDeps = (opts: {
 			mailbox: {
 				get: opts.mailboxGet,
 			},
+			mailboxLock: { releaseLock: async () => {} },
 			secrets: {},
 		}),
 		buildLifecycleDeps: () => ({}),
@@ -425,6 +426,7 @@ const buildRunningSyncDeps = (
 				},
 			},
 			mailboxLock: {
+				releaseLock: async () => {},
 				withMailboxLock: async (
 					_mailboxId: string,
 					_operation: string,

package/src/handlers/sync-messages.ts

@@ -32,7 +32,7 @@ import {
 	isMailSyncDisabled,
 	isUnsyncableHost,
 } from "../account-check.js";
-import { emitEvent } from "../emit.js";
+import { emitEvent, SYNC_MESSAGES_PENDING } from "../emit.js";
 import type {
 	FlagPushEvent,
 	SyncMessageBodyEvent,
@@ -101,6 +101,13 @@ export const syncMessages = async (
 		secrets,
 	} = await deps.getClient();
 
+	await mailboxLockService.releaseLock(
+		event.accountId,
+		event.mailboxId,
+		SYNC_MESSAGES_PENDING,
+		event.eventId,
+	);
+
 	// A deleted account never has its DDB row purged in lockstep with the queued
 	// SYNC_MESSAGES triggers, so a trigger can outlive its account. The lookup
 	// then returns null (or throws a named NotFoundError), which can never succeed

package/src/mail-sync-selection.sqlite.test.ts

@@ -17,6 +17,7 @@ import {
 import {
 	AccountRepo,
 	DrizzleMessageRepository,
+	MailboxLockRepo,
 	MailboxRepo,
 } from "@remit/drizzle-service";
 import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
@@ -47,6 +48,7 @@ const silentLogger: Logger = (() => {
 let accounts: AccountRepo;
 let mailboxes: MailboxRepo;
 let messages: DrizzleMessageRepository;
+let locks: MailboxLockRepo;
 let client: RemitClient;
 let close: () => void;
 
@@ -56,10 +58,12 @@ beforeEach(() => {
 	accounts = new AccountRepo(store.db);
 	mailboxes = new MailboxRepo(store.db);
 	messages = new DrizzleMessageRepository(store.db as never);
+	locks = new MailboxLockRepo(store.db);
 	client = {
 		account: accounts,
 		mailbox: mailboxes,
 		message: messages,
+		mailboxLock: locks,
 	} as unknown as RemitClient;
 	setClient(client);
 });

package/src/pending-sync.sqlite.test.ts

@@ -0,0 +1,167 @@
+import assert from "node:assert/strict";
+import { afterEach, before, beforeEach, describe, it } from "node:test";
+import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
+import {
+	_resetForTest,
+	type RemitClient,
+	setClient,
+} from "@remit/backend/client";
+import { MailboxLockRepo } from "@remit/drizzle-service";
+import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
+import { mockClient } from "aws-sdk-client-mock";
+import type * as Emit from "./emit.js";
+import type { SyncMessagesEvent } from "./events.js";
+import { emitMailboxResync, emitMoveResync } from "./handlers/message-move.js";
+import { emitSyncMessagesEvents } from "./handlers/sync-mailboxes.js";
+
+const ACCOUNT = "account-1366";
+const FOLDERS = [
+	{ mailboxId: "mbx-inbox", fullPath: "INBOX" },
+	{ mailboxId: "mbx-sent", fullPath: "Sent" },
+	{ mailboxId: "mbx-archive", fullPath: "Archive" },
+];
+
+const sqsMock = mockClient(SQSClient);
+
+let emitEvent: typeof Emit.emitEvent;
+let SYNC_MESSAGES_PENDING: typeof Emit.SYNC_MESSAGES_PENDING;
+let locks: MailboxLockRepo;
+let closeStore: () => void;
+
+type Queued = { mailboxId: string; eventId: string };
+
+const queued = (): Queued[] =>
+	sqsMock.commandCalls(SendMessageCommand).map((call) => {
+		const body = JSON.parse(call.args[0].input.MessageBody ?? "{}") as Queued;
+		return { mailboxId: body.mailboxId, eventId: body.eventId };
+	});
+
+const inboxSync = (): Omit<SyncMessagesEvent, "eventId" | "timestamp"> => ({
+	type: "SYNC_MESSAGES",
+	accountId: ACCOUNT,
+	mailboxId: "mbx-inbox",
+});
+
+const fanOut = (): Promise<void> =>
+	emitSyncMessagesEvents(ACCOUNT, FOLDERS, emitEvent);
+
+before(async () => {
+	const fifo = (name: string) =>
+		`https://sqs.eu-west-1.amazonaws.com/0/test-${name}.fifo`;
+	process.env.SQS_QUEUE_URL_MAILBOXES = fifo("mailboxes");
+	process.env.SQS_QUEUE_URL_MESSAGES = fifo("messages");
+	process.env.SQS_QUEUE_URL_FLAGS = fifo("flags");
+	({ emitEvent, SYNC_MESSAGES_PENDING } = await import("./emit.js"));
+});
+
+beforeEach(() => {
+	sqsMock.reset();
+	sqsMock.on(SendMessageCommand).resolves({});
+	const store = createShippedSqliteDb();
+	closeStore = store.close;
+	locks = new MailboxLockRepo(store.db);
+	setClient({ mailboxLock: locks } as unknown as RemitClient);
+});
+
+afterEach(() => {
+	_resetForTest();
+	closeStore();
+});
+
+describe("SYNC_MESSAGES under explicit refreshes (#1366)", () => {
+	it("queues one event per folder however many rounds arrive while none has started", async () => {
+		for (let round = 0; round < 25; round += 1) await fanOut();
+
+		assert.deepEqual(
+			queued()
+				.map((event) => event.mailboxId)
+				.sort(),
+			["mbx-archive", "mbx-inbox", "mbx-sent"],
+		);
+	});
+
+	it("queues a folder again once its pending sync has started", async () => {
+		await fanOut();
+		const inbox = queued().find((event) => event.mailboxId === "mbx-inbox");
+		if (!inbox) throw new Error("INBOX was not queued");
+
+		await locks.releaseLock(
+			ACCOUNT,
+			inbox.mailboxId,
+			SYNC_MESSAGES_PENDING,
+			inbox.eventId,
+		);
+		await fanOut();
+
+		assert.equal(queued().length, FOLDERS.length + 1);
+		assert.equal(queued().at(-1)?.mailboxId, "mbx-inbox");
+	});
+
+	it("keeps a folder's pending marker when a stale event starts", async () => {
+		await fanOut();
+		await locks.releaseLock(
+			ACCOUNT,
+			"mbx-inbox",
+			SYNC_MESSAGES_PENDING,
+			"an-event-from-before-the-marker",
+		);
+		await fanOut();
+
+		assert.equal(queued().length, FOLDERS.length);
+	});
+
+	it("takes the same marker for a move's resync and a delete's resync", async () => {
+		await emitMoveResync(emitEvent, {
+			accountId: ACCOUNT,
+			sourceMailboxId: "mbx-inbox",
+			destinationMailboxId: "mbx-sent",
+		});
+		await emitMailboxResync(emitEvent, {
+			accountId: ACCOUNT,
+			mailboxId: "mbx-inbox",
+		});
+		await fanOut();
+
+		assert.equal(queued().length, FOLDERS.length);
+	});
+
+	it("takes over a marker whose event was lost once it has expired", async () => {
+		await locks.tryAcquireLock(
+			"mbx-inbox",
+			"SYNC_MESSAGES_PENDING",
+			ACCOUNT,
+			"lost-event",
+			-1,
+		);
+
+		await emitEvent(inboxSync());
+
+		assert.equal(queued().length, 1);
+	});
+
+	it("releases the marker when the send fails, so the next round is not swallowed", async () => {
+		sqsMock
+			.on(SendMessageCommand)
+			.rejectsOnce(new Error("queue unreachable"))
+			.resolves({});
+
+		await assert.rejects(emitEvent(inboxSync()), /queue unreachable/);
+		await emitEvent(inboxSync());
+
+		assert.equal(queued().length, 2);
+	});
+
+	it("fails the operation when the marker store fails", async () => {
+		setClient({
+			mailboxLock: {
+				get: async () => null,
+				tryAcquireLock: async () => {
+					throw new Error("marker store down");
+				},
+			},
+		} as unknown as RemitClient);
+
+		await assert.rejects(emitEvent(inboxSync()), /marker store down/);
+		assert.equal(sqsMock.commandCalls(SendMessageCommand).length, 0);
+	});
+});