@remit/imap-worker 0.0.23 → 0.0.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.23",
+  "version": "0.0.24",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/handlers/mailbox-management.test.ts

@@ -363,3 +363,109 @@ describe("processMailboxManagement — MAILBOX_DELETE", () => {
 		assert.equal(h.disconnectCount, 1, "the scope is still disconnected");
 	});
 });
+
+describe("processMailboxManagement — a tagged NO the server means as success (#339)", () => {
+	beforeEach(() => {
+		h = fresh();
+	});
+
+	/**
+	 * Dovecot answers `NO [NONEXISTENT] Mailbox doesn't exist`, which ImapFlow
+	 * raises as a bare "Command failed" carrying the code on the error. Reading
+	 * only the message treated an already-absent folder as a failure: the row was
+	 * marked failed and the event left poisoning the account's queue.
+	 */
+	it("treats it as the delete already having happened, and drops the local row", async () => {
+		h.connection.deleteMailbox = async () => {
+			throw Object.assign(new Error("Command failed"), {
+				serverResponseCode: "NONEXISTENT",
+				responseText: "Mailbox doesn't exist: Archive",
+			});
+		};
+
+		await assert.doesNotReject(
+			processMailboxManagement(deleteEvent, noopLog, deps()),
+		);
+		assert.deepEqual(called("mailbox.delete")[0]?.args, ["acc-1", "mbx-1"]);
+		assert.equal(called("mailbox.update").length, 0);
+	});
+
+	it("still marks failed and rethrows when the server fails for any other reason", async () => {
+		h.connection.deleteMailbox = async () => {
+			throw Object.assign(new Error("Command failed"), {
+				serverResponseCode: "SERVERBUG",
+				responseText: "Internal error",
+			});
+		};
+
+		await assert.rejects(
+			processMailboxManagement(deleteEvent, noopLog, deps()),
+			/Command failed/,
+		);
+		assert.deepEqual(lastUpdate(), { syncStatus: "failed" });
+	});
+
+	/**
+	 * Dovecot answers `NO [ALREADYEXISTS]` when the folder is already there — a
+	 * folder another client made, or a redelivered create. That is the create
+	 * having happened. Reading it as a failure marked the row `failed` and
+	 * rethrew, holding back every later sync on the account's FIFO group.
+	 */
+	it("reads ALREADYEXISTS on a CREATE as the folder already being there", async () => {
+		h.connection.createMailbox = async () => {
+			throw Object.assign(new Error("Command failed"), {
+				serverResponseCode: "ALREADYEXISTS",
+				responseText: "Mailbox already exists: Archive",
+			});
+		};
+
+		await assert.doesNotReject(
+			processMailboxManagement(createEvent, noopLog, deps()),
+		);
+		assert.deepEqual(lastUpdate(), { syncStatus: "synced" });
+	});
+
+	it("still marks failed and rethrows when a CREATE fails for any other reason", async () => {
+		h.connection.createMailbox = async () => {
+			throw Object.assign(new Error("Command failed"), {
+				serverResponseCode: "SERVERBUG",
+				responseText: "Internal error",
+			});
+		};
+
+		await assert.rejects(
+			processMailboxManagement(createEvent, noopLog, deps()),
+			/Command failed/,
+		);
+		assert.deepEqual(lastUpdate(), { syncStatus: "failed" });
+	});
+
+	it("reads NONEXISTENT on a RENAME as the source folder being gone", async () => {
+		h.connection.renameMailbox = async () => {
+			throw Object.assign(new Error("Command failed"), {
+				serverResponseCode: "NONEXISTENT",
+				responseText: "Mailbox doesn't exist: Archive",
+			});
+		};
+
+		await assert.doesNotReject(
+			processMailboxManagement(renameEvent, noopLog, deps()),
+		);
+		assert.deepEqual(called("mailbox.delete")[0]?.args, ["acc-1", "mbx-1"]);
+	});
+
+	it("still rolls back and rethrows when a RENAME fails for any other reason", async () => {
+		h.connection.renameMailbox = async () => {
+			throw Object.assign(new Error("Command failed"), {
+				serverResponseCode: "SERVERBUG",
+				responseText: "Internal error",
+			});
+		};
+
+		await assert.rejects(
+			processMailboxManagement(renameEvent, noopLog, deps()),
+			/Command failed/,
+		);
+		assert.equal(lastUpdate()?.syncStatus, "failed");
+	});
+});

package/src/handlers/mailbox-management.ts

@@ -28,6 +28,44 @@ const defaultDeps: MailboxManagementDeps = {
 	createConnectionScope: createConnectionScopeWithCredentials,
 };
 
+const stringField = (value: unknown, key: string): string => {
+	if (!(value instanceof Object)) return "";
+	const field = Reflect.get(value, key);
+	return typeof field === "string" ? field : "";
+};
+
+/**
+ * Read a tagged-NO outcome out of an IMAP failure.
+ *
+ * The two outcomes below are each a folder operation finding the server already
+ * in the state it was asked for — the operation having happened, not failing.
+ * Reading them as failures marks the row `failed` and rethrows, and since folder
+ * management shares the account's per-account FIFO group with mailbox sync, that
+ * un-acked rethrow holds back every later sync for the account.
+ *
+ * Both places the server can say so are read. `message` carries it when the
+ * client raises the error itself; RFC 5530's response code and its text carry it
+ * when the server does. ImapFlow surfaces a tagged NO as a bare "Command failed"
+ * with the code on the error, which is why matching the message alone never
+ * caught a real Dovecot answer.
+ */
+const saidByServer = (error: Error): string =>
+	`${error.message} ${stringField(error, "responseText")}`;
+
+/** The folder is not on the server: a delete has nothing left to do. */
+const isMailboxAbsentUpstream = (error: unknown): boolean => {
+	if (!(error instanceof Error)) return false;
+	if (stringField(error, "serverResponseCode") === "NONEXISTENT") return true;
+	return /not found|does ?n.?t exist/i.test(saidByServer(error));
+};
+
+/** The folder is already on the server: a create has nothing left to do. */
+const isMailboxPresentUpstream = (error: unknown): boolean => {
+	if (!(error instanceof Error)) return false;
+	if (stringField(error, "serverResponseCode") === "ALREADYEXISTS") return true;
+	return /already exists/i.test(saidByServer(error));
+};
+
 /**
  * Pinned invariant for the whole-chain terminal guards below.
  *
@@ -112,10 +150,7 @@ const handleCreate = async (
 					})
 					.catch(async (error) => {
 						// Check if mailbox already exists (idempotent)
-						if (
-							error instanceof Error &&
-							error.message.includes("already exists")
-						) {
+						if (isMailboxPresentUpstream(error)) {
 							log.info(
 								{ accountId, mailboxId, path },
 								"Mailbox already exists, marking as synced",
@@ -223,7 +258,7 @@ const handleRename = async (
 					})
 					.catch(async (error) => {
 						// If source not found, delete local mailbox
-						if (error instanceof Error && error.message.includes("not found")) {
+						if (isMailboxAbsentUpstream(error)) {
 							log.info(
 								{ accountId, mailboxId, oldPath },
 								"Source mailbox not found, deleting local",
@@ -321,7 +356,7 @@ const handleDelete = async (
 					})
 					.catch(async (error) => {
 						// If mailbox not found, it's already deleted (idempotent)
-						if (error instanceof Error && error.message.includes("not found")) {
+						if (isMailboxAbsentUpstream(error)) {
 							log.info(
 								{ accountId, mailboxId, path },
 								"Mailbox not found on IMAP, deleting local",

package/src/handlers/sync-message-body.ts

@@ -5,6 +5,7 @@ import {
 	BodySyncService,
 	guardConnectionCursor,
 	isCursorRebuildNeeded,
+	isFolderOffServer,
 	MailboxCursorPausedError,
 	PlacementMoveService,
 	QuarantineService,
@@ -197,10 +198,13 @@ export const syncMessageBody = async (
 					if (isNotFoundError(error)) return null;
 					throw error;
 				});
-			if (!mailbox) {
+			// A folder still `pending` is terminal for the same reason: the batch
+			// was cut before the folder reached the server, so there is nothing
+			// there to fetch from and no retry that changes it.
+			if (!mailbox || isFolderOffServer(mailbox)) {
 				log.warn(
 					{ accountId, mailboxId, eventId: event.eventId },
-					"Skipping SYNC_MESSAGE_BODY: mailbox no longer exists (deleted)",
+					"Skipping SYNC_MESSAGE_BODY: the server does not hold this folder",
 				);
 				return;
 			}

package/src/handlers/sync-messages.test.ts

@@ -319,7 +319,9 @@ describe("syncMessages — terminal handling of events for a deleted mailbox (is
 			0,
 			"a deleted mailbox must short-circuit before the OAuth/connection lifecycle",
 		);
-		const skip = warns.find((w) => w.msg.includes("mailbox no longer exists"));
+		const skip = warns.find((w) =>
+			w.msg.includes("the server does not hold this folder"),
+		);
 		assert.ok(skip, "expected a WARN naming the skipped deleted mailbox");
 		assert.equal(skip.fields.accountId, "acc-1");
 		assert.equal(skip.fields.mailboxId, "mbx-gone");
@@ -352,6 +354,30 @@ describe("syncMessages — terminal handling of events for a deleted mailbox (is
 		assert.equal(harness.lifecycleCalls, 1);
 	});
 
+	for (const syncStatus of ["pending", "deleting"]) {
+		it(`acks a SYNC_MESSAGES event for a \`${syncStatus}\` mailbox — the row exists, the server folder does not`, async () => {
+			const { log, warns } = buildLogger();
+			const harness = buildSyncDeps({
+				mailboxGet: async () => ({
+					mailboxId: "mbx-unsettled",
+					fullPath: "Archive",
+					syncStatus,
+				}),
+			});
+
+			await assert.doesNotReject(
+				syncMessages(syncEvent("mbx-unsettled"), log, harness.deps),
+			);
+
+			assert.equal(harness.lifecycleCalls, 0);
+			assert.ok(
+				warns.find((w) =>
+					w.msg.includes("the server does not hold this folder"),
+				),
+			);
+		});
+	}
+
 	it("a deleted-mailbox event does not stall the group — a following live-mailbox event still processes", async () => {
 		const { log } = buildLogger();
 		const gone = buildSyncDeps({
@@ -370,3 +396,151 @@ describe("syncMessages — terminal handling of events for a deleted mailbox (is
 		assert.equal(live.lifecycleCalls, 1);
 	});
 });
+
+/**
+ * A deps factory that runs the lifecycle and the mailbox lock for real, so the
+ * sync body — and the failure handling around it — is what gets exercised.
+ * `mailboxGet` is called per lookup, which is how a delete landing mid-round is
+ * expressed: the guard's read succeeds, a later one does not.
+ */
+const buildRunningSyncDeps = (
+	mailboxGet: (call: number) => Promise<unknown>,
+): {
+	deps: SyncMessagesDeps;
+	accountUpdates: Array<Record<string, unknown>>;
+} => {
+	const accountUpdates: Array<Record<string, unknown>> = [];
+	let calls = 0;
+	const deps = {
+		getClient: async () => ({
+			account: {
+				get: async () => liveAccount,
+				update: async (_id: string, input: Record<string, unknown>) => {
+					accountUpdates.push(input);
+				},
+			},
+			mailbox: {
+				get: async () => {
+					calls += 1;
+					return mailboxGet(calls);
+				},
+			},
+			mailboxLock: {
+				withMailboxLock: async (
+					_mailboxId: string,
+					_operation: string,
+					_accountId: string,
+					run: () => Promise<void>,
+				) => {
+					await run();
+					return { executed: true };
+				},
+			},
+			flagPush: { listByMailboxId: async () => [] },
+			secrets: {},
+		}),
+		buildLifecycleDeps: () => ({}),
+		withOAuthLifecycle: async (
+			_lifecycleDeps: unknown,
+			_account: AccountItem,
+			_log: Logger,
+			run: (credentials: unknown) => Promise<void>,
+		) => {
+			await run({ type: "password", password: "pw" });
+		},
+	} as unknown as SyncMessagesDeps;
+	return { deps, accountUpdates };
+};
+
+describe("syncMessages — a delete that lands mid-round (issue #339)", () => {
+	it("resolves terminally when the mailbox went away while the sync was in flight, and records no error phase", async () => {
+		const { log, warns } = buildLogger();
+		const harness = buildRunningSyncDeps(async (call) => {
+			if (call === 1) return { mailboxId: "mbx-1", fullPath: "Archive" };
+			throw new NotFoundError("Mailbox not found: mbx-1");
+		});
+
+		await assert.doesNotReject(
+			syncMessages(syncEvent("mbx-1"), log, harness.deps),
+		);
+
+		const resolved = warns.find((w) =>
+			w.msg.includes("left the server while the sync was in flight"),
+		);
+		assert.ok(
+			resolved,
+			"expected a WARN naming the folder that left mid-round",
+		);
+		assert.equal(resolved.fields.mailboxId, "mbx-1");
+		assert.deepEqual(
+			harness.accountUpdates,
+			[],
+			"one folder leaving must not put the account into an error phase",
+		);
+	});
+
+	it("resolves terminally when the failure is on the server and the row is only marked deleting", async () => {
+		const { log, warns } = buildLogger();
+		const harness = buildRunningSyncDeps(async (call) => {
+			if (call === 1) return { mailboxId: "mbx-1", fullPath: "Archive" };
+			if (call === 2) throw new Error("IMAP SEARCH failed in mailbox null");
+			return {
+				mailboxId: "mbx-1",
+				fullPath: "Archive",
+				syncStatus: "deleting",
+			};
+		});
+
+		await assert.doesNotReject(
+			syncMessages(syncEvent("mbx-1"), log, harness.deps),
+		);
+
+		assert.ok(
+			warns.find((w) =>
+				w.msg.includes("left the server while the sync was in flight"),
+			),
+		);
+		assert.deepEqual(harness.accountUpdates, []);
+	});
+
+	/**
+	 * The classifying read must not become the reported failure. A throttled or
+	 * unreachable repository during the catch previously propagated in place of
+	 * the real IMAP error, and the `syncPhase: error` write below it never ran —
+	 * the let-it-crash contract lost both the diagnosis and the state.
+	 */
+	it("reports the real failure, and still records the error phase, when the classifying read itself fails", async () => {
+		const { log } = buildLogger();
+		const harness = buildRunningSyncDeps(async (call) => {
+			if (call === 1) return { mailboxId: "mbx-1", fullPath: "Archive" };
+			if (call === 2) throw new Error("IMAP SEARCH failed in mailbox null");
+			throw new Error("dynamodb throttled");
+		});
+
+		await assert.rejects(
+			syncMessages(syncEvent("mbx-1"), log, harness.deps),
+			/IMAP SEARCH failed in mailbox null/,
+		);
+		assert.equal(harness.accountUpdates.length, 1);
+		assert.equal(harness.accountUpdates[0]?.syncPhase, "error");
+		assert.match(
+			String(harness.accountUpdates[0]?.lastError),
+			/IMAP SEARCH failed in mailbox null/,
+		);
+	});
+
+	it("still fails loudly when the mailbox is live — the failure is the account's, not a deletion", async () => {
+		const { log } = buildLogger();
+		const harness = buildRunningSyncDeps(async (call) => {
+			if (call === 2) throw new Error("connection reset by peer");
+			return { mailboxId: "mbx-1", fullPath: "Archive" };
+		});
+
+		await assert.rejects(
+			syncMessages(syncEvent("mbx-1"), log, harness.deps),
+			/connection reset by peer/,
+		);
+		assert.equal(harness.accountUpdates.length, 1);
+		assert.equal(harness.accountUpdates[0]?.syncPhase, "error");
+	});
+});

package/src/handlers/sync-messages.ts

@@ -18,6 +18,7 @@ import { type Logger, MetricUnit, metrics } from "@remit/logger-lambda";
 import { RefreshTokenError } from "@remit/mail-oauth-service";
 import {
 	createManagedConnectionFactory,
+	isMailboxNotOnServer,
 	MailConnectionError,
 	type MailCredentials,
 	MessageSyncService,
@@ -132,23 +133,18 @@ export const syncMessages = async (
 		return;
 	}
 
-	// A SYNC_MESSAGES trigger can outlive the mailbox it targets. Deleting a
-	// mailbox that has held mail leaves already-queued events — a `hasMore`
-	// next-batch, a periodic sync tick — pointing at a row that is now gone. The
-	// lookup then throws a named NotFoundError that can never succeed on retry,
-	// and the account's per-group FIFO ordering lets that one poison message
-	// stall the whole account's message pipeline forever (issue #287). A mailbox
-	// the user deliberately deleted is an expected terminal outcome, not an infra
-	// failure: ack the event with a WARN. Any other error — a transient read, a
-	// NotFoundError from elsewhere — still propagates to be retried.
-	const mailboxExists = await mailboxService
-		.get(event.accountId, event.mailboxId)
-		.then(() => true)
-		.catch((err) => {
-			if (isNotFoundError(err)) return false;
-			throw err;
-		});
-	if (!mailboxExists) {
+	// A SYNC_MESSAGES trigger can address a folder the server does not hold: the
+	// fan-out enqueues one for a folder whose own create has not landed yet, and
+	// a `hasMore` next-batch or a periodic tick outlives a folder the user
+	// deleted. Such an event can never succeed on retry, and the account's
+	// per-group FIFO ordering lets that one poison message stall the
+	// whole account's message pipeline for the full visibility window (issue
+	// #287, #339). A folder the server does not hold — not yet created, or being
+	// deleted — is an expected terminal outcome, not an infra failure: ack the
+	// event with a WARN.
+	if (
+		await isMailboxNotOnServer(mailboxService, event.accountId, event.mailboxId)
+	) {
 		log.warn(
 			{
 				accountId: event.accountId,
@@ -156,7 +152,7 @@ export const syncMessages = async (
 				eventId: event.eventId,
 				event: event.type,
 			},
-			"Skipping SYNC_MESSAGES: mailbox no longer exists (deleted)",
+			"Skipping SYNC_MESSAGES: the server does not hold this folder",
 		);
 		return;
 	}
@@ -203,6 +199,39 @@ export const syncMessages = async (
 						) {
 							throw err;
 						}
+						// The guard above is a check-then-act: a delete asked for while
+						// this round was in flight lands inside it, and the round then
+						// fails somewhere deeper — on the row (NotFoundError from the
+						// service's own re-read) or on the server (the SELECTed folder
+						// vanished, so the next command runs against no mailbox). Both are
+						// the same expected outcome as an event that arrived after the
+						// delete, so they resolve the same way rather than head-of-line
+						// blocking the account's FIFO group (issue #339). Recording an
+						// error phase is skipped too: the account is healthy, one of its
+						// folders is not there.
+						//
+						// This read only classifies the failure already in hand, so it must
+						// never become the failure that is reported. A read that cannot answer
+						// leaves the round on the loud path below, with the real error and the
+						// state write that goes with it intact.
+						const folderIsGone = await isMailboxNotOnServer(
+							mailboxService,
+							event.accountId,
+							event.mailboxId,
+						).catch(() => false);
+						if (folderIsGone) {
+							log.warn(
+								{
+									accountId: event.accountId,
+									mailboxId: event.mailboxId,
+									eventId: event.eventId,
+									event: event.type,
+									error: err instanceof Error ? err.message : String(err),
+								},
+								"Resolving SYNC_MESSAGES: the folder left the server while the sync was in flight",
+							);
+							return;
+						}
 						// Record the terminal error phase before crashing (let-it-crash:
 						// record state, then rethrow so the event is retried/DLQ'd).
 						const message = err instanceof Error ? err.message : String(err);