@remit/imap-worker 0.0.54 → 0.0.55

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.54",
+  "version": "0.0.55",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/handlers/message-delete-terminal.ts

@@ -0,0 +1,119 @@
+import type { IMessageRepository } from "@remit/data-ports";
+import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
+import {
+	type IImapConnection,
+	isMessageGoneFromOpenMailbox,
+	reconcileStaleMessage,
+	type StaleMessageReconcileDeps,
+} from "@remit/mailbox-service";
+
+export interface MessageDeleteTerminalLogger {
+	info(obj: Record<string, unknown>, msg: string): void;
+	error(obj: Record<string, unknown>, msg: string): void;
+}
+
+export interface ResolveExhaustedMessageDeleteDeps
+	extends StaleMessageReconcileDeps {
+	messageService: Pick<IMessageRepository, "delete" | "update">;
+	log: MessageDeleteTerminalLogger;
+}
+
+export interface ResolveExhaustedMessageDeleteInput {
+	accountId: string;
+	accountConfigId: string;
+	messageId: string;
+	uid: number;
+	sourceMailboxPath: string;
+	getConnection: () => Promise<IImapConnection>;
+}
+
+export type MessageDeleteTerminalOutcome = "reconciled" | "broken";
+
+export interface ResolveExhaustedMessageDeleteResult {
+	outcome: MessageDeleteTerminalOutcome;
+}
+
+/**
+ * Resolve a MESSAGE_DELETE failure that has exhausted its redelivery budget
+ * into exactly one of two terminal outcomes, mirroring
+ * `resolveExhaustedMessageMoveFailure` for the same failure taxonomy (issue
+ * #655) — no third, softer outcome.
+ *
+ * 1. RECONCILED (expected) — the message no longer exists at the delete's
+ *    source on IMAP, confirmed by {@link isMessageGoneFromOpenMailbox} rather
+ *    than by a FETCH coming back empty. The move to Trash landed server-side,
+ *    or a foreign client moved or expunged the message; from here those are
+ *    indistinguishable and have the same answer. The stale rows are deleted via
+ *    {@link reconcileStaleMessage} and the caller resyncs the affected folders,
+ *    so whichever folder actually holds the message re-projects it with the
+ *    server's own UID. Metric only, no alarm — routine.
+ * 2. BROKEN — the message is still at the source, so the delete never took
+ *    effect, but it keeps failing: broken code or a broken account, not a
+ *    transient blip. The row's mailbox and uid are left exactly as they stand,
+ *    because reverting the optimistic move on this ambiguity is what PR #652
+ *    was pulled for.
+ *
+ *    `status` does settle, to `active`. It is not a claim about where the
+ *    message is — only that this row is no longer mid-mutation. Leaving it
+ *    `moving` makes `isPlacementUnsettled` true forever, and every later delete
+ *    of that message then waits on a mutation that has already terminated, so
+ *    the user is left with mail they cannot delete.
+ *
+ * A server that cannot be reached at exhaustion time never reaches either
+ * verdict: the probe throws and the record dead-letters with the row untouched.
+ * Absence is only ever concluded from an answer the server gave.
+ */
+export const resolveExhaustedMessageDeleteFailure = async (
+	deps: ResolveExhaustedMessageDeleteDeps,
+	input: ResolveExhaustedMessageDeleteInput,
+): Promise<ResolveExhaustedMessageDeleteResult> => {
+	const {
+		accountId,
+		accountConfigId,
+		messageId,
+		uid,
+		sourceMailboxPath,
+		getConnection,
+	} = input;
+
+	const connection = await getConnection();
+	await connection.openBox(sourceMailboxPath, true);
+
+	if (await isMessageGoneFromOpenMailbox(connection, uid)) {
+		const { threadMessagesDeleted } = await reconcileStaleMessage(
+			deps,
+			accountConfigId,
+			messageId,
+		);
+		deps.log.info(
+			{
+				metric: "message_delete_stale_row_reconciled",
+				accountId,
+				accountConfigId,
+				messageId,
+				uid,
+				sourceMailboxPath,
+				threadMessagesDeleted,
+			},
+			"Message no longer at its delete source after retry exhaustion (the delete landed server-side, or an external delete or move); stale row reconciled",
+		);
+		return { outcome: "reconciled" };
+	}
+
+	await deps.messageService.update(messageId, {
+		status: MessageStatus.active,
+		syncStatus: MessageSyncStatus.failed,
+	});
+	deps.log.error(
+		{
+			alert: "message_delete_failed",
+			accountId,
+			accountConfigId,
+			messageId,
+			uid,
+			sourceMailboxPath,
+		},
+		"Delete could not be pushed to IMAP after retry exhaustion; the message is still at its source — row settled out of `moving` and left in place for operator investigation",
+	);
+	return { outcome: "broken" };
+};

package/src/handlers/message-delete.test.ts

@@ -2,15 +2,50 @@ import assert from "node:assert";
 import { beforeEach, describe, it, mock } from "node:test";
 import type { ThreadMessageItem } from "@remit/data-ports";
 import type { Logger } from "@remit/logger-lambda";
+import { renderMetrics, resetMetrics } from "@remit/logger-lambda";
 import type { MessageDeleteEvent } from "../events.js";
 import {
 	buildThreadMessageTrashUpdate,
 	buildThreadMessageUndelete,
 	deleteAllThreadMessagesForMessage,
+	getMessageDeleteMaxAttempts,
 	handleMessageDelete,
+	MESSAGE_DELETE_MAX_ATTEMPTS,
 	type MessageDeleteDeps,
 } from "./message-delete.js";
 
+describe("getMessageDeleteMaxAttempts — env-derived threshold (#980)", () => {
+	it("parses the injected env var", () => {
+		assert.equal(
+			getMessageDeleteMaxAttempts({ MESSAGE_DELETE_MAX_ATTEMPTS: "3" }),
+			3,
+		);
+		assert.equal(
+			getMessageDeleteMaxAttempts({ MESSAGE_DELETE_MAX_ATTEMPTS: "5" }),
+			5,
+		);
+	});
+
+	it("defaults to the mailbox queue's own maxReceiveCount when unset", () => {
+		assert.equal(getMessageDeleteMaxAttempts({}), 3);
+	});
+
+	it("defaults on a non-numeric or non-positive value", () => {
+		assert.equal(
+			getMessageDeleteMaxAttempts({ MESSAGE_DELETE_MAX_ATTEMPTS: "nope" }),
+			3,
+		);
+		assert.equal(
+			getMessageDeleteMaxAttempts({ MESSAGE_DELETE_MAX_ATTEMPTS: "0" }),
+			3,
+		);
+	});
+
+	it("MESSAGE_DELETE_MAX_ATTEMPTS is a concrete, positive number at module load", () => {
+		assert.ok(MESSAGE_DELETE_MAX_ATTEMPTS > 0);
+	});
+});
+
 const sourceMailboxId = "source-mailbox-id-aaaaaaaaa";
 const trashMailboxId = "trash-mailbox-id-aaaaaaaaa";
 
@@ -358,6 +393,7 @@ const deps = (): MessageDeleteDeps =>
 					if (h.threadMessageUpdateError) throw h.threadMessageUpdateError;
 				},
 				delete: record("threadMessage.delete"),
+				deleteMany: record("threadMessage.deleteMany"),
 			},
 			mailbox: {
 				get: async () => {
@@ -375,6 +411,7 @@ const deps = (): MessageDeleteDeps =>
 			_log: unknown,
 			cb: (credentials: unknown) => Promise<void>,
 		) => cb({}),
+		emitEvent: record("emitEvent"),
 		createConnectionScope: () => ({
 			getConnection: async () => {
 				h.getConnectionCount += 1;
@@ -413,13 +450,26 @@ const permanentEvent: MessageDeleteEvent = {
 const called = (method: string): Call[] =>
 	h.calls.filter((c) => c.method === method);
 
+// Label order in the rendered text is prom-client's, not ours.
+const imapFailures = async (operation: string): Promise<number> => {
+	const line = (await renderMetrics())
+		.split("\n")
+		.find(
+			(candidate) =>
+				candidate.startsWith("remit_imap_failures_total{") &&
+				candidate.includes(`operation="${operation}"`),
+		);
+	return line ? Number(line.slice(line.lastIndexOf(" ") + 1)) : 0;
+};
+
 describe("handleMessageDelete", () => {
 	beforeEach(() => {
 		h = fresh();
+		resetMetrics();
 	});
 
 	it("moves to trash, rewrites the uid, and flips the thread row to deleted", async () => {
-		await handleMessageDelete(moveEvent, noopLog, deps());
+		await handleMessageDelete(moveEvent, noopLog, 1, deps());
 
 		assert.deepEqual(called("message.updateUid")[0]?.args, [
 			"msg-1",
@@ -445,7 +495,7 @@ describe("handleMessageDelete", () => {
 			sourceNoLongerHoldsTheUid();
 			h.destinationSearchUids = [77];
 
-			await handleMessageDelete(moveEvent, noopLog, deps());
+			await handleMessageDelete(moveEvent, noopLog, 1, deps());
 
 			assert.deepEqual(called("message.updateUid")[0]?.args, [
 				"msg-1",
@@ -474,7 +524,7 @@ describe("handleMessageDelete", () => {
 				return { uidvalidity: 1 };
 			}) as Connection["openBox"];
 
-			await handleMessageDelete(moveEvent, noopLog, deps());
+			await handleMessageDelete(moveEvent, noopLog, 1, deps());
 
 			assert.deepEqual(
 				opened,
@@ -490,23 +540,31 @@ describe("handleMessageDelete", () => {
 			]);
 		});
 
-		it("leaves local state alone when the probe finds nothing — never reverts, never deletes (#655)", async () => {
+		// Issue #980. Within the budget the row is marked unsettled and the
+		// event is re-thrown so the queue redelivers it, exactly as
+		// `handleMessageMove` does; nothing is reverted and nothing is deleted,
+		// because a MOVE that ran server-side but dropped before the tagged OK
+		// is indistinguishable from one that never ran (#655, PR #652).
+		it("throws to redeliver while the budget still has attempts left", async () => {
 			h.connection.moveMessages = async () => ({ uidMap: new Map() });
 			sourceNoLongerHoldsTheUid();
 			h.destinationSearchUids = [];
 
-			await handleMessageDelete(moveEvent, noopLog, deps());
+			await assert.rejects(
+				handleMessageDelete(moveEvent, noopLog, 1, deps()),
+				/unconfirmed/,
+			);
 
 			assert.equal(called("message.updateUid").length, 0);
 			assert.equal(
 				called("message.delete").length,
 				0,
-				"an unconfirmed move must never delete the local row",
+				"an unconfirmed move must never delete the local row inside the budget",
 			);
 			assert.equal(
-				called("threadMessage.delete").length,
+				called("threadMessage.deleteMany").length,
 				0,
-				"an unconfirmed move must never delete the listing rows",
+				"an unconfirmed move must never delete the listing rows inside the budget",
 			);
 			assert.equal(
 				called("threadMessage.update").length,
@@ -518,14 +576,42 @@ describe("handleMessageDelete", () => {
 					?.syncStatus,
 				"failed",
 			);
+			assert.equal(h.disconnectCount, 1);
 		});
 
-		it("does not probe when the row carries no Message-ID header", async () => {
+		it("settles once at the ceiling: reconciles the stale rows and resyncs both folders", async () => {
+			h.connection.moveMessages = async () => ({ uidMap: new Map() });
+			sourceNoLongerHoldsTheUid();
+			h.destinationSearchUids = [];
+
+			await handleMessageDelete(moveEvent, noopLog, 3, deps());
+
+			assert.equal(
+				called("threadMessage.deleteMany").length,
+				1,
+				"the ceiling settles the row rather than leaving it claiming a uid the server does not have",
+			);
+			assert.equal(called("message.delete").length, 1);
+			assert.deepEqual(
+				called("emitEvent").map((c) => c.args[0]),
+				[
+					{ type: "SYNC_MESSAGES", accountId: "acc-1", mailboxId: "src-mbx" },
+					{ type: "SYNC_MESSAGES", accountId: "acc-1", mailboxId: "trash-mbx" },
+				],
+				"both folders re-project from IMAP so the message reappears where it actually is",
+			);
+			assert.equal(h.disconnectCount, 1);
+		});
+
+		// A message with no Message-ID header on a non-UIDPLUS server can never
+		// be confirmed at the destination, so redelivering it only burns the
+		// budget on an answer that cannot change.
+		it("settles on the first attempt when the row carries no Message-ID header", async () => {
 			h.connection.moveMessages = async () => ({ uidMap: new Map() });
 			sourceNoLongerHoldsTheUid();
 			h.messageRow = {};
 
-			await handleMessageDelete(moveEvent, noopLog, deps());
+			await handleMessageDelete(moveEvent, noopLog, 1, deps());
 
 			assert.equal(
 				called("connection.search").filter((c) =>
@@ -535,12 +621,33 @@ describe("handleMessageDelete", () => {
 			);
 			assert.equal(called("message.updateUid").length, 0);
 			assert.equal(
-				(called("message.update")[0]?.args[1] as { syncStatus?: string })
-					?.syncStatus,
-				"failed",
+				called("message.delete").length,
+				1,
+				"an unprobeable row settles instead of spending the whole budget",
+			);
+			assert.equal(called("emitEvent").length, 2);
+			assert.equal(
+				await imapFailures("MESSAGE_DELETE_TRASH_MOVE_UNCONFIRMED"),
+				1,
 			);
 		});
 
+		// The row this event names was already deleted, so there is nothing to
+		// settle and nothing to show. Re-throwing would loop a NotFoundError on
+		// the account's per-group FIFO and block every later mailbox event.
+		it("acks without touching the database when the local row is already gone", async () => {
+			h.connection.moveMessages = async () => ({ uidMap: new Map() });
+			sourceNoLongerHoldsTheUid();
+			h.messageRow = undefined;
+
+			await handleMessageDelete(moveEvent, noopLog, 1, deps());
+
+			assert.equal(called("message.update").length, 0);
+			assert.equal(called("message.delete").length, 0);
+			assert.equal(called("threadMessage.deleteMany").length, 0);
+			assert.equal(called("emitEvent").length, 0);
+		});
+
 		// `searchMailboxByMessageId` returns the LOWEST matching uid, and one
 		// Message-ID can have several server copies in one account while
 		// `deriveMessageId` gives them one local row. A source that still holds
@@ -550,7 +657,10 @@ describe("handleMessageDelete", () => {
 			h.connection.moveMessages = async () => ({ uidMap: new Map() });
 			h.destinationSearchUids = [100];
 
-			await handleMessageDelete(moveEvent, noopLog, deps());
+			await assert.rejects(
+				handleMessageDelete(moveEvent, noopLog, 1, deps()),
+				/unconfirmed/,
+			);
 
 			assert.equal(
 				called("message.updateUid").length,
@@ -566,22 +676,109 @@ describe("handleMessageDelete", () => {
 			);
 		});
 
-		// The MOVE has already run by the time the probe goes out, so a probe
-		// that cannot answer counts as not-confirmed. Throwing would redeliver
-		// on the account's per-group FIFO and re-MOVE a uid the source no longer
-		// holds, head-of-line blocking every other delete on the account.
-		it("treats an unanswerable probe as unconfirmed rather than throwing", async () => {
+		// A source that still holds the uid at the ceiling means the MOVE never
+		// took effect. Local state is left exactly as it stands — the row is the
+		// only record that this delete is still owed, and a revert races a MOVE
+		// that may yet have landed (#652, #655).
+		it("leaves a message still at its source untouched at the ceiling and never re-throws", async () => {
+			h.connection.moveMessages = async () => ({ uidMap: new Map() });
+			h.destinationSearchUids = [100];
+
+			await handleMessageDelete(moveEvent, noopLog, 3, deps());
+
+			assert.equal(called("message.delete").length, 0);
+			assert.equal(called("threadMessage.deleteMany").length, 0);
+			assert.equal(called("threadMessage.update").length, 0);
+			assert.equal(called("emitEvent").length, 0);
+
+			// The row's mailbox and uid stay put, but `status` must leave
+			// `moving`: `isPlacementUnsettled` reads exactly that value, so a row
+			// left mid-mutation makes every later delete of this message wait on
+			// a mutation that has already terminated.
+			assert.deepEqual(called("message.update").at(-1)?.args[1], {
+				status: "active",
+				syncStatus: "failed",
+			});
+			assert.equal(await imapFailures("MESSAGE_DELETE_EXHAUSTED"), 1);
+		});
+
+		// Issue #980, the failure the budget exists for: every redelivery
+		// re-MOVEs a uid the source no longer holds and throws identically. The
+		// ceiling lives in the error catch, so a throwing `moveMessages` settles
+		// there rather than running past the budget into the dead-letter queue.
+		describe("a re-MOVE that throws", () => {
+			const moveThrows = (): void => {
+				h.connection.moveMessages = async () => {
+					throw new Error(
+						"NO [TRYAGAIN] UID MOVE failed: no matching messages",
+					);
+				};
+			};
+
+			it("re-throws inside the budget so the queue redelivers", async () => {
+				moveThrows();
+
+				await assert.rejects(
+					handleMessageDelete(moveEvent, noopLog, 1, deps()),
+					/UID MOVE failed/,
+				);
+
+				assert.equal(called("message.delete").length, 0);
+				assert.equal(called("threadMessage.deleteMany").length, 0);
+				assert.equal(
+					(called("message.update")[0]?.args[1] as { syncStatus?: string })
+						?.syncStatus,
+					"failed",
+				);
+			});
+
+			it("settles at the ceiling instead of dead-lettering undiagnosed", async () => {
+				moveThrows();
+				sourceNoLongerHoldsTheUid();
+
+				await handleMessageDelete(moveEvent, noopLog, 3, deps());
+
+				assert.equal(
+					called("threadMessage.deleteMany").length,
+					1,
+					"the ceiling must diagnose the failure, not hand it to the DLQ",
+				);
+				assert.equal(called("message.delete").length, 1);
+				assert.equal(called("emitEvent").length, 2);
+			});
+
+			it("settles a still-present message at the ceiling out of `moving`", async () => {
+				moveThrows();
+
+				await handleMessageDelete(moveEvent, noopLog, 3, deps());
+
+				assert.deepEqual(called("message.update").at(-1)?.args[1], {
+					status: "active",
+					syncStatus: "failed",
+				});
+				assert.equal(await imapFailures("MESSAGE_DELETE_EXHAUSTED"), 1);
+			});
+		});
+
+		// A probe the server refused says nothing either way, so it counts as
+		// unconfirmed and spends an attempt. The budget is what makes that safe:
+		// without a ceiling the redeliveries would re-MOVE a uid the source no
+		// longer holds forever and head-of-line block the account's deletes.
+		it("treats an unanswerable probe as unconfirmed and spends an attempt", async () => {
 			h.connection.moveMessages = async () => ({ uidMap: new Map() });
 			h.connection.openBox = (async (_path: string, readOnly?: boolean) => {
 				if (readOnly) throw new Error("NO [SERVERBUG] EXAMINE failed");
 				return { uidvalidity: 1 };
 			}) as Connection["openBox"];
 
-			await handleMessageDelete(moveEvent, noopLog, deps());
+			await assert.rejects(
+				handleMessageDelete(moveEvent, noopLog, 1, deps()),
+				/unconfirmed/,
+			);
 
 			assert.equal(called("message.updateUid").length, 0);
 			assert.equal(called("message.delete").length, 0);
-			assert.equal(called("threadMessage.delete").length, 0);
+			assert.equal(called("threadMessage.deleteMany").length, 0);
 			assert.equal(called("threadMessage.update").length, 0);
 			assert.equal(
 				(called("message.update")[0]?.args[1] as { syncStatus?: string })
@@ -606,7 +803,7 @@ describe("handleMessageDelete", () => {
 				operation,
 			} as unknown as MessageDeleteEvent;
 
-			await handleMessageDelete(malformed, noopLog, deps());
+			await handleMessageDelete(malformed, noopLog, 1, deps());
 
 			assert.equal(
 				called("connection.deleteMessages").length,
@@ -643,7 +840,7 @@ describe("handleMessageDelete", () => {
 			destinationMailboxPath: undefined,
 		} as MessageDeleteEvent;
 
-		await handleMessageDelete(destinationless, noopLog, deps());
+		await handleMessageDelete(destinationless, noopLog, 1, deps());
 
 		assert.equal(called("connection.deleteMessages").length, 0);
 		assert.equal(called("message.delete").length, 0);
@@ -654,7 +851,7 @@ describe("handleMessageDelete", () => {
 	});
 
 	it("expunges on the server and removes every thread row before the message row", async () => {
-		await handleMessageDelete(permanentEvent, noopLog, deps());
+		await handleMessageDelete(permanentEvent, noopLog, 1, deps());
 
 		assert.deepEqual(called("connection.deleteMessages")[0]?.args, [[10]]);
 		assert.equal(called("threadMessage.delete").length, 2);
@@ -673,7 +870,7 @@ describe("handleMessageDelete", () => {
 		};
 		sourceNoLongerHoldsTheUid();
 
-		await handleMessageDelete(permanentEvent, noopLog, deps());
+		await handleMessageDelete(permanentEvent, noopLog, 1, deps());
 
 		assert.equal(called("message.delete").length, 1);
 		assert.equal(called("threadMessage.delete").length, 2);
@@ -694,7 +891,7 @@ describe("handleMessageDelete", () => {
 		sourceNoLongerHoldsTheUid();
 
 		await assert.rejects(
-			handleMessageDelete(moveEvent, noopLog, deps()),
+			handleMessageDelete(moveEvent, noopLog, 1, deps()),
 			/not found/,
 		);
 
@@ -722,7 +919,7 @@ describe("handleMessageDelete", () => {
 		};
 
 		await assert.rejects(
-			handleMessageDelete(moveEvent, noopLog, deps()),
+			handleMessageDelete(moveEvent, noopLog, 1, deps()),
 			/NONEXISTENT/,
 		);
 
@@ -744,7 +941,7 @@ describe("handleMessageDelete", () => {
 		h.connection.fetchMessages = async () => [];
 
 		await assert.rejects(
-			handleMessageDelete(permanentEvent, noopLog, deps()),
+			handleMessageDelete(permanentEvent, noopLog, 1, deps()),
 			/NONEXISTENT/,
 		);
 
@@ -769,7 +966,7 @@ describe("handleMessageDelete", () => {
 		};
 
 		await assert.rejects(
-			handleMessageDelete(permanentEvent, noopLog, deps()),
+			handleMessageDelete(permanentEvent, noopLog, 1, deps()),
 			/NONEXISTENT mailbox does not exist/,
 		);
 
@@ -790,7 +987,7 @@ describe("handleMessageDelete", () => {
 			throw new Error("TRYCREATE: no such mailbox");
 		};
 
-		await handleMessageDelete(moveEvent, noopLog, deps());
+		await handleMessageDelete(moveEvent, noopLog, 1, deps());
 
 		assert.equal(called("connection.createMailbox").length, 0);
 		assert.deepEqual(called("message.updateUid")[0]?.args, [
@@ -815,7 +1012,7 @@ describe("handleMessageDelete", () => {
 			schemaVersion: undefined,
 		} as unknown as MessageDeleteEvent;
 
-		await handleMessageDelete(unversioned, noopLog, deps());
+		await handleMessageDelete(unversioned, noopLog, 1, deps());
 
 		assert.equal(h.getConnectionCount, 0);
 		assert.equal(called("connection.deleteMessages").length, 0);
@@ -837,7 +1034,7 @@ describe("handleMessageDelete", () => {
 			schemaVersion: undefined,
 		} as unknown as MessageDeleteEvent;
 
-		await handleMessageDelete(unversioned, noopLog, deps());
+		await handleMessageDelete(unversioned, noopLog, 1, deps());
 
 		assert.deepEqual(
 			called("threadMessage.update").map((c) => c.args[1]),
@@ -856,7 +1053,7 @@ describe("handleMessageDelete", () => {
 			schemaVersion: undefined,
 		} as unknown as MessageDeleteEvent;
 
-		await handleMessageDelete(unversioned, noopLog, deps());
+		await handleMessageDelete(unversioned, noopLog, 1, deps());
 
 		assert.equal(called("connection.deleteMessages").length, 0);
 		assert.deepEqual(called("message.delete")[0]?.args, ["msg-1"]);
@@ -873,7 +1070,7 @@ describe("handleMessageDelete", () => {
 		};
 
 		await assert.rejects(
-			handleMessageDelete(moveEvent, noopLog, deps()),
+			handleMessageDelete(moveEvent, noopLog, 1, deps()),
 			/server exploded/,
 		);
 
@@ -887,7 +1084,7 @@ describe("handleMessageDelete", () => {
 	it("pauses quietly when openBox trips a UIDVALIDITY mismatch", async () => {
 		h.connection.openBox = async () => ({ uidvalidity: 999 });
 
-		await handleMessageDelete(moveEvent, noopLog, deps());
+		await handleMessageDelete(moveEvent, noopLog, 1, deps());
 
 		assert.equal(
 			(called("mailbox.update")[0]?.args[2] as { cursorState?: string })
@@ -904,7 +1101,7 @@ describe("handleMessageDelete", () => {
 			cursorState: "rebuilding",
 		};
 
-		await handleMessageDelete(moveEvent, noopLog, deps());
+		await handleMessageDelete(moveEvent, noopLog, 1, deps());
 
 		assert.equal(h.getConnectionCount, 0);
 	});
@@ -914,7 +1111,7 @@ describe("handleMessageDelete", () => {
 			name: "NotFoundError",
 		});
 
-		await handleMessageDelete(moveEvent, noopLog, deps());
+		await handleMessageDelete(moveEvent, noopLog, 1, deps());
 
 		assert.equal(h.getConnectionCount, 0);
 		assert.equal(called("message.updateUid").length, 0);
@@ -928,7 +1125,7 @@ describe("handleMessageDelete", () => {
 			deletedAt: Date.now(),
 		};
 
-		await handleMessageDelete(moveEvent, noopLog, deps());
+		await handleMessageDelete(moveEvent, noopLog, 1, deps());
 
 		assert.equal(h.getConnectionCount, 0);
 	});
@@ -937,7 +1134,7 @@ describe("handleMessageDelete", () => {
 		h.account = null;
 
 		await assert.rejects(
-			handleMessageDelete(moveEvent, noopLog, deps()),
+			handleMessageDelete(moveEvent, noopLog, 1, deps()),
 			/not found/,
 		);
 	});

package/src/handlers/message-delete.ts

@@ -7,6 +7,7 @@ import type {
 import { isCurrentSchemaVersion } from "@remit/data-ports/mutation-events";
 import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
 import type { Logger } from "@remit/logger-lambda";
+import { recordImapFailure } from "@remit/logger-lambda";
 import {
 	guardConnectionCursor,
 	type IImapConnection,
@@ -16,11 +17,34 @@ import {
 } from "@remit/mailbox-service";
 import { isAccountDeleted } from "../account-check.js";
 import { createConnectionScopeWithCredentials } from "../connection-scope.js";
+import { emitEvent } from "../emit.js";
 import type { MessageDeleteEvent } from "../events.js";
 import { isNotFoundError } from "../is-not-found.js";
 import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
 import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
-import { searchMailboxByMessageId } from "./message-move.js";
+import { resolveExhaustedMessageDeleteFailure } from "./message-delete-terminal.js";
+import { emitMoveResync, searchMailboxByMessageId } from "./message-move.js";
+
+/**
+ * Fallback when `MESSAGE_DELETE_MAX_ATTEMPTS` is unset (local dev, unit tests).
+ * Matches the `maxReceiveCount` the redrive policy of the queue `emit.ts`
+ * routes MESSAGE_DELETE onto uses (`remit-message-mgmt`,
+ * `deploy/vps/queues.json`), same pattern as `MESSAGE_MOVE_MAX_ATTEMPTS`.
+ */
+const DEFAULT_MESSAGE_DELETE_MAX_ATTEMPTS = 3;
+
+export const getMessageDeleteMaxAttempts = (
+	processEnv: NodeJS.ProcessEnv = process.env,
+): number => {
+	const raw = processEnv.MESSAGE_DELETE_MAX_ATTEMPTS;
+	if (!raw) return DEFAULT_MESSAGE_DELETE_MAX_ATTEMPTS;
+	const parsed = Number.parseInt(raw, 10);
+	return Number.isFinite(parsed) && parsed > 0
+		? parsed
+		: DEFAULT_MESSAGE_DELETE_MAX_ATTEMPTS;
+};
+
+export const MESSAGE_DELETE_MAX_ATTEMPTS = getMessageDeleteMaxAttempts();
 
 /**
  * Delete every ThreadMessage row that points at this messageId.
@@ -143,9 +167,17 @@ export const buildThreadMessageUndelete = (
  * expunges by that uid. It also closes the second half of #912: an empty
  * `uidMap` can mean the MOVE matched nothing at all.
  *
- * A row with no `messageIdHeader` has nothing to probe with, so it stays
- * unconfirmed rather than guessing.
+ * A row with no `messageIdHeader`, and a row that is already deleted, have
+ * nothing to probe with; they are distinct verdicts because no redelivery can
+ * change either answer.
  */
+export type TrashMoveConfirmation =
+	| { outcome: "confirmed"; uid: number }
+	| { outcome: "still-at-source" }
+	| { outcome: "row-gone" }
+	| { outcome: "unprobeable" }
+	| { outcome: "unconfirmed" };
+
 const confirmTrashMoveUid = async (
 	sourceConnection: IImapConnection,
 	destinationConnection: IImapConnection,
@@ -154,18 +186,24 @@ const confirmTrashMoveUid = async (
 	sourceMailboxPath: string,
 	destinationMailboxPath: string,
 	uid: number,
-): Promise<number | null> => {
+): Promise<TrashMoveConfirmation> => {
 	await sourceConnection.openBox(sourceMailboxPath, true);
-	if (!(await isMessageGoneFromOpenMailbox(sourceConnection, uid))) return null;
+	if (!(await isMessageGoneFromOpenMailbox(sourceConnection, uid))) {
+		return { outcome: "still-at-source" };
+	}
 
 	const [message] = await messageService.get([messageId]);
-	if (!message?.messageIdHeader) return null;
+	if (!message) return { outcome: "row-gone" };
+	if (!message.messageIdHeader) return { outcome: "unprobeable" };
 
-	return searchMailboxByMessageId(
+	const probedUid = await searchMailboxByMessageId(
 		destinationConnection,
 		destinationMailboxPath,
 		message.messageIdHeader,
 	);
+	return probedUid === null
+		? { outcome: "unconfirmed" }
+		: { outcome: "confirmed", uid: probedUid };
 };
 
 export interface MessageDeleteDeps {
@@ -173,6 +211,7 @@ export interface MessageDeleteDeps {
 	buildLifecycleDeps: typeof buildLifecycleDeps;
 	withOAuthLifecycle: typeof withOAuthLifecycle;
 	createConnectionScope: typeof createConnectionScopeWithCredentials;
+	emitEvent: typeof emitEvent;
 }
 
 const defaultDeps: MessageDeleteDeps = {
@@ -180,15 +219,22 @@ const defaultDeps: MessageDeleteDeps = {
 	buildLifecycleDeps,
 	withOAuthLifecycle,
 	createConnectionScope: createConnectionScopeWithCredentials,
+	emitEvent,
 };
 
 /**
  * Handle MESSAGE_DELETE events.
  * Either moves to Trash (IMAP MOVE) or permanently deletes (IMAP DELETE).
+ *
+ * A failing delete retries on redelivery until `receiveCount` reaches
+ * {@link MESSAGE_DELETE_MAX_ATTEMPTS}, at which point
+ * {@link resolveExhaustedMessageDeleteFailure} asks IMAP where the message
+ * actually is and settles the row into one terminal outcome (issue #980).
  */
 export const handleMessageDelete = async (
 	event: MessageDeleteEvent,
 	log: Logger,
+	receiveCount = 1,
 	deps: MessageDeleteDeps = defaultDeps,
 ): Promise<void> => {
 	const {
@@ -196,6 +242,7 @@ export const handleMessageDelete = async (
 		buildLifecycleDeps,
 		withOAuthLifecycle,
 		createConnectionScope: createConnectionScopeWithCredentials,
+		emitEvent,
 	} = deps;
 
 	const {
@@ -292,6 +339,80 @@ export const handleMessageDelete = async (
 		}
 	};
 
+	const settleExhaustedDelete = async (
+		accountConfigId: string,
+		getConnection: () => Promise<IImapConnection>,
+	): Promise<void> => {
+		const { outcome } = await resolveExhaustedMessageDeleteFailure(
+			{ messageService, threadMessageService, log },
+			{
+				accountId,
+				accountConfigId,
+				messageId,
+				uid,
+				sourceMailboxPath: mailboxPath,
+				getConnection,
+			},
+		);
+
+		if (outcome === "broken") {
+			recordImapFailure("MESSAGE_DELETE_EXHAUSTED", "other");
+			return;
+		}
+
+		if (destinationMailboxId) {
+			await emitMoveResync(emitEvent, {
+				accountId,
+				sourceMailboxId: mailboxId,
+				destinationMailboxId,
+			});
+		}
+	};
+
+	/**
+	 * Decide what an unconfirmed move to Trash does with this delivery. The
+	 * settle itself is `resolveExhaustedMessageDeleteFailure`, reached through
+	 * the attempt budget in the catch below — the two verdicts handled here are
+	 * the ones a redelivery could never answer.
+	 */
+	const settleUnconfirmedTrashMove = async (
+		confirmation: Exclude<TrashMoveConfirmation, { outcome: "confirmed" }>,
+		accountConfigId: string,
+		getConnection: () => Promise<IImapConnection>,
+	): Promise<void> => {
+		const context = {
+			accountId,
+			accountConfigId,
+			messageId,
+			uid,
+			mailboxPath,
+			receiveCount,
+			confirmation: confirmation.outcome,
+		};
+
+		if (confirmation.outcome === "row-gone") {
+			log.warn(
+				context,
+				"Move to trash unconfirmed and the local row is already gone; nothing left to settle",
+			);
+			return;
+		}
+
+		if (confirmation.outcome === "unprobeable") {
+			recordImapFailure("MESSAGE_DELETE_TRASH_MOVE_UNCONFIRMED", "other");
+			log.info(
+				context,
+				"Move to trash carries no Message-ID header to probe the destination with; settling on the source's answer alone",
+			);
+			await settleExhaustedDelete(accountConfigId, getConnection);
+			return;
+		}
+
+		throw new Error(
+			`Move to trash unconfirmed for message ${messageId} (attempt ${receiveCount}/${MESSAGE_DELETE_MAX_ATTEMPTS})`,
+		);
+	};
+
 	if (!isCurrentSchemaVersion(event.schemaVersion)) {
 		await abandonDelete(
 			"Refused to delete: event was minted under an unknown contract",
@@ -385,42 +506,35 @@ export const handleMessageDelete = async (
 						// perfectly successful MOVE with no COPYUID entry, so an empty
 						// map is UNCONFIRMED, never evidence the move failed: the server
 						// is asked before any verdict, exactly as `handleMessageMove` and
-						// `attemptMove` do. Reading the empty map as a failure left the
-						// message in Trash under a uid nothing local knew, while the row
-						// kept the SOURCE folder's uid — which Empty Trash then decides
-						// by (issues #979, #665).
-						//
-						// A probe that cannot answer counts as not-confirmed, never as a
-						// throw. The MOVE has already run by this point, so throwing here
-						// would redeliver on the account's per-group FIFO and re-MOVE a
-						// uid the source no longer holds — head-of-line blocking the
-						// whole account's deletes over a transient NO, a renamed folder
-						// or a SEARCH the server refused.
-						const newUid =
-							result.uidMap.get(uid) ??
-							(await confirmTrashMoveUid(
-								connection,
-								rawConnection,
-								messageService,
-								messageId,
-								mailboxPath,
-								destinationMailboxPath,
-								uid,
-							).catch((probeError: unknown) => {
-								log.warn(
-									{
-										messageId,
-										uid,
-										mailboxPath,
-										destinationMailboxPath,
-										probeError,
-									},
-									"Could not confirm the move to trash; keeping local rows",
-								);
-								return null;
-							}));
-
-						if (newUid) {
+						// `attemptMove` do (issues #979, #665). A probe the server refused
+						// says nothing either way and counts as unconfirmed.
+						const copyUid = result.uidMap.get(uid);
+						const confirmation: TrashMoveConfirmation = copyUid
+							? { outcome: "confirmed", uid: copyUid }
+							: await confirmTrashMoveUid(
+									connection,
+									rawConnection,
+									messageService,
+									messageId,
+									mailboxPath,
+									destinationMailboxPath,
+									uid,
+								).catch((probeError: unknown) => {
+									log.warn(
+										{
+											messageId,
+											uid,
+											mailboxPath,
+											destinationMailboxPath,
+											probeError,
+										},
+										"Could not confirm the move to trash; keeping local rows",
+									);
+									return { outcome: "unconfirmed" } as const;
+								});
+
+						if (confirmation.outcome === "confirmed") {
+							const newUid = confirmation.uid;
 							// Update message with new UID in Trash
 							await messageService.updateUid(
 								messageId,
@@ -448,39 +562,14 @@ export const handleMessageDelete = async (
 							}
 
 							log.info({ messageId, newUid }, "Message moved to trash");
-						} else {
-							// Unconfirmed, not failed. A MOVE that ran server-side but
-							// dropped before the tagged OK is indistinguishable from one
-							// that never ran, so local state is left exactly as it stands:
-							// nothing is reverted and nothing is deleted. Reverting on that
-							// ambiguity is the blind revert #655 recorded when it was
-							// pulled from PR #652.
-							//
-							// This marks and returns rather than throwing. Throwing is the
-							// shape `handleMessageMove` uses, but it can only carry a
-							// budget: this handler has no `receiveCount`, no MAX_ATTEMPTS
-							// and no exhaustion path, so every redelivery on the account's
-							// per-group FIFO would re-MOVE a uid the source no longer
-							// holds, fail identically, and stall that account's whole
-							// delete pipeline (#287, #289, #290) until the queue's
-							// maxReceiveCount dead-letters it — leaving the row in this
-							// same state, minus the pipeline. Issue #980 wires the budget;
-							// the throw belongs with it, not ahead of it.
-							log.error(
-								{
-									alert: "message_delete_trash_move_unconfirmed",
-									accountId,
-									messageId,
-									uid,
-									mailboxPath,
-									destinationMailboxPath,
-								},
-								"Move to trash unconfirmed: no COPYUID entry, and the server did not confirm the message at the destination; local rows left as they stand",
-							);
-							await messageService.update(messageId, {
-								syncStatus: MessageSyncStatus.failed,
-							});
+							return;
 						}
+
+						await settleUnconfirmedTrashMove(
+							confirmation,
+							account.accountConfigId,
+							scope.getConnection,
+						);
 					} else {
 						// Permanent delete — reached only by `operation === "permanent_delete"`.
 						await connection.deleteMessages([uid]);
@@ -567,11 +656,28 @@ export const handleMessageDelete = async (
 						return;
 					}
 
-					// Mark as failed for other errors
-					await messageService.update(messageId, {
-						syncStatus: MessageSyncStatus.failed,
-					});
-					throw error;
+					if (receiveCount < MESSAGE_DELETE_MAX_ATTEMPTS) {
+						// Transient failure — connections drop. No alarm; redelivery
+						// retries, and `failed` marks the row unsettled meanwhile. It is
+						// not a terminal signal: only the resolver below settles anything.
+						await messageService.update(messageId, {
+							syncStatus: MessageSyncStatus.failed,
+						});
+						throw error;
+					}
+
+					// Budget exhausted. Settling here is what keeps a throwing
+					// `moveMessages` inside the ceiling: re-MOVEing a uid the source no
+					// longer holds fails identically on every redelivery, which is the
+					// failure this budget exists for.
+					await settleExhaustedDelete(
+						account.accountConfigId,
+						scope.getConnection,
+					);
+					log.error(
+						{ accountId, messageId, uid, mailboxPath, error: errorMessage },
+						"Delete retry exhausted; settled into a terminal outcome",
+					);
 				})
 				.finally(() => scope.disconnect());
 		},

package/src/processor.ts

@@ -38,7 +38,7 @@ export const processEvent = async (
 		case "MAILBOX_DELETE":
 			return processMailboxManagement(event, log);
 		case "MESSAGE_DELETE":
-			return handleMessageDelete(event, log);
+			return handleMessageDelete(event, log, receiveCount);
 		case "MESSAGE_MOVE":
 			return handleMessageMove(event, log, receiveCount);
 		case "PLACEMENT_MOVE_PUSH":