@remit/imap-worker 0.0.52 → 0.0.54

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.52",
+  "version": "0.0.54",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/handlers/message-delete.test.ts

@@ -260,6 +260,8 @@ interface Harness {
 	mailboxError?: Error;
 	connection: Connection;
 	threadMessageUpdateError?: Error;
+	messageRow: { messageIdHeader?: string } | undefined;
+	destinationSearchUids: number[];
 	threadMessage: Record<string, unknown> | null;
 	allThreadMessages: Record<string, unknown>[];
 	getConnectionCount: number;
@@ -274,6 +276,16 @@ const record =
 		h.calls.push({ method, args });
 	};
 
+const MESSAGE_ID_HEADER = "<trashed-message@example.com>";
+
+// The source-presence probe (`isMessageGoneFromOpenMailbox`) and the
+// destination probe both go through `connection.search`; only the criterion
+// form tells them apart.
+const isMessageIdSearch = (criteria: unknown): boolean =>
+	Array.isArray(criteria) &&
+	Array.isArray(criteria[0]) &&
+	criteria[0][0] === "HEADER";
+
 const buildConnection = (): Connection => ({
 	openBox: async () => ({ uidvalidity: 1 }),
 	moveMessages: async () => ({ uidMap: new Map([[10, 20]]) }),
@@ -288,7 +300,7 @@ const buildConnection = (): Connection => ({
 	fetchMessages: async (uids: number[]) => uids.map((uid) => ({ uid })),
 	search: async (...args: unknown[]) => {
 		h.calls.push({ method: "connection.search", args });
-		return [10];
+		return isMessageIdSearch(args[0]) ? h.destinationSearchUids : [10];
 	},
 });
 
@@ -296,7 +308,7 @@ const sourceNoLongerHoldsTheUid = (): void => {
 	h.connection.fetchMessages = async () => [];
 	h.connection.search = async (...args: unknown[]) => {
 		h.calls.push({ method: "connection.search", args });
-		return [];
+		return isMessageIdSearch(args[0]) ? h.destinationSearchUids : [];
 	};
 };
 
@@ -304,6 +316,8 @@ const fresh = (): Harness => ({
 	calls: [],
 	account: { accountId: "acc-1", accountConfigId: "cfg-1" },
 	mailbox: { mailboxId: "src-mbx", uidValidity: 1, cursorState: undefined },
+	messageRow: { messageIdHeader: MESSAGE_ID_HEADER },
+	destinationSearchUids: [],
 	connection: buildConnection(),
 	threadMessage: {
 		...baseThreadMessage,
@@ -328,6 +342,10 @@ const deps = (): MessageDeleteDeps =>
 				},
 			},
 			message: {
+				get: async (messageIds: string[]) => {
+					h.calls.push({ method: "message.get", args: [messageIds] });
+					return h.messageRow ? [h.messageRow] : [];
+				},
 				updateUid: record("message.updateUid"),
 				update: record("message.update"),
 				delete: record("message.delete"),
@@ -417,17 +435,161 @@ describe("handleMessageDelete", () => {
 		assert.equal(h.disconnectCount, 1);
 	});
 
-	it("marks the message failed when the MOVE returns no new uid", async () => {
-		h.connection.moveMessages = async () => ({ uidMap: new Map() });
+	// UIDPLUS is an extension: a server without it answers a perfectly
+	// successful MOVE with no COPYUID entry. An empty uidMap is therefore
+	// UNCONFIRMED, and the destination is asked by Message-ID before any
+	// verdict — the rule every sibling handler already carries. Issue #979.
+	describe("no COPYUID entry on the move to trash", () => {
+		it("settles the row on the probed uid when the message left the source and is at the destination (non-UIDPLUS server, genuine success)", async () => {
+			h.connection.moveMessages = async () => ({ uidMap: new Map() });
+			sourceNoLongerHoldsTheUid();
+			h.destinationSearchUids = [77];
 
-		await handleMessageDelete(moveEvent, noopLog, deps());
+			await handleMessageDelete(moveEvent, noopLog, deps());
 
-		assert.equal(called("message.updateUid").length, 0);
-		assert.equal(
-			(called("message.update")[0]?.args[1] as { syncStatus?: string })
-				?.syncStatus,
-			"failed",
-		);
+			assert.deepEqual(called("message.updateUid")[0]?.args, [
+				"msg-1",
+				77,
+				"trash-mbx",
+			]);
+			assert.deepEqual(called("threadMessage.update")[0]?.args[2], {
+				uid: 77,
+				mailboxId: "trash-mbx",
+				isDeleted: true,
+			});
+			assert.equal(
+				called("message.update").length,
+				0,
+				"a settled move never marks the row failed",
+			);
+		});
+
+		it("probes the DESTINATION mailbox, read-only, by Message-ID", async () => {
+			h.connection.moveMessages = async () => ({ uidMap: new Map() });
+			sourceNoLongerHoldsTheUid();
+			h.destinationSearchUids = [77];
+			const opened: unknown[][] = [];
+			h.connection.openBox = (async (...args: unknown[]) => {
+				opened.push(args);
+				return { uidvalidity: 1 };
+			}) as Connection["openBox"];
+
+			await handleMessageDelete(moveEvent, noopLog, deps());
+
+			assert.deepEqual(
+				opened,
+				[
+					["INBOX", false],
+					["INBOX", true],
+					["Trash", true],
+				],
+				"the source is re-asked read-only, then the destination is EXAMINEd — neither probe may SELECT a box writable",
+			);
+			assert.deepEqual(called("connection.search").at(-1)?.args[0], [
+				["HEADER", "Message-ID", MESSAGE_ID_HEADER],
+			]);
+		});
+
+		it("leaves local state alone when the probe finds nothing — never reverts, never deletes (#655)", async () => {
+			h.connection.moveMessages = async () => ({ uidMap: new Map() });
+			sourceNoLongerHoldsTheUid();
+			h.destinationSearchUids = [];
+
+			await handleMessageDelete(moveEvent, noopLog, deps());
+
+			assert.equal(called("message.updateUid").length, 0);
+			assert.equal(
+				called("message.delete").length,
+				0,
+				"an unconfirmed move must never delete the local row",
+			);
+			assert.equal(
+				called("threadMessage.delete").length,
+				0,
+				"an unconfirmed move must never delete the listing rows",
+			);
+			assert.equal(
+				called("threadMessage.update").length,
+				0,
+				"an unconfirmed move must never revert the listing row to the source",
+			);
+			assert.equal(
+				(called("message.update")[0]?.args[1] as { syncStatus?: string })
+					?.syncStatus,
+				"failed",
+			);
+		});
+
+		it("does not probe when the row carries no Message-ID header", async () => {
+			h.connection.moveMessages = async () => ({ uidMap: new Map() });
+			sourceNoLongerHoldsTheUid();
+			h.messageRow = {};
+
+			await handleMessageDelete(moveEvent, noopLog, deps());
+
+			assert.equal(
+				called("connection.search").filter((c) =>
+					JSON.stringify(c.args).includes("HEADER"),
+				).length,
+				0,
+			);
+			assert.equal(called("message.updateUid").length, 0);
+			assert.equal(
+				(called("message.update")[0]?.args[1] as { syncStatus?: string })
+					?.syncStatus,
+				"failed",
+			);
+		});
+
+		// `searchMailboxByMessageId` returns the LOWEST matching uid, and one
+		// Message-ID can have several server copies in one account while
+		// `deriveMessageId` gives them one local row. A source that still holds
+		// the uid proves the MOVE did not happen, so any hit at the destination
+		// is a different copy.
+		it("never takes a destination hit while the source still holds the uid (duplicate Message-ID)", async () => {
+			h.connection.moveMessages = async () => ({ uidMap: new Map() });
+			h.destinationSearchUids = [100];
+
+			await handleMessageDelete(moveEvent, noopLog, deps());
+
+			assert.equal(
+				called("message.updateUid").length,
+				0,
+				"an earlier copy's uid must never settle this row",
+			);
+			assert.equal(called("threadMessage.update").length, 0);
+			assert.equal(called("message.delete").length, 0);
+			assert.equal(
+				(called("message.update")[0]?.args[1] as { syncStatus?: string })
+					?.syncStatus,
+				"failed",
+			);
+		});
+
+		// The MOVE has already run by the time the probe goes out, so a probe
+		// that cannot answer counts as not-confirmed. Throwing would redeliver
+		// on the account's per-group FIFO and re-MOVE a uid the source no longer
+		// holds, head-of-line blocking every other delete on the account.
+		it("treats an unanswerable probe as unconfirmed rather than throwing", async () => {
+			h.connection.moveMessages = async () => ({ uidMap: new Map() });
+			h.connection.openBox = (async (_path: string, readOnly?: boolean) => {
+				if (readOnly) throw new Error("NO [SERVERBUG] EXAMINE failed");
+				return { uidvalidity: 1 };
+			}) as Connection["openBox"];
+
+			await handleMessageDelete(moveEvent, noopLog, deps());
+
+			assert.equal(called("message.updateUid").length, 0);
+			assert.equal(called("message.delete").length, 0);
+			assert.equal(called("threadMessage.delete").length, 0);
+			assert.equal(called("threadMessage.update").length, 0);
+			assert.equal(
+				(called("message.update")[0]?.args[1] as { syncStatus?: string })
+					?.syncStatus,
+				"failed",
+			);
+			assert.equal(h.disconnectCount, 1);
+		});
 	});
 
 	// The queue handler `JSON.parse`s the body and casts it to WorkerEvent with

package/src/handlers/message-delete.ts

@@ -1,5 +1,6 @@
 import { getClient } from "@remit/backend/client";
 import type {
+	IMessageRepository,
 	IThreadMessageRepository,
 	ThreadMessageItem,
 } from "@remit/data-ports";
@@ -8,6 +9,7 @@ import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
 import type { Logger } from "@remit/logger-lambda";
 import {
 	guardConnectionCursor,
+	type IImapConnection,
 	isCursorRebuildNeeded,
 	isMessageGoneFromOpenMailbox,
 	MailboxCursorPausedError,
@@ -18,6 +20,7 @@ import type { MessageDeleteEvent } from "../events.js";
 import { isNotFoundError } from "../is-not-found.js";
 import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
 import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
+import { searchMailboxByMessageId } from "./message-move.js";
 
 /**
  * Delete every ThreadMessage row that points at this messageId.
@@ -122,6 +125,49 @@ export const buildThreadMessageUndelete = (
 	composites: currentComposites(threadMessage),
 });
 
+/**
+ * Settle a move to Trash the server left unconfirmed, by asking it two
+ * read-only questions instead of one. Both handles wrap the SAME connection but
+ * are scoped to their own mailbox: a `guardConnectionCursor` wrap binds its
+ * checks to the ONE mailbox snapshot it was built with, so the destination must
+ * never be opened through the source's guard.
+ *
+ * The source is asked first, and a source that still holds the uid ends it: the
+ * MOVE did not happen, so nothing at the destination can be this message.
+ * Skipping that question is not safe, because `searchMailboxByMessageId`
+ * returns the LOWEST matching uid rather than the one that just arrived, and
+ * one Message-ID can have several server copies in one account (a sieve
+ * `fileinto` + `keep`, a multi-label store, a resend) while
+ * `deriveMessageId` is folder-independent and gives them one local row. An
+ * ungated probe can hand back an earlier copy's uid, and Empty Trash then
+ * expunges by that uid. It also closes the second half of #912: an empty
+ * `uidMap` can mean the MOVE matched nothing at all.
+ *
+ * A row with no `messageIdHeader` has nothing to probe with, so it stays
+ * unconfirmed rather than guessing.
+ */
+const confirmTrashMoveUid = async (
+	sourceConnection: IImapConnection,
+	destinationConnection: IImapConnection,
+	messageService: Pick<IMessageRepository, "get">,
+	messageId: string,
+	sourceMailboxPath: string,
+	destinationMailboxPath: string,
+	uid: number,
+): Promise<number | null> => {
+	await sourceConnection.openBox(sourceMailboxPath, true);
+	if (!(await isMessageGoneFromOpenMailbox(sourceConnection, uid))) return null;
+
+	const [message] = await messageService.get([messageId]);
+	if (!message?.messageIdHeader) return null;
+
+	return searchMailboxByMessageId(
+		destinationConnection,
+		destinationMailboxPath,
+		message.messageIdHeader,
+	);
+};
+
 export interface MessageDeleteDeps {
 	getClient: typeof getClient;
 	buildLifecycleDeps: typeof buildLifecycleDeps;
@@ -324,15 +370,57 @@ export const handleMessageDelete = async (
 							"Refused to delete: move to trash carries no destination mailbox",
 							"message_delete_missing_destination",
 						);
-					} else if (operation === "move_to_trash" && destinationMailboxPath) {
+					} else if (
+						operation === "move_to_trash" &&
+						destinationMailboxPath &&
+						destinationMailboxId
+					) {
 						// Move to Trash
 						const result = await connection.moveMessages(
 							[uid],
 							destinationMailboxPath,
 						);
-						const newUid = result.uidMap.get(uid);
 
-						if (newUid && destinationMailboxId) {
+						// UIDPLUS is an extension. A server without it answers a
+						// perfectly successful MOVE with no COPYUID entry, so an empty
+						// map is UNCONFIRMED, never evidence the move failed: the server
+						// is asked before any verdict, exactly as `handleMessageMove` and
+						// `attemptMove` do. Reading the empty map as a failure left the
+						// message in Trash under a uid nothing local knew, while the row
+						// kept the SOURCE folder's uid — which Empty Trash then decides
+						// by (issues #979, #665).
+						//
+						// A probe that cannot answer counts as not-confirmed, never as a
+						// throw. The MOVE has already run by this point, so throwing here
+						// would redeliver on the account's per-group FIFO and re-MOVE a
+						// uid the source no longer holds — head-of-line blocking the
+						// whole account's deletes over a transient NO, a renamed folder
+						// or a SEARCH the server refused.
+						const newUid =
+							result.uidMap.get(uid) ??
+							(await confirmTrashMoveUid(
+								connection,
+								rawConnection,
+								messageService,
+								messageId,
+								mailboxPath,
+								destinationMailboxPath,
+								uid,
+							).catch((probeError: unknown) => {
+								log.warn(
+									{
+										messageId,
+										uid,
+										mailboxPath,
+										destinationMailboxPath,
+										probeError,
+									},
+									"Could not confirm the move to trash; keeping local rows",
+								);
+								return null;
+							}));
+
+						if (newUid) {
 							// Update message with new UID in Trash
 							await messageService.updateUid(
 								messageId,
@@ -361,9 +449,33 @@ export const handleMessageDelete = async (
 
 							log.info({ messageId, newUid }, "Message moved to trash");
 						} else {
+							// Unconfirmed, not failed. A MOVE that ran server-side but
+							// dropped before the tagged OK is indistinguishable from one
+							// that never ran, so local state is left exactly as it stands:
+							// nothing is reverted and nothing is deleted. Reverting on that
+							// ambiguity is the blind revert #655 recorded when it was
+							// pulled from PR #652.
+							//
+							// This marks and returns rather than throwing. Throwing is the
+							// shape `handleMessageMove` uses, but it can only carry a
+							// budget: this handler has no `receiveCount`, no MAX_ATTEMPTS
+							// and no exhaustion path, so every redelivery on the account's
+							// per-group FIFO would re-MOVE a uid the source no longer
+							// holds, fail identically, and stall that account's whole
+							// delete pipeline (#287, #289, #290) until the queue's
+							// maxReceiveCount dead-letters it — leaving the row in this
+							// same state, minus the pipeline. Issue #980 wires the budget;
+							// the throw belongs with it, not ahead of it.
 							log.error(
-								{ messageId, uid },
-								"Failed to get new UID after move to trash",
+								{
+									alert: "message_delete_trash_move_unconfirmed",
+									accountId,
+									messageId,
+									uid,
+									mailboxPath,
+									destinationMailboxPath,
+								},
+								"Move to trash unconfirmed: no COPYUID entry, and the server did not confirm the message at the destination; local rows left as they stand",
 							);
 							await messageService.update(messageId, {
 								syncStatus: MessageSyncStatus.failed,

package/src/handlers/message-move.test.ts

@@ -3,6 +3,7 @@ import { afterEach, before, describe, it, mock } from "node:test";
 import { getClient, type RemitClient, setClient } from "@remit/backend/client";
 import type { AccountItem, ThreadMessageItem } from "@remit/data-ports";
 import type { Logger } from "@remit/logger-lambda";
+import type { IImapConnection } from "@remit/mailbox-service";
 import type { MessageMoveEvent } from "../events.js";
 import {
 	buildThreadMessageMoveUpdate,
@@ -11,6 +12,7 @@ import {
 	handleMessageMove,
 	MESSAGE_MOVE_MAX_ATTEMPTS,
 	moveThenResync,
+	searchMailboxByMessageId,
 } from "./message-move.js";
 
 const silentLogger = (() => {
@@ -340,3 +342,85 @@ describe("handleMessageMove — the move's own pending state gates every attempt
 		assert.equal(mailboxGet.mock.calls.length, 0);
 	});
 });
+
+describe("searchMailboxByMessageId — the probe that binds a move to a UID (#912)", () => {
+	const isMessageIdCriterion = (
+		criterion: unknown,
+	): criterion is [string, string, string] =>
+		Array.isArray(criterion) &&
+		criterion.length === 3 &&
+		typeof criterion[0] === "string" &&
+		criterion[0].toUpperCase() === "HEADER" &&
+		typeof criterion[1] === "string" &&
+		criterion[1].toLowerCase() === "message-id" &&
+		typeof criterion[2] === "string";
+
+	const buildDestination = (
+		messages: Array<{ uid: number; messageIdHeader: string }>,
+	): IImapConnection => {
+		const searchAll = () => messages.map((row) => row.uid);
+		return {
+			openBox: async () => ({}) as never,
+			search: async (criteria: unknown[]) => {
+				const criterion = criteria.find(isMessageIdCriterion);
+				if (criterion === undefined) return searchAll();
+				return messages
+					.filter((row) => row.messageIdHeader === criterion[2])
+					.map((row) => row.uid);
+			},
+		} as unknown as IImapConnection;
+	};
+
+	it("asks the folder by Message-ID rather than by an interpolated string", async () => {
+		const sent: unknown[][] = [];
+		const destination = {
+			openBox: async () => ({}) as never,
+			search: async (criteria: unknown[]) => {
+				sent.push(criteria);
+				return [];
+			},
+		} as unknown as IImapConnection;
+
+		await searchMailboxByMessageId(
+			destination,
+			"Archive",
+			'<a"b@example.com>\r\nUID 1',
+		);
+
+		assert.deepStrictEqual(sent, [
+			[["HEADER", "Message-ID", '<a"b@example.com>\r\nUID 1']],
+		]);
+	});
+
+	it("answers null, not the lowest UID, when the folder holds no such message", async () => {
+		const destination = buildDestination([
+			{ uid: 11, messageIdHeader: "<stranger-a@example.com>" },
+			{ uid: 12, messageIdHeader: "<stranger-b@example.com>" },
+		]);
+
+		assert.strictEqual(
+			await searchMailboxByMessageId(
+				destination,
+				"Archive",
+				"<moved@example.com>",
+			),
+			null,
+		);
+	});
+
+	it("answers the UID of the message carrying that Message-ID", async () => {
+		const destination = buildDestination([
+			{ uid: 11, messageIdHeader: "<stranger-a@example.com>" },
+			{ uid: 12, messageIdHeader: "<moved@example.com>" },
+		]);
+
+		assert.strictEqual(
+			await searchMailboxByMessageId(
+				destination,
+				"Archive",
+				"<moved@example.com>",
+			),
+			12,
+		);
+	});
+});

package/src/handlers/message-move.ts

@@ -78,7 +78,7 @@ export const searchMailboxByMessageId = async (
 ): Promise<number | null> => {
 	await connection.openBox(mailboxPath, true);
 	const uids = await connection.search([
-		`HEADER Message-ID "${messageIdHeader}"`,
+		["HEADER", "Message-ID", messageIdHeader],
 	]);
 	return uids[0] ?? null;
 };