@remit/imap-worker 0.0.68 → 0.0.70

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.68",
+  "version": "0.0.70",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/handlers/empty-trash.test.ts

@@ -33,6 +33,10 @@ interface LocalMessage {
 	messageId: string;
 	uid: number;
 	status: string;
+	syncStatus?: string;
+	mailboxId?: string;
+	originalMailboxId?: string;
+	originalUid?: number;
 }
 
 type TrashMailbox = { mailboxId: string; fullPath: string };
@@ -77,6 +81,29 @@ const deleting = (messageId: string, uid: number): LocalMessage => ({
 	status: "deleting",
 });
 
+/** Mid-move into Trash: the folder is written, the uid is still the source's. */
+const movingIntoTrash = (messageId: string, uid: number): LocalMessage => ({
+	messageId,
+	uid,
+	status: "moving",
+	syncStatus: "pending",
+	mailboxId: "trash-mbx",
+	originalMailboxId: "inbox-mbx",
+	originalUid: uid,
+});
+
+/** The same row between a failed attempt and its redelivery. */
+const retryingIntoTrash = (messageId: string, uid: number): LocalMessage => ({
+	...movingIntoTrash(messageId, uid),
+	syncStatus: "failed",
+});
+
+/** The same row after an empty marked the folder, `status` overwritten. */
+const markedMidMove = (messageId: string, uid: number): LocalMessage => ({
+	...movingIntoTrash(messageId, uid),
+	status: "deleting",
+});
+
 const fresh = (): Harness => ({
 	calls: [],
 	account: { accountId: "acc-1", accountConfigId: "cfg-1" },
@@ -417,3 +444,114 @@ describe("handleEmptyTrash", () => {
 		assert.equal(h.disconnectCount, 1);
 	});
 });
+
+/**
+ * Issue #1217. The sweep matched every row the Trash listing returned against
+ * the expunged uids. A row whose move into Trash has not settled carries the
+ * SOURCE folder's uid, so that match answers for whatever Trash holds at that
+ * uid — a different message, whose Message and thread rows the sweep then
+ * deletes while the mail it named is still sitting in the folder it never left.
+ */
+describe("handleEmptyTrash and an unsettled placement", () => {
+	beforeEach(() => {
+		h = fresh();
+	});
+
+	it("deletes no row of a message whose move into Trash has not settled", async () => {
+		h.localMessages = [
+			deleting("msg-1", 10),
+			movingIntoTrash("msg-moving", 10),
+		];
+
+		await handleEmptyTrash(event, noopLog, deps());
+
+		assert.deepEqual(
+			called("message.delete").map((c) => c.args[0]),
+			["msg-1"],
+			"uid 10 is msg-1's; the in-flight row only borrows it from the inbox",
+		);
+		assert.equal(called("threadMessage.delete").length, 1);
+	});
+
+	it("leaves a row whose move is mid-retry rather than sweeping it", async () => {
+		// `syncStatus: failed` under `status: moving` is an ordinary transient
+		// attempt about to be redelivered (`data-ports/message-settlement.ts`),
+		// not a give-up. Either way the pair is a lie while it stands, and the
+		// uid is somebody else's.
+		h.localMessages = [retryingIntoTrash("msg-retrying", 11)];
+
+		await handleEmptyTrash(event, noopLog, deps());
+
+		assert.equal(called("message.delete").length, 0);
+		assert.equal(called("threadMessage.delete").length, 0);
+		assert.deepEqual(revertedMessageIds(), []);
+	});
+
+	it("hands back, rather than binds, a mid-move row an earlier mark rewrote", async () => {
+		// Defence in depth for the shape `emptyTrash` used to write: the mark
+		// overwrote `status` and the row reached here reading settled while its
+		// uid still belonged to the inbox. The uid is not bound — but the mark is
+		// cleared, because nothing else is coming to clear it and a row left
+		// `deleting` is hidden from every listing for good.
+		h.localMessages = [deleting("msg-1", 10), markedMidMove("msg-marked", 11)];
+
+		await handleEmptyTrash(event, noopLog, deps());
+
+		assert.deepEqual(
+			called("message.delete").map((c) => c.args[0]),
+			["msg-1"],
+		);
+		assert.deepEqual(revertedMessageIds(), ["msg-marked"]);
+		assert.deepEqual(undeletedThreadMessageIds(), ["tm-msg-marked"]);
+	});
+
+	it("sweeps a settled row whose Trash uid matches the one it left behind", async () => {
+		// Two folders count uids independently, so a move can land on the same
+		// number it started from. `updateUid` drops `originalUid` when it settles,
+		// which is what keeps this row apart from one still mid-move — without it
+		// the sweep would expunge the message on the server and then refuse its
+		// rows, leaving mail that no longer exists hidden in Trash for good.
+		h.localMessages = [
+			{
+				messageId: "msg-collided",
+				uid: 10,
+				status: "deleting",
+				syncStatus: "pending",
+				mailboxId: "trash-mbx",
+				originalMailboxId: "inbox-mbx",
+			},
+		];
+
+		await handleEmptyTrash(event, noopLog, deps());
+
+		assert.deepEqual(
+			called("message.delete").map((c) => c.args[0]),
+			["msg-collided"],
+		);
+		assert.deepEqual(revertedMessageIds(), []);
+	});
+
+	it("sweeps a settled row that reached Trash by a move", async () => {
+		// The confirmed pair: `updateUid` wrote Trash's own uid over the
+		// source's, leaving `originalUid` behind as history. Refusing this would
+		// leave the folder marked for a deletion that already happened.
+		h.localMessages = [
+			{
+				messageId: "msg-settled",
+				uid: 11,
+				status: "deleting",
+				syncStatus: "pending",
+				mailboxId: "trash-mbx",
+				originalMailboxId: "inbox-mbx",
+				originalUid: 42,
+			},
+		];
+
+		await handleEmptyTrash(event, noopLog, deps());
+
+		assert.deepEqual(
+			called("message.delete").map((c) => c.args[0]),
+			["msg-settled"],
+		);
+	});
+});

package/src/handlers/empty-trash.ts

@@ -5,6 +5,7 @@ import { isCurrentSchemaVersion } from "@remit/data-ports/mutation-events";
 import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
 import type { Logger } from "@remit/logger-lambda";
 import {
+	carriesForeignUid,
 	guardConnectionCursor,
 	isCursorRebuildNeeded,
 	MailboxCursorPausedError,
@@ -248,12 +249,29 @@ export const handleEmptyTrash = async (
 					const expunged = new Set(uids);
 					const localMessages =
 						await messageService.listAllByMailbox(trashMailboxId);
-					let deletedCount = 0;
 
-					for (const message of localMessages) {
-						if (!expunged.has(message.uid)) continue;
-						deletedCount += 1;
+					// Reconciles, never waits (imap-mutations R2). A row whose move
+					// into Trash has not settled names this folder while still
+					// carrying the SOURCE folder's uid, so matching it against the
+					// expunge answers for whatever Trash held at that uid — another
+					// message, deleted here in both its rows. Waiting is not open to
+					// this handler the way it is to the API-side mutators: every event
+					// of an account shares one FIFO group, so the move that would
+					// settle the row cannot run until this returns, and the ceiling
+					// would be spent to reach the same answer.
+					//
+					// `carriesForeignUid`, not the placement binding: the binding reads
+					// `status`, and `status` is exactly what an operation marking this
+					// folder overwrites. This is the second half of the gate
+					// `emptyTrash` applies before it marks anything.
+					const swept = localMessages.filter(
+						(message) =>
+							!carriesForeignUid(message) && expunged.has(message.uid),
+					);
+					const sweptIds = new Set(swept.map((message) => message.messageId));
+					const deletedCount = swept.length;
 
+					for (const message of swept) {
 						await messageService.delete(message.messageId);
 
 						const threadMessage = await threadMessageService.findByMessageId(
@@ -268,12 +286,16 @@ export const handleEmptyTrash = async (
 						}
 					}
 
-					// Everything the SEARCH did not name is still marked for a deletion
-					// that will never come: mail another client already emptied, mail
-					// that reached Trash after the SEARCH, or a redelivery finishing a
-					// partial sweep. All three must come back rather than sit invisible.
+					// Everything this sweep did not carry through is still marked for a
+					// deletion that will never come: mail another client already
+					// emptied, mail that reached Trash after the SEARCH, a redelivery
+					// finishing a partial sweep, and a row whose uid the sweep would
+					// not bind. All of them must come back rather than sit invisible —
+					// the last one especially, since nothing else is coming to clear
+					// its mark, and a row hidden in a folder it never left is worse
+					// than one the next sync re-projects.
 					const revertedCount = await handBackMarkedRows(
-						localMessages.filter((message) => !expunged.has(message.uid)),
+						localMessages.filter((message) => !sweptIds.has(message.messageId)),
 					);
 
 					log.info(

package/src/handlers/message-copy.test.ts

@@ -157,6 +157,7 @@ const deps = (): MessageCopyDeps =>
 			_log: unknown,
 			cb: (credentials: unknown) => Promise<void>,
 		) => cb({}),
+		emitEvent: record("emitEvent"),
 		createConnectionScope: () => ({
 			getConnection: async () => {
 				h.getConnectionCount += 1;
@@ -369,7 +370,12 @@ describe("handleMessageCopy", () => {
 		assert.equal(called("message.update").length, 0);
 	});
 
-	it("skips the copy without opening a connection when the cursor is rebuilding", async () => {
+	// Issue #1203. Both pauses are reached before `copyMessages`, so the COPY
+	// was never issued. Acking left the optimistic row at `uid: 0`/`moving`
+	// with nothing to re-enqueue it and no folder set for the cursor rebuild to
+	// find it in, which is `holdsCopyOf`'s row-existence answer turning into a
+	// copy that can never settle.
+	it("reconciles the optimistic copy row away when the cursor is rebuilding", async () => {
 		h.mailbox = {
 			mailboxId: "src-mbx",
 			uidValidity: 1,
@@ -380,9 +386,99 @@ describe("handleMessageCopy", () => {
 
 		assert.equal(h.getConnectionCount, 0);
 		assert.equal(called("message.updateUid").length, 0);
+		assert.deepEqual(called("message.delete")[0]?.args, ["new-msg"]);
+		assert.equal(called("threadMessage.deleteMany").length, 1);
+	});
+
+	// A redelivery's earlier attempt may have copied: the tagged OK can be lost
+	// with the connection. Deleting this row on the pause alone would leave the
+	// landed copy with no row of its own — `holdsCopyOf` answers on row
+	// existence, so no sync makes a second one — and the next destination sync
+	// would repoint the SOURCE row out of the folder the server still holds it
+	// in. The pause is on the source; the destination is asked before anything
+	// is deleted.
+	it("asks the destination before reconciling a redelivered copy onto a paused cursor", async () => {
+		h.mailbox = {
+			mailboxId: "src-mbx",
+			uidValidity: 1,
+			cursorState: "rebuilding",
+		};
+		h.destinationHolds = [42];
+
+		await handleMessageCopy(event, noopLog, 2, deps());
+
+		assert.ok(askedTheDestination(), "the destination was asked for the copy");
+		assert.deepEqual(called("message.updateUid")[0]?.args, [
+			"new-msg",
+			42,
+			"dst-mbx",
+		]);
+		assert.equal(
+			called("message.delete").length,
+			0,
+			"a copy the destination confirms is never reconciled away",
+		);
+		assert.equal(h.disconnectCount, 1);
+	});
+
+	it("settles a redelivered copy broken when the destination cannot be asked", async () => {
+		h.mailbox = {
+			mailboxId: "src-mbx",
+			uidValidity: 1,
+			cursorState: "rebuilding",
+		};
+		h.copyRow = { ...unsettledCopyRow(), messageIdHeader: undefined };
+
+		await handleMessageCopy(event, noopLog, 2, deps());
+
+		assert.equal(
+			called("message.delete").length,
+			0,
+			"silence is never grounds to delete a row that may describe real mail",
+		);
+		assert.equal(
+			(called("message.update")[0]?.args[1] as { status?: string })?.status,
+			"deleted",
+		);
+	});
+
+	// A paused settle is a settle, and this handler had no way to say so: it
+	// carried no `emitEvent` at all, so the destination — which is not paused
+	// and has no rebuild coming — was never told to re-project. The move and
+	// delete handlers have run this resync since #1203.
+	it("resyncs both folders after a paused copy settles", async () => {
+		h.mailbox = {
+			mailboxId: "src-mbx",
+			uidValidity: 1,
+			cursorState: "rebuilding",
+		};
+
+		await handleMessageCopy(event, noopLog, 1, deps());
+
+		assert.deepEqual(
+			called("emitEvent").map((c) => c.args[0]),
+			[
+				{ type: "SYNC_MESSAGES", accountId: "acc-1", mailboxId: "src-mbx" },
+				{ type: "SYNC_MESSAGES", accountId: "acc-1", mailboxId: "dst-mbx" },
+			],
+		);
+	});
+
+	it("reconciles a redelivered copy away when the destination denies holding it", async () => {
+		h.mailbox = {
+			mailboxId: "src-mbx",
+			uidValidity: 1,
+			cursorState: "rebuilding",
+		};
+		h.destinationHolds = [];
+
+		await handleMessageCopy(event, noopLog, 2, deps());
+
+		assert.ok(askedTheDestination());
+		assert.deepEqual(called("message.delete")[0]?.args, ["new-msg"]);
 	});
 
-	it("pauses quietly when openBox trips a UIDVALIDITY mismatch", async () => {
+	it("reconciles the optimistic copy row away when openBox trips a UIDVALIDITY mismatch", async () => {
 		h.connection.openBox = async () => ({ uidvalidity: 999 });
 
 		await handleMessageCopy(event, noopLog, 1, deps());
@@ -394,6 +490,7 @@ describe("handleMessageCopy", () => {
 			"the mismatch trips the mailbox cursor",
 		);
 		assert.equal(called("message.updateUid").length, 0);
+		assert.deepEqual(called("message.delete")[0]?.args, ["new-msg"]);
 		assert.equal(h.disconnectCount, 1);
 	});
 

package/src/handlers/message-copy.ts

@@ -12,17 +12,22 @@ import {
 import { attemptBudget } from "@remit/sqs-client/attempt-budget";
 import { isAccountDeleted } from "../account-check.js";
 import { createConnectionScopeWithCredentials } from "../connection-scope.js";
+import { emitEvent } from "../emit.js";
 import type { MessageCopyEvent } from "../events.js";
 import { isNotFoundError } from "../is-not-found.js";
 import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
 import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
-import { searchMailboxForHighestMessageIdUid } from "./message-move.js";
+import {
+	emitMoveResync,
+	searchMailboxForHighestMessageIdUid,
+} from "./message-move.js";
 
 export interface MessageCopyDeps {
 	getClient: typeof getClient;
 	buildLifecycleDeps: typeof buildLifecycleDeps;
 	withOAuthLifecycle: typeof withOAuthLifecycle;
 	createConnectionScope: typeof createConnectionScopeWithCredentials;
+	emitEvent: typeof emitEvent;
 }
 
 const defaultDeps: MessageCopyDeps = {
@@ -30,6 +35,7 @@ const defaultDeps: MessageCopyDeps = {
 	buildLifecycleDeps,
 	withOAuthLifecycle,
 	createConnectionScope: createConnectionScopeWithCredentials,
+	emitEvent,
 };
 
 export const getMessageCopyMaxAttempts = (
@@ -83,6 +89,7 @@ export const handleMessageCopy = async (
 		buildLifecycleDeps,
 		withOAuthLifecycle,
 		createConnectionScope: createConnectionScopeWithCredentials,
+		emitEvent,
 	} = deps;
 
 	const {
@@ -172,17 +179,6 @@ export const handleMessageCopy = async (
 				return;
 			}
 
-			// Cheap frugal skip (epic #1281 invariant 6): a mailbox already known
-			// paused never even opens a connection. Optimization only — the
-			// guardConnectionCursor openBox wrap below is the structural guarantee.
-			if (isCursorRebuildNeeded(mailbox.cursorState)) {
-				log.info(
-					{ accountId, sourceMessageId, mailboxId: sourceMailboxId },
-					"Mailbox cursor not normal; pausing outbound copy this round",
-				);
-				return;
-			}
-
 			// The copy row carries the source's Message-ID header, so the
 			// destination can be asked where this copy is. Always on the UNGUARDED
 			// handle: a `guardConnectionCursor` wrap is bound to the SOURCE mailbox
@@ -317,6 +313,78 @@ export const handleMessageCopy = async (
 
 			const scope = createConnectionScopeWithCredentials(account, credentials);
 
+			// A paused cursor settles rather than acks: acking leaves the optimistic
+			// row at `uid: 0`/`moving` with nothing to re-enqueue it and no folder
+			// set for the cursor rebuild to find it in (issue #1203). Like every
+			// other verdict on this row, it reconciles rather than waits (R2,
+			// docs/architecture/imap-mutations.md) — the settle below IS the
+			// reconciliation path, since the destination is not paused and no
+			// rebuild is coming for it.
+			//
+			// What may be concluded from the pause depends on what was issued, and
+			// `receiveCount` alone does not say: `commandIssued` is what this round
+			// did, a redelivery is what an earlier one may have done and lost with
+			// its connection. Where neither says the server may have been asked, the
+			// row records something the server was never told about and reconciling
+			// it away is exact. Otherwise this row may be the only record real mail
+			// will ever have, and deleting it would let the next destination sync
+			// repoint the SOURCE row out of the folder the server still holds it in.
+			// The pause is on the source and says nothing about the destination,
+			// which is probed on the unguarded handle anyway, so the destination is
+			// asked before anything is deleted and silence settles broken.
+			//
+			// A settle is a settle, so both folders re-read their counts from IMAP,
+			// the same resync the move and delete handlers run: the destination is
+			// not paused, so nothing else is coming to re-project it.
+			const settlePausedCopy = async (
+				commandIssued: boolean,
+			): Promise<void> => {
+				const mayHaveIssued = commandIssued || receiveCount > 1;
+
+				if (!mayHaveIssued) {
+					await settleNeverLanded(
+						"source mailbox cursor paused before the copy",
+					);
+				} else {
+					const probe = await probeDestination(await scope.getConnection());
+					if (probe.kind === "confirmed") {
+						await settleCopied(probe.uid);
+					} else if (probe.kind === "absent") {
+						await settleNeverLanded(
+							"redelivered onto a paused source cursor, absent from destination",
+						);
+					} else {
+						await settleBroken(
+							"redelivered onto a paused source cursor with no Message-ID to ask the destination with",
+						);
+					}
+				}
+
+				await emitMoveResync(emitEvent, {
+					accountId,
+					sourceMailboxId,
+					destinationMailboxId,
+				});
+			};
+
+			// Cheap frugal skip (epic #1281 invariant 6): a mailbox already known
+			// paused never even opens a connection on a first delivery. Optimization
+			// only — the guardConnectionCursor openBox wrap below is the structural
+			// guarantee.
+			if (isCursorRebuildNeeded(mailbox.cursorState)) {
+				log.info(
+					{ accountId, sourceMessageId, mailboxId: sourceMailboxId },
+					"Mailbox cursor not normal; pausing outbound copy this round",
+				);
+				await settlePausedCopy(false).finally(() => scope.disconnect());
+				return;
+			}
+
+			// Flipped the instant the COPY leaves, so every settle downstream of it
+			// knows the server was asked. IMAP COPY is not idempotent, so this is
+			// the flag that must never be inferred.
+			let commandIssued = false;
+
 			await scope
 				.getConnection()
 				.then(async (rawConnection) => {
@@ -354,6 +422,7 @@ export const handleMessageCopy = async (
 					await connection.openBox(sourceMailboxPath, true);
 
 					// Execute IMAP COPY
+					commandIssued = true;
 					const result = await connection.copyMessages(
 						[uid],
 						destinationMailboxPath,
@@ -395,6 +464,7 @@ export const handleMessageCopy = async (
 							},
 							"Mailbox cursor not normal; pausing outbound copy this round",
 						);
+						await settlePausedCopy(commandIssued);
 						return;
 					}
 

package/src/handlers/message-delete-terminal.ts

@@ -2,6 +2,7 @@ import type {
 	IMessageRepository,
 	IThreadMessageRepository,
 } from "@remit/data-ports";
+import { MessageSyncStatus } from "@remit/domain-enums";
 import {
 	type IImapConnection,
 	isMessageGoneFromOpenMailbox,
@@ -17,7 +18,7 @@ export interface MessageDeleteTerminalLogger {
 
 export interface ResolveExhaustedMessageDeleteDeps
 	extends StaleMessageReconcileDeps {
-	messageService: Pick<IMessageRepository, "delete" | "updateUid">;
+	messageService: Pick<IMessageRepository, "delete" | "updateForMove">;
 	threadMessageService: Pick<
 		IThreadMessageRepository,
 		"findAllByMessageId" | "deleteMany" | "update"
@@ -130,6 +131,10 @@ export const resolveExhaustedMessageDeleteFailure = async (
 		messageId,
 		sourceMailboxId,
 		uid,
+		// The probe above just confirmed the pair, so the row is a faithful
+		// projection of the source again. The delete's own failure is the alert,
+		// not a `failed` left on a row nothing is still trying to delete.
+		syncStatus: MessageSyncStatus.synced,
 	});
 
 	return { outcome: "broken" };