@remit/imap-worker 0.0.71 → 0.0.73

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.71",
+  "version": "0.0.73",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -39,7 +39,6 @@
     "@remit/logger-lambda": "*",
     "@remit/mailbox-service": "*",
     "@remit/search-index-worker": "*",
-    "@remit/search-service": "*",
     "@remit/sqs-client": "*",
     "@remit/backend": "*",
     "@remit/domain-enums": "*",

package/src/handlers/append-sent-message.ts

@@ -4,6 +4,7 @@ import {
 	APPENDED_UID_UNREPORTED,
 	isSentCopyFiled,
 } from "@remit/data-ports";
+import { isNotFoundError } from "@remit/data-ports/errors";
 import { OutboxMessageStatus } from "@remit/domain-enums";
 import type { Logger } from "@remit/logger-lambda";
 import {
@@ -14,7 +15,6 @@ import { attemptBudget } from "@remit/sqs-client/attempt-budget";
 import { isAccountDeleted } from "../account-check.js";
 import { createConnectionScopeWithCredentials } from "../connection-scope.js";
 import type { AppendSentMessageEvent } from "../events.js";
-import { isNotFoundError } from "../is-not-found.js";
 import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
 import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
 

package/src/handlers/empty-trash.test.ts

@@ -45,6 +45,8 @@ interface Harness {
 	localMessages: LocalMessage[];
 	threadMessage: boolean;
 	messagesWithoutListingRow: string[];
+	/** Message ids whose placement another lane has changed under the sweep. */
+	transitionsLost: string[];
 	getConnectionCount: number;
 	disconnectCount: number;
 }
@@ -88,6 +90,25 @@ const retryingIntoTrash = (messageId: string, uid: number): LocalMessage => ({
 	syncStatus: "failed",
 });
 
+/**
+ * A row whose move into Trash SETTLED, on a database written before #1217
+ * taught `updateUid` to clear `originalUid`. Two folders count uids
+ * independently, so Trash handing back the source's own number is ordinary —
+ * and the pair it leaves is indistinguishable from a move still in flight.
+ */
+const settledWithStaleOriginalUid = (
+	messageId: string,
+	uid: number,
+): LocalMessage => ({
+	messageId,
+	uid,
+	status: "deleting",
+	syncStatus: "pending",
+	mailboxId: "trash-mbx",
+	originalMailboxId: "inbox-mbx",
+	originalUid: uid,
+});
+
 /** The same row after an empty marked the folder, `status` overwritten. */
 const markedMidMove = (messageId: string, uid: number): LocalMessage => ({
 	...movingIntoTrash(messageId, uid),
@@ -106,6 +127,7 @@ const fresh = (): Harness => ({
 	localMessages: [deleting("msg-1", 10), deleting("msg-2", 11)],
 	threadMessage: true,
 	messagesWithoutListingRow: [],
+	transitionsLost: [],
 	getConnectionCount: 0,
 	disconnectCount: 0,
 });
@@ -140,6 +162,18 @@ const deps = (): EmptyTrashDeps =>
 				listAllByMailbox: async () => h.localMessages,
 				delete: record("message.delete"),
 				update: record("message.update"),
+				transitionPlacement: async (
+					messageId: string,
+					expected: unknown,
+					next: unknown,
+				) => {
+					const won = !h.transitionsLost.includes(messageId);
+					h.calls.push({
+						method: "message.transitionPlacement",
+						args: [messageId, expected, next, won],
+					});
+					return won ? { messageId } : undefined;
+				},
 			},
 			threadMessage: {
 				findByMessageId: async (_cfg: string, messageId: string) =>
@@ -193,13 +227,16 @@ const event: EmptyTrashEvent = {
 const called = (method: string): Call[] =>
 	h.calls.filter((c) => c.method === method);
 
+// Only a transition that WON reverted anything; a lost predicate is an offered
+// write that the row refused.
 const revertedMessageIds = (): string[] =>
-	called("message.update")
+	called("message.transitionPlacement")
 		.filter(
 			(c) =>
-				(c.args[1] as { status?: string; syncStatus?: string }).status ===
+				c.args[3] === true &&
+				(c.args[2] as { status?: string; syncStatus?: string }).status ===
 					"active" &&
-				(c.args[1] as { syncStatus?: string }).syncStatus === "synced",
+				(c.args[2] as { syncStatus?: string }).syncStatus === "synced",
 		)
 		.map((c) => c.args[0] as string);
 
@@ -498,6 +535,84 @@ describe("handleEmptyTrash and an unsettled placement", () => {
 		assert.deepEqual(undeletedThreadMessageIds(), ["tm-msg-marked"]);
 	});
 
+	it("hands nothing back for a row another lane settled after the sweep read it", async () => {
+		// The hand-back is a transition off the row as this sweep read it
+		// (imap-mutations R3). PLACEMENT_MOVE_PUSH rides a standard queue that
+		// this account's FIFO group does not order, so it can settle a row
+		// mid-sweep; writing `active` + `synced` over that would declare a
+		// mutation settled that nobody confirmed, and take the listing row with
+		// it.
+		h.localMessages = [deleting("msg-1", 10), markedMidMove("msg-raced", 11)];
+		h.transitionsLost = ["msg-raced"];
+
+		await handleEmptyTrash(event, noopLogger, deps());
+
+		assert.deepEqual(
+			called("message.transitionPlacement").map((c) => c.args[0]),
+			["msg-raced"],
+			"the predicate was still offered — the row just lost it",
+		);
+		assert.deepEqual(revertedMessageIds(), []);
+		assert.deepEqual(
+			undeletedThreadMessageIds(),
+			[],
+			"a lost predicate writes nothing at all, listing rows included",
+		);
+	});
+
+	it("predicates the hand-back on the placement the sweep actually read", async () => {
+		h.localMessages = [markedMidMove("msg-marked", 11)];
+
+		await handleEmptyTrash(event, noopLogger, deps());
+
+		const [call] = called("message.transitionPlacement");
+		assert.deepEqual(call?.args[1], {
+			status: "deleting",
+			mailboxId: "trash-mbx",
+			uid: 11,
+		});
+	});
+
+	// Issue #1230. The exclusion used to run after `deleteMessages`, so a row
+	// the sweep refused had already lost its server copy: mail that no longer
+	// exists, visible in Trash, with nothing able to clear it — pressing Empty
+	// Trash again skips it too, because the server no longer lists the uid.
+	it("never expunges a uid whose row it will refuse to remove", async () => {
+		h.localMessages = [
+			deleting("msg-1", 10),
+			settledWithStaleOriginalUid("msg-stale", 11),
+		];
+
+		await handleEmptyTrash(event, noopLogger, deps());
+
+		assert.deepEqual(
+			called("connection.deleteMessages")[0]?.args,
+			[[10]],
+			"uid 11 survives on the server, because its row survives locally",
+		);
+		assert.deepEqual(
+			called("message.delete").map((c) => c.args[0]),
+			["msg-1"],
+		);
+		assert.deepEqual(
+			revertedMessageIds(),
+			["msg-stale"],
+			"and the refused row is handed back rather than left marked",
+		);
+	});
+
+	it("issues no expunge at all when every uid in the folder is refused", async () => {
+		h.localMessages = [
+			settledWithStaleOriginalUid("msg-stale-a", 10),
+			settledWithStaleOriginalUid("msg-stale-b", 11),
+		];
+
+		await handleEmptyTrash(event, noopLogger, deps());
+
+		assert.equal(called("connection.deleteMessages").length, 0);
+		assert.equal(called("message.delete").length, 0);
+	});
+
 	it("sweeps a settled row whose Trash uid matches the one it left behind", async () => {
 		// Two folders count uids independently, so a move can land on the same
 		// number it started from. `updateUid` drops `originalUid` when it settles,

package/src/handlers/empty-trash.ts

@@ -1,10 +1,12 @@
 import { getClient } from "@remit/backend/client";
 import type { MessageItem } from "@remit/data-ports";
+import { isNotFoundError } from "@remit/data-ports/errors";
 import { trashMailboxAt } from "@remit/data-ports/folder-role";
 import { isCurrentSchemaVersion } from "@remit/data-ports/mutation-events";
 import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
 import type { Logger } from "@remit/logger-lambda";
 import {
+	buildThreadMessageUndelete,
 	carriesForeignUid,
 	guardConnectionCursor,
 	isCursorRebuildNeeded,
@@ -13,10 +15,8 @@ import {
 import { isAccountDeleted } from "../account-check.js";
 import { createConnectionScopeWithCredentials } from "../connection-scope.js";
 import type { EmptyTrashEvent } from "../events.js";
-import { isNotFoundError } from "../is-not-found.js";
 import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
 import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
-import { buildThreadMessageUndelete } from "./thread-message-rows.js";
 
 export interface EmptyTrashDeps {
 	getClient: typeof getClient;
@@ -89,7 +89,10 @@ export const handleEmptyTrash = async (
 	// an unsettled move is rewritten by its own MESSAGE_DELETE. Not `failed`:
 	// the mail is intact, and saying otherwise about a whole folder is a lie.
 	const handBackMarkedRows = async (
-		rows: readonly { messageId: string; status: MessageItem["status"] }[],
+		rows: readonly Pick<
+			MessageItem,
+			"messageId" | "status" | "mailboxId" | "uid"
+		>[],
 	): Promise<number> => {
 		let revertedCount = 0;
 		for (const message of rows) {
@@ -105,10 +108,30 @@ export const handleEmptyTrash = async (
 			);
 			if (threadMessages.length === 0) continue;
 
-			await messageService.update(message.messageId, {
-				status: MessageStatus.active,
-				syncStatus: MessageSyncStatus.synced,
-			});
+			// The hand-back is a transition off the row as this sweep read it
+			// (imap-mutations R3), because the snapshot it walks holds nothing:
+			// between the listing and here another lane can settle the row onto a
+			// placement of its own, and writing `active` + `synced` over that would
+			// declare a mutation settled that nobody confirmed.
+			const restored = await messageService.transitionPlacement(
+				message.messageId,
+				{
+					status: MessageStatus.deleting,
+					mailboxId: message.mailboxId,
+					uid: message.uid,
+				},
+				{
+					status: MessageStatus.active,
+					syncStatus: MessageSyncStatus.synced,
+				},
+			);
+			if (!restored) {
+				log.info(
+					{ accountId, messageId: message.messageId },
+					"Empty Trash left a row alone: its placement changed after the sweep read it",
+				);
+				continue;
+			}
 			for (const threadMessage of threadMessages) {
 				const args = buildThreadMessageUndelete(threadMessage);
 				await threadMessageService.update(
@@ -234,39 +257,75 @@ export const handleEmptyTrash = async (
 						return;
 					}
 
-					const uids = await connection.search(["ALL"]);
+					// One read, before the expunge, decides both what is destroyed on
+					// the server and what is removed locally — so the two sets are the
+					// same set by construction and no message can be expunged that the
+					// sweep then declines to remove (issue #1230). Reading afterwards
+					// put the exclusion downstream of the destruction: a refused row
+					// had already lost its server copy, could not be refused into
+					// existence again, and nothing routine repaired it.
+					//
+					// Reconciles, never waits (imap-mutations R2). A row whose move
+					// into Trash has not settled names this folder while still
+					// carrying the SOURCE folder's uid, so matching it against the
+					// expunge answers for whatever Trash held at that uid — another
+					// message, deleted here in both its rows. Waiting is not open to
+					// this handler the way it is to the API-side mutators: a move on
+					// this account's own FIFO group cannot run until this returns, so
+					// the ceiling would be spent to reach the same answer. It buys
+					// nothing against the other lane either — PLACEMENT_MOVE_PUSH
+					// rides a standard queue with no group at all
+					// (`deploy/vps/queues.json`), so it is ordered against nothing
+					// here and can settle a row mid-sweep. That is why the writes
+					// below are transitions rather than snapshot writes (R3).
+					//
+					// `carriesForeignUid`, not the placement binding: the binding reads
+					// `status`, and `status` is exactly what an operation marking this
+					// folder overwrites. This is the same predicate `emptyTrash`
+					// applies before it marks anything — one gate, one answer, now
+					// asked once and honoured on both sides of the expunge.
+					const localMessages =
+						await messageService.listAllByMailbox(trashMailboxId);
+					const sweepable = localMessages.filter(
+						(message) => !carriesForeignUid(message),
+					);
+
+					// A refused row spares its uid only where no row the sweep will
+					// carry claims the same number. The two cases the predicate cannot
+					// tell apart part here: a row genuinely mid-move borrows a uid that
+					// belongs to a different message in this folder, and sparing that
+					// number would strand the message that owns it; a settled row whose
+					// `originalUid` a pre-#1217 build left behind owns its uid outright,
+					// and expunging it destroys mail the sweep then refuses to remove.
+					const sweepableUids = new Set(
+						sweepable.map((message) => message.uid),
+					);
+					const refusedUids = new Set(
+						localMessages
+							.filter(
+								(message) =>
+									carriesForeignUid(message) && !sweepableUids.has(message.uid),
+							)
+							.map((message) => message.uid),
+					);
+
+					const uids = (await connection.search(["ALL"])).filter(
+						(uid) => !refusedUids.has(uid),
+					);
 
 					if (uids.length > 0) {
 						await connection.deleteMessages(uids);
 						log.info(
-							{ count: uids.length },
+							{ count: uids.length, refused: refusedUids.size },
 							"Deleted messages from IMAP trash",
 						);
 					}
 
-					// Local cleanup follows the expunge, uid by uid. What was expunged
-					// is a fact this connection observed, and only those rows go.
+					// What was expunged is a fact this connection observed, and only
+					// those rows go.
 					const expunged = new Set(uids);
-					const localMessages =
-						await messageService.listAllByMailbox(trashMailboxId);
-
-					// Reconciles, never waits (imap-mutations R2). A row whose move
-					// into Trash has not settled names this folder while still
-					// carrying the SOURCE folder's uid, so matching it against the
-					// expunge answers for whatever Trash held at that uid — another
-					// message, deleted here in both its rows. Waiting is not open to
-					// this handler the way it is to the API-side mutators: every event
-					// of an account shares one FIFO group, so the move that would
-					// settle the row cannot run until this returns, and the ceiling
-					// would be spent to reach the same answer.
-					//
-					// `carriesForeignUid`, not the placement binding: the binding reads
-					// `status`, and `status` is exactly what an operation marking this
-					// folder overwrites. This is the second half of the gate
-					// `emptyTrash` applies before it marks anything.
-					const swept = localMessages.filter(
-						(message) =>
-							!carriesForeignUid(message) && expunged.has(message.uid),
+					const swept = sweepable.filter((message) =>
+						expunged.has(message.uid),
 					);
 					const sweptIds = new Set(swept.map((message) => message.messageId));
 					const deletedCount = swept.length;

package/src/handlers/flag-push-placement-race.test.ts

@@ -26,7 +26,8 @@ import type {
 	MessageFlagPushItem,
 	MessageItem,
 	MessagePlacementMoveItem,
-	UpdateMessageMoveInput,
+	PlacementPredicate,
+	PlacementTransitionInput,
 } from "@remit/data-ports";
 import type { Logger } from "@remit/logger-lambda";
 import {
@@ -87,11 +88,24 @@ const buildHarness = (): Harness => {
 			assert.equal(messageId, MESSAGE_ID);
 			return row;
 		},
-		updateForMove: async (
+		transitionPlacement: async (
 			_messageId: string,
-			input: UpdateMessageMoveInput,
+			expected: PlacementPredicate,
+			next: PlacementTransitionInput,
 		) => {
-			Object.assign(row, input);
+			if (expected.status !== undefined && expected.status !== row.status) {
+				return undefined;
+			}
+			if (expected.uid !== undefined && expected.uid !== row.uid) {
+				return undefined;
+			}
+			if (
+				expected.mailboxId !== undefined &&
+				expected.mailboxId !== row.mailboxId
+			) {
+				return undefined;
+			}
+			Object.assign(row, next);
 			return row;
 		},
 	} as unknown as IMessageRepository;

package/src/handlers/flag-push.ts

@@ -1,4 +1,5 @@
 import { getClient } from "@remit/backend/client";
+import { isNotFoundError } from "@remit/data-ports/errors";
 import type { Logger } from "@remit/logger-lambda";
 import { recordImapFailure } from "@remit/logger-lambda";
 import {
@@ -12,7 +13,6 @@ import { attemptBudget } from "@remit/sqs-client/attempt-budget";
 import { isAccountDeleted } from "../account-check.js";
 import { createConnectionScopeWithCredentials } from "../connection-scope.js";
 import type { FlagPushEvent } from "../events.js";
-import { isNotFoundError } from "../is-not-found.js";
 import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
 import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
 

package/src/handlers/mailbox-management.ts

@@ -1,4 +1,5 @@
 import { getClient } from "@remit/backend/client";
+import { isNotFoundError } from "@remit/data-ports/errors";
 import { MailboxSyncStatus } from "@remit/domain-enums";
 import type { Logger } from "@remit/logger-lambda";
 import { MailboxManagementService } from "@remit/mailbox-service";
@@ -10,7 +11,6 @@ import type {
 	MailboxManagementEvent,
 	MailboxRenameEvent,
 } from "../events.js";
-import { isNotFoundError } from "../is-not-found.js";
 import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
 import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
 

package/src/handlers/message-copy.test.ts

@@ -112,6 +112,10 @@ const deps = (): MessageCopyDeps =>
 				},
 				updateUid: record("message.updateUid"),
 				update: record("message.update"),
+				transitionPlacement: async (...args: unknown[]) => {
+					h.calls.push({ method: "message.transitionPlacement", args });
+					return { messageId: "new-msg" };
+				},
 				delete: record("message.delete"),
 			},
 			threadMessage: {
@@ -191,11 +195,9 @@ describe("handleMessageCopy", () => {
 			20,
 			"dst-mbx",
 		]);
-		const statusUpdate = called("message.update")[0];
-		assert.equal(
-			(statusUpdate?.args[1] as { syncStatus?: string })?.syncStatus,
-			"synced",
-		);
+		// `updateUid` writes the confirmed uid, `active` and `synced` in one
+		// statement, so there is no second, unpredicated write behind it.
+		assert.equal(called("message.update").length, 0);
 		assert.equal(called("threadMessage.update").length, 1);
 		assert.equal(h.disconnectCount, 1, "the scope is always disconnected");
 	});
@@ -212,11 +214,7 @@ describe("handleMessageCopy", () => {
 			42,
 			"dst-mbx",
 		]);
-		assert.equal(
-			(called("message.update")[0]?.args[1] as { syncStatus?: string })
-				?.syncStatus,
-			"synced",
-		);
+		assert.equal(called("message.update").length, 0);
 		assert.equal(called("threadMessage.update").length, 1);
 	});
 
@@ -263,11 +261,15 @@ describe("handleMessageCopy", () => {
 
 		assert.equal(copies, 1, "the COPY is issued once");
 		assert.equal(called("search").length, 0, "there is nothing to ask with");
-		const update = called("message.update")[0];
-		assert.equal((update?.args[1] as { status?: string })?.status, "deleted");
-		assert.equal(
-			(update?.args[1] as { syncStatus?: string })?.syncStatus,
-			"failed",
+		const [givenUp] = called("message.transitionPlacement");
+		assert.deepEqual(
+			givenUp?.args[2],
+			{
+				status: "deleted",
+				syncStatus: "abandoned",
+				abandonedMutation: "copy",
+			},
+			"a copy that gave up carries the give-up value and names itself",
 		);
 		assert.equal(called("message.delete").length, 0, "no row is thrown away");
 	});
@@ -302,8 +304,8 @@ describe("handleMessageCopy", () => {
 
 		assert.equal(copies, 1, "the redelivery issues no second COPY");
 		assert.equal(called("search").length, 0);
-		const settled = called("message.update").at(-1);
-		assert.equal((settled?.args[1] as { status?: string })?.status, "deleted");
+		const settled = called("message.transitionPlacement").at(-1);
+		assert.equal((settled?.args[2] as { status?: string })?.status, "deleted");
 	});
 
 	it("acks a copy that already settled without touching IMAP", async () => {
@@ -327,6 +329,7 @@ describe("handleMessageCopy", () => {
 
 		assert.equal(h.getConnectionCount, 0);
 		assert.equal(called("message.update").length, 0);
+		assert.equal(called("message.transitionPlacement").length, 0);
 	});
 
 	it("returns early without connecting when the account is soft-deleted", async () => {
@@ -358,6 +361,7 @@ describe("handleMessageCopy", () => {
 		assert.equal(h.getConnectionCount, 0);
 		assert.equal(called("message.updateUid").length, 0);
 		assert.equal(called("message.update").length, 0);
+		assert.equal(called("message.transitionPlacement").length, 0);
 	});
 
 	// Issue #1203. Both pauses are reached before `copyMessages`, so the COPY
@@ -427,7 +431,11 @@ describe("handleMessageCopy", () => {
 			"silence is never grounds to delete a row that may describe real mail",
 		);
 		assert.equal(
-			(called("message.update")[0]?.args[1] as { status?: string })?.status,
+			(
+				called("message.transitionPlacement")[0]?.args[2] as {
+					status?: string;
+				}
+			)?.status,
 			"deleted",
 		);
 	});
@@ -505,11 +513,33 @@ describe("handleMessageCopy", () => {
 
 		await handleMessageCopy(event, noopLogger, 1, deps());
 
-		const update = called("message.update")[0];
-		assert.equal((update?.args[1] as { status?: string })?.status, "deleted");
+		const [givenUp] = called("message.transitionPlacement");
+		assert.equal((givenUp?.args[2] as { status?: string })?.status, "deleted");
+		assert.equal(
+			(givenUp?.args[2] as { abandonedMutation?: string })?.abandonedMutation,
+			"copy",
+		);
 		assert.equal(called("createMailbox").length, 0);
 	});
 
+	// The guard asks whether THIS copy is still outstanding, not whether anything
+	// is. A row a delete has claimed is `deleting`, and reading that as unsettled
+	// let a redelivery COPY the message a second time — COPY has no source-side
+	// effect to make that a no-op, so the duplicate is permanent.
+	it("skips a redelivered copy whose row a delete has claimed", async () => {
+		let copies = 0;
+		h.connection.copyMessages = async () => {
+			copies += 1;
+			return { uidMap: new Map([[10, 20]]) };
+		};
+		h.copyRow = { ...unsettledCopyRow(), status: "deleting" };
+
+		await handleMessageCopy(event, noopLogger, 2, deps());
+
+		assert.equal(copies, 0, "no second copy is issued");
+		assert.equal(called("message.updateUid").length, 0);
+	});
+
 	it("marks failed and rethrows on an unclassified IMAP error within the budget", async () => {
 		h.connection.copyMessages = async () => {
 			throw new Error("server exploded");
@@ -520,11 +550,12 @@ describe("handleMessageCopy", () => {
 			/server exploded/,
 		);
 
-		const update = called("message.update")[0];
-		assert.equal(
-			(update?.args[1] as { syncStatus?: string })?.syncStatus,
-			"failed",
-		);
+		// The attempt marker is a transition too, predicated on this copy still
+		// being outstanding: a row another mutation has claimed keeps its own
+		// placement rather than picking up this attempt's failure.
+		const [marker] = called("message.transitionPlacement");
+		assert.deepEqual(marker?.args[1], { status: "moving" });
+		assert.deepEqual(marker?.args[2], { syncStatus: "failed" });
 	});
 
 	it("settles on the destination UID instead of dead-lettering the last attempt", async () => {