@remit/mailbox-service 0.0.91 → 0.0.92

package/src/message-move.ts

@@ -29,17 +29,28 @@ import { deriveCopyMessageId } from "@remit/data-ports/id";
 import { MUTATION_EVENT_SCHEMA_VERSION } from "@remit/data-ports/mutation-events";
 import {
 	CanonicalMailboxRole,
+	MessageMutation,
 	MessageStatus,
 	MessageSyncStatus,
 } from "@remit/domain-enums";
 import { createQueueProducer } from "@remit/sqs-client/producer";
 import {
-	bindsForeignUid,
-	type PlacementBinding,
+	carriesForeignUid,
 	placementBindingOf,
 	waitForPlacementToSettle,
 } from "./placement-settled.js";
 
+/**
+ * What a delete batch did not claim. A row whose placement changed between the
+ * read that decided this delete and the mark that recorded it belongs to
+ * whoever changed it (docs/architecture/imap-mutations.md R3), so this delete
+ * leaves it alone — and says which, because the client removed it optimistically
+ * and a row that reappears unexplained is a failure the user cannot act on.
+ */
+export interface DeleteMessagesOutcome {
+	refusedMessageIds: string[];
+}
+
 /**
  * Event types for message move/delete operations.
  * These match the worker event types in remit-imap-worker/events.ts.
@@ -288,9 +299,8 @@ export class MessageMoveService {
 		messageId: string,
 		accountId: string,
 		options: DeleteOptions = { toTrash: true },
-	): Promise<void> => {
-		await this.deleteMessages(accountConfigId, [messageId], accountId, options);
-	};
+	): Promise<DeleteMessagesOutcome> =>
+		this.deleteMessages(accountConfigId, [messageId], accountId, options);
 
 	/**
 	 * Delete multiple messages using batch operations.
@@ -305,12 +315,12 @@ export class MessageMoveService {
 		messageIds: string[],
 		accountId: string,
 		options: DeleteOptions = { toTrash: true },
-	): Promise<void> => {
-		if (messageIds.length === 0) return;
+	): Promise<DeleteMessagesOutcome> => {
+		if (messageIds.length === 0) return { refusedMessageIds: [] };
 
 		// Batch get all messages
 		const rows = await this.messageService.get(messageIds);
-		if (rows.length === 0) return;
+		if (rows.length === 0) return { refusedMessageIds: [] };
 
 		// Ahead of the Trash gates below, which compare `mailboxId` against the
 		// Trash folder: an in-flight move makes that comparison answer for a
@@ -381,20 +391,36 @@ export class MessageMoveService {
 		}
 
 		// Group messages by operation type
-		const moveToTrashMessages: Array<{
+		type DeleteEntry = {
 			messageId: string;
 			message: { mailboxId: string; uid: number };
 			sourceMailbox: { mailboxId: string; fullPath: string };
-		}> = [];
-		const permanentDeleteMessages: Array<{
-			messageId: string;
-			message: { mailboxId: string; uid: number };
-			sourceMailbox: { mailboxId: string; fullPath: string };
-		}> = [];
+		};
+		const moveToTrashMessages: DeleteEntry[] = [];
+		const permanentDeleteMessages: DeleteEntry[] = [];
+
+		// Rows whose placement changed between the read above and the mark below,
+		// so this delete never claimed them. Reported rather than skipped in
+		// silence: the client removed them optimistically, and a row that
+		// reappears with nothing said is the dead-button failure again (#1229).
+		const refused: string[] = [];
 
 		for (const message of messages) {
 			const sourceMailbox = mailboxMap.get(message.mailboxId);
-			if (!sourceMailbox) continue;
+			if (!sourceMailbox) {
+				// The folder lookup is scoped to one account, so a selection that
+				// spans accounts — the daily brief lists every account's mail —
+				// leaves this row's mailbox unresolved. Nothing about it was
+				// written and no event was enqueued, so it is a refusal like any
+				// other; dropping it silently answered "deleted" over a message
+				// still sitting where it was.
+				refused.push(message.messageId);
+				this.log.info(
+					{ accountId, messageId: message.messageId },
+					"Refused delete: this message's folder does not belong to this account",
+				);
+				continue;
+			}
 
 			const isInTrash =
 				trashMailbox !== null && message.mailboxId === trashMailbox.mailboxId;
@@ -428,14 +454,41 @@ export class MessageMoveService {
 					accountConfigId,
 				);
 			for (const { messageId, message, sourceMailbox } of moveToTrashMessages) {
-				// Update local state optimistically
-				await this.messageService.updateForMove(messageId, {
-					mailboxId: trashMailbox.mailboxId,
-					status: MessageStatus.moving,
-					syncStatus: MessageSyncStatus.pending,
-					originalMailboxId: sourceMailbox.mailboxId,
-					originalUid: message.uid,
-				});
+				// The optimistic write is a transition off the row this call read
+				// (imap-mutations R3). Several folder lookups stand between the read
+				// and here, and PLACEMENT_MOVE_PUSH runs on a queue this account's
+				// FIFO group does not order, so the row can settle onto a different
+				// folder and uid in between. Writing anyway would stamp `moving` and
+				// the OLD uid over the new placement, and the event enqueued below
+				// would name a uid the source no longer holds — a delete aimed at
+				// whatever message has since taken that number.
+				const marked = await this.messageService.transitionPlacement(
+					messageId,
+					{
+						// `active`, not whatever the row was read as: a row already
+						// `moving` or `deleting` is mid-mutation, and marking it again
+						// would enqueue a second event against the same uid.
+						status: MessageStatus.active,
+						mailboxId: message.mailboxId,
+						uid: message.uid,
+					},
+					{
+						mailboxId: trashMailbox.mailboxId,
+						status: MessageStatus.moving,
+						syncStatus: MessageSyncStatus.pending,
+						abandonedMutation: MessageMutation.none,
+						originalMailboxId: sourceMailbox.mailboxId,
+						originalUid: message.uid,
+					},
+				);
+				if (!marked) {
+					refused.push(messageId);
+					this.log.info(
+						{ accountId, messageId },
+						"Refused delete: this message's placement changed after it was read",
+					);
+					continue;
+				}
 
 				// Update ThreadMessage
 				await this.updateThreadMessageForMove(
@@ -481,11 +534,36 @@ export class MessageMoveService {
 			message,
 			sourceMailbox,
 		} of permanentDeleteMessages) {
-			// Update local state optimistically
-			await this.messageService.update(messageId, {
-				status: MessageStatus.deleting,
-				syncStatus: MessageSyncStatus.pending,
-			});
+			// A transition off the row this call read, like the trash path above
+			// (imap-mutations R3). The expunge below is aimed at a specific uid in
+			// a specific folder, so a row that has moved since the read must not be
+			// marked for it: a restore out of Trash settling in between would be
+			// overwritten with `deleting`, and the event would carry the uid Trash
+			// no longer holds.
+			const marked = await this.messageService.transitionPlacement(
+				messageId,
+				{
+					// `active` for the same reason as the trash path above: a second
+					// press on a row already `deleting` would enqueue a duplicate
+					// expunge rather than refusing.
+					status: MessageStatus.active,
+					mailboxId: message.mailboxId,
+					uid: message.uid,
+				},
+				{
+					status: MessageStatus.deleting,
+					syncStatus: MessageSyncStatus.pending,
+					abandonedMutation: MessageMutation.none,
+				},
+			);
+			if (!marked) {
+				refused.push(messageId);
+				this.log.info(
+					{ accountId, messageId },
+					"Refused permanent delete: this message's placement changed after it was read",
+				);
+				continue;
+			}
 
 			// Delete ThreadMessage rows up-front (one row per mailbox copy).
 			// The IMAP worker also deletes them once the IMAP DELETE succeeds —
@@ -517,6 +595,8 @@ export class MessageMoveService {
 
 		// Batch send events to SQS
 		await this.enqueueEventsBatch(events);
+
+		return { refusedMessageIds: refused };
 	};
 
 	/**
@@ -591,14 +671,41 @@ export class MessageMoveService {
 			trashMailbox && message.mailboxId === trashMailbox.mailboxId,
 		);
 
-		// Update local state optimistically
-		await this.messageService.updateForMove(messageId, {
-			mailboxId: destinationMailboxId,
-			status: MessageStatus.moving,
-			syncStatus: MessageSyncStatus.pending,
-			originalMailboxId: sourceMailbox.mailboxId,
-			originalUid: message.uid,
-		});
+		// The optimistic write is a transition off the row this call read
+		// (imap-mutations R3). Three mailbox lookups stand between the read and
+		// here, and PLACEMENT_MOVE_PUSH runs on a queue this account's FIFO group
+		// does not order, so the row can settle onto a different folder and uid in
+		// between. A blind write would stamp `moving` and the OLD uid over that
+		// placement, leaving a pair `carriesForeignUid` calls consistent, and the
+		// event below would send the worker after whatever now holds the old uid.
+		const moved = await this.messageService.transitionPlacement(
+			messageId,
+			{
+				status: MessageStatus.active,
+				mailboxId: message.mailboxId,
+				uid: message.uid,
+			},
+			{
+				mailboxId: destinationMailboxId,
+				status: MessageStatus.moving,
+				syncStatus: MessageSyncStatus.pending,
+				abandonedMutation: MessageMutation.none,
+				originalMailboxId: sourceMailbox.mailboxId,
+				originalUid: message.uid,
+			},
+		);
+		if (!moved) {
+			this.log.error(
+				{ accountId, messageId, destinationMailboxId },
+				"Refused: this message's placement changed while the move was being prepared",
+			);
+			throw new MessagePlacementUnsettledError(
+				`Message ${messageId} was not moved: something else changed where it is while this move was being prepared`,
+				accountId,
+				messageId,
+				"in_flight",
+			);
+		}
 
 		// Update ThreadMessage
 		await this.updateThreadMessageForMove(
@@ -928,18 +1035,45 @@ export class MessageMoveService {
 		// says so: its own MESSAGE_MOVE then reads the row as settled and returns
 		// without moving anything, and the worker's expunge binds that borrowed
 		// uid against whatever Trash really holds at it. Left as it stands, that
-		// move runs ahead of this expunge in the account's FIFO group and settles
-		// the row, so the message is in Trash by the time the sweep reaches it and
-		// goes with the rest of this same press — only the count below is one
-		// short of what the sweep ends up removing.
-		const messages = rows.filter((message) => !bindsForeignUid(message));
-
-		// Mark all as deleting locally
-		for (const message of messages) {
-			await this.messageService.update(message.messageId, {
-				status: MessageStatus.deleting,
-				syncStatus: MessageSyncStatus.pending,
-			});
+		// move settles the row on its own, so the message is in Trash by the time
+		// the sweep reaches it and goes with the rest of this same press — only
+		// the count below is one short of what the sweep ends up removing.
+		//
+		// `carriesForeignUid`, the same predicate the sweep applies (#1217).
+		// The binding form reads `status` too, and `status` is exactly what this
+		// mark is about to overwrite, so the two halves of one gate would have
+		// disagreed the moment the mark landed.
+		const candidates = rows.filter((message) => !carriesForeignUid(message));
+
+		// Each mark is a transition off the row as it was just read
+		// (imap-mutations R3), not a blind write onto a snapshot. The rows are
+		// held by nothing between the listing and here, and PLACEMENT_MOVE_PUSH
+		// runs on a standard queue that this account's FIFO group does not order,
+		// so a row can move out from under the sweep. A lost predicate means
+		// another lane won: the row is left alone and not counted, rather than
+		// marked `deleting` for an expunge that would bind somebody else's uid.
+		const messages: MessageItem[] = [];
+		for (const message of candidates) {
+			const marked = await this.messageService.transitionPlacement(
+				message.messageId,
+				{
+					status: message.status,
+					mailboxId: message.mailboxId,
+					uid: message.uid,
+				},
+				{
+					status: MessageStatus.deleting,
+					syncStatus: MessageSyncStatus.pending,
+				},
+			);
+			if (!marked) {
+				this.log.info(
+					{ accountId, messageId: message.messageId },
+					"Empty Trash skipped a message whose placement changed while the folder was being read",
+				);
+				continue;
+			}
+			messages.push(marked);
 			await this.updateThreadMessageDeleted(
 				accountConfigId,
 				message.messageId,
@@ -952,6 +1086,7 @@ export class MessageMoveService {
 				accountId,
 				trashMailboxId: trashMailbox.mailboxId,
 				count: messages.length,
+				skipped: rows.length - messages.length,
 			},
 			"Marked all trash messages for deletion (local)",
 		);
@@ -976,17 +1111,16 @@ export class MessageMoveService {
 	private refusePlacement = (
 		message: MessageItem,
 		accountId: string,
-		binding: Exclude<PlacementBinding, "consistent">,
 	): never => {
 		this.log.error(
-			{ accountId, messageId: message.messageId, binding },
+			{ accountId, messageId: message.messageId },
 			"Refused: this message's folder and uid do not name the same message",
 		);
 		throw new MessagePlacementUnsettledError(
 			`Message ${message.messageId} was not acted on: its folder and uid do not name the same message`,
 			accountId,
 			message.messageId,
-			binding === "abandoned" ? "unverified" : "in_flight",
+			"in_flight",
 		);
 	};
 
@@ -994,13 +1128,7 @@ export class MessageMoveService {
 		message: MessageItem,
 		accountId: string,
 	): Promise<MessageItem> => {
-		const binding = placementBindingOf(message);
-		if (binding === "consistent") return message;
-
-		// An abandoned move is never coming back, so the ceiling would be spent
-		// only to reach the same answer.
-		if (binding === "abandoned")
-			return this.refusePlacement(message, accountId, binding);
+		if (placementBindingOf(message) === "consistent") return message;
 
 		const settled = await waitForPlacementToSettle(
 			this.messageService,
@@ -1010,9 +1138,8 @@ export class MessageMoveService {
 				pollMs: this.moveSettlePollMs,
 			},
 		);
-		const settledBinding = placementBindingOf(settled);
-		if (settledBinding === "consistent") return settled;
-		return this.refusePlacement(settled, accountId, settledBinding);
+		if (placementBindingOf(settled) === "consistent") return settled;
+		return this.refusePlacement(settled, accountId);
 	};
 
 	/**

package/src/message-sync-repoint.test.ts

@@ -336,13 +336,33 @@ describe("repointsOnSighting", () => {
 		);
 	});
 
-	it("refuses a row whose move failed and has not been re-tried", () => {
+	/**
+	 * The row `abandonDelete` hands back: reader refused the delete, put the
+	 * message back where the server still has it, and nothing else is coming
+	 * for the row. Reader shares its mailboxes, so the user moving that same
+	 * message in another client is ordinary — and before R3 the sighting was
+	 * refused on `syncStatus` alone and the row never followed the move.
+	 */
+	it("accepts a row whose delete was abandoned, so a move made elsewhere still lands", () => {
+		assert.equal(
+			repointsOnSighting(
+				JUNK,
+				storedIn(INBOX, {
+					status: MessageStatus.active,
+					syncStatus: MessageSyncStatus.abandoned,
+				}),
+			),
+			true,
+		);
+	});
+
+	it("accepts a row left `failed` by a transient attempt that has since settled", () => {
 		assert.equal(
 			repointsOnSighting(
 				JUNK,
 				storedIn(INBOX, { syncStatus: MessageSyncStatus.failed }),
 			),
-			false,
+			true,
 		);
 	});
 

package/src/message-sync.ts

@@ -30,7 +30,6 @@ import {
 	MailboxCursorState,
 	MessageKeywordFlag,
 	MessageStatus,
-	MessageSyncStatus,
 	MessageSystemFlag,
 	StarColor,
 } from "@remit/domain-enums";
@@ -149,22 +148,22 @@ export const addressSightingIn = (
  * reconciles rather than waits (imap-mutations R2): it never blocks on a
  * mutation, it re-reads the row on the next sighting after the move settles.
  *
- * In flight is read off `status`, never off `syncStatus` (#1096). Every
+ * In flight is `status`, and `status` alone (#1096, imap-mutations R3). Every
  * outbound mutation writes `moving` or `deleting` and only settling returns the
- * row to `active`, while an ordinary inbound row stays `pending` for its whole
- * life because nothing on the sync path promotes it — so demanding `synced`
- * here meant ordinary mail never re-pointed at all. `failed` is still refused:
- * a mutation that gave up leaves it set on a row put back to `active`, and
- * nothing routine settles that row afterwards.
+ * row to `active`, so `active` is precisely the set nothing is coming for —
+ * whatever `syncStatus` says about how the row got there. A row put back by
+ * `abandonDelete` carries `abandoned`, and it needs this repair more than any
+ * other row does: nothing routine settles it, and reader shares its mailboxes,
+ * so refusing it left a message whose delete was refused unable to follow a
+ * move the user then made in another client.
  */
 export const repointsOnSighting = (
 	mailbox: MailboxItem,
-	message: Pick<MessageItem, "mailboxId" | "status" | "syncStatus">,
+	message: Pick<MessageItem, "mailboxId" | "status">,
 ): boolean => {
 	if (message.mailboxId === mailbox.mailboxId) return false;
 	if (isVirtualCopyMailbox(mailbox)) return false;
-	if (message.status !== MessageStatus.active) return false;
-	return message.syncStatus !== MessageSyncStatus.failed;
+	return message.status === MessageStatus.active;
 };
 
 /**