@remit/imap-worker 0.0.86 → 0.0.87

package/src/handlers/message-delete.sqlite.test.ts

@@ -0,0 +1,291 @@
+import assert from "node:assert/strict";
+import { after, before, describe, it } from "node:test";
+import type { RemitClient } from "@remit/backend/client";
+import {
+	MailboxCursorState,
+	MailboxSpecialUse,
+	MessageMutation,
+	MessageStatus,
+	MessageSyncStatus,
+} from "@remit/domain-enums";
+import type { MessageDeleteEvent } from "../events.js";
+import { mail, openStore, type World } from "../test-helpers/test-world.js";
+import { handleMessageDelete } from "./message-delete.js";
+
+let store: Awaited<ReturnType<typeof openStore>>;
+
+before(async () => {
+	store = await openStore();
+});
+
+after(() => store.close());
+
+interface Arranged {
+	w: World;
+	messageId: string;
+	uid: number;
+	messageIdHeader: string;
+	inboxId: string;
+	trashId: string;
+	event: MessageDeleteEvent;
+}
+
+/**
+ * A message in INBOX, deleted through the service the API calls, so the row
+ * the handler finds and the event it gets are the ones a person's delete
+ * leaves behind.
+ */
+const arrange = async (
+	options: { permanent?: boolean; olderCopyInTrash?: boolean } = {},
+): Promise<Arranged> => {
+	const w = await store.world();
+	const inbox = await w.folder("INBOX");
+	const trash = await w.folder("Trash", {
+		specialUse: MailboxSpecialUse.Trash,
+	});
+	const seeded = await w.message(inbox);
+	if (options.olderCopyInTrash)
+		w.imap.deliver("Trash", mail(seeded.messageIdHeader));
+
+	await w.client.messageMove.deleteMessage(
+		w.account.accountConfigId,
+		seeded.messageId,
+		w.account.accountId,
+		options.permanent ? { permanent: true } : { toTrash: true },
+	);
+	const event = w
+		.queued()
+		.find(
+			(body) => body.type === "MESSAGE_DELETE",
+		) as unknown as MessageDeleteEvent;
+	assert.ok(event, "the delete enqueued a MESSAGE_DELETE");
+
+	return {
+		w,
+		messageId: seeded.messageId,
+		uid: seeded.uid,
+		messageIdHeader: seeded.messageIdHeader,
+		inboxId: inbox.mailboxId,
+		trashId: trash.mailboxId,
+		event,
+	};
+};
+
+const run = (a: Arranged, receiveCount = 1, event = a.event) =>
+	handleMessageDelete(event, a.w.log, receiveCount, a.w.scope);
+
+const resyncs = (w: World): unknown[] =>
+	w
+		.queued()
+		.filter((body) => body.type === "SYNC_MESSAGES")
+		.map((body) => body.mailboxId);
+
+const client = (): RemitClient => store.client;
+
+describe("a move to Trash", () => {
+	it("lands in Trash and settles the row on the uid the server gave it", async () => {
+		const a = await arrange();
+
+		await run(a);
+
+		assert.deepEqual(a.w.imap.uids("INBOX"), []);
+		const [trashUid] = a.w.imap.uids("Trash", a.messageIdHeader);
+		const row = await a.w.row(a.messageId);
+		assert.equal(row?.mailboxId, a.trashId);
+		assert.equal(row?.uid, trashUid);
+		assert.equal(row?.status, MessageStatus.active);
+		assert.deepEqual(await a.w.threadRows(a.messageId), [
+			{ mailboxId: a.trashId, uid: trashUid, isDeleted: true },
+		]);
+	});
+
+	// Issue #1122. Without UIDPLUS the destination is asked by Message-ID, and
+	// Trash already holds an older copy: binding the row to it lets Empty Trash
+	// expunge the wrong one and leaves the fresh copy unreachable.
+	it("settles on the copy this move delivered when the server sends no COPYUID", async () => {
+		const a = await arrange({ olderCopyInTrash: true });
+		a.w.imap.uidplus = false;
+
+		await run(a);
+
+		const copies = a.w.imap.uids("Trash", a.messageIdHeader);
+		assert.equal(copies.length, 2);
+		assert.equal((await a.w.row(a.messageId))?.uid, Math.max(...copies));
+	});
+
+	// One Message-ID can have several server copies. A source that still holds
+	// the uid proves the MOVE never ran, so the older copy in Trash is not this
+	// message and must never settle the row.
+	it("never settles on an older copy in Trash while the source still holds the message", async () => {
+		const a = await arrange({ olderCopyInTrash: true });
+		a.w.imap.uidplus = false;
+		a.w.imap.ignoreNext.add("MOVE");
+
+		await assert.rejects(run(a), /unconfirmed/);
+
+		const row = await a.w.row(a.messageId);
+		assert.equal(row?.uid, a.uid);
+		assert.equal(row?.status, MessageStatus.moving);
+		assert.equal(row?.syncStatus, MessageSyncStatus.failed);
+	});
+
+	it("issues nothing when a redelivery finds the delete already settled", async () => {
+		const a = await arrange();
+		await run(a);
+		const settled = await a.w.row(a.messageId);
+		const commands = a.w.imap.log.length;
+
+		await run(a);
+
+		assert.equal(
+			a.w.imap.log.length,
+			commands,
+			"no IMAP command on redelivery",
+		);
+		assert.deepEqual(await a.w.row(a.messageId), settled);
+	});
+
+	it("marks the row failed and redelivers when the connection drops mid-move", async () => {
+		const a = await arrange();
+		a.w.imap.failNext.MOVE = new Error("Connection reset");
+
+		await assert.rejects(run(a), /Connection reset/);
+
+		const row = await a.w.row(a.messageId);
+		assert.equal(row?.status, MessageStatus.moving);
+		assert.equal(row?.syncStatus, MessageSyncStatus.failed);
+		assert.deepEqual(a.w.imap.uids("INBOX"), [a.uid]);
+	});
+
+	// Issue #980. At the ceiling the server is asked where the message is, and
+	// a message that never left INBOX puts the row back there.
+	it("puts the row back on its source at the ceiling when the message never left", async () => {
+		const a = await arrange();
+		a.w.imap.failNext.MOVE = new Error("Connection reset");
+
+		await run(a, 3);
+
+		const row = await a.w.row(a.messageId);
+		assert.equal(row?.mailboxId, a.inboxId);
+		assert.equal(row?.uid, a.uid);
+		assert.notEqual(row?.status, MessageStatus.moving);
+		assert.deepEqual(resyncs(a.w), [a.inboxId, a.trashId]);
+	});
+
+	it("hands the row back without creating a Trash the server does not have", async () => {
+		const a = await arrange();
+		a.w.imap.removeFolder("Trash");
+
+		await run(a);
+
+		assert.deepEqual(a.w.imap.commands("CREATE"), []);
+		const row = await a.w.row(a.messageId);
+		assert.equal(row?.mailboxId, a.inboxId);
+		assert.equal(row?.uid, a.uid);
+		assert.equal(row?.syncStatus, MessageSyncStatus.abandoned);
+		assert.equal(row?.abandonedMutation, MessageMutation.delete);
+	});
+
+	it("hands the row back without connecting while the source cursor is rebuilding", async () => {
+		const a = await arrange();
+		await client().mailbox.update(a.w.account.accountId, a.inboxId, {
+			cursorState: MailboxCursorState.rebuilding,
+		});
+
+		await run(a);
+
+		assert.equal(a.w.imap.connections, 0);
+		const row = await a.w.row(a.messageId);
+		assert.equal(row?.mailboxId, a.inboxId);
+		assert.equal(row?.status, MessageStatus.active);
+		assert.equal(
+			a.w.logs.filter(
+				(line) => line.fields.metric === "message_delete_dropped_on_pause",
+			).length,
+			1,
+		);
+		assert.deepEqual(resyncs(a.w), [a.inboxId, a.trashId]);
+	});
+
+	// The folder was deleted and recreated on the server: uid 1 now names
+	// whatever the server put there. Nothing may be moved by it.
+	it("moves nothing when the source's UIDVALIDITY changed under it", async () => {
+		const a = await arrange();
+		a.w.imap.setUidValidity("INBOX", 2);
+
+		await run(a);
+
+		assert.deepEqual(a.w.imap.commands("MOVE"), []);
+		assert.equal((await a.w.row(a.messageId))?.mailboxId, a.inboxId);
+		const inbox = await client().mailbox.get(a.w.account.accountId, a.inboxId);
+		assert.notEqual(inbox.cursorState, MailboxCursorState.normal);
+	});
+
+	it("moves from the folder's current path when it was renamed after enqueue", async () => {
+		const a = await arrange();
+		a.w.imap.renameFolder("INBOX", "Old/Inbox");
+		await client().mailbox.update(a.w.account.accountId, a.inboxId, {
+			fullPath: "Old/Inbox",
+		});
+
+		await run(a);
+
+		assert.deepEqual(a.w.imap.uids("Old/Inbox"), []);
+		assert.equal(a.w.imap.uids("Trash", a.messageIdHeader).length, 1);
+	});
+
+	it("acks without connecting when the source mailbox was deleted", async () => {
+		const a = await arrange();
+		await client().mailbox.delete(a.w.account.accountId, a.inboxId);
+
+		await run(a);
+
+		assert.equal(a.w.imap.connections, 0);
+	});
+
+	it("refuses an event minted under an unknown contract and hands the row back", async () => {
+		const a = await arrange();
+
+		await run(a, 1, {
+			...a.event,
+			schemaVersion: 99,
+		} as unknown as MessageDeleteEvent);
+
+		assert.equal(a.w.imap.connections, 0);
+		const row = await a.w.row(a.messageId);
+		assert.equal(row?.mailboxId, a.inboxId);
+		assert.equal(row?.syncStatus, MessageSyncStatus.abandoned);
+	});
+});
+
+describe("a permanent delete", () => {
+	it("expunges the message and removes every local row", async () => {
+		const a = await arrange({ permanent: true });
+
+		await run(a);
+
+		assert.deepEqual(a.w.imap.uids("INBOX"), []);
+		assert.equal(await a.w.row(a.messageId), undefined);
+		assert.deepEqual(await a.w.threadRows(a.messageId), []);
+	});
+
+	it("cleans up locally when another client already expunged it", async () => {
+		const a = await arrange({ permanent: true });
+		a.w.imap.empty("INBOX");
+		a.w.imap.failNext.EXPUNGE = new Error("Message not found (NONEXISTENT)");
+
+		await run(a);
+
+		assert.equal(await a.w.row(a.messageId), undefined);
+	});
+
+	it("keeps the row when the server still holds the message after a failed expunge", async () => {
+		const a = await arrange({ permanent: true });
+		a.w.imap.failNext.EXPUNGE = new Error("Message not found (NONEXISTENT)");
+
+		await assert.rejects(run(a), /NONEXISTENT/);
+
+		assert.equal((await a.w.row(a.messageId))?.status, MessageStatus.deleting);
+		assert.deepEqual(a.w.imap.uids("INBOX"), [a.uid]);
+	});
+});

package/src/handlers/message-delete.ts

@@ -10,8 +10,8 @@ import {
 	MessageStatus,
 	MessageSyncStatus,
 } from "@remit/domain-enums";
-import type { Logger } from "@remit/logger-lambda";
-import { recordImapFailure } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
+import { recordImapFailure } from "@remit/logger-lambda/metrics";
 import {
 	buildThreadMessageTrashUpdate,
 	guardConnectionCursor,
@@ -37,6 +37,7 @@ import {
 	probePausedPlacement,
 	searchMailboxForHighestMessageIdUid,
 } from "./message-move.js";
+import { probeableMessageIdHeader } from "./probe-message-id.js";
 
 interface FolderPaths {
 	source: string;
@@ -109,6 +110,8 @@ const confirmTrashMoveUid = async (
 	sourceConnection: IImapConnection,
 	destinationConnection: IImapConnection,
 	messageService: Pick<IMessageRepository, "get">,
+	threadMessageService: Pick<IThreadMessageRepository, "findByMessageId">,
+	accountConfigId: string,
 	messageId: string,
 	sourceMailboxPath: string,
 	destinationMailboxPath: string,
@@ -121,34 +124,23 @@ const confirmTrashMoveUid = async (
 
 	const [message] = await messageService.get([messageId]);
 	if (!message) return { outcome: "row-gone" };
-	if (!message.messageIdHeader) return { outcome: "unprobeable" };
+	const messageIdHeader = await probeableMessageIdHeader(
+		threadMessageService,
+		accountConfigId,
+		messageId,
+	);
+	if (!messageIdHeader) return { outcome: "unprobeable" };
 
 	const probedUid = await searchMailboxForHighestMessageIdUid(
 		destinationConnection,
 		destinationMailboxPath,
-		message.messageIdHeader,
+		messageIdHeader,
 	);
 	return probedUid === null
 		? { outcome: "unconfirmed" }
 		: { outcome: "confirmed", uid: probedUid };
 };
 
-export interface MessageDeleteDeps {
-	getClient: typeof getClient;
-	buildLifecycleDeps: typeof buildLifecycleDeps;
-	withOAuthLifecycle: typeof withOAuthLifecycle;
-	createConnectionScope: typeof createConnectionScopeWithCredentials;
-	emitEvent: typeof emitEvent;
-}
-
-const defaultDeps: MessageDeleteDeps = {
-	getClient,
-	buildLifecycleDeps,
-	withOAuthLifecycle,
-	createConnectionScope: createConnectionScopeWithCredentials,
-	emitEvent,
-};
-
 /**
  * Handle MESSAGE_DELETE events.
  * Either moves to Trash (IMAP MOVE) or permanently deletes (IMAP DELETE).
@@ -162,16 +154,8 @@ export const handleMessageDelete = async (
 	event: MessageDeleteEvent,
 	log: Logger,
 	receiveCount = 1,
-	deps: MessageDeleteDeps = defaultDeps,
+	createConnectionScope = createConnectionScopeWithCredentials,
 ): Promise<void> => {
-	const {
-		getClient,
-		buildLifecycleDeps,
-		withOAuthLifecycle,
-		createConnectionScope: createConnectionScopeWithCredentials,
-		emitEvent,
-	} = deps;
-
 	const {
 		account: accountService,
 		message: messageService,
@@ -446,8 +430,11 @@ export const handleMessageDelete = async (
 
 		const placement: PausedPlacement = isProbeableTrashMove
 			? await probePausedPlacement(await getRawConnection(), {
-					messageIdHeader: (await messageService.get([messageId]))[0]
-						?.messageIdHeader,
+					messageIdHeader: await probeableMessageIdHeader(
+						threadMessageService,
+						account.accountConfigId,
+						messageId,
+					),
 					sourceMailboxPath: paths.source,
 					destinationMailboxPath,
 				})
@@ -498,6 +485,11 @@ export const handleMessageDelete = async (
 		const settled = await resolveExhaustedMessageDeleteFailure(
 			{ messageService, threadMessageService, log },
 			{
+				placement: {
+					status: message.status,
+					mailboxId: message.mailboxId,
+					uid: message.uid,
+				},
 				accountId,
 				accountConfigId,
 				messageId,
@@ -642,7 +634,7 @@ export const handleMessageDelete = async (
 				destination: destinationMailboxPath,
 			};
 
-			const scope = createConnectionScopeWithCredentials(account, credentials);
+			const scope = createConnectionScope(account, credentials);
 
 			// The terminal resolver opens the source itself, and its answer now
 			// WRITES a placement, so it has to reach IMAP through the same
@@ -737,6 +729,8 @@ export const handleMessageDelete = async (
 									connection,
 									rawConnection,
 									messageService,
+									threadMessageService,
+									account.accountConfigId,
 									messageId,
 									mailboxPath,
 									destinationMailboxPath,

package/src/handlers/message-move-terminal.ts

@@ -1,14 +1,13 @@
 import type {
 	IMessageRepository,
 	IThreadMessageRepository,
+	MessageItem,
 } from "@remit/data-ports";
 import { MessageMutation, MessageSyncStatus } from "@remit/domain-enums";
 import {
 	type IImapConnection,
 	isMessageGoneFromOpenMailbox,
-	reconcileStaleMessage,
 	restoreSourcePlacement,
-	type StaleMessageReconcileDeps,
 } from "@remit/mailbox-service";
 
 export interface MessageMoveTerminalLogger {
@@ -16,9 +15,11 @@ export interface MessageMoveTerminalLogger {
 	error(obj: Record<string, unknown>, msg: string): void;
 }
 
-export interface ResolveExhaustedMessageMoveDeps
-	extends StaleMessageReconcileDeps {
-	messageService: Pick<IMessageRepository, "delete" | "transitionPlacement">;
+export interface ResolveExhaustedMessageMoveDeps {
+	messageService: Pick<
+		IMessageRepository,
+		"deleteIfPlacement" | "transitionPlacement"
+	>;
 	threadMessageService: Pick<
 		IThreadMessageRepository,
 		"findAllByMessageId" | "deleteMany" | "update"
@@ -27,6 +28,8 @@ export interface ResolveExhaustedMessageMoveDeps
 }
 
 export interface ResolveExhaustedMessageMoveInput {
+	/** Where the row stood when the caller read it; the delete holds only while it still does. */
+	placement: Pick<MessageItem, "status" | "mailboxId" | "uid">;
 	accountId: string;
 	accountConfigId: string;
 	messageId: string;
@@ -58,7 +61,7 @@ export interface ResolveExhaustedMessageMoveResult {
  *    the connection dropped before the tagged OK was read, or a foreign client
  *    moved or expunged the message; from here those are indistinguishable and
  *    have the same answer. The stale Message/ThreadMessage rows are deleted
- *    via {@link reconcileStaleMessage} and the caller resyncs both folders, so
+ *    via `deleteIfPlacement` and the caller resyncs both folders, so
  *    whichever folder actually holds the message re-projects it with the
  *    server's own UID. Metric only, no alarm — routine.
  * 2. BROKEN — the message is still at the source, so the move never took
@@ -110,10 +113,9 @@ export const resolveExhaustedMessageMoveFailure = async (
 	await connection.openBox(sourceMailboxPath, true);
 
 	if (await isMessageGoneFromOpenMailbox(connection, uid)) {
-		const { threadMessagesDeleted } = await reconcileStaleMessage(
-			deps,
-			accountConfigId,
+		const reconciled = await deps.messageService.deleteIfPlacement(
 			messageId,
+			input.placement,
 		);
 		deps.log.info(
 			{
@@ -123,7 +125,7 @@ export const resolveExhaustedMessageMoveFailure = async (
 				messageId,
 				uid,
 				sourceMailboxPath,
-				threadMessagesDeleted,
+				reconciled,
 			},
 			"Message no longer at its move source after retry exhaustion (move landed server-side, or an external delete or move); stale row reconciled",
 		);