@remit/imap-worker 0.0.47 → 0.0.49

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.47",
+  "version": "0.0.49",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/handlers/append-sent-message.test.ts

@@ -66,6 +66,7 @@ const fresh = (): Harness => ({
 		references: ["parent@example.com"],
 		inReplyTo: "parent@example.com",
 		sentAt: 1700000000000,
+		appendedUid: 0,
 	},
 	sentMailbox: { mailboxId: "sent-mbx", fullPath: "INBOX/Sent" },
 	append: async (path, raw, flags) => {
@@ -125,6 +126,49 @@ const event: AppendSentMessageEvent = {
 const called = (method: string): Call[] =>
 	h.calls.filter((c) => c.method === method);
 
+const patches = (): unknown[] =>
+	called("outboxMessage.update").map((c) => c.args[2]);
+
+type BackendClient = Awaited<ReturnType<AppendSentMessageDeps["getClient"]>>;
+
+const depsWithFailingDelete = (): AppendSentMessageDeps => {
+	const base = deps();
+	return {
+		...base,
+		getClient: async (): Promise<BackendClient> => {
+			const client = await base.getClient();
+			return {
+				...client,
+				outboxMessage: {
+					...client.outboxMessage,
+					delete: async (): Promise<void> => {
+						throw new Error("storage down");
+					},
+				},
+			};
+		},
+	};
+};
+
+const depsWithFailingUpdate = (): AppendSentMessageDeps => {
+	const base = deps();
+	return {
+		...base,
+		getClient: async (): Promise<BackendClient> => {
+			const client = await base.getClient();
+			return {
+				...client,
+				outboxMessage: {
+					...client.outboxMessage,
+					update: async (): Promise<never> => {
+						throw new Error("storage down");
+					},
+				},
+			};
+		},
+	};
+};
+
 describe("handleAppendSentMessage", () => {
 	beforeEach(() => {
 		h = fresh();
@@ -268,24 +312,138 @@ describe("handleAppendSentMessage", () => {
 	});
 
 	it("leaves the row alone when the APPEND landed but the delete did not", async () => {
-		const failingDeps = deps();
-		const client = await failingDeps.getClient();
-		(
-			client as unknown as { outboxMessage: { delete: () => Promise<void> } }
-		).outboxMessage.delete = async () => {
-			throw new Error("storage down");
-		};
-
 		await assert.rejects(
-			handleAppendSentMessage(event, noopLog, APPEND_SENT_MAX_ATTEMPTS, {
-				...failingDeps,
-				getClient: async () => client,
-			} as AppendSentMessageDeps),
+			handleAppendSentMessage(event, noopLog, 1, depsWithFailingDelete()),
 			/storage down/,
 		);
 
 		// The copy is in Sent, so the message is findable — settling it as unfiled
 		// would say the opposite.
-		assert.equal(called("outboxMessage.update").length, 0);
+		assert.deepEqual(patches(), [{ appendedUid: 55 }]);
+	});
+
+	it("stops redelivering a landed APPEND at the budget instead of filing another copy", async () => {
+		await handleAppendSentMessage(
+			event,
+			noopLog,
+			APPEND_SENT_MAX_ATTEMPTS,
+			depsWithFailingDelete(),
+		);
+
+		assert.equal(called("connection.append").length, 1);
+		assert.deepEqual(patches(), [{ appendedUid: 55 }]);
+	});
+
+	it("records the uid the APPEND produced before it deletes the row", async () => {
+		await handleAppendSentMessage(event, noopLog, 1, deps());
+
+		// Order is the whole of it: a uid written after the delete is a uid a
+		// redelivery never sees, and the redelivery is what files the second copy.
+		const order = h.calls.map((c) => c.method);
+		assert.deepEqual(patches(), [{ appendedUid: 55 }]);
+		assert.ok(
+			order.indexOf("outboxMessage.update") <
+				order.indexOf("outboxMessage.delete"),
+		);
+	});
+
+	it("records a copy the server filed but named no uid for", async () => {
+		// UIDPLUS is an extension. Without it the APPEND succeeds and reports
+		// nothing, and "filed" still has to be told apart from "not filed".
+		h.append = async () => ({ uid: 0, uidValidity: 0 });
+
+		await handleAppendSentMessage(event, noopLog, 1, deps());
+
+		assert.deepEqual(patches(), [{ appendedUid: -1 }]);
+		assert.equal(called("outboxMessage.delete").length, 1);
+	});
+
+	it("files nothing a second time when a redelivery finds a recorded uid (#858)", async () => {
+		h.outbox = { ...h.outbox, appendedUid: 55 };
+
+		await handleAppendSentMessage(event, noopLog, 1, deps());
+
+		// The copy is already in the user's Sent folder. All this redelivery owes
+		// is the delete the last attempt could not make.
+		assert.equal(called("connection.append").length, 0);
+		assert.equal(h.disconnectCount, 0);
+		assert.deepEqual(called("outboxMessage.delete")[0]?.args, [
+			"cfg-1",
+			"out-1",
+		]);
+		assert.deepEqual(called("outboxAttachment.discardAll")[0]?.args, [
+			"cfg-1",
+			"acc-1",
+			"out-1",
+		]);
+	});
+
+	it("retries the delete a redelivery owes while the queue has attempts left", async () => {
+		h.outbox = { ...h.outbox, appendedUid: 55 };
+
+		await assert.rejects(
+			handleAppendSentMessage(event, noopLog, 1, depsWithFailingDelete()),
+			/storage down/,
+		);
+
+		assert.equal(called("connection.append").length, 0);
+		assert.deepEqual(patches(), []);
+	});
+
+	it("keeps the row when the uid cannot be recorded", async () => {
+		await assert.rejects(
+			handleAppendSentMessage(event, noopLog, 1, depsWithFailingUpdate()),
+			/storage down/,
+		);
+
+		// The other half of "written before the delete, never after". A row
+		// deleted without its uid is a row a redelivery reads as never filed.
+		assert.equal(called("outboxMessage.delete").length, 0);
+		assert.equal(called("outboxAttachment.discardAll").length, 0);
+	});
+
+	it("acks an event whose outbox row is already gone", async () => {
+		// Its own last attempt deleted it, or the boot repair dropped it. Throwing
+		// the same NotFoundError on every redelivery only dead-letters the record.
+		const notFound = Object.assign(new Error("gone"), {
+			name: "NotFoundError",
+		});
+		const base = deps();
+		const failing: AppendSentMessageDeps = {
+			...base,
+			getClient: async (): Promise<BackendClient> => {
+				const client = await base.getClient();
+				return {
+					...client,
+					outboxMessage: {
+						...client.outboxMessage,
+						get: async () => {
+							throw notFound;
+						},
+					},
+				} as BackendClient;
+			},
+		};
+
+		await handleAppendSentMessage(event, noopLog, 1, failing);
+
+		assert.equal(called("connection.append").length, 0);
+		assert.deepEqual(patches(), []);
+	});
+
+	it("gives up on that delete at the budget rather than dead-lettering it", async () => {
+		h.outbox = { ...h.outbox, appendedUid: 55 };
+
+		await handleAppendSentMessage(
+			event,
+			noopLog,
+			APPEND_SENT_MAX_ATTEMPTS,
+			depsWithFailingDelete(),
+		);
+
+		// The row stays `sent` and hidden, which the boot-time repair drops on the
+		// strength of the same recorded uid. Settling it as unfiled would tell the
+		// user a message sitting in Sent was never filed.
+		assert.deepEqual(patches(), []);
 	});
 });

package/src/handlers/append-sent-message.ts

@@ -1,4 +1,9 @@
 import { getClient } from "@remit/backend/client";
+import {
+	APPENDED_UID_NONE,
+	APPENDED_UID_UNREPORTED,
+	isSentCopyFiled,
+} from "@remit/data-ports";
 import { OutboxMessageStatus } from "@remit/domain-enums";
 import type { Logger } from "@remit/logger-lambda";
 import {
@@ -8,6 +13,7 @@ import {
 import { isAccountDeleted } from "../account-check.js";
 import { createConnectionScopeWithCredentials } from "../connection-scope.js";
 import type { AppendSentMessageEvent } from "../events.js";
+import { isNotFoundError } from "../is-not-found.js";
 import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
 import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
 
@@ -17,6 +23,9 @@ const UNFILED_NO_SENT_MAILBOX =
 const UNFILED_SIGNED_OUT =
 	"Sent, but not filed: this account has to be signed in again before a copy can be stored in Sent.";
 
+const ROW_SURVIVED_ITS_DELETE =
+	"Sent message was filed but its outbox row survived its delete and stays hidden until the boot-time repair";
+
 const unfiledAppendRefused = (fullPath: string, error: unknown): string =>
 	`Sent, but not filed: the mail server refused to store a copy in ${fullPath} (${error instanceof Error ? error.message : String(error)}).`;
 
@@ -90,10 +99,23 @@ export const handleAppendSentMessage = async (
 		return;
 	}
 
-	const outbox = await outboxMessageService.get(
-		account.accountConfigId,
-		outboxMessageId,
-	);
+	// A row this event names can be gone by the time the event is redelivered:
+	// its own last attempt deleted it, or the boot repair dropped it on the
+	// strength of a recorded uid. Either way the work is done, and re-throwing
+	// the same NotFoundError on every redelivery only dead-letters it.
+	const outbox = await outboxMessageService
+		.get(account.accountConfigId, outboxMessageId)
+		.catch((error: unknown) => {
+			if (isNotFoundError(error)) return null;
+			throw error;
+		});
+	if (!outbox) {
+		log.warn(
+			{ accountId, outboxMessageId },
+			"Skipping APPEND_SENT_MESSAGE: the outbox row is already gone",
+		);
+		return;
+	}
 	if (outbox.status !== OutboxMessageStatus.sent) {
 		log.info(
 			{ outboxMessageId, status: outbox.status },
@@ -121,13 +143,58 @@ export const handleAppendSentMessage = async (
 		);
 	};
 
+	// The copy is in Sent, so the row is a leftover and deleting it is all that
+	// is left to do. Both steps are idempotent against a row that is already
+	// gone, so nothing here needs to know how far the last attempt got.
+	//
+	// Files first, row second, the same order outbox-queue.ts discards a draft
+	// in. Nothing but this row points at those objects: a row that outlives its
+	// files is hidden and the boot repair drops it, while files that outlive
+	// their row name a message nothing can reach and are vouched for forever.
+	const dropOutboxRow = async (): Promise<void> => {
+		await outboxAttachmentService.discardAll(
+			account.accountConfigId,
+			accountId,
+			outboxMessageId,
+		);
+		await outboxMessageService.delete(account.accountConfigId, outboxMessageId);
+		log.info(
+			{ outboxMessageId },
+			"Deleted outbox row after successful APPEND to Sent",
+		);
+	};
+
+	// Everything this recovers from happened after the copy reached Sent, so
+	// re-appending is off the table and the delete is the only step left to
+	// retry. Below the redrive budget that retry is worth having; at it the
+	// record would dead-letter, so the row is left at `sent` for the repair.
+	const retryDropWithinBudget = (error: unknown): void => {
+		if (receiveCount < APPEND_SENT_MAX_ATTEMPTS) throw error;
+		log.error({ accountId, outboxMessageId, error }, ROW_SURVIVED_ITS_DELETE);
+	};
+
+	// A redelivery of an event whose APPEND already landed. The copy is in the
+	// user's Sent folder and the only step still owed is the delete that failed
+	// last time; starting from the top would file a second copy of a message the
+	// user sent once (#858). Ahead of the Sent-mailbox lookup on purpose — a
+	// message that is already filed does not need one, and an account that lost
+	// its Sent appointment in between must not settle as unfiled.
+	if (isSentCopyFiled(outbox)) {
+		log.info(
+			{ outboxMessageId, appendedUid: outbox.appendedUid },
+			"Sent copy was already filed by an earlier attempt, skipping the APPEND",
+		);
+		await dropOutboxRow().catch(retryDropWithinBudget);
+		return;
+	}
+
 	const sentMailbox = await mailboxSpecialUseService.findSentMailbox(accountId);
 	if (!sentMailbox) {
 		await settleUnfiled(UNFILED_NO_SENT_MAILBOX);
 		return;
 	}
 
-	let appended = false;
+	let appendedUid = APPENDED_UID_NONE;
 
 	const failure = await withOAuthLifecycle(
 		buildLifecycleDeps(secrets, accountService),
@@ -157,33 +224,25 @@ export const handleAppendSentMessage = async (
 						"Appended sent message to Sent mailbox",
 					);
 
-					appended = true;
+					// A server without UIDPLUS files the copy and names no uid for
+					// it, which is still a filed copy and has to read as one.
+					appendedUid = result.uid > 0 ? result.uid : APPENDED_UID_UNREPORTED;
 				})
 				.finally(() => scope.disconnect());
 
-			// The message now lives in the IMAP Sent folder. Drop the outbox row so
-			// the user does not see it twice in the UI (Outbox + Sent). Issue #178.
-			//
-			// The row goes first, and nothing that can fail may come before it. The
-			// APPEND above has already happened; anything between it and this delete
-			// that throws leaves the row for the job to retry, and the retry appends
-			// a second copy to the user's Sent folder. A storage error is not worth
-			// a duplicate message — the attachment objects that outlive their row
-			// are exactly what the sweep collects, so losing this delete costs a
-			// sweep and nothing else.
-			await outboxMessageService.delete(
-				account.accountConfigId,
-				outboxMessageId,
-			);
-			await outboxAttachmentService.discardAll(
+			// The idempotency key, and the only reason a redelivery is safe. It is
+			// written before the delete and never after it: from here on every step
+			// can fail and be retried without the user gaining a second copy, and
+			// the one window that still can is this single row update.
+			await outboxMessageService.update(
 				account.accountConfigId,
-				accountId,
 				outboxMessageId,
+				{ appendedUid },
 			);
-			log.info(
-				{ outboxMessageId },
-				"Deleted outbox row after successful APPEND to Sent",
-			);
+
+			// The message now lives in the IMAP Sent folder. Drop the outbox row so
+			// the user does not see it twice in the UI (Outbox + Sent). Issue #178.
+			await dropOutboxRow();
 		},
 	).then(
 		() => null,
@@ -193,16 +252,27 @@ export const handleAppendSentMessage = async (
 			// attempt to pick up. At the budget the record would dead-letter, and a
 			// dead-lettered APPEND is exactly how a delivered message goes missing.
 			//
-			// Once the APPEND itself has landed the copy is in Sent whatever else
-			// failed, so that case keeps the plain retry semantics.
-			if (appended || receiveCount < APPEND_SENT_MAX_ATTEMPTS) throw error;
+			// The budget still binds after a landed APPEND, though no longer to
+			// hold off a duplicate: the recorded uid does that, and the retries
+			// the budget allows re-drive the delete alone.
+			if (receiveCount < APPEND_SENT_MAX_ATTEMPTS) throw error;
 			return error;
 		},
 	);
 
-	// The APPEND landed: the copy is in Sent even if the row delete that follows
-	// it failed, so there is nothing to settle and a retry may still run.
-	if (appended) return;
+	// The APPEND landed: the copy is in Sent whatever failed after it, so there
+	// is nothing to settle as unfiled. A row that outlives its delete holds
+	// `sent`, which every view hides, until the migrator's boot-time stranded-row
+	// repair drops it — the next container start, not sooner (#824).
+	if (isSentCopyFiled({ appendedUid })) {
+		if (failure) {
+			log.error(
+				{ accountId, outboxMessageId, reason: String(failure) },
+				ROW_SURVIVED_ITS_DELETE,
+			);
+		}
+		return;
+	}
 
 	// A terminal auth failure returns here without throwing — withOAuthLifecycle
 	// flips the account to reauth_required and ACKs the record, which without