@remit/imap-worker 0.0.80 → 0.0.82

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.80",
+  "version": "0.0.82",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/handlers/append-sent-message.test.ts

@@ -1,6 +1,7 @@
 import assert from "node:assert/strict";
 import { beforeEach, describe, it } from "node:test";
 import { noopLogger } from "@remit/logger-lambda/noop-logger";
+import { OutboxAttachmentUnavailableError } from "@remit/mailbox-service/outbox-attachment-content";
 import type { AppendSentMessageEvent } from "../events.js";
 import {
 	APPEND_SENT_MAX_ATTEMPTS,
@@ -31,6 +32,11 @@ interface Harness {
 	 * without ever running the work. */
 	ackWithoutWork: boolean;
 	disconnectCount: number;
+	attachments: Array<{
+		filename: string;
+		contentType: string;
+		content: Buffer;
+	}>;
 }
 
 let h: Harness;
@@ -65,6 +71,7 @@ const fresh = (): Harness => ({
 	},
 	ackWithoutWork: false,
 	disconnectCount: 0,
+	attachments: [],
 });
 
 const deps = (): AppendSentMessageDeps =>
@@ -83,6 +90,10 @@ const deps = (): AppendSentMessageDeps =>
 			},
 			outboxAttachment: {
 				discardAll: record("outboxAttachment.discardAll"),
+				contentsFor: async (...args: unknown[]) => {
+					h.calls.push({ method: "outboxAttachment.contentsFor", args });
+					return h.attachments;
+				},
 			},
 			mailboxSpecialUse: {
 				findSentMailbox: async () => h.sentMailbox,
@@ -190,6 +201,85 @@ describe("handleAppendSentMessage", () => {
 		]);
 	});
 
+	it("files the Sent copy with the same files the wire copy carried", async () => {
+		h.attachments = [
+			{
+				filename: "numbers.csv",
+				contentType: "text/csv",
+				content: Buffer.from("q1,q2\n1,2\n"),
+			},
+		];
+
+		await handleAppendSentMessage(event, noopLogger, 1, deps());
+
+		assert.deepEqual(called("outboxAttachment.contentsFor")[0]?.args, [
+			"cfg-1",
+			"acc-1",
+			"out-1",
+		]);
+		const raw = String(called("connection.append")[0]?.args[1] as Buffer);
+		assert.match(raw, /^Content-Type: multipart\/mixed;/m);
+		assert.match(
+			raw,
+			/^Content-Disposition: attachment; filename=numbers\.csv$/m,
+		);
+		assert.match(raw, /^Content-Type: text\/csv; name=numbers\.csv$/m);
+	});
+
+	const unreadable = new OutboxAttachmentUnavailableError(
+		"numbers.csv",
+		"nothing is stored at key/numbers",
+	);
+
+	const depsWithUnreadableFile = (): AppendSentMessageDeps => {
+		const base = deps();
+		return {
+			...base,
+			getClient: async (): Promise<BackendClient> => {
+				const client = await base.getClient();
+				client.outboxAttachment.contentsFor = async () => {
+					throw unreadable;
+				};
+				return client;
+			},
+		};
+	};
+
+	it("retries a copy whose files could not be read, below the budget", async () => {
+		const failing = depsWithUnreadableFile();
+
+		await assert.rejects(
+			() => handleAppendSentMessage(event, noopLogger, 1, failing),
+			unreadable,
+		);
+		assert.equal(called("connection.append").length, 0);
+		assert.deepEqual(patches(), [], "the row is left for the retry");
+	});
+
+	it("settles a delivered message whose files cannot be read as sent, not filed, at the budget", async () => {
+		const failing = depsWithUnreadableFile();
+
+		await handleAppendSentMessage(
+			event,
+			noopLogger,
+			APPEND_SENT_MAX_ATTEMPTS,
+			failing,
+		);
+		assert.deepEqual(patches(), [
+			{
+				status: "unfiled",
+				lastError:
+					'Sent, but not filed: the copy for INBOX/Sent could not be built because "numbers.csv" could not be read: nothing is stored at key/numbers.',
+			},
+		]);
+		assert.equal(called("connection.append").length, 0);
+		assert.equal(
+			called("outboxMessage.delete").length,
+			0,
+			"the row stays in the Outbox with its reason",
+		);
+	});
+
 	it("builds the message from the outbox row's own headers", async () => {
 		await handleAppendSentMessage(event, noopLogger, 1, deps());
 

package/src/handlers/append-sent-message.ts

@@ -7,6 +7,7 @@ import {
 import { isNotFoundError } from "@remit/data-ports/errors";
 import { OutboxMessageStatus } from "@remit/domain-enums";
 import type { Logger } from "@remit/logger-lambda";
+import { OutboxAttachmentUnavailableError } from "@remit/mailbox-service/outbox-attachment-content";
 import {
 	buildMailMessage,
 	renderRawMessage,
@@ -28,7 +29,9 @@ const ROW_SURVIVED_ITS_DELETE =
 	"Sent message was filed but its outbox row survived its delete and stays hidden until the boot-time repair";
 
 const unfiledAppendRefused = (fullPath: string, error: unknown): string =>
-	`Sent, but not filed: the mail server refused to store a copy in ${fullPath} (${error instanceof Error ? error.message : String(error)}).`;
+	error instanceof OutboxAttachmentUnavailableError
+		? `Sent, but not filed: the copy for ${fullPath} could not be built because ${error.message}.`
+		: `Sent, but not filed: the mail server refused to store a copy in ${fullPath} (${error instanceof Error ? error.message : String(error)}).`;
 
 export const getAppendSentMaxAttempts = (
 	processEnv: NodeJS.ProcessEnv = process.env,
@@ -188,12 +191,22 @@ export const handleAppendSentMessage = async (
 		account,
 		log,
 		async (credentials) => {
+			// Inside the retried unit: a file that cannot be read is retried below
+			// the budget and settles `unfiled` at it, rather than dead-lettering a
+			// delivered message out of every view.
+			const attachments = await outboxAttachmentService.contentsFor(
+				account.accountConfigId,
+				accountId,
+				outboxMessageId,
+			);
 			const scope = createConnectionScopeWithCredentials(account, credentials);
 
 			await scope
 				.getConnection()
 				.then(async (connection) => {
-					const rawMessage = await renderRawMessage(buildMailMessage(outbox));
+					const rawMessage = await renderRawMessage(
+						buildMailMessage(outbox, attachments),
+					);
 
 					const result = await connection.append(
 						sentMailbox.fullPath,

package/src/handlers/mailbox-management.test.ts

@@ -711,6 +711,23 @@ describe("processMailboxManagement — a tagged NO the server means as success (
 		assert.equal(called("mailbox.transition").length, 0);
 	});
 
+	it("marks failed and acks when the server refuses the DELETE with NOPERM", async () => {
+		recordDeleteIntent();
+		h.connection.deleteMailbox = async () => {
+			throw Object.assign(new Error("Command failed"), {
+				serverResponseCode: "NOPERM",
+				responseText: "Permission denied",
+			});
+		};
+
+		await assert.doesNotReject(
+			processMailboxManagement(deleteEvent, noopLogger, deps()),
+		);
+		assert.equal(lastSettle().to, "failed");
+		assert.equal(lastSettle().set?.syncFailureReason, "Permission denied");
+		assert.equal(called("mailbox.deleteMailboxWithMail").length, 0);
+	});
+
 	it("still marks failed and rethrows when the server fails for any other reason", async () => {
 		recordDeleteIntent();
 		h.connection.deleteMailbox = async () => {
@@ -747,6 +764,21 @@ describe("processMailboxManagement — a tagged NO the server means as success (
 		assert.equal(lastSettle().to, "synced");
 	});
 
+	it("marks failed with the server's reason and acks when a CREATE is refused", async () => {
+		h.connection.createMailbox = async () => {
+			throw Object.assign(new Error("Command failed"), {
+				serverResponseCode: "NOPERM",
+				responseText: "Permission denied",
+			});
+		};
+
+		await assert.doesNotReject(
+			processMailboxManagement(createEvent, noopLogger, deps()),
+		);
+		assert.equal(lastSettle().to, "failed");
+		assert.equal(lastSettle().set?.syncFailureReason, "Permission denied");
+	});
+
 	it("still marks failed and rethrows when a CREATE fails for any other reason", async () => {
 		h.connection.createMailbox = async () => {
 			throw Object.assign(new Error("Command failed"), {
@@ -781,6 +813,25 @@ describe("processMailboxManagement — a tagged NO the server means as success (
 		]);
 	});
 
+	it("marks failed with the server's reason and acks when a RENAME is refused", async () => {
+		recordRenameIntent();
+		h.connection.renameMailbox = async () => {
+			throw Object.assign(new Error("Command failed"), {
+				serverResponseCode: "CANNOT",
+				responseText: "Can't rename mailbox with its children",
+			});
+		};
+
+		await assert.doesNotReject(
+			processMailboxManagement(renameEvent, noopLogger, deps()),
+		);
+		assert.equal(lastSettle().to, "failed");
+		assert.equal(
+			lastSettle().set?.syncFailureReason,
+			"Can't rename mailbox with its children",
+		);
+	});
+
 	it("still refuses the intent and rethrows when a RENAME fails for any other reason", async () => {
 		recordRenameIntent();
 		h.connection.renameMailbox = async () => {

package/src/handlers/mailbox-management.ts

@@ -9,7 +9,9 @@ import {
 	FolderRenameSettleError,
 	isMailboxAbsentUpstream,
 	isMailboxPresentUpstream,
+	isMailboxRefusedUpstream,
 	MailboxManagementService,
+	upstreamFailureReason,
 } from "@remit/mailbox-service";
 import { isAccountDeleted } from "../account-check.js";
 import { createConnectionScopeWithCredentials } from "../connection-scope.js";
@@ -47,12 +49,13 @@ const recordCreateOutcome = async (
 	accountId: string,
 	mailboxId: string,
 	to: (typeof MailboxSyncStatus)[keyof typeof MailboxSyncStatus],
+	syncFailureReason = "",
 ): Promise<void> => {
 	const written = await mailboxService.transition(accountId, mailboxId, {
 		from: [MailboxSyncStatus.pending],
 		wherePendingPath: null,
 		to,
-		set: {},
+		set: { syncFailureReason },
 	});
 	if (!written) throw new NotFoundError(`Mailbox not found: ${mailboxId}`);
 };
@@ -167,7 +170,15 @@ const handleCreate = async (
 								accountId,
 								mailboxId,
 								MailboxSyncStatus.failed,
+								upstreamFailureReason(error),
 							);
+							if (isMailboxRefusedUpstream(error)) {
+								log.error(
+									{ accountId, mailboxId, path, intent: "create", error },
+									"Mail server refused the create",
+								);
+								return;
+							}
 							throw error;
 						}
 					})
@@ -333,7 +344,22 @@ const handleRename = async (
 							mailboxId,
 							oldPath,
 							newPath,
+							upstreamFailureReason(error),
 						);
+						if (isMailboxRefusedUpstream(error)) {
+							log.error(
+								{
+									accountId,
+									mailboxId,
+									oldPath,
+									newPath,
+									intent: "rename",
+									error,
+								},
+								"Mail server refused the rename",
+							);
+							return;
+						}
 						throw error;
 					})
 					.finally(() => scope.disconnect());
@@ -431,7 +457,11 @@ const handleDelete = async (
 							// worker's backstop is unreachable from it; if something else
 							// reaches it, the honest outcome is `failed`, not `synced` —
 							// the folder was never deleted and nothing was undone.
-							await managementService.failDelete(accountId, mailboxId);
+							await managementService.failDelete(
+								accountId,
+								mailboxId,
+								upstreamFailureReason(error),
+							);
 							if (
 								error instanceof Error &&
 								error.message.includes("Cannot delete INBOX")
@@ -443,6 +473,13 @@ const handleDelete = async (
 								// Don't rethrow — no retry can make this succeed.
 								return;
 							}
+							if (isMailboxRefusedUpstream(error)) {
+								log.error(
+									{ accountId, mailboxId, path, intent: "delete", error },
+									"Mail server refused the delete",
+								);
+								return;
+							}
 							throw error;
 						}
 					})