@remit/imap-worker 0.0.85 → 0.0.87

package/src/handlers/empty-trash.sqlite.test.ts

@@ -0,0 +1,209 @@
+import assert from "node:assert/strict";
+import { after, before, describe, it } from "node:test";
+import type { MailboxItem } from "@remit/data-ports";
+import {
+	MailboxCursorState,
+	MailboxSpecialUse,
+	MessageStatus,
+} from "@remit/domain-enums";
+import type { EmptyTrashEvent } from "../events.js";
+import {
+	openStore,
+	type SeededMessage,
+	type World,
+} from "../test-helpers/test-world.js";
+import { handleEmptyTrash } from "./empty-trash.js";
+
+let store: Awaited<ReturnType<typeof openStore>>;
+
+before(async () => {
+	store = await openStore();
+});
+
+after(() => store.close());
+
+interface Arranged {
+	w: World;
+	inbox: MailboxItem;
+	trash: MailboxItem;
+	trashed: SeededMessage[];
+}
+
+const arrange = async (count = 2): Promise<Arranged> => {
+	const w = await store.world();
+	const inbox = await w.folder("INBOX");
+	const trash = await w.folder("Trash", {
+		specialUse: MailboxSpecialUse.Trash,
+	});
+	const trashed: SeededMessage[] = [];
+	for (let i = 0; i < count; i += 1) trashed.push(await w.message(trash));
+	return { w, inbox, trash, trashed };
+};
+
+/** Empty Trash through the service the API calls, and hand back its event. */
+const emptyTrash = async (a: Arranged): Promise<EmptyTrashEvent> => {
+	await a.w.client.messageMove.emptyTrash(
+		a.w.account.accountConfigId,
+		a.w.account.accountId,
+	);
+	const event = a.w
+		.queued()
+		.find((body) => body.type === "EMPTY_TRASH") as unknown as EmptyTrashEvent;
+	assert.ok(event, "Empty Trash enqueued an EMPTY_TRASH");
+	return event;
+};
+
+const run = (a: Arranged, event: EmptyTrashEvent) =>
+	handleEmptyTrash(event, a.w.log, a.w.scope);
+
+const visible = async (a: Arranged): Promise<boolean[]> =>
+	Promise.all(
+		a.trashed.map(async (message) => {
+			const row = await a.w.row(message.messageId);
+			const listing = await a.w.threadRows(message.messageId);
+			return (
+				row?.status === MessageStatus.active &&
+				listing.length > 0 &&
+				listing.every((tm) => !tm.isDeleted)
+			);
+		}),
+	);
+
+describe("Empty Trash", () => {
+	it("expunges every message on the server and removes both local rows", async () => {
+		const a = await arrange();
+		const event = await emptyTrash(a);
+
+		await run(a, event);
+
+		assert.deepEqual(a.w.imap.uids("Trash"), []);
+		for (const message of a.trashed) {
+			assert.equal(await a.w.row(message.messageId), undefined);
+			assert.deepEqual(await a.w.threadRows(message.messageId), []);
+		}
+	});
+
+	it("hands every row back when another client emptied the Trash first", async () => {
+		const a = await arrange();
+		const event = await emptyTrash(a);
+		a.w.imap.empty("Trash");
+
+		await run(a, event);
+
+		assert.deepEqual(await visible(a), [true, true]);
+	});
+
+	it("keeps the row of a message the expunge never reached", async () => {
+		const a = await arrange();
+		const event = await emptyTrash(a);
+		const [kept, gone] = a.trashed;
+		a.w.imap.expunge("Trash", kept?.uid as number);
+
+		await run(a, event);
+
+		assert.equal(await a.w.row(gone?.messageId ?? ""), undefined);
+		assert.equal(
+			(await a.w.row(kept?.messageId ?? ""))?.status,
+			MessageStatus.active,
+		);
+	});
+
+	// A move into Trash that has not settled names Trash while still carrying
+	// INBOX's uid. Matching it against the expunge would delete whatever Trash
+	// holds at that number.
+	it("leaves a message whose move into Trash has not settled", async () => {
+		const a = await arrange(1);
+		const moving = await a.w.message(a.inbox);
+		await a.w.client.messageMove.deleteMessage(
+			a.w.account.accountConfigId,
+			moving.messageId,
+			a.w.account.accountId,
+		);
+		const event = await emptyTrash(a);
+
+		await run(a, event);
+
+		const row = await a.w.row(moving.messageId);
+		assert.equal(row?.status, MessageStatus.moving);
+		assert.deepEqual(a.w.imap.uids("INBOX"), [moving.uid]);
+		assert.equal(await a.w.row(a.trashed[0]?.messageId ?? ""), undefined);
+	});
+
+	it("expunges nothing and hands the rows back when the folder at the path is not the one emptied", async () => {
+		const a = await arrange();
+		const event = await emptyTrash(a);
+		a.w.imap.setUidValidity("Trash", 99);
+
+		await run(a, event);
+
+		assert.deepEqual(a.w.imap.commands("EXPUNGE"), []);
+		assert.deepEqual(await visible(a), [true, true]);
+	});
+
+	it("expunges nothing when the Trash role no longer rests on confirmed evidence", async () => {
+		const a = await arrange();
+		const event = await emptyTrash(a);
+		await a.w.client.mailboxSpecialUse.deleteByMailboxId(a.trash.mailboxId);
+
+		await run(a, event);
+
+		assert.equal(a.w.imap.connections, 0);
+		assert.deepEqual(await visible(a), [true, true]);
+	});
+
+	it("hands the rows back without connecting while the cursor is rebuilding", async () => {
+		const a = await arrange();
+		const event = await emptyTrash(a);
+		await a.w.client.mailbox.update(a.w.account.accountId, a.trash.mailboxId, {
+			cursorState: MailboxCursorState.rebuilding,
+		});
+
+		await run(a, event);
+
+		assert.equal(a.w.imap.connections, 0);
+		assert.deepEqual(await visible(a), [true, true]);
+	});
+
+	it("refuses an event minted under an unknown contract and hands the rows back", async () => {
+		const a = await arrange();
+		const event = await emptyTrash(a);
+
+		await run(a, { ...event, schemaVersion: 99 } as unknown as EmptyTrashEvent);
+
+		assert.equal(a.w.imap.connections, 0);
+		assert.deepEqual(await visible(a), [true, true]);
+	});
+
+	it("acks without connecting when the Trash mailbox was deleted", async () => {
+		const a = await arrange();
+		const event = await emptyTrash(a);
+		await a.w.client.mailbox.delete(a.w.account.accountId, a.trash.mailboxId);
+
+		await run(a, event);
+
+		assert.equal(a.w.imap.connections, 0);
+	});
+
+	it("empties Trash under its current path when the folder was renamed after enqueue", async () => {
+		const a = await arrange();
+		const event = await emptyTrash(a);
+		a.w.imap.renameFolder("Trash", "Deleted Items");
+		await a.w.client.mailbox.update(a.w.account.accountId, a.trash.mailboxId, {
+			fullPath: "Deleted Items",
+		});
+
+		await run(a, event);
+
+		assert.deepEqual(a.w.imap.uids("Deleted Items"), []);
+	});
+
+	it("rethrows a failed expunge so the event is retried", async () => {
+		const a = await arrange();
+		const event = await emptyTrash(a);
+		a.w.imap.failNext.EXPUNGE = new Error("Connection reset");
+
+		await assert.rejects(run(a, event), /Connection reset/);
+
+		assert.equal(a.w.imap.uids("Trash").length, 2);
+	});
+});

package/src/handlers/empty-trash.ts

@@ -3,7 +3,7 @@ import type { MessageItem } from "@remit/data-ports";
 import { trashMailboxAt } from "@remit/data-ports/folder-role";
 import { isAcceptedSchemaVersion } from "@remit/data-ports/mutation-events";
 import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
-import type { Logger } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
 import {
 	buildThreadMessageUndelete,
 	carriesForeignUid,
@@ -18,20 +18,6 @@ import { findMailboxRow } from "../mailbox-row.js";
 import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
 import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
 
-export interface EmptyTrashDeps {
-	getClient: typeof getClient;
-	buildLifecycleDeps: typeof buildLifecycleDeps;
-	withOAuthLifecycle: typeof withOAuthLifecycle;
-	createConnectionScope: typeof createConnectionScopeWithCredentials;
-}
-
-const defaultDeps: EmptyTrashDeps = {
-	getClient,
-	buildLifecycleDeps,
-	withOAuthLifecycle,
-	createConnectionScope: createConnectionScopeWithCredentials,
-};
-
 /**
  * Handle EMPTY_TRASH events.
  * Permanently deletes all messages in the Trash mailbox.
@@ -46,15 +32,8 @@ const defaultDeps: EmptyTrashDeps = {
 export const handleEmptyTrash = async (
 	event: EmptyTrashEvent,
 	log: Logger,
-	deps: EmptyTrashDeps = defaultDeps,
+	createConnectionScope = createConnectionScopeWithCredentials,
 ): Promise<void> => {
-	const {
-		getClient,
-		buildLifecycleDeps,
-		withOAuthLifecycle,
-		createConnectionScope: createConnectionScopeWithCredentials,
-	} = deps;
-
 	const {
 		account: accountService,
 		message: messageService,
@@ -219,7 +198,7 @@ export const handleEmptyTrash = async (
 				return;
 			}
 
-			const scope = createConnectionScopeWithCredentials(account, credentials);
+			const scope = createConnectionScope(account, credentials);
 
 			await scope
 				.getConnection()

package/src/handlers/flag-push.ts

@@ -1,7 +1,7 @@
 import { getClient } from "@remit/backend/client";
 import { isNotFoundError } from "@remit/data-ports/errors";
-import type { Logger } from "@remit/logger-lambda";
-import { recordImapFailure } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
+import { recordImapFailure } from "@remit/logger-lambda/metrics";
 import {
 	guardConnectionCursor,
 	isCursorRebuildNeeded,
@@ -30,20 +30,7 @@ export const FLAG_PUSH_MAX_ATTEMPTS = getFlagPushMaxAttempts();
  * cycle one SQS round trip per sync tick forever instead of surfacing the
  * stall.
  */
-const DEFAULT_FLAG_PUSH_DEFER_MAX_MS = 10 * 60 * 1000;
-
-export const getFlagPushDeferMaxMs = (
-	processEnv: NodeJS.ProcessEnv = process.env,
-): number => {
-	const raw = processEnv.FLAG_PUSH_DEFER_MAX_MS;
-	if (!raw) return DEFAULT_FLAG_PUSH_DEFER_MAX_MS;
-	const parsed = Number.parseInt(raw, 10);
-	return Number.isFinite(parsed) && parsed > 0
-		? parsed
-		: DEFAULT_FLAG_PUSH_DEFER_MAX_MS;
-};
-
-export const FLAG_PUSH_DEFER_MAX_MS = getFlagPushDeferMaxMs();
+export const FLAG_PUSH_DEFER_MAX_MS = 10 * 60 * 1000;
 
 /**
  * Handle FLAG_PUSH events (issue #1273, epic #1281). Drains ONE pending
@@ -75,6 +62,7 @@ export const handleFlagPush = async (
 	event: FlagPushEvent,
 	log: Logger,
 	receiveCount = 1,
+	createConnectionScope = createConnectionScopeWithCredentials,
 ): Promise<void> => {
 	const {
 		account: accountService,
@@ -202,7 +190,7 @@ export const handleFlagPush = async (
 		account,
 		log,
 		async (credentials) => {
-			const scope = createConnectionScopeWithCredentials(account, credentials);
+			const scope = createConnectionScope(account, credentials);
 
 			await scope
 				.getConnection()
@@ -261,8 +249,13 @@ export const handleFlagPush = async (
 					// two terminal outcomes (epic invariant 3) instead of
 					// dead-lettering with no diagnosis.
 					const { outcome } = await resolveExhaustedFlagPushFailure(
-						{ markerService, messageService, threadMessageService, log },
+						{ markerService, messageService, log },
 						{
+							placement: {
+								status: message.status,
+								mailboxId: message.mailboxId,
+								uid: message.uid,
+							},
 							accountId,
 							accountConfigId,
 							messageId,

package/src/handlers/mailbox-management.sqlite.test.ts

@@ -0,0 +1,265 @@
+import assert from "node:assert/strict";
+import { after, before, describe, it } from "node:test";
+import type { MailboxItem } from "@remit/data-ports";
+import { MailboxSyncStatus } from "@remit/domain-enums";
+import { imapError } from "@remit/mailbox-service/test-helpers/fake-imap";
+import type { MailboxManagementEvent } from "../events.js";
+import { openStore, type World } from "../test-helpers/test-world.js";
+import { processMailboxManagement } from "./mailbox-management.js";
+
+let store: Awaited<ReturnType<typeof openStore>>;
+
+before(async () => {
+	store = await openStore();
+});
+
+after(() => store.close());
+
+const lastEvent = (w: World, type: string): MailboxManagementEvent => {
+	const event = w
+		.queued()
+		.filter((body) => body.type === type)
+		.at(-1) as unknown as MailboxManagementEvent | undefined;
+	assert.ok(event, `a ${type} was enqueued`);
+	return event;
+};
+
+const run = (w: World, event: MailboxManagementEvent) =>
+	processMailboxManagement(event, w.log, w.scope);
+
+const mailbox = (
+	w: World,
+	mailboxId: string,
+): Promise<MailboxItem | undefined> =>
+	w.client.mailbox
+		.listAllByAccount(w.account.accountId)
+		.then((rows) => rows.find((row) => row.mailboxId === mailboxId));
+
+/** A folder created through the service the API calls, not yet on the server. */
+const requestCreate = async (
+	w: World,
+	path: string,
+	subscribe?: boolean,
+): Promise<{ mailboxId: string; event: MailboxManagementEvent }> => {
+	const created = await w.client.mailboxQueue.createMailbox(
+		{
+			accountId: w.account.accountId,
+			namespacePrefix: "",
+			hierarchyDelimiter: "/",
+			fullPath: path,
+			uidValidity: 0,
+			uidNext: 0,
+			highestModseq: "0",
+			messageCount: 0,
+			unseenCount: 0,
+			deletedCount: 0,
+			totalSize: 0,
+			lastSyncUid: 0,
+			highWaterMarkUid: 0,
+			lastMessageSyncAt: 0,
+		},
+		w.account.accountId,
+		subscribe,
+	);
+	assert.equal(created.outcome, "Created");
+	if (created.outcome !== "Created") throw new Error("path taken");
+	return {
+		mailboxId: created.mailbox.mailboxId,
+		event: lastEvent(w, "MAILBOX_CREATE"),
+	};
+};
+
+describe("creating a folder", () => {
+	it("creates it on the server and settles the row on the server's UIDVALIDITY", async () => {
+		const w = await store.world();
+		const { mailboxId, event } = await requestCreate(w, "Receipts");
+
+		await run(w, event);
+
+		assert.ok(w.imap.hasFolder("Receipts"));
+		const row = await mailbox(w, mailboxId);
+		assert.equal(row?.syncStatus, MailboxSyncStatus.synced);
+		assert.equal(row?.uidValidity, w.imap.uidValidityOf("Receipts"));
+	});
+
+	it("subscribes only when the request asks for it", async () => {
+		const w = await store.world();
+		const quiet = await requestCreate(w, "Quiet");
+		const loud = await requestCreate(w, "Loud", true);
+
+		await run(w, quiet.event);
+		await run(w, loud.event);
+
+		assert.deepEqual(w.imap.commands("SUBSCRIBE"), ["SUBSCRIBE Loud"]);
+	});
+
+	it("settles a folder the server already has as created", async () => {
+		const w = await store.world();
+		const { mailboxId, event } = await requestCreate(w, "Receipts");
+		w.imap.addFolder("Receipts", { uidValidity: 7 });
+
+		await run(w, event);
+
+		assert.equal(
+			(await mailbox(w, mailboxId))?.syncStatus,
+			MailboxSyncStatus.synced,
+		);
+	});
+
+	it("marks the folder failed with the server's reason and acks a refused create", async () => {
+		const w = await store.world();
+		const { mailboxId, event } = await requestCreate(w, "Receipts");
+		w.imap.failNext.CREATE = imapError("NOPERM", "Permission denied");
+
+		await run(w, event);
+
+		const row = await mailbox(w, mailboxId);
+		assert.equal(row?.syncStatus, MailboxSyncStatus.failed);
+		assert.equal(row?.syncFailureReason, "Permission denied");
+	});
+
+	it("marks the folder failed and redelivers when the create fails for another reason", async () => {
+		const w = await store.world();
+		const { mailboxId, event } = await requestCreate(w, "Receipts");
+		w.imap.failNext.CREATE = new Error("Connection reset");
+
+		await assert.rejects(run(w, event), /Connection reset/);
+
+		assert.equal(
+			(await mailbox(w, mailboxId))?.syncStatus,
+			MailboxSyncStatus.failed,
+		);
+	});
+
+	it("acks when the folder row was deleted before the create ran", async () => {
+		const w = await store.world();
+		const { mailboxId, event } = await requestCreate(w, "Receipts");
+		await w.client.mailbox.delete(w.account.accountId, mailboxId);
+
+		await run(w, event);
+	});
+});
+
+describe("renaming a folder", () => {
+	const arrange = async () => {
+		const w = await store.world();
+		const work = await w.folder("Work");
+		const child = await w.folder("Work/Clients");
+		const message = await w.message(child);
+		await w.client.mailboxQueue.renameMailbox(
+			work.mailboxId,
+			"Projects",
+			w.account.accountId,
+		);
+		return { w, work, child, message, event: lastEvent(w, "MAILBOX_RENAME") };
+	};
+
+	it("renames on the server and settles the folder and its children onto the new path", async () => {
+		const { w, work, child, event } = await arrange();
+
+		await run(w, event);
+
+		assert.ok(w.imap.hasFolder("Projects/Clients"));
+		assert.equal((await mailbox(w, work.mailboxId))?.fullPath, "Projects");
+		const settledChild = await mailbox(w, child.mailboxId);
+		assert.equal(settledChild?.fullPath, "Projects/Clients");
+		assert.equal(settledChild?.syncStatus, MailboxSyncStatus.synced);
+	});
+
+	it("removes the folder, its children and their mail when the source is gone on the server", async () => {
+		const { w, work, child, message, event } = await arrange();
+		w.imap.removeFolder("Work");
+		w.imap.removeFolder("Work/Clients");
+
+		await run(w, event);
+
+		assert.equal(await mailbox(w, work.mailboxId), undefined);
+		assert.equal(await mailbox(w, child.mailboxId), undefined);
+		assert.equal(await w.row(message.messageId), undefined);
+	});
+
+	it("keeps the confirmed path and acks a refused rename", async () => {
+		const { w, work, event } = await arrange();
+		w.imap.failNext.RENAME = imapError("NOPERM", "Permission denied");
+
+		await run(w, event);
+
+		const row = await mailbox(w, work.mailboxId);
+		assert.equal(row?.fullPath, "Work");
+		assert.equal(row?.syncStatus, MailboxSyncStatus.failed);
+		assert.ok(w.imap.hasFolder("Work"));
+	});
+
+	it("issues no RENAME when a redelivery finds the rename already settled", async () => {
+		const { w, event } = await arrange();
+		await run(w, event);
+
+		await run(w, event);
+
+		assert.equal(w.imap.commands("RENAME").length, 1);
+	});
+});
+
+describe("deleting a folder", () => {
+	const arrange = async () => {
+		const w = await store.world();
+		const old = await w.folder("Old");
+		const message = await w.message(old);
+		await w.client.mailboxQueue.deleteMailbox(
+			old.mailboxId,
+			w.account.accountId,
+		);
+		return { w, old, message, event: lastEvent(w, "MAILBOX_DELETE") };
+	};
+
+	it("deletes it on the server and removes the folder and its mail", async () => {
+		const { w, old, message, event } = await arrange();
+
+		await run(w, event);
+
+		assert.equal(w.imap.hasFolder("Old"), false);
+		assert.equal(await mailbox(w, old.mailboxId), undefined);
+		assert.equal(await w.row(message.messageId), undefined);
+	});
+
+	it("removes the folder and its mail when the server no longer has it", async () => {
+		const { w, old, message, event } = await arrange();
+		w.imap.removeFolder("Old");
+
+		await run(w, event);
+
+		assert.equal(await mailbox(w, old.mailboxId), undefined);
+		assert.equal(await w.row(message.messageId), undefined);
+	});
+
+	it("marks the folder failed, keeps its mail and acks a refused delete", async () => {
+		const { w, old, message, event } = await arrange();
+		w.imap.failNext.DELETE = imapError("NOPERM", "Permission denied");
+
+		await run(w, event);
+
+		assert.equal(
+			(await mailbox(w, old.mailboxId))?.syncStatus,
+			MailboxSyncStatus.failed,
+		);
+		assert.ok(await w.row(message.messageId));
+		assert.ok(w.imap.hasFolder("Old"));
+	});
+
+	it("issues no DELETE for a folder that is not being deleted", async () => {
+		const w = await store.world();
+		const kept = await w.folder("Kept");
+
+		await run(w, {
+			type: "MAILBOX_DELETE",
+			eventId: "evt-stale",
+			timestamp: Date.now(),
+			accountId: w.account.accountId,
+			mailboxId: kept.mailboxId,
+			path: "Kept",
+		} as MailboxManagementEvent);
+
+		assert.deepEqual(w.imap.commands("DELETE"), []);
+		assert.ok(await mailbox(w, kept.mailboxId));
+	});
+});