@remit/mailbox-service 0.0.111 → 0.0.112

package/src/placement.test.ts

@@ -0,0 +1,404 @@
+import assert from "node:assert/strict";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import type {
+	AccountItem,
+	CreateMessageInput,
+	MailboxItem,
+	MessageItem,
+} from "@remit/data-ports";
+import { MessagePlacementUnsettledError } from "@remit/data-ports/errors";
+import { deriveAddressId } from "@remit/data-ports/id";
+import { MessageMoveService } from "./message-move.js";
+import { PlacementMoveService } from "./placement-move.js";
+import { resolveExhaustedPlacementMoveFailure } from "./placement-move-terminal.js";
+import { eml, FakeImapServer } from "./test-helpers/fake-imap.js";
+import {
+	arriveAt,
+	beforeNext,
+	captureQueue,
+	openStore,
+	type QueueCapture,
+	type QueuedEvent,
+	type Store,
+	seedAccount,
+	seedMailbox,
+	seedMessage,
+	silentLog,
+} from "./test-helpers/test-store.js";
+
+let store: Store;
+let capture: QueueCapture;
+let queue: QueuedEvent[];
+let imap: FakeImapServer;
+let account: AccountItem;
+let inbox: MailboxItem;
+let archive: MailboxItem;
+let junk: MailboxItem;
+let trash: MailboxItem;
+
+beforeEach(async () => {
+	capture = captureQueue();
+	queue = capture.sent;
+	store = await openStore();
+	imap = new FakeImapServer();
+	account = await seedAccount(store);
+	inbox = await seedMailbox(store, account, "INBOX");
+	archive = await seedMailbox(store, account, "Archive");
+	junk = await seedMailbox(store, account, "Junk", { specialUse: "Junk" });
+	trash = await seedMailbox(store, account, "Trash", { specialUse: "Trash" });
+	for (const mailbox of await store.mailbox.listAllByAccount(
+		account.accountId,
+	)) {
+		imap.addFolder(mailbox.fullPath);
+	}
+});
+
+afterEach(async () => {
+	capture.restore();
+	await store.close();
+});
+
+const filing = () =>
+	new PlacementMoveService({
+		messageService: store.message,
+		threadMessageService: store.threadMessage,
+		markerService: store.placementMove,
+		addressService: store.address,
+		mailboxSpecialUseService: store.mailboxSpecialUse,
+		mailboxService: store.mailbox,
+		sqsQueueUrl: "https://sqs.test/message-mgmt",
+		moveSettleTimeoutMs: 200,
+		moveSettlePollMs: 10,
+	});
+
+const userMoves = () =>
+	new MessageMoveService({
+		messageService: store.message,
+		mailboxService: store.mailbox,
+		mailboxSpecialUseService: store.mailboxSpecialUse,
+		threadMessageService: store.threadMessage,
+		addressService: store.address,
+		sqsQueueUrl: "http://localhost:9324/000000000000/remit-messages.fifo",
+		moveSettleTimeoutMs: 200,
+		moveSettlePollMs: 10,
+	});
+
+const rowIn = async (
+	mailbox: MailboxItem,
+	uid: number,
+	message: Partial<CreateMessageInput> = {},
+) =>
+	(await seedMessage(store, account, mailbox, uid, { message })).message
+		.messageId;
+
+const row = (messageId: string) => store.message.get(messageId);
+
+const listing = async (messageId: string) =>
+	(
+		await store.threadMessage.findAllByMessageId(
+			account.accountConfigId,
+			messageId,
+		)
+	)[0];
+
+const file = (messageId: string, destination: MailboxItem) =>
+	filing().moveMessage(
+		account.accountConfigId,
+		messageId,
+		destination.mailboxId,
+		account.accountId,
+	);
+
+const confirmMove = async (messageId: string, uid: number) => {
+	const marker = await store.placementMove.find(messageId);
+	await store.message.updateUid(
+		messageId,
+		uid,
+		marker?.destinationMailboxId ?? "",
+	);
+	await store.placementMove.delete(messageId);
+};
+
+const settleElsewhere = (messageId: string) => () =>
+	store.message.updateUid(messageId, 907, archive.mailboxId);
+
+describe("a classification filing", () => {
+	it("moves the row and its listing locally, leaves a queued marker and enqueues the push", async () => {
+		const messageId = await rowIn(inbox, 42);
+
+		await file(messageId, junk);
+
+		const moved = await row(messageId);
+		assert.equal(moved.mailboxId, junk.mailboxId);
+		assert.equal(moved.status, "moving");
+		assert.equal(moved.originalUid, 42);
+		assert.equal((await listing(messageId)).mailboxId, junk.mailboxId);
+		assert.equal((await store.placementMove.find(messageId))?.state, "queued");
+		assert.deepEqual(
+			queue.map((event) => event.body.type),
+			["PLACEMENT_MOVE_PUSH"],
+		);
+	});
+
+	it("withholds the sender of a message it files into Junk", async () => {
+		const messageId = await arriveAt(
+			store,
+			imap,
+			account,
+			inbox,
+			eml({ From: "Spam <spam@example.com>" }),
+		);
+
+		await file(messageId, junk);
+
+		const sender = await store.address.getAddress(
+			account.accountConfigId,
+			deriveAddressId(account.accountConfigId, "spam@example.com"),
+		);
+		assert.equal(sender.flags.junkOnly?.value, true);
+	});
+
+	it("does nothing for a message already in the destination", async () => {
+		const messageId = await rowIn(junk, 42);
+
+		await file(messageId, junk);
+
+		assert.equal(await store.placementMove.find(messageId), null);
+		assert.equal(queue.length, 0);
+	});
+
+	it("binds a second filing to the confirmed row once the first move settles", async () => {
+		const messageId = await rowIn(inbox, 42);
+		await file(messageId, archive);
+
+		setTimeout(() => void confirmMove(messageId, 907), 30);
+		await file(messageId, junk);
+
+		const marker = await store.placementMove.find(messageId);
+		assert.equal(marker?.sourceMailboxId, archive.mailboxId);
+		assert.equal((await row(messageId)).originalUid, 907);
+	});
+
+	it("leaves the first move standing when it never settles", async () => {
+		const messageId = await rowIn(inbox, 42);
+		await file(messageId, archive);
+
+		await assert.rejects(file(messageId, junk), /has not settled/);
+
+		const marker = await store.placementMove.find(messageId);
+		assert.equal(marker?.sourceMailboxId, inbox.mailboxId);
+		assert.equal(marker?.destinationMailboxId, archive.mailboxId);
+	});
+
+	it("drives a marker whose push never went out forward without moving again", async () => {
+		const messageId = await rowIn(inbox, 42);
+		await file(messageId, archive);
+		await store.placementMove.updateState(messageId, "pending");
+
+		await file(messageId, archive);
+
+		assert.equal((await store.placementMove.find(messageId))?.state, "queued");
+		assert.equal((await row(messageId)).originalUid, 42);
+		assert.equal(queue.length, 2);
+	});
+
+	it("skips a message a delete has claimed without waiting", async () => {
+		const messageId = await rowIn(inbox, 42);
+		await userMoves().deleteMessages(
+			account.accountConfigId,
+			[messageId],
+			account.accountId,
+			{ permanent: true },
+		);
+		assert.equal((await row(messageId)).status, "deleting");
+		queue.length = 0;
+
+		const startedAt = Date.now();
+		await file(messageId, junk);
+
+		assert.ok(Date.now() - startedAt < 100);
+		assert.equal(await store.placementMove.find(messageId), null);
+		assert.equal(queue.length, 0);
+	});
+
+	it("writes nothing, listing included, when the row settles elsewhere mid-decision", async () => {
+		const messageId = await rowIn(inbox, 42);
+		beforeNext(store.placementMove, "put", settleElsewhere(messageId));
+
+		await file(messageId, junk);
+
+		const after = await row(messageId);
+		assert.equal(after.mailboxId, archive.mailboxId);
+		assert.equal(after.uid, 907);
+		assert.equal((await listing(messageId)).mailboxId, inbox.mailboxId);
+		assert.equal(await store.placementMove.find(messageId), null);
+		assert.equal(queue.length, 0);
+	});
+});
+
+describe("a user mutation writes only against the placement it read", () => {
+	it("refuses a move when the row settles elsewhere mid-decision", async () => {
+		const messageId = await rowIn(inbox, 42);
+		beforeNext(store.mailbox, "get", settleElsewhere(messageId));
+
+		await assert.rejects(
+			userMoves().moveMessage(
+				account.accountConfigId,
+				messageId,
+				junk.mailboxId,
+				account.accountId,
+			),
+			MessagePlacementUnsettledError,
+		);
+
+		const after = await row(messageId);
+		assert.equal(after.mailboxId, archive.mailboxId);
+		assert.equal(after.status, "active");
+		assert.equal(queue.length, 0);
+	});
+
+	for (const source of ["INBOX", "Trash"] as const) {
+		it(`reports a delete out of ${source} it could not claim`, async () => {
+			const messageId = await rowIn(source === "Trash" ? trash : inbox, 42);
+			beforeNext(
+				store.mailboxSpecialUse,
+				"resolveTrashRole",
+				settleElsewhere(messageId),
+			);
+
+			const { refusedMessageIds } = await userMoves().deleteMessages(
+				account.accountConfigId,
+				[messageId],
+				account.accountId,
+			);
+
+			assert.deepEqual(refusedMessageIds, [messageId]);
+			assert.equal((await row(messageId)).mailboxId, archive.mailboxId);
+			assert.equal(queue.length, 0);
+		});
+	}
+
+	it("reports a row whose folder belongs to another account", async () => {
+		const other = await seedAccount(store);
+		const foreign = await seedMailbox(store, other, "INBOX");
+		const messageId = await rowIn(foreign, 42);
+
+		const { refusedMessageIds } = await userMoves().deleteMessages(
+			account.accountConfigId,
+			[messageId],
+			account.accountId,
+		);
+
+		assert.deepEqual(refusedMessageIds, [messageId]);
+		assert.equal(queue.length, 0);
+	});
+});
+
+describe("a filing that exhausted its retries", () => {
+	const resolve = (
+		messageId: string,
+		uid: number,
+		source: MailboxItem,
+		placement: Pick<MessageItem, "status" | "mailboxId" | "uid">,
+	) =>
+		resolveExhaustedPlacementMoveFailure(
+			{
+				messageService: store.message,
+				threadMessageService: store.threadMessage,
+				markerService: store.placementMove,
+				log: silentLog,
+			},
+			{
+				placement,
+				accountId: account.accountId,
+				accountConfigId: account.accountConfigId,
+				messageId,
+				uid,
+				sourceMailboxId: source.mailboxId,
+				sourceMailboxPath: source.fullPath,
+				getConnection: async () => imap.connection(),
+			},
+		);
+
+	const filedFromInbox = async () => {
+		const uid = imap.deliver("INBOX", eml({}));
+		const messageId = await rowIn(inbox, uid);
+		await file(messageId, junk);
+		const read = await row(messageId);
+		return {
+			messageId,
+			uid,
+			placement: {
+				status: read.status,
+				mailboxId: read.mailboxId,
+				uid: read.uid,
+			},
+		};
+	};
+
+	it("removes the row of a message gone from the folder it left", async () => {
+		const { messageId, uid, placement } = await filedFromInbox();
+		imap.expunge("INBOX", uid);
+
+		const { outcome } = await resolve(messageId, uid, inbox, placement);
+
+		assert.equal(outcome, "reconciled");
+		await assert.rejects(row(messageId));
+		assert.equal(await store.placementMove.find(messageId), null);
+	});
+
+	it("puts a message still in its source back there instead of leaving it moving", async () => {
+		const { messageId, uid, placement } = await filedFromInbox();
+
+		const { outcome } = await resolve(messageId, uid, inbox, placement);
+
+		assert.equal(outcome, "broken");
+		const restored = await row(messageId);
+		assert.equal(restored.mailboxId, inbox.mailboxId);
+		assert.equal(restored.uid, uid);
+		assert.equal(restored.status, "active");
+		assert.equal(restored.abandonedMutation, "none");
+		assert.equal((await listing(messageId)).mailboxId, inbox.mailboxId);
+		assert.equal(await store.placementMove.find(messageId), null);
+	});
+
+	it("does not read a dropped FETCH row as the message being gone", async () => {
+		const { messageId, uid, placement } = await filedFromInbox();
+		const connection = imap.connection();
+		connection.fetchMessages = async () => [];
+
+		const { outcome } = await resolveExhaustedPlacementMoveFailure(
+			{
+				messageService: store.message,
+				threadMessageService: store.threadMessage,
+				markerService: store.placementMove,
+				log: silentLog,
+			},
+			{
+				placement,
+				accountId: account.accountId,
+				accountConfigId: account.accountConfigId,
+				messageId,
+				uid,
+				sourceMailboxId: inbox.mailboxId,
+				sourceMailboxPath: "INBOX",
+				getConnection: async () => connection,
+			},
+		);
+
+		assert.equal(outcome, "broken");
+		assert.ok(await row(messageId));
+	});
+
+	it("leaves a row another lane already settled to that lane", async () => {
+		const { messageId, uid, placement } = await filedFromInbox();
+		await confirmMove(messageId, 5);
+
+		const { outcome } = await resolve(messageId, uid, inbox, placement);
+
+		assert.equal(outcome, "broken");
+		const after = await row(messageId);
+		assert.equal(after.mailboxId, junk.mailboxId);
+		assert.equal((await listing(messageId)).mailboxId, junk.mailboxId);
+	});
+});

package/src/quarantine.test.ts

@@ -1,65 +1,39 @@
 import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import type {
-	IMailboxSpecialUseRepository,
-	IQuarantineRepository,
-	MessageData,
-	MessageItem,
-	QuarantineItem,
-	QuarantineUpsertInput,
-} from "@remit/data-ports";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import type { AccountItem, MailboxItem } from "@remit/data-ports";
 import {
-	QuarantinedUids,
 	QuarantineService,
 	resolveMailboxRole,
 	shapeFromMessageData,
 } from "./quarantine.js";
+import { eml, FakeImapServer } from "./test-helpers/fake-imap.js";
+import {
+	arriveAt,
+	openStore,
+	type Store,
+	seedAccount,
+	seedMailbox,
+	silentLog,
+} from "./test-helpers/test-store.js";
+
+let store: Store;
+let account: AccountItem;
+let spam: MailboxItem;
+let service: QuarantineService;
+
+beforeEach(async () => {
+	store = await openStore();
+	account = await seedAccount(store);
+	spam = await seedMailbox(store, account, "Spam", { specialUse: "Junk" });
+	service = new QuarantineService(
+		store.quarantine,
+		store.mailboxSpecialUse,
+		"sha-abc",
+		silentLog,
+	);
+});
 
-const noopLog = { info: () => {}, warn: () => {} };
-
-const buildService = (
-	specialUse: string[] = [],
-): {
-	service: QuarantineService;
-	writes: QuarantineUpsertInput[];
-	listed: string[];
-} => {
-	const writes: QuarantineUpsertInput[] = [];
-	const listed: string[] = [];
-	const repository = {
-		listByAccountConfigId: async (accountConfigId: string) => {
-			listed.push(accountConfigId);
-			return [] as QuarantineItem[];
-		},
-		upsert: async (input: QuarantineUpsertInput) => {
-			writes.push(input);
-		},
-	} satisfies IQuarantineRepository;
-
-	const mailboxSpecialUseService = {
-		listByMailboxId: async () => specialUse.map((use) => ({ specialUse: use })),
-	} as unknown as IMailboxSpecialUseRepository;
-
-	return {
-		service: new QuarantineService(
-			repository,
-			mailboxSpecialUseService,
-			"sha-abc",
-			noopLog,
-		),
-		writes,
-		listed,
-	};
-};
-
-const context = {
-	accountId: "acct-1",
-	accountConfigId: "cfg-1",
-	mailboxId: "mbx-1",
-	mailboxPath: "INBOX",
-	uidValidity: 1_712_000_000,
-	attempts: 2,
-};
+afterEach(() => store.close());
 
 const failure = {
 	stage: "BodyParse" as const,
@@ -67,47 +41,48 @@ const failure = {
 	message: "the parser said no",
 };
 
+const record = (uid: number, mailboxPath = "Spam") =>
+	service.record(
+		{
+			accountId: account.accountId,
+			accountConfigId: account.accountConfigId,
+			mailboxId: spam.mailboxId,
+			mailboxPath,
+			uidValidity: 1_712_000_000,
+			attempts: 2,
+		},
+		uid,
+		failure,
+	);
+
+const rows = () =>
+	store.quarantine.listByAccountConfigId(account.accountConfigId);
+
 describe("QuarantineService.record", () => {
-	it("stamps the worker build, not the client's", async () => {
-		const { service, writes } = buildService();
-		await service.record(context, 40217, failure);
-		assert.equal(writes[0]?.workerVersion, "sha-abc");
+	it("keeps one record per uid however often the message is set aside", async () => {
+		await record(40217);
+		await record(40217);
+
+		const recorded = (await rows()).filter((row) => row.uid === 40217);
+		assert.equal(recorded.length, 1);
+		assert.equal(recorded[0].uidValidity, 1_712_000_000);
+		assert.equal(recorded[0].workerVersion, "sha-abc");
+		assert.equal(recorded[0].failurePartPath, undefined);
 	});
 
-	it("names the message by mailbox, uidValidity and uid", async () => {
-		const { service, writes } = buildService();
-		await service.record(context, 40217, failure);
-		assert.deepEqual(
-			{
-				mailboxId: writes[0]?.mailboxId,
-				uidValidity: writes[0]?.uidValidity,
-				uid: writes[0]?.uid,
-			},
-			{ mailboxId: "mbx-1", uidValidity: 1_712_000_000, uid: 40217 },
-		);
-	});
+	it("reads the folder's role off the server's SPECIAL-USE", async () => {
+		await record(7);
 
-	it("supplies no quarantineId, so the random-id default is unreachable", async () => {
-		const { service, writes } = buildService();
-		await service.record(context, 40217, failure);
-		assert.ok(!("quarantineId" in (writes[0] as object)));
+		const [row] = (await rows()).filter((entry) => entry.uid === 7);
+		assert.equal(row.mailboxRole, "Junk");
 	});
 
-	it("omits an absent diagnostic rather than writing a null through", async () => {
-		const { service, writes } = buildService();
-		await service.record(
-			{ ...context, mailboxPath: "Clients/Acme" },
-			1,
-			failure,
-		);
-		assert.ok(!("failurePartPath" in (writes[0] as object)));
-		assert.ok(!("mailboxRole" in (writes[0] as object)));
-	});
+	it("loads what it recorded as the set a sync round skips", async () => {
+		await record(8);
 
-	it("reads the folder's role off the server's SPECIAL-USE", async () => {
-		const { service, writes } = buildService(["Junk"]);
-		await service.record({ ...context, mailboxPath: "Spam" }, 1, failure);
-		assert.equal(writes[0]?.mailboxRole, "Junk");
+		const loaded = await service.load(account.accountConfigId);
+		assert.ok(loaded.has(spam.mailboxId, 1_712_000_000, 8));
+		assert.ok(!loaded.has(spam.mailboxId, 1_712_000_001, 8));
 	});
 });
 
@@ -125,67 +100,34 @@ describe("resolveMailboxRole", () => {
 	});
 });
 
-describe("QuarantinedUids", () => {
-	const entries = [
-		{ mailboxId: "mbx-1", uidValidity: 10, uid: 5 },
-	] as QuarantineItem[];
-
-	it("matches a uid on the same mailbox and UIDVALIDITY", () => {
-		assert.equal(new QuarantinedUids(entries).has("mbx-1", 10, 5), true);
-	});
-
-	it("does not match the same uid on a new UIDVALIDITY, which is a different message", () => {
-		assert.equal(new QuarantinedUids(entries).has("mbx-1", 11, 5), false);
-	});
-
-	it("does not match the same uid in another mailbox", () => {
-		assert.equal(new QuarantinedUids(entries).has("mbx-2", 10, 5), false);
-	});
-});
-
 describe("shapeFromMessageData", () => {
-	const message = {
-		rfc822Size: 2048,
-		messageIdHeader: "<xyz@example.com>",
-	} as MessageItem;
-
-	const data = {
-		bodyPart: [
-			{
-				bodyPartId: "bp-0",
-				partPath: "0",
-				mediaType: "multipart",
-				mediaSubtype: "alternative",
-				transferEncoding: "7bit",
-			},
-			{
-				bodyPartId: "bp-1",
-				partPath: "1",
-				mediaType: "text",
-				mediaSubtype: "plain",
-				transferEncoding: "quoted-printable",
-			},
-		],
-		bodyPartParameter: [
-			{
-				bodyPartId: "bp-0",
-				parameterName: "charset",
-				parameterValue: "iso-8859-1",
-			},
-		],
-	} as unknown as MessageData;
-
-	it("rebuilds the tree from the rows metadata sync already wrote", () => {
-		assert.deepEqual(shapeFromMessageData(message, data).structure, [
-			{ depth: 0, contentType: "multipart/alternative" },
-			{ depth: 1, contentType: "text/plain" },
-		]);
-	});
+	it("describes the message from the rows metadata sync wrote", async () => {
+		const imap = new FakeImapServer();
+		imap.addFolder("Spam");
+		const raw = eml({
+			"Message-ID": "<xyz@example.com>",
+			"Content-Type": "text/plain; charset=iso-8859-1",
+			"Content-Transfer-Encoding": "quoted-printable",
+		});
+		const messageId = await arriveAt(store, imap, account, spam, raw);
+
+		const shape = shapeFromMessageData(
+			await store.message.get(messageId),
+			await store.envelope.getMessageData(messageId),
+			(
+				await store.threadMessage.findByMessageId(
+					account.accountConfigId,
+					messageId,
+				)
+			)?.messageIdHeader,
+		);
 
-	it("takes the root part's declared charset and encoding", () => {
-		const shape = shapeFromMessageData(message, data);
+		assert.equal(shape.structure.length, 1);
+		assert.equal(shape.structure[0].depth, 0);
+		assert.match(shape.structure[0].contentType, /^text\/plain$/i);
 		assert.equal(shape.charset, "iso-8859-1");
-		assert.equal(shape.transferEncoding, "7bit");
-		assert.equal(shape.contentType, "multipart/alternative");
+		assert.equal(shape.transferEncoding?.toLowerCase(), "quoted-printable");
+		assert.equal(shape.sizeBytes, raw.length);
+		assert.ok(shape.messageIdHash);
 	});
 });

package/src/quarantine.ts

@@ -76,8 +76,9 @@ const EMPTY_SHAPE: QuarantineMessageShape = { structure: [] };
  * has just refused.
  */
 export const shapeFromMessageData = (
-	message: MessageItem,
+	message: Pick<MessageItem, "rfc822Size">,
 	data: MessageData,
+	messageIdHeader: string | undefined,
 ): QuarantineMessageShape => {
 	const parts = [...data.bodyPart].sort((a, b) =>
 		a.partPath.localeCompare(b.partPath, "en", { numeric: true }),
@@ -100,7 +101,7 @@ export const shapeFromMessageData = (
 			depth: partPathDepth(part.partPath),
 			contentType: `${part.mediaType}/${part.mediaSubtype}`,
 		})),
-		...hashOf(message.messageIdHeader),
+		...hashOf(messageIdHeader),
 	};
 };