@remit/mailbox-service 0.0.79 → 0.0.81

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.79",
+  "version": "0.0.81",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/message-sync-flags.test.ts

@@ -19,7 +19,6 @@ import type { ImapEnvelope } from "./types.js";
 // in another client arrived unstarred and never appeared in Flagged. The
 // server's \Flagged keyword is the star; these cover the mapping on create.
 
-const ACCOUNT_ID = "acct-1";
 const ACCOUNT_CONFIG_ID = "cfg-1";
 const MAILBOX_ID = "mbx-1";
 const MESSAGE_ID = "msg-1";
@@ -66,7 +65,6 @@ type CreateThreadForMessage = (
 	threadMessageService: IThreadMessageRepository,
 	messageId: string,
 	mailboxId: string,
-	accountId: string,
 	accountConfigId: string,
 	uid: number,
 	internalDate: number,
@@ -99,7 +97,6 @@ const createThreadWithFlags = async (
 		repo,
 		MESSAGE_ID,
 		MAILBOX_ID,
-		ACCOUNT_ID,
 		ACCOUNT_CONFIG_ID,
 		42,
 		now,

package/src/message-sync-repoint.test.ts

@@ -0,0 +1,322 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	IAddressRepository,
+	IEnvelopeRepository,
+	IMailboxRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+	MailboxItem,
+	MessageItem,
+	ThreadMessageItem,
+	UpdateThreadMessageInput,
+} from "@remit/data-ports";
+import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
+import type { ManagedConnectionFactory } from "./connection-factory.js";
+import {
+	type AccountFolderRoles,
+	MessageSyncService,
+	repointsOnSighting,
+} from "./message-sync.js";
+import { folderRoles, NO_JUNK_ROLES } from "./test-helpers/folder-roles.js";
+import type { ImapEnvelope, ImapMessage } from "./types.js";
+
+const stub = <T>(): T => ({}) as T;
+
+const envelope: ImapEnvelope = {
+	date: new Date(0).toISOString(),
+	messageId: "<root@example.com>",
+	subject: "Subject",
+	from: [{ name: "Pharma Deals", mailbox: "sales", host: "pharma.example" }],
+	sender: [],
+	replyTo: [],
+	to: [{ name: "", mailbox: "victim", host: "ischen.nl" }],
+	cc: [],
+	bcc: [],
+	inReplyTo: "",
+};
+
+const mailboxAt = (fullPath: string, specialUse?: string[]): MailboxItem =>
+	({
+		mailboxId: `mbx-${fullPath}`,
+		fullPath,
+		hierarchyDelimiter: "/",
+		...(specialUse ? { specialUse } : {}),
+	}) as MailboxItem;
+
+const INBOX = mailboxAt("INBOX");
+const JUNK = mailboxAt("INBOX/Rubbish");
+const TRASH = mailboxAt("Trash");
+const ALL_MAIL = mailboxAt("[Gmail]/All Mail", ["All"]);
+
+const storedIn = (
+	mailbox: MailboxItem,
+	overrides: Partial<MessageItem> = {},
+): MessageItem =>
+	({
+		mailboxId: mailbox.mailboxId,
+		uid: 7,
+		status: MessageStatus.active,
+		syncStatus: MessageSyncStatus.synced,
+		...overrides,
+	}) as MessageItem;
+
+const threadRowIn = (mailbox: MailboxItem): ThreadMessageItem =>
+	({
+		threadMessageId: "tm-1",
+		accountConfigId: "cfg-1",
+		mailboxId: mailbox.mailboxId,
+		uid: 7,
+		sentDate: 0,
+		isRead: false,
+		isDeleted: mailbox === TRASH,
+		hasStars: false,
+		hasAttachment: false,
+	}) as ThreadMessageItem;
+
+interface Observed {
+	repointedTo: Array<{ mailboxId: string; uid: number }>;
+	threadUpdates: UpdateThreadMessageInput[];
+	reconciled: string[];
+	owned: boolean;
+}
+
+/**
+ * Sync one message out of `sighting`, against a database that already holds it
+ * under `stored`. The account's Junk folder is `JUNK` and its Trash is `TRASH`,
+ * so the sighting's own role follows from which mailbox it is.
+ */
+const sync = async (
+	sighting: MailboxItem,
+	stored: MessageItem,
+	threadRow: ThreadMessageItem | null = threadRowIn(INBOX),
+	copiesHere: MessageItem[] = [],
+): Promise<Observed> => {
+	const observed: Observed = {
+		repointedTo: [],
+		threadUpdates: [],
+		reconciled: [],
+		owned: false,
+	};
+
+	const messageService = {
+		upsertWithStatus: async () => ({ item: stored, created: false }),
+		get: async () => copiesHere,
+		updateUid: async (_id: string, uid: number, mailboxId: string) => {
+			observed.repointedTo.push({ mailboxId, uid });
+			return stored;
+		},
+	} as unknown as IMessageRepository;
+
+	const threadMessageService = {
+		findByMessageId: async () => threadRow,
+		update: async (
+			_config: string,
+			_id: string,
+			input: UpdateThreadMessageInput,
+		) => {
+			observed.threadUpdates.push(input);
+			return threadRow as ThreadMessageItem;
+		},
+		create: async (input: unknown) => input as ThreadMessageItem,
+	} as unknown as IThreadMessageRepository;
+
+	const envelopeService = {
+		upsertEnvelope: async () => {},
+		upsertBodyParts: async () => {},
+	} as unknown as IEnvelopeRepository;
+
+	const addressService = {
+		upsertCorrespondentAddress: async () => {},
+		upsertJunkAddress: async () => {},
+		upsertAddress: async () => {},
+		upsertEnvelopeAddress: async () => {},
+		reconcileJunkOnlyForMessage: async (messageId: string) => {
+			observed.reconciled.push(messageId);
+		},
+	} as unknown as IAddressRepository;
+
+	const service = new MessageSyncService(
+		stub<ManagedConnectionFactory>(),
+		stub<IMailboxRepository>(),
+		folderRoles({
+			junkMailboxId: JUNK.mailboxId,
+			trashMailboxId: TRASH.mailboxId,
+		}),
+		messageService,
+		envelopeService,
+		addressService,
+		threadMessageService,
+	);
+
+	const msg = {
+		uid: 42,
+		seq: 1,
+		size: 100,
+		internalDate: new Date(0),
+		flags: [],
+		envelope,
+	} as unknown as ImapMessage;
+
+	const result = (await (
+		service as unknown as {
+			saveMessage: (
+				mailbox: MailboxItem,
+				accountId: string,
+				accountConfigId: string,
+				msg: ImapMessage,
+				roles: AccountFolderRoles,
+			) => Promise<{ owned: boolean }>;
+		}
+	).saveMessage(sighting, "acct-1", "cfg-1", msg, {
+		junkMailboxId: JUNK.mailboxId,
+		trashMailboxId: TRASH.mailboxId,
+		configJunkRoles: NO_JUNK_ROLES,
+	})) as { owned: boolean };
+
+	observed.owned = result.owned;
+	return observed;
+};
+
+describe("which folder a message the database already holds lives in", () => {
+	it("follows the last move when another client files it into Junk", async () => {
+		const observed = await sync(JUNK, storedIn(INBOX));
+
+		assert.deepEqual(observed.repointedTo, [
+			{ mailboxId: JUNK.mailboxId, uid: 42 },
+		]);
+	});
+
+	it("follows the last move back when another client rescues it", async () => {
+		const observed = await sync(INBOX, storedIn(JUNK), threadRowIn(JUNK));
+
+		assert.deepEqual(observed.repointedTo, [
+			{ mailboxId: INBOX.mailboxId, uid: 42 },
+		]);
+	});
+
+	it("takes the UID of the folder that issued it, never the old one", async () => {
+		const observed = await sync(JUNK, storedIn(INBOX, { uid: 7 }));
+
+		assert.equal(observed.repointedTo[0].uid, 42);
+	});
+
+	it("leaves a row this folder already owns alone", async () => {
+		const observed = await sync(INBOX, storedIn(INBOX));
+
+		assert.deepEqual(observed.repointedTo, []);
+		assert.equal(observed.owned, true);
+	});
+
+	it("holds a row whose own move has not settled", async () => {
+		const observed = await sync(
+			INBOX,
+			storedIn(JUNK, {
+				status: MessageStatus.moving,
+				syncStatus: MessageSyncStatus.pending,
+			}),
+		);
+
+		assert.deepEqual(observed.repointedTo, []);
+		assert.equal(observed.owned, false);
+	});
+
+	it("declines a sighting in a folder that copies every message", async () => {
+		const observed = await sync(ALL_MAIL, storedIn(INBOX));
+
+		assert.deepEqual(observed.repointedTo, []);
+		assert.equal(observed.owned, false);
+	});
+
+	it("declines a sighting of a copy the user put in this folder", async () => {
+		const observed = await sync(JUNK, storedIn(INBOX), threadRowIn(INBOX), [
+			storedIn(JUNK),
+		]);
+
+		assert.deepEqual(observed.repointedTo, []);
+		assert.deepEqual(observed.threadUpdates, []);
+	});
+
+	it("hands the re-pointed message to this folder's body sync", async () => {
+		const observed = await sync(JUNK, storedIn(INBOX));
+
+		assert.equal(observed.owned, true);
+	});
+});
+
+describe("what a re-pointed message does to the listing that renders it", () => {
+	it("moves the row the listing and the folder counts read", async () => {
+		const observed = await sync(JUNK, storedIn(INBOX));
+
+		assert.deepEqual(observed.threadUpdates, [
+			{ mailboxId: JUNK.mailboxId, uid: 42, isDeleted: false },
+		]);
+	});
+
+	it("marks the row deleted when the move was into Trash", async () => {
+		const observed = await sync(TRASH, storedIn(INBOX));
+
+		assert.equal(observed.threadUpdates[0].isDeleted, true);
+	});
+
+	it("clears the deleted mark when the move was out of Trash", async () => {
+		const observed = await sync(INBOX, storedIn(TRASH), threadRowIn(TRASH));
+
+		assert.equal(observed.threadUpdates[0].isDeleted, false);
+	});
+
+	it("writes nothing when no row renders the message yet", async () => {
+		const observed = await sync(JUNK, storedIn(INBOX), null);
+
+		assert.deepEqual(observed.threadUpdates, []);
+		assert.deepEqual(observed.repointedTo, [
+			{ mailboxId: JUNK.mailboxId, uid: 42 },
+		]);
+	});
+});
+
+describe("what a re-pointed message does to its sender's standing", () => {
+	it("re-asks the question when the move was into Junk", async () => {
+		const observed = await sync(JUNK, storedIn(INBOX));
+
+		assert.equal(observed.reconciled.length, 1);
+	});
+
+	it("re-asks it again when the move was back out of Junk", async () => {
+		const observed = await sync(INBOX, storedIn(JUNK), threadRowIn(JUNK));
+
+		assert.equal(observed.reconciled.length, 1);
+	});
+
+	it("asks nothing when the pointer did not move", async () => {
+		const observed = await sync(INBOX, storedIn(INBOX));
+
+		assert.deepEqual(observed.reconciled, []);
+	});
+});
+
+describe("repointsOnSighting", () => {
+	it("refuses a Gmail virtual folder the server never flagged", () => {
+		assert.equal(
+			repointsOnSighting(mailboxAt("[Gmail]/All Mail"), storedIn(INBOX)),
+			false,
+		);
+	});
+
+	it("accepts a folder the user named after a virtual one", () => {
+		assert.equal(
+			repointsOnSighting(mailboxAt("Starred ideas"), storedIn(INBOX)),
+			true,
+		);
+	});
+
+	it("refuses a row whose move failed and has not been re-tried", () => {
+		assert.equal(
+			repointsOnSighting(
+				JUNK,
+				storedIn(INBOX, { syncStatus: MessageSyncStatus.failed }),
+			),
+			false,
+		);
+	});
+});

package/src/message-sync.ts

@@ -10,6 +10,7 @@ import type {
 	IThreadMessageRepository,
 	IUnitOfWork,
 	MailboxItem,
+	MessageItem,
 	ThreadMessageItem,
 } from "@remit/data-ports";
 import { storedDisplayName } from "@remit/data-ports/display-name";
@@ -17,15 +18,19 @@ import type { JunkRoleMailboxes } from "@remit/data-ports/folder-role";
 import {
 	deriveAddressId,
 	deriveBodyPartId,
+	deriveCopyMessageId,
 	deriveEnvelopeId,
 	deriveMessageIdFromSource,
 	deriveThreadId,
 	isValidMessageId,
 } from "@remit/data-ports/id";
+import { isVirtualCopyMailbox } from "@remit/data-ports/virtual-copy";
 import {
 	AddressRole,
 	MailboxCursorState,
 	MessageKeywordFlag,
+	MessageStatus,
+	MessageSyncStatus,
 	MessageSystemFlag,
 	StarColor,
 } from "@remit/domain-enums";
@@ -128,6 +133,33 @@ export const addressSightingIn = (
 	return "correspondent";
 };
 
+/**
+ * Whether a sighting of an already-stored message in this mailbox re-points the
+ * row at this folder (#859). A message lives in one folder and the model never
+ * holds it in two, so a sighting somewhere else is not a rival claim to the
+ * row — it is a move another client made under reader. The last move wins, in
+ * both directions, and junk withholding derives from the pointer that results.
+ *
+ * Two sightings are not moves. Gmail's All Mail, Starred and Important hold
+ * every message that also lives in a real folder, so they are views rather than
+ * filing locations; following one would empty every other folder's listing. And
+ * a row whose own mutation has not settled is in flight, not stale
+ * (imap-mutations R1): the server's answer predates the move reader is still
+ * waiting on, so taking it would undo the optimistic write. That sighting is
+ * reconciled by the next one after the mutation settles.
+ */
+export const repointsOnSighting = (
+	mailbox: MailboxItem,
+	message: Pick<MessageItem, "mailboxId" | "status" | "syncStatus">,
+): boolean => {
+	if (message.mailboxId === mailbox.mailboxId) return false;
+	if (isVirtualCopyMailbox(mailbox)) return false;
+	return (
+		message.status === MessageStatus.active &&
+		message.syncStatus === MessageSyncStatus.synced
+	);
+};
+
 /**
  * The name an envelope carries for an address, as it should be stored: any
  * claim to be some other address removed (issue #826), the rest of the name
@@ -194,8 +226,9 @@ export interface SyncedMessage {
 
 /**
  * Per-message save outcome. `owned` is true when the row was created by this
- * sync or already belongs to the current mailbox; false for a residual
- * cross-mailbox collision whose stored row points at a different mailbox.
+ * sync, already belongs to the current mailbox, or was re-pointed at it by this
+ * sighting; false for a collision this sync declined to follow — a virtual-copy
+ * folder, or a row whose own mutation has not settled.
  */
 interface SaveMessageResult extends SyncedMessage {
 	owned: boolean;
@@ -1443,9 +1476,35 @@ export class MessageSyncService {
 				envelopeId,
 				rootBodyPartId,
 			});
-			owned = created || item.mailboxId === mailboxId;
 
-			if (sighting === "junk") {
+			// The server has just named this folder and this UID for a message the
+			// database holds under another folder. `upsertWithStatus` left the row
+			// alone, so the pointer is repaired here — mailbox and UID together,
+			// because a UID only means anything inside the folder that issued it.
+			const repointed =
+				!created &&
+				repointsOnSighting(mailbox, item) &&
+				!(await this.holdsCopyOf(repos.message, messageId, mailboxId));
+			if (repointed) {
+				await repos.message.updateUid(messageId, msg.uid, mailboxId);
+				await this.repointThreadMessage(
+					repos.threadMessage,
+					accountConfigId,
+					messageId,
+					mailboxId,
+					msg.uid,
+					sighting === "discarded",
+				);
+			}
+
+			owned = created || item.mailboxId === mailboxId || repointed;
+
+			// Withholding reads the pointer, so it is re-derived whenever the
+			// pointer could have changed the answer: a save into Junk, and a
+			// re-point in either direction. The reconcile both applies and lifts,
+			// so a sender whose only mail was moved out of Junk is restored by the
+			// same call that would have withheld them (#859).
+			if (sighting === "junk" || repointed) {
 				await repos.address.reconcileJunkOnlyForMessage(
 					messageId,
 					roles.configJunkRoles,
@@ -1456,7 +1515,6 @@ export class MessageSyncService {
 				repos.threadMessage,
 				messageId,
 				mailboxId,
-				accountId,
 				accountConfigId,
 				msg.uid,
 				msg.internalDate.getTime(),
@@ -1480,6 +1538,69 @@ export class MessageSyncService {
 		return { messageId, uid: msg.uid, owned };
 	}
 
+	/**
+	 * Whether this folder already holds a copy the user made of this message
+	 * (#75). A copy is a second row under its own derived id, so the sighting is
+	 * that copy standing where it was put — the original has not moved, and
+	 * following it would take the mail out of the folder the user copied it from.
+	 */
+	private async holdsCopyOf(
+		messageService: IMessageRepository,
+		messageId: string,
+		mailboxId: string,
+	): Promise<boolean> {
+		const rows = await messageService.get([
+			deriveCopyMessageId(messageId, mailboxId),
+		]);
+		return rows.length > 0;
+	}
+
+	/**
+	 * Follow a re-pointed Message with the ThreadMessage row that renders it.
+	 * The listing and every folder count read that row, so leaving it behind
+	 * would show the message in the folder it just left and hide it from the one
+	 * it landed in. `isDeleted` follows the same rule the move path uses — true
+	 * in Trash, false anywhere else — so a message another client rescues out of
+	 * Trash stops being filtered out of listings.
+	 */
+	private async repointThreadMessage(
+		threadMessageService: IThreadMessageRepository,
+		accountConfigId: string,
+		messageId: string,
+		mailboxId: string,
+		uid: number,
+		isDeleted: boolean,
+	): Promise<void> {
+		const row = await threadMessageService.findByMessageId(
+			accountConfigId,
+			messageId,
+		);
+		if (row === null) return;
+		if (
+			row.mailboxId === mailboxId &&
+			row.uid === uid &&
+			row.isDeleted === isDeleted
+		) {
+			return;
+		}
+
+		await threadMessageService.update(
+			row.accountConfigId,
+			row.threadMessageId,
+			{ mailboxId, uid, isDeleted },
+			{
+				composites: {
+					mailboxId: row.mailboxId,
+					sentDate: row.sentDate,
+					isRead: row.isRead,
+					isDeleted: row.isDeleted,
+					hasStars: row.hasStars,
+					hasAttachment: row.hasAttachment,
+				},
+			},
+		);
+	}
+
 	private async saveAddresses(
 		addressService: IAddressRepository,
 		messageId: string,
@@ -1569,7 +1690,6 @@ export class MessageSyncService {
 		threadMessageService: IThreadMessageRepository,
 		messageId: string,
 		mailboxId: string,
-		accountId: string,
 		accountConfigId: string,
 		uid: number,
 		internalDate: number,
@@ -1603,7 +1723,7 @@ export class MessageSyncService {
 		}
 
 		// Derive threadId from the root Message-ID (deterministic)
-		const threadId = deriveThreadId(accountId, rootMessageIdHeader);
+		const threadId = deriveThreadId(accountConfigId, rootMessageIdHeader);
 
 		const isRead = flags.includes(MessageSystemFlag.Seen);