@remit/mailbox-service 0.0.95 → 0.0.97

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.95",
+  "version": "0.0.97",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/external-move.test.ts

@@ -0,0 +1,369 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	IMailboxRepository,
+	IMessageRepository,
+	MailboxItem,
+	MessageItem,
+} from "@remit/data-ports";
+import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
+import {
+	confirmDepartures,
+	placementKey,
+	sightingContestsPlacement,
+} from "./external-move.js";
+import type { IImapConnection } from "./types.js";
+
+const mailboxAt = (
+	fullPath: string,
+	specialUse?: string[],
+	uidValidity = 1,
+): MailboxItem =>
+	({
+		mailboxId: `mbx-${fullPath}`,
+		fullPath,
+		uidValidity,
+		hierarchyDelimiter: "/",
+		...(specialUse ? { specialUse } : {}),
+	}) as MailboxItem;
+
+const INBOX = mailboxAt("INBOX");
+const LABEL = mailboxAt("Receipts");
+const JUNK = mailboxAt("INBOX/Rubbish");
+const ALL_MAIL = mailboxAt("[Gmail]/All Mail", ["All"]);
+
+const storedIn = (
+	mailbox: MailboxItem,
+	overrides: Partial<MessageItem> = {},
+): MessageItem =>
+	({
+		messageId: "msg-1",
+		mailboxId: mailbox.mailboxId,
+		uid: 7,
+		status: MessageStatus.active,
+		syncStatus: MessageSyncStatus.pending,
+		...overrides,
+	}) as MessageItem;
+
+interface Probe {
+	opened: string[];
+	searched: string[];
+	departed: Set<string>;
+	settled: Set<string>;
+}
+
+/**
+ * Ask whether the messages this batch sighted in `sightedIn` have left the
+ * folders their rows point at, against a server where `held` names the uids each
+ * folder still holds and `served` the UIDVALIDITY it answers on SELECT.
+ */
+const probe = async (
+	sightedIn: MailboxItem,
+	rows: MessageItem[],
+	held: Record<string, number[]>,
+	served: Record<string, number> = {},
+	sources: MailboxItem[] = [INBOX],
+	unreachable: string[] = [],
+): Promise<Probe> => {
+	const opened: string[] = [];
+	const searched: string[] = [];
+
+	const connection = {
+		openBox: async (fullPath: string) => {
+			if (unreachable.includes(fullPath)) {
+				throw new Error(`no such mailbox on the server: ${fullPath}`);
+			}
+			opened.push(fullPath);
+			const source = sources.find((box) => box.fullPath === fullPath);
+			return { uidvalidity: served[fullPath] ?? source?.uidValidity ?? 1 };
+		},
+		search: async (criteria: unknown[]) => {
+			const [[, uids]] = criteria as Array<[string, string]>;
+			searched.push(uids);
+			const current = opened[opened.length - 1];
+			const asked = uids.split(",").map(Number);
+			return (held[current] ?? []).filter((uid) => asked.includes(uid));
+		},
+	} as unknown as Pick<IImapConnection, "openBox" | "search">;
+
+	const verdicts = await confirmDepartures(
+		{
+			connection,
+			mailboxService: {
+				get: async (_accountId: string, mailboxId: string) => {
+					const source = sources.find((box) => box.mailboxId === mailboxId);
+					if (!source) throw new Error(`no such mailbox: ${mailboxId}`);
+					return source;
+				},
+			} as unknown as Pick<IMailboxRepository, "get">,
+			messageService: {
+				get: async () => rows,
+			} as unknown as Pick<IMessageRepository, "get">,
+			log: { info: () => {}, warn: () => {} },
+		},
+		"acct-1",
+		sightedIn,
+		rows.map((row) => row.messageId),
+	);
+
+	return { opened, searched, ...verdicts };
+};
+
+describe("whether a sighting in a second folder is a move", () => {
+	/**
+	 * The reported bug (#1146). A Gmail user label, a Sieve `fileinto` beside a
+	 * `keep` and a Sent copy a list echoes back all put one message in two real
+	 * folders. Reading the second folder's sighting as a move took the message
+	 * out of the Inbox on the round that enumerated the label.
+	 */
+	it("is not a move while the folder the row points at still holds it", async () => {
+		const observed = await probe(LABEL, [storedIn(INBOX)], {
+			INBOX: [7],
+		});
+
+		assert.deepEqual([...observed.departed], []);
+	});
+
+	it("is a move once the folder the row points at has let it go", async () => {
+		const observed = await probe(LABEL, [storedIn(INBOX)], {
+			INBOX: [3, 11],
+		});
+
+		assert.deepEqual(
+			[...observed.departed],
+			[placementKey("msg-1", INBOX.mailboxId, 7)],
+		);
+	});
+
+	it("names the placement it was answered for, so a row that moved since is left alone", async () => {
+		const observed = await probe(JUNK, [storedIn(INBOX, { uid: 42 })], {
+			INBOX: [],
+		});
+
+		assert.equal(
+			observed.departed.has(placementKey("msg-1", INBOX.mailboxId, 7)),
+			false,
+		);
+		assert.equal(
+			observed.departed.has(placementKey("msg-1", INBOX.mailboxId, 42)),
+			true,
+		);
+	});
+
+	it("asks each source folder once for the whole batch", async () => {
+		const observed = await probe(
+			LABEL,
+			[
+				storedIn(INBOX, { messageId: "msg-1", uid: 7 }),
+				storedIn(INBOX, { messageId: "msg-2", uid: 8 }),
+				storedIn(INBOX, { messageId: "msg-3", uid: 9 }),
+			],
+			{ INBOX: [7] },
+		);
+
+		assert.deepEqual(observed.opened, ["INBOX"]);
+		assert.deepEqual(observed.searched, ["7,8,9"]);
+	});
+
+	/**
+	 * The answer needed is about the contested uids. A SEARCH ALL would pull a
+	 * page of uids per batch out of every source folder an account has, which on
+	 * a large Inbox is traffic proportional to the mailbox rather than to the
+	 * question.
+	 */
+	it("asks about the contested uids, not the whole source folder", async () => {
+		const observed = await probe(LABEL, [storedIn(INBOX, { uid: 42 })], {
+			INBOX: [1, 2, 42],
+		});
+
+		assert.deepEqual(observed.searched, ["42"]);
+		assert.deepEqual([...observed.departed], []);
+	});
+
+	it("asks nothing at all when no row is contested", async () => {
+		const observed = await probe(INBOX, [storedIn(INBOX)], { INBOX: [7] });
+
+		assert.deepEqual(observed.opened, []);
+		assert.deepEqual([...observed.departed], []);
+	});
+
+	it("asks nothing for a folder that holds a copy of every message", async () => {
+		const observed = await probe(ALL_MAIL, [storedIn(INBOX)], { INBOX: [] });
+
+		assert.deepEqual(observed.opened, []);
+		assert.deepEqual([...observed.departed], []);
+	});
+
+	it("asks nothing for a row whose own mutation has not settled", async () => {
+		const observed = await probe(
+			LABEL,
+			[storedIn(INBOX, { status: MessageStatus.moving })],
+			{ INBOX: [] },
+		);
+
+		assert.deepEqual(observed.opened, []);
+		assert.deepEqual([...observed.departed], []);
+	});
+
+	/**
+	 * All Mail, Starred and Important hold a copy of every message and release
+	 * nothing, so a row that landed on one — its sync ran before the real
+	 * folder's — would answer "still here" forever and freeze there. Reachable on
+	 * any Gmail account whose mail takes more than one batch.
+	 */
+	it("frees a row whose source is a folder that copies every message", async () => {
+		const observed = await probe(
+			LABEL,
+			[storedIn(ALL_MAIL)],
+			{ "[Gmail]/All Mail": [7] },
+			{},
+			[ALL_MAIL],
+		);
+
+		assert.deepEqual(observed.opened, []);
+		assert.deepEqual(
+			[...observed.departed],
+			[placementKey("msg-1", ALL_MAIL.mailboxId, 7)],
+		);
+	});
+
+	/**
+	 * Reader shares its mailboxes, so another client renaming or deleting the
+	 * source folder between the row being written and this round is ordinary. It
+	 * decides nothing about these rows, and it may not fail the sync of the
+	 * unrelated folder that happened to sight them.
+	 */
+	it("leaves a source folder it cannot reach undecided rather than failing the round", async () => {
+		const observed = await probe(
+			LABEL,
+			[storedIn(INBOX)],
+			{ INBOX: [] },
+			{},
+			[INBOX],
+			["INBOX"],
+		);
+
+		assert.deepEqual([...observed.departed], []);
+		assert.deepEqual([...observed.settled], []);
+	});
+
+	it("settles a decided sighting, so nothing holds the watermark for it", async () => {
+		const observed = await probe(LABEL, [storedIn(INBOX)], { INBOX: [7] });
+
+		assert.deepEqual(
+			[...observed.settled],
+			[placementKey("msg-1", INBOX.mailboxId, 7)],
+		);
+	});
+
+	/**
+	 * A uid means nothing without the axis it was issued on (RFC 9051 2.3.1.1),
+	 * so a source that answers a different UIDVALIDITY is evidence of nothing.
+	 * The cursor rebuild re-keys that folder; until it has, the sighting waits.
+	 */
+	it("refuses a source folder that has been re-keyed under it", async () => {
+		const observed = await probe(
+			LABEL,
+			[storedIn(INBOX)],
+			{ INBOX: [] },
+			{
+				INBOX: 99,
+			},
+		);
+
+		assert.deepEqual([...observed.departed], []);
+		assert.deepEqual([...observed.settled], []);
+	});
+});
+
+describe("sightingContestsPlacement", () => {
+	it("refuses a Gmail virtual folder the server never flagged", () => {
+		assert.equal(
+			sightingContestsPlacement(mailboxAt("[Gmail]/All Mail"), storedIn(INBOX)),
+			false,
+		);
+	});
+
+	it("accepts a folder the user named after a virtual one", () => {
+		assert.equal(
+			sightingContestsPlacement(mailboxAt("Starred ideas"), storedIn(INBOX)),
+			true,
+		);
+	});
+
+	it("accepts an ordinary inbound row the sync path left pending", () => {
+		assert.equal(
+			sightingContestsPlacement(
+				JUNK,
+				storedIn(INBOX, { syncStatus: MessageSyncStatus.pending }),
+			),
+			true,
+		);
+	});
+
+	it("accepts a row a settled mutation marked synced", () => {
+		assert.equal(
+			sightingContestsPlacement(
+				JUNK,
+				storedIn(INBOX, { syncStatus: MessageSyncStatus.synced }),
+			),
+			true,
+		);
+	});
+
+	/**
+	 * The row `abandonDelete` hands back: reader refused the delete, put the
+	 * message back where the server still has it, and nothing else is coming
+	 * for the row. Reader shares its mailboxes, so the user moving that same
+	 * message in another client is ordinary — and before R3 the sighting was
+	 * refused on `syncStatus` alone and the row never followed the move.
+	 */
+	it("accepts a row whose delete was abandoned, so a move made elsewhere still lands", () => {
+		assert.equal(
+			sightingContestsPlacement(
+				JUNK,
+				storedIn(INBOX, {
+					status: MessageStatus.active,
+					syncStatus: MessageSyncStatus.abandoned,
+				}),
+			),
+			true,
+		);
+	});
+
+	it("accepts a row left `failed` by a transient attempt that has since settled", () => {
+		assert.equal(
+			sightingContestsPlacement(
+				JUNK,
+				storedIn(INBOX, { syncStatus: MessageSyncStatus.failed }),
+			),
+			true,
+		);
+	});
+
+	it("refuses a row whose own move is still in flight", () => {
+		assert.equal(
+			sightingContestsPlacement(
+				JUNK,
+				storedIn(INBOX, {
+					status: MessageStatus.moving,
+					syncStatus: MessageSyncStatus.pending,
+				}),
+			),
+			false,
+		);
+	});
+
+	it("refuses a row whose own delete is still in flight", () => {
+		assert.equal(
+			sightingContestsPlacement(
+				JUNK,
+				storedIn(INBOX, {
+					status: MessageStatus.deleting,
+					syncStatus: MessageSyncStatus.pending,
+				}),
+			),
+			false,
+		);
+	});
+});

package/src/external-move.ts

@@ -0,0 +1,272 @@
+import type {
+	IMailboxRepository,
+	IMessageRepository,
+	MailboxItem,
+	MessageItem,
+} from "@remit/data-ports";
+import { isVirtualCopyMailbox } from "@remit/data-ports/virtual-copy";
+import { MessageStatus } from "@remit/domain-enums";
+import type { IImapConnection } from "./types.js";
+
+export interface ExternalMoveLogger {
+	info(obj: Record<string, unknown>, msg: string): void;
+	warn(obj: Record<string, unknown>, msg: string): void;
+}
+
+/**
+ * Whether a sighting of an already-stored message in this mailbox CONTESTS the
+ * placement the row holds (#859). Contesting is not deciding: the sighting is a
+ * candidate move, and {@link confirmDepartures} is what turns it into one.
+ *
+ * Three sightings do not even contest. The same mailbox is the row standing
+ * where it already is. Gmail's All Mail, Starred and Important hold every
+ * message that also lives in a real folder, so they are views rather than filing
+ * locations; following one would empty every other folder's listing. And a row
+ * whose own mutation has not settled is in flight, not stale
+ * (imap-mutations R1): the server's answer predates the move reader is still
+ * waiting on, so taking it would undo the optimistic write. The sync path
+ * reconciles rather than waits (imap-mutations R2): it never blocks on a
+ * mutation, it re-reads the row on the next sighting after the move settles.
+ *
+ * In flight is `status`, and `status` alone (#1096, imap-mutations R3). Every
+ * outbound mutation writes `moving` or `deleting` and only settling returns the
+ * row to `active`, so `active` is precisely the set nothing is coming for —
+ * whatever `syncStatus` says about how the row got there. A row put back by
+ * `abandonDelete` carries `abandoned`, and it needs this repair more than any
+ * other row does: nothing routine settles it, and reader shares its mailboxes,
+ * so refusing it left a message whose delete was refused unable to follow a
+ * move the user then made in another client.
+ */
+export const sightingContestsPlacement = (
+	mailbox: MailboxItem,
+	message: Pick<MessageItem, "mailboxId" | "status">,
+): boolean => {
+	if (message.mailboxId === mailbox.mailboxId) return false;
+	if (isVirtualCopyMailbox(mailbox)) return false;
+	return message.status === MessageStatus.active;
+};
+
+/**
+ * The exact placement a verdict was reached for. The probe reads the row outside
+ * the transaction that acts on it, so the verdict names the mailbox and uid it
+ * was reached for; a row that moved in between matches no verdict and is left
+ * unsettled.
+ */
+export const placementKey = (
+	messageId: string,
+	mailboxId: string,
+	uid: number,
+): string => `${messageId}:${mailboxId}:${uid}`;
+
+export interface DepartureVerdicts {
+	/**
+	 * Placements the source folder no longer holds. A sighting elsewhere of one
+	 * of these is a move, and re-points the row.
+	 */
+	departed: Set<string>;
+	/**
+	 * Placements a verdict was reached for at all, departed or still held. A
+	 * contested sighting outside this set was not decided — the source could not
+	 * be asked, or the row has moved since it was — and its uid is held back from
+	 * the watermark so the next round serves the sighting again. Without that a
+	 * single unanswerable round would leave the row mispointed until a cursor
+	 * rebuild, which is the silent never-repair this whole change is about.
+	 */
+	settled: Set<string>;
+}
+
+export interface DepartureProbeDeps {
+	connection: Pick<IImapConnection, "openBox" | "search">;
+	mailboxService: Pick<IMailboxRepository, "get">;
+	messageService: Pick<IMessageRepository, "get">;
+	log: ExternalMoveLogger;
+}
+
+/**
+ * Which of this batch's messages have really left the folder their row points at
+ * (#1146).
+ *
+ * A message sighted in a second real folder has not necessarily moved. Living in
+ * two folders at once is ordinary mail: a Gmail user label, a Sieve `fileinto`
+ * beside a `keep`, a Sent copy a mailing list echoes back. Reading every such
+ * sighting as a move takes the message out of the folder it is still in, and
+ * whichever folder the sync round enumerates last wins — which is how a labelled
+ * message disappeared from the Inbox on the next sync.
+ *
+ * What separates a move from a copy is the source folder: a move empties it, a
+ * copy leaves it alone. So the source is asked, and only a source that no longer
+ * holds the uid the row claims makes the sighting a move. A move another client
+ * made is still followed — that is the case where the source really is empty —
+ * and reader stops fighting a second client over mail neither of them moved.
+ *
+ * The question is asked with a UID SEARCH rather than a FETCH, for the reason
+ * `isMessageGoneFromOpenMailbox` documents: imapflow drops rows on back-to-back
+ * FETCHes (#408), and absence read off a FETCH would file live mail as departed.
+ * `IImapConnection.search` throws on a failed SEARCH, so an empty answer means
+ * the server matched nothing. The SEARCH names the contested uids rather than
+ * ALL: the answer needed is about those uids, and a folder's whole uid set is a
+ * page of traffic per batch for an account with a large Inbox.
+ *
+ * One SEARCH per source folder per batch, and no IMAP at all when nothing is
+ * contested — which is every round on an account whose folders do not overlap.
+ */
+export const confirmDepartures = async (
+	deps: DepartureProbeDeps,
+	accountId: string,
+	sightedIn: MailboxItem,
+	messageIds: string[],
+): Promise<DepartureVerdicts> => {
+	const verdicts: DepartureVerdicts = {
+		departed: new Set<string>(),
+		settled: new Set<string>(),
+	};
+	if (messageIds.length === 0) return verdicts;
+
+	const stored = await deps.messageService.get(messageIds);
+	// A row carrying no uid names no placement the source can be asked about, so
+	// it reaches no verdict here and its sighting is served again next round. It
+	// is not reachable from a sync — `upsertWithStatus` writes the uid the FETCH
+	// returned — and a folder that somehow produces one stalls its cursor and
+	// raises the alert rather than repairing a row against nothing.
+	const contested = stored.filter(
+		(row) => row.uid > 0 && sightingContestsPlacement(sightedIn, row),
+	);
+	if (contested.length === 0) return verdicts;
+
+	const bySource = new Map<string, MessageItem[]>();
+	for (const row of contested) {
+		const group = bySource.get(row.mailboxId);
+		if (group === undefined) {
+			bySource.set(row.mailboxId, [row]);
+			continue;
+		}
+		group.push(row);
+	}
+
+	for (const [sourceMailboxId, rows] of bySource) {
+		await askSource(
+			deps,
+			accountId,
+			sightedIn,
+			sourceMailboxId,
+			rows,
+			verdicts,
+		);
+	}
+
+	return verdicts;
+};
+
+/**
+ * What one source folder said about the rows pointed at it. Only `holds` decides
+ * anything; the other three are the folder declining to be evidence, each for
+ * its own reason and each leaving the sightings to come back next round.
+ */
+type SourceAnswer =
+	| { kind: "holds"; path: string; held: Set<number> }
+	| { kind: "copiesEverything"; path: string }
+	| { kind: "rekeyed"; path: string; stored: number; served: number }
+	| { kind: "unreachable"; reason: string };
+
+const readSource = async (
+	deps: DepartureProbeDeps,
+	accountId: string,
+	sourceMailboxId: string,
+	rows: MessageItem[],
+): Promise<SourceAnswer> => {
+	const source = await deps.mailboxService.get(accountId, sourceMailboxId);
+
+	// A row whose source is All Mail, Starred or Important is not filed there:
+	// those folders hold a copy of everything and never release anything, so
+	// asking them would answer "still here" forever and freeze the row on a view.
+	if (isVirtualCopyMailbox(source)) {
+		return { kind: "copiesEverything", path: source.fullPath };
+	}
+
+	const box = await deps.connection.openBox(source.fullPath, true);
+
+	// A source folder on a different UIDVALIDITY axis holds no uid these rows can
+	// be compared against, so its message set is evidence of nothing (RFC 9051
+	// 2.3.1.1). The cursor rebuild re-keys that folder; until it has, the
+	// sightings stay unsettled and come back next round.
+	if (box.uidvalidity !== source.uidValidity) {
+		return {
+			kind: "rekeyed",
+			path: source.fullPath,
+			stored: source.uidValidity,
+			served: box.uidvalidity,
+		};
+	}
+
+	const held = await deps.connection.search([
+		["UID", rows.map((row) => row.uid).join(",")],
+	]);
+	return { kind: "holds", path: source.fullPath, held: new Set(held) };
+};
+
+const askSource = async (
+	deps: DepartureProbeDeps,
+	accountId: string,
+	sightedIn: MailboxItem,
+	sourceMailboxId: string,
+	rows: MessageItem[],
+	verdicts: DepartureVerdicts,
+): Promise<void> => {
+	const settle = (row: MessageItem, departed: boolean): void => {
+		const key = placementKey(row.messageId, row.mailboxId, row.uid);
+		verdicts.settled.add(key);
+		if (departed) verdicts.departed.add(key);
+	};
+
+	// A source folder reader cannot reach decides nothing, and does not fail the
+	// round either. Another client renaming or deleting it between the row being
+	// written and now is ordinary — reader shares its mailboxes — and the sync of
+	// the unrelated folder that sighted these messages must survive it.
+	const answer = await readSource(
+		deps,
+		accountId,
+		sourceMailboxId,
+		rows,
+	).catch<SourceAnswer>((error: unknown) => ({
+		kind: "unreachable",
+		reason: error instanceof Error ? error.message : String(error),
+	}));
+
+	const observed = {
+		sourceMailboxId,
+		sightedMailboxId: sightedIn.mailboxId,
+		contested: rows.length,
+	};
+
+	if (answer.kind === "unreachable") {
+		deps.log.warn(
+			{ ...observed, error: answer.reason },
+			"Could not ask the source folder whether it still holds these messages; leaving the sightings unsettled this round",
+		);
+		return;
+	}
+
+	if (answer.kind === "rekeyed") {
+		deps.log.warn(
+			{
+				...observed,
+				sourceMailboxPath: answer.path,
+				storedUidValidity: answer.stored,
+				servedUidValidity: answer.served,
+			},
+			"Source folder is on a different UIDVALIDITY axis; leaving these sightings unsettled this round",
+		);
+		return;
+	}
+
+	if (answer.kind === "copiesEverything") {
+		for (const row of rows) settle(row, true);
+		deps.log.info(
+			{ ...observed, sourceMailboxPath: answer.path },
+			"Row points at a folder that copies every message; the sighting in a real folder is where it lives",
+		);
+		return;
+	}
+
+	for (const row of rows) settle(row, !answer.held.has(row.uid));
+};

package/src/imapflow-connection.ts

@@ -1053,6 +1053,15 @@ export class ImapFlowConnection {
 		}
 
 		const result = await client.mailboxRename(oldPath, newPath);
+		// `mailboxRename` resolves `undefined` when the connection is not
+		// AUTHENTICATED or SELECTED — RENAME was never issued. Dereferencing it
+		// raised a `TypeError` where the settle now reads the resolved path, and a
+		// `TypeError` is not an outcome the rename settle can classify.
+		if (!result) {
+			throw new Error(
+				`Rename of "${oldPath}" was not issued: the IMAP connection is not ready`,
+			);
+		}
 		return {
 			path: result.path,
 			newPath: result.newPath,

package/src/index.ts

@@ -193,6 +193,12 @@ export {
 	MailboxSyncService,
 	type SyncAccountInfo,
 } from "./mailbox-sync.js";
+export {
+	FolderGoneUpstreamError,
+	FolderRenameSettleError,
+	isMailboxAbsentUpstream,
+	isMailboxPresentUpstream,
+} from "./mailbox-upstream.js";
 export {
 	denormalizeMessageCategory,
 	type MessageCategoryDenormalizeDeps,