@remit/mailbox-service 0.0.94 → 0.0.96

package/src/external-move.ts

@@ -0,0 +1,272 @@
+import type {
+	IMailboxRepository,
+	IMessageRepository,
+	MailboxItem,
+	MessageItem,
+} from "@remit/data-ports";
+import { isVirtualCopyMailbox } from "@remit/data-ports/virtual-copy";
+import { MessageStatus } from "@remit/domain-enums";
+import type { IImapConnection } from "./types.js";
+
+export interface ExternalMoveLogger {
+	info(obj: Record<string, unknown>, msg: string): void;
+	warn(obj: Record<string, unknown>, msg: string): void;
+}
+
+/**
+ * Whether a sighting of an already-stored message in this mailbox CONTESTS the
+ * placement the row holds (#859). Contesting is not deciding: the sighting is a
+ * candidate move, and {@link confirmDepartures} is what turns it into one.
+ *
+ * Three sightings do not even contest. The same mailbox is the row standing
+ * where it already is. Gmail's All Mail, Starred and Important hold every
+ * message that also lives in a real folder, so they are views rather than filing
+ * locations; following one would empty every other folder's listing. And a row
+ * whose own mutation has not settled is in flight, not stale
+ * (imap-mutations R1): the server's answer predates the move reader is still
+ * waiting on, so taking it would undo the optimistic write. The sync path
+ * reconciles rather than waits (imap-mutations R2): it never blocks on a
+ * mutation, it re-reads the row on the next sighting after the move settles.
+ *
+ * In flight is `status`, and `status` alone (#1096, imap-mutations R3). Every
+ * outbound mutation writes `moving` or `deleting` and only settling returns the
+ * row to `active`, so `active` is precisely the set nothing is coming for —
+ * whatever `syncStatus` says about how the row got there. A row put back by
+ * `abandonDelete` carries `abandoned`, and it needs this repair more than any
+ * other row does: nothing routine settles it, and reader shares its mailboxes,
+ * so refusing it left a message whose delete was refused unable to follow a
+ * move the user then made in another client.
+ */
+export const sightingContestsPlacement = (
+	mailbox: MailboxItem,
+	message: Pick<MessageItem, "mailboxId" | "status">,
+): boolean => {
+	if (message.mailboxId === mailbox.mailboxId) return false;
+	if (isVirtualCopyMailbox(mailbox)) return false;
+	return message.status === MessageStatus.active;
+};
+
+/**
+ * The exact placement a verdict was reached for. The probe reads the row outside
+ * the transaction that acts on it, so the verdict names the mailbox and uid it
+ * was reached for; a row that moved in between matches no verdict and is left
+ * unsettled.
+ */
+export const placementKey = (
+	messageId: string,
+	mailboxId: string,
+	uid: number,
+): string => `${messageId}:${mailboxId}:${uid}`;
+
+export interface DepartureVerdicts {
+	/**
+	 * Placements the source folder no longer holds. A sighting elsewhere of one
+	 * of these is a move, and re-points the row.
+	 */
+	departed: Set<string>;
+	/**
+	 * Placements a verdict was reached for at all, departed or still held. A
+	 * contested sighting outside this set was not decided — the source could not
+	 * be asked, or the row has moved since it was — and its uid is held back from
+	 * the watermark so the next round serves the sighting again. Without that a
+	 * single unanswerable round would leave the row mispointed until a cursor
+	 * rebuild, which is the silent never-repair this whole change is about.
+	 */
+	settled: Set<string>;
+}
+
+export interface DepartureProbeDeps {
+	connection: Pick<IImapConnection, "openBox" | "search">;
+	mailboxService: Pick<IMailboxRepository, "get">;
+	messageService: Pick<IMessageRepository, "get">;
+	log: ExternalMoveLogger;
+}
+
+/**
+ * Which of this batch's messages have really left the folder their row points at
+ * (#1146).
+ *
+ * A message sighted in a second real folder has not necessarily moved. Living in
+ * two folders at once is ordinary mail: a Gmail user label, a Sieve `fileinto`
+ * beside a `keep`, a Sent copy a mailing list echoes back. Reading every such
+ * sighting as a move takes the message out of the folder it is still in, and
+ * whichever folder the sync round enumerates last wins — which is how a labelled
+ * message disappeared from the Inbox on the next sync.
+ *
+ * What separates a move from a copy is the source folder: a move empties it, a
+ * copy leaves it alone. So the source is asked, and only a source that no longer
+ * holds the uid the row claims makes the sighting a move. A move another client
+ * made is still followed — that is the case where the source really is empty —
+ * and reader stops fighting a second client over mail neither of them moved.
+ *
+ * The question is asked with a UID SEARCH rather than a FETCH, for the reason
+ * `isMessageGoneFromOpenMailbox` documents: imapflow drops rows on back-to-back
+ * FETCHes (#408), and absence read off a FETCH would file live mail as departed.
+ * `IImapConnection.search` throws on a failed SEARCH, so an empty answer means
+ * the server matched nothing. The SEARCH names the contested uids rather than
+ * ALL: the answer needed is about those uids, and a folder's whole uid set is a
+ * page of traffic per batch for an account with a large Inbox.
+ *
+ * One SEARCH per source folder per batch, and no IMAP at all when nothing is
+ * contested — which is every round on an account whose folders do not overlap.
+ */
+export const confirmDepartures = async (
+	deps: DepartureProbeDeps,
+	accountId: string,
+	sightedIn: MailboxItem,
+	messageIds: string[],
+): Promise<DepartureVerdicts> => {
+	const verdicts: DepartureVerdicts = {
+		departed: new Set<string>(),
+		settled: new Set<string>(),
+	};
+	if (messageIds.length === 0) return verdicts;
+
+	const stored = await deps.messageService.get(messageIds);
+	// A row carrying no uid names no placement the source can be asked about, so
+	// it reaches no verdict here and its sighting is served again next round. It
+	// is not reachable from a sync — `upsertWithStatus` writes the uid the FETCH
+	// returned — and a folder that somehow produces one stalls its cursor and
+	// raises the alert rather than repairing a row against nothing.
+	const contested = stored.filter(
+		(row) => row.uid > 0 && sightingContestsPlacement(sightedIn, row),
+	);
+	if (contested.length === 0) return verdicts;
+
+	const bySource = new Map<string, MessageItem[]>();
+	for (const row of contested) {
+		const group = bySource.get(row.mailboxId);
+		if (group === undefined) {
+			bySource.set(row.mailboxId, [row]);
+			continue;
+		}
+		group.push(row);
+	}
+
+	for (const [sourceMailboxId, rows] of bySource) {
+		await askSource(
+			deps,
+			accountId,
+			sightedIn,
+			sourceMailboxId,
+			rows,
+			verdicts,
+		);
+	}
+
+	return verdicts;
+};
+
+/**
+ * What one source folder said about the rows pointed at it. Only `holds` decides
+ * anything; the other three are the folder declining to be evidence, each for
+ * its own reason and each leaving the sightings to come back next round.
+ */
+type SourceAnswer =
+	| { kind: "holds"; path: string; held: Set<number> }
+	| { kind: "copiesEverything"; path: string }
+	| { kind: "rekeyed"; path: string; stored: number; served: number }
+	| { kind: "unreachable"; reason: string };
+
+const readSource = async (
+	deps: DepartureProbeDeps,
+	accountId: string,
+	sourceMailboxId: string,
+	rows: MessageItem[],
+): Promise<SourceAnswer> => {
+	const source = await deps.mailboxService.get(accountId, sourceMailboxId);
+
+	// A row whose source is All Mail, Starred or Important is not filed there:
+	// those folders hold a copy of everything and never release anything, so
+	// asking them would answer "still here" forever and freeze the row on a view.
+	if (isVirtualCopyMailbox(source)) {
+		return { kind: "copiesEverything", path: source.fullPath };
+	}
+
+	const box = await deps.connection.openBox(source.fullPath, true);
+
+	// A source folder on a different UIDVALIDITY axis holds no uid these rows can
+	// be compared against, so its message set is evidence of nothing (RFC 9051
+	// 2.3.1.1). The cursor rebuild re-keys that folder; until it has, the
+	// sightings stay unsettled and come back next round.
+	if (box.uidvalidity !== source.uidValidity) {
+		return {
+			kind: "rekeyed",
+			path: source.fullPath,
+			stored: source.uidValidity,
+			served: box.uidvalidity,
+		};
+	}
+
+	const held = await deps.connection.search([
+		["UID", rows.map((row) => row.uid).join(",")],
+	]);
+	return { kind: "holds", path: source.fullPath, held: new Set(held) };
+};
+
+const askSource = async (
+	deps: DepartureProbeDeps,
+	accountId: string,
+	sightedIn: MailboxItem,
+	sourceMailboxId: string,
+	rows: MessageItem[],
+	verdicts: DepartureVerdicts,
+): Promise<void> => {
+	const settle = (row: MessageItem, departed: boolean): void => {
+		const key = placementKey(row.messageId, row.mailboxId, row.uid);
+		verdicts.settled.add(key);
+		if (departed) verdicts.departed.add(key);
+	};
+
+	// A source folder reader cannot reach decides nothing, and does not fail the
+	// round either. Another client renaming or deleting it between the row being
+	// written and now is ordinary — reader shares its mailboxes — and the sync of
+	// the unrelated folder that sighted these messages must survive it.
+	const answer = await readSource(
+		deps,
+		accountId,
+		sourceMailboxId,
+		rows,
+	).catch<SourceAnswer>((error: unknown) => ({
+		kind: "unreachable",
+		reason: error instanceof Error ? error.message : String(error),
+	}));
+
+	const observed = {
+		sourceMailboxId,
+		sightedMailboxId: sightedIn.mailboxId,
+		contested: rows.length,
+	};
+
+	if (answer.kind === "unreachable") {
+		deps.log.warn(
+			{ ...observed, error: answer.reason },
+			"Could not ask the source folder whether it still holds these messages; leaving the sightings unsettled this round",
+		);
+		return;
+	}
+
+	if (answer.kind === "rekeyed") {
+		deps.log.warn(
+			{
+				...observed,
+				sourceMailboxPath: answer.path,
+				storedUidValidity: answer.stored,
+				servedUidValidity: answer.served,
+			},
+			"Source folder is on a different UIDVALIDITY axis; leaving these sightings unsettled this round",
+		);
+		return;
+	}
+
+	if (answer.kind === "copiesEverything") {
+		for (const row of rows) settle(row, true);
+		deps.log.info(
+			{ ...observed, sourceMailboxPath: answer.path },
+			"Row points at a folder that copies every message; the sighting in a real folder is where it lives",
+		);
+		return;
+	}
+
+	for (const row of rows) settle(row, !answer.held.has(row.uid));
+};

package/src/heuristics/classifyPlacement.test.ts

@@ -1,20 +1,21 @@
 /**
  * `classifyPlacement` had no dedicated unit test — only the DKIM/DMARC paths
  * were exercised indirectly through realistic-mail fixtures elsewhere. Issue
- * #300 (RFC 039 Decision 3/3a) adds `senderBlocked` as a confident demote
- * independent of every DKIM/DMARC/provider signal, plus a `setAt` tie-break
- * against `vip`/`wellknown` (Decision 3a, `resolveBlockedVsTrust`). These tests
- * cover both the new branch and the pre-existing DKIM/DMARC branches, so a
- * regression in either shows up here.
+ * #300 (RFC 039 Decision 3/3a) added the user's placement instruction as a
+ * confident move independent of every DKIM/DMARC/provider signal, plus a
+ * `setAt` tie-break against `vip`/`wellknown` (Decision 3a). Issue #605 adds
+ * the other direction, `neverSpam`. These tests cover both instruction
+ * branches and the pre-existing DKIM/DMARC branches, so a regression in any of
+ * them shows up here.
  */
 
 import assert from "node:assert/strict";
 import { describe, it } from "node:test";
 import type { MessageItem } from "@remit/data-ports";
-import { SenderTrust } from "@remit/domain-enums";
+import { SenderOverride, SenderTrust } from "@remit/domain-enums";
 import {
 	classifyPlacement,
-	resolveBlockedVsTrust,
+	resolveSenderPlacement,
 } from "./classifyPlacement.js";
 
 const baseMessage = (overrides: Partial<MessageItem> = {}): MessageItem =>
@@ -28,7 +29,7 @@ const baseMessage = (overrides: Partial<MessageItem> = {}): MessageItem =>
 	}) as unknown as MessageItem;
 
 describe("classifyPlacement", () => {
-	describe("senderBlocked (RFC 039 Decision 3)", () => {
+	describe("SenderOverride.Blocked (RFC 039 Decision 3)", () => {
 		it("demotes an inbox message from a blocked sender, independent of DKIM/DMARC", () => {
 			const message = baseMessage({
 				providerSpam: undefined,
@@ -38,7 +39,7 @@ describe("classifyPlacement", () => {
 				message,
 				"inbox",
 				SenderTrust.Unknown,
-				true,
+				SenderOverride.Blocked,
 			);
 			assert.deepEqual(verdict, {
 				action: "move-to-junk",
@@ -52,7 +53,7 @@ describe("classifyPlacement", () => {
 				baseMessage(),
 				"other",
 				SenderTrust.Unknown,
-				true,
+				SenderOverride.Blocked,
 			);
 			assert.equal(verdict.action, "move-to-junk");
 			assert.equal(verdict.confidence, "confident");
@@ -63,17 +64,17 @@ describe("classifyPlacement", () => {
 				baseMessage(),
 				"junk",
 				SenderTrust.Unknown,
-				true,
+				SenderOverride.Blocked,
 			);
 			assert.notEqual(verdict.action, "move-to-junk");
 		});
 
-		it("does not demote when the sender is not blocked, all else equal", () => {
+		it("does not demote when the sender carries no instruction, all else equal", () => {
 			const verdict = classifyPlacement(
 				baseMessage({ providerSpam: undefined, authResult: undefined }),
 				"inbox",
 				SenderTrust.Unknown,
-				false,
+				SenderOverride.None,
 			);
 			assert.deepEqual(verdict, {
 				action: "leave",
@@ -87,7 +88,7 @@ describe("classifyPlacement", () => {
 				baseMessage({ movedByRemit: true }),
 				"inbox",
 				SenderTrust.Unknown,
-				true,
+				SenderOverride.Blocked,
 			);
 			assert.deepEqual(verdict, {
 				action: "leave",
@@ -97,70 +98,221 @@ describe("classifyPlacement", () => {
 		});
 	});
 
-	describe("resolveBlockedVsTrust (Decision 3a tie-break)", () => {
+	describe("SenderOverride.NeverSpam (issue #605)", () => {
+		it("rescues mail the provider rated clean from a domain that publishes no DMARC", () => {
+			const verdict = classifyPlacement(
+				baseMessage({
+					providerSpam: { classified: false },
+					authResult: { dmarc: "None" },
+				}),
+				"junk",
+				SenderTrust.Unknown,
+				SenderOverride.NeverSpam,
+			);
+			assert.deepEqual(verdict, {
+				action: "move-to-inbox",
+				confidence: "confident",
+				reasons: ["sender=never-spam"],
+			});
+		});
+
+		it("rescues before the missing-signals return, so a message with no parsed signals still comes out of junk", () => {
+			const verdict = classifyPlacement(
+				baseMessage({ providerSpam: undefined, authResult: undefined }),
+				"junk",
+				SenderTrust.Unknown,
+				SenderOverride.NeverSpam,
+			);
+			assert.equal(verdict.action, "move-to-inbox");
+			assert.equal(verdict.confidence, "confident");
+		});
+
+		it("leaves a never-spam sender's mail filed in a user folder where it is", () => {
+			const verdict = classifyPlacement(
+				baseMessage(),
+				"other",
+				SenderTrust.Unknown,
+				SenderOverride.NeverSpam,
+			);
+			assert.equal(verdict.action, "leave");
+		});
+
+		// The demote branch is reached only from `inbox`, so a spoof already in
+		// Junk never meets it. Without the guard on the rescue, the grant would
+		// carry mail impersonating the granted address out of Junk.
+		it("refuses to rescue a dkim-mismatch + dmarc-fail spoof sitting in junk", () => {
+			const verdict = classifyPlacement(
+				baseMessage({
+					providerSpam: { classified: true },
+					authResult: { dmarc: "Fail" },
+					authenticity: { fromDomain: "example.com", dkimMismatch: true },
+				}),
+				"junk",
+				SenderTrust.Unknown,
+				SenderOverride.NeverSpam,
+			);
+			assert.notEqual(verdict.action, "move-to-inbox");
+		});
+
+		it("refuses to rescue on a hard DMARC failure even with DKIM aligned", () => {
+			const verdict = classifyPlacement(
+				baseMessage({
+					providerSpam: { classified: true },
+					authResult: { dmarc: "Fail" },
+				}),
+				"junk",
+				SenderTrust.Unknown,
+				SenderOverride.NeverSpam,
+			);
+			assert.notEqual(verdict.action, "move-to-inbox");
+		});
+
+		it("refuses to rescue on a DKIM mismatch even where DMARC did not fail", () => {
+			const verdict = classifyPlacement(
+				baseMessage({
+					providerSpam: { classified: true },
+					authResult: { dmarc: "None" },
+					authenticity: { fromDomain: "example.com", dkimMismatch: true },
+				}),
+				"junk",
+				SenderTrust.Unknown,
+				SenderOverride.NeverSpam,
+			);
+			assert.notEqual(verdict.action, "move-to-inbox");
+		});
+
+		it("still demotes a never-spam sender's inbox mail on dkim-mismatch + dmarc-fail", () => {
+			const verdict = classifyPlacement(
+				baseMessage({
+					authResult: { dmarc: "Fail" },
+					authenticity: { fromDomain: "example.com", dkimMismatch: true },
+				}),
+				"inbox",
+				SenderTrust.Unknown,
+				SenderOverride.NeverSpam,
+			);
+			assert.deepEqual(verdict, {
+				action: "move-to-junk",
+				confidence: "confident",
+				reasons: ["dkim-mismatch", "dmarc=fail", "sender=untrusted"],
+			});
+		});
+
+		it("still leaves an already-Remit-moved message alone", () => {
+			const verdict = classifyPlacement(
+				baseMessage({ movedByRemit: true }),
+				"junk",
+				SenderTrust.Unknown,
+				SenderOverride.NeverSpam,
+			);
+			assert.deepEqual(verdict, {
+				action: "leave",
+				confidence: "confident",
+				reasons: ["already-moved-by-remit"],
+			});
+		});
+	});
+
+	describe("resolveSenderPlacement (Decision 3a tie-break)", () => {
 		it("blocked wins when set after vip", () => {
-			const result = resolveBlockedVsTrust(
+			const result = resolveSenderPlacement(
 				{ trust: SenderTrust.Vip, setAt: 1_000 },
 				{ blocked: true, setAt: 5_000 },
+				false,
 			);
 			assert.deepEqual(result, {
 				senderTrust: SenderTrust.Unknown,
-				senderBlocked: true,
+				senderOverride: SenderOverride.Blocked,
 			});
 		});
 
 		it("vip wins when set after blocked", () => {
-			const result = resolveBlockedVsTrust(
+			const result = resolveSenderPlacement(
 				{ trust: SenderTrust.Vip, setAt: 5_000 },
 				{ blocked: true, setAt: 1_000 },
+				false,
 			);
 			assert.deepEqual(result, {
 				senderTrust: SenderTrust.Vip,
-				senderBlocked: false,
+				senderOverride: SenderOverride.None,
 			});
 		});
 
 		it("breaks a same-second tie in favor of blocked", () => {
-			const result = resolveBlockedVsTrust(
+			const result = resolveSenderPlacement(
 				{ trust: SenderTrust.Wellknown, setAt: 1_000 },
 				{ blocked: true, setAt: 1_400 },
+				false,
 			);
 			assert.deepEqual(result, {
 				senderTrust: SenderTrust.Unknown,
-				senderBlocked: true,
+				senderOverride: SenderOverride.Blocked,
 			});
 		});
 
 		it("blocked applies outright when there is no competing trust flag", () => {
-			const result = resolveBlockedVsTrust(
+			const result = resolveSenderPlacement(
 				{ trust: SenderTrust.Unknown },
 				{ blocked: true, setAt: 1_000 },
+				false,
 			);
 			assert.deepEqual(result, {
 				senderTrust: SenderTrust.Unknown,
-				senderBlocked: true,
+				senderOverride: SenderOverride.Blocked,
 			});
 		});
 
-		it("passes trust through unchanged when the sender isn't blocked", () => {
-			const result = resolveBlockedVsTrust(
+		it("passes trust through unchanged when the sender carries no instruction", () => {
+			const result = resolveSenderPlacement(
 				{ trust: SenderTrust.Wellknown, setAt: 1_000 },
 				{ blocked: false },
+				false,
 			);
 			assert.deepEqual(result, {
 				senderTrust: SenderTrust.Wellknown,
-				senderBlocked: false,
+				senderOverride: SenderOverride.None,
 			});
 		});
+
+		it("reads never-spam as the instruction, leaving trust untouched", () => {
+			const result = resolveSenderPlacement(
+				{ trust: SenderTrust.Unknown },
+				{ blocked: false },
+				true,
+			);
+			assert.deepEqual(result, {
+				senderTrust: SenderTrust.Unknown,
+				senderOverride: SenderOverride.NeverSpam,
+			});
+		});
+
+		it("does not promote a never-spam sender's trust to vip or wellknown", () => {
+			const result = resolveSenderPlacement(
+				{ trust: SenderTrust.Vip, setAt: 1_000 },
+				{ blocked: false },
+				true,
+			);
+			assert.equal(result.senderTrust, SenderTrust.Vip);
+			assert.equal(result.senderOverride, SenderOverride.NeverSpam);
+		});
+
+		it("resolves a row that somehow carries both instructions to blocked", () => {
+			const result = resolveSenderPlacement(
+				{ trust: SenderTrust.Unknown },
+				{ blocked: true, setAt: 1_000 },
+				true,
+			);
+			assert.equal(result.senderOverride, SenderOverride.Blocked);
+		});
 	});
 
-	describe("existing DKIM/DMARC paths (unaffected by senderBlocked=false)", () => {
+	describe("existing DKIM/DMARC paths (unaffected by SenderOverride.None)", () => {
 		it("rescues a trusted sender's mail from junk on provider-spam + dmarc-pass", () => {
 			const verdict = classifyPlacement(
 				baseMessage({ providerSpam: { classified: true } }),
 				"junk",
 				SenderTrust.Vip,
-				false,
+				SenderOverride.None,
 			);
 			assert.equal(verdict.action, "move-to-inbox");
 			assert.equal(verdict.confidence, "confident");
@@ -171,7 +323,7 @@ describe("classifyPlacement", () => {
 				baseMessage({ providerSpam: { classified: true } }),
 				"junk",
 				SenderTrust.Unknown,
-				false,
+				SenderOverride.None,
 			);
 			assert.equal(verdict.action, "leave");
 			assert.equal(verdict.confidence, "unsure");
@@ -185,7 +337,7 @@ describe("classifyPlacement", () => {
 				}),
 				"inbox",
 				SenderTrust.Unknown,
-				false,
+				SenderOverride.None,
 			);
 			assert.deepEqual(verdict, {
 				action: "move-to-junk",
@@ -202,7 +354,7 @@ describe("classifyPlacement", () => {
 				}),
 				"inbox",
 				SenderTrust.Unknown,
-				false,
+				SenderOverride.None,
 			);
 			assert.deepEqual(verdict, {
 				action: "leave",