@remit/mailbox-service 0.0.94 → 0.0.96

package/src/heuristics/classifyPlacement.ts

@@ -1,8 +1,10 @@
 import type { MessageItem } from "@remit/data-ports";
-import { SenderTrust } from "@remit/domain-enums";
+import { SenderOverride, SenderTrust } from "@remit/domain-enums";
 
 type SenderTrustValue = (typeof SenderTrust)[keyof typeof SenderTrust];
 
+type SenderOverrideValue = (typeof SenderOverride)[keyof typeof SenderOverride];
+
 export type FolderPlacement = "inbox" | "junk" | "other";
 
 export type PlacementAction = "move-to-inbox" | "move-to-junk" | "leave";
@@ -39,7 +41,7 @@ export interface SenderBlockedSignal {
  * fixed, arbitrary order: `blocked` before `vip` before `wellknown` — a
  * determinism backstop, not a meaningful signal.
  */
-export const resolveBlockedVsTrust = (
+const resolveBlockedVsTrust = (
 	trust: SenderTrustSignal,
 	blocked: SenderBlockedSignal,
 ): { senderTrust: SenderTrustValue; senderBlocked: boolean } => {
@@ -60,6 +62,32 @@ export const resolveBlockedVsTrust = (
 	return { senderTrust: trust.trust, senderBlocked: false };
 };
 
+/**
+ * Resolve the two axes {@link classifyPlacement} takes: the trust signal, and
+ * the user's standing instruction about placement.
+ *
+ * `blocked` and `neverSpam` are two directly contradictory instructions and
+ * cannot both stand — `AddressRepository.mergeFlags` drops the loser at write
+ * time. Reading a row that carries both anyway (written before that invariant)
+ * resolves to `Blocked`, the same direction the `blocked`/`vip` same-second tie
+ * already breaks in: the demote is the recoverable mistake.
+ */
+export const resolveSenderPlacement = (
+	trust: SenderTrustSignal,
+	blocked: SenderBlockedSignal,
+	neverSpam: boolean,
+): {
+	senderTrust: SenderTrustValue;
+	senderOverride: SenderOverrideValue;
+} => {
+	const { senderTrust, senderBlocked } = resolveBlockedVsTrust(trust, blocked);
+	if (senderBlocked)
+		return { senderTrust, senderOverride: SenderOverride.Blocked };
+	if (neverSpam)
+		return { senderTrust, senderOverride: SenderOverride.NeverSpam };
+	return { senderTrust, senderOverride: SenderOverride.None };
+};
+
 /**
  * Tier 0 deterministic placement verdict (RFC 031, "Confident moves").
  *
@@ -69,17 +97,32 @@ export const resolveBlockedVsTrust = (
  * recall-biased — a confident move only fires when cheap, deterministic signals
  * agree; everything else is left in place for a later LLM tier.
  *
- * `senderBlocked` (RFC 039 Decision 3) is a confident demote independent of
- * every DKIM/DMARC/provider signal below — a user's explicit block is not a
- * heuristic. Already-tie-broken against `vip`/`wellknown` by
- * {@link resolveBlockedVsTrust} in the caller; this function itself does not
+ * `senderOverride` (RFC 039 Decision 3, issue #605) is the user's standing
+ * instruction about this sender's placement, and it outranks every
+ * DKIM/DMARC/provider signal below — an explicit block or never-spam grant is
+ * not a heuristic. Already-tie-broken against `vip`/`wellknown` by
+ * {@link resolveSenderPlacement} in the caller; this function itself does not
  * compare `setAt`.
+ *
+ * `NeverSpam` is checked above the `missing-signals` return and above the
+ * `providerSpam.classified` gate on purpose: the senders who need the grant are
+ * the ones the provider rated clean and whose domain publishes no DMARC record,
+ * so a check inside the rescue branch below would never fire for them. It
+ * rescues only from `junk` — a message a standing filter put in a user folder
+ * is not a spam question.
+ *
+ * It refuses outright on a `dmarc === "Fail"` or a DKIM mismatch. The demote
+ * branch below is reached only from `inbox`, so a spoof already sitting in Junk
+ * would never meet it: without this guard the grant would launder mail out of
+ * Junk on exactly the two signals that identify someone impersonating the
+ * address the user granted. A domain that publishes no DMARC record never
+ * scores `Fail`, so the case the grant exists for is untouched.
  */
 export const classifyPlacement = (
 	message: MessageItem,
 	placement: FolderPlacement,
 	senderTrust: SenderTrustValue,
-	senderBlocked: boolean,
+	senderOverride: SenderOverrideValue,
 ): PlacementVerdict => {
 	if (message.movedByRemit === true) {
 		return {
@@ -89,7 +132,7 @@ export const classifyPlacement = (
 		};
 	}
 
-	if (senderBlocked && placement !== "junk") {
+	if (senderOverride === SenderOverride.Blocked && placement !== "junk") {
 		return {
 			action: "move-to-junk",
 			confidence: "confident",
@@ -97,6 +140,19 @@ export const classifyPlacement = (
 		};
 	}
 
+	if (
+		senderOverride === SenderOverride.NeverSpam &&
+		placement === "junk" &&
+		message.authResult?.dmarc !== "Fail" &&
+		message.authenticity?.dkimMismatch !== true
+	) {
+		return {
+			action: "move-to-inbox",
+			confidence: "confident",
+			reasons: ["sender=never-spam"],
+		};
+	}
+
 	if (!message.providerSpam || !message.authResult) {
 		return {
 			action: "leave",

package/src/message-sync-changedsince.test.ts

@@ -265,11 +265,18 @@ const buildHarness = (options: HarnessOptions): Harness => {
 		},
 	) as IAddressRepository;
 
+	// The rows sync reads back before it saves, to ask whether a message sighted
+	// here has left the folder its row points at (#1146). This harness stores
+	// none, so nothing is contested and the probe issues no IMAP.
+	const messageRepository = {
+		get: async () => [],
+	} as unknown as IMessageRepository;
+
 	const service = new MessageSyncService(
 		connectionFactory,
 		mailboxService,
 		noFolderRoles,
-		{} as IMessageRepository,
+		messageRepository,
 		{} as IEnvelopeRepository,
 		addressRepository,
 		threadMessageService,

package/src/message-sync-quarantine.test.ts

@@ -174,6 +174,13 @@ const buildHarness = (options: {
 		},
 	} satisfies IQuarantineRepository;
 
+	// The rows sync reads back before it saves, to ask whether a message sighted
+	// here has left the folder its row points at (#1146). This harness stores
+	// none, so nothing is contested and the probe issues no IMAP.
+	const messageRepository = {
+		get: async () => [],
+	} as unknown as IMessageRepository;
+
 	const service = new MessageSyncService(
 		{
 			getConnection: () => connection,
@@ -181,7 +188,7 @@ const buildHarness = (options: {
 		} as ManagedConnectionFactory,
 		mailboxService,
 		noFolderRoles,
-		{} as IMessageRepository,
+		messageRepository,
 		{} as IEnvelopeRepository,
 		{} as IAddressRepository,
 		threadMessageService,

package/src/message-sync-repoint.test.ts

@@ -11,13 +11,11 @@ import type {
 	ThreadMessageItem,
 	UpdateThreadMessageInput,
 } from "@remit/data-ports";
+import { deriveMessageId } from "@remit/data-ports/id";
 import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
 import type { ManagedConnectionFactory } from "./connection-factory.js";
-import {
-	type AccountFolderRoles,
-	MessageSyncService,
-	repointsOnSighting,
-} from "./message-sync.js";
+import { type DepartureVerdicts, placementKey } from "./external-move.js";
+import { type AccountFolderRoles, MessageSyncService } from "./message-sync.js";
 import { folderRoles, NO_JUNK_ROLES } from "./test-helpers/folder-roles.js";
 import type { ImapEnvelope, ImapMessage } from "./types.js";
 
@@ -85,8 +83,22 @@ interface Observed {
 	threadUpdates: UpdateThreadMessageInput[];
 	reconciled: string[];
 	owned: boolean;
+	unresolved: boolean;
 }
 
+/**
+ * What `confirmDepartures` had answered about the stored placement by the time
+ * the batch was saved (#1146).
+ *
+ * - `departed` — the source folder no longer holds it: a move another client made.
+ * - `held` — the source folder still holds it: the labelled message, in two
+ *   folders at once.
+ * - `none` — no verdict at all: the source could not be asked this round.
+ * - `stale` — a verdict, but for a placement this row no longer has, because it
+ *   moved between the probe and the save.
+ */
+type Verdict = "departed" | "held" | "none" | "stale";
+
 /**
  * Sync one message out of `sighting`, against a database that already holds it
  * under `stored`. The account's Junk folder is `JUNK` and its Trash is `TRASH`,
@@ -97,12 +109,14 @@ const sync = async (
 	stored: MessageItem,
 	threadRow: ThreadMessageItem | null = threadRowIn(INBOX),
 	copiesHere: MessageItem[] = [],
+	verdict: Verdict = "departed",
 ): Promise<Observed> => {
 	const observed: Observed = {
 		repointedTo: [],
 		threadUpdates: [],
 		reconciled: [],
 		owned: false,
+		unresolved: false,
 	};
 
 	const messageService = {
@@ -164,6 +178,19 @@ const sync = async (
 		envelope,
 	} as unknown as ImapMessage;
 
+	const answeredFor = placementKey(
+		deriveMessageId("acct-1", envelope.messageId),
+		stored.mailboxId,
+		verdict === "stale" ? stored.uid + 1 : stored.uid,
+	);
+	const departures: DepartureVerdicts = {
+		departed:
+			verdict === "held" || verdict === "none"
+				? new Set()
+				: new Set([answeredFor]),
+		settled: verdict === "none" ? new Set() : new Set([answeredFor]),
+	};
+
 	const result = (await (
 		service as unknown as {
 			saveMessage: (
@@ -172,15 +199,24 @@ const sync = async (
 				accountConfigId: string,
 				msg: ImapMessage,
 				roles: AccountFolderRoles,
-			) => Promise<{ owned: boolean }>;
+				departures: DepartureVerdicts,
+			) => Promise<{ owned: boolean; unresolvedSighting: boolean }>;
 		}
-	).saveMessage(sighting, "acct-1", "cfg-1", msg, {
-		junkMailboxId: JUNK.mailboxId,
-		trashMailboxId: TRASH.mailboxId,
-		configJunkRoles: NO_JUNK_ROLES,
-	})) as { owned: boolean };
+	).saveMessage(
+		sighting,
+		"acct-1",
+		"cfg-1",
+		msg,
+		{
+			junkMailboxId: JUNK.mailboxId,
+			trashMailboxId: TRASH.mailboxId,
+			configJunkRoles: NO_JUNK_ROLES,
+		},
+		departures,
+	)) as { owned: boolean; unresolvedSighting: boolean };
 
 	observed.owned = result.owned;
+	observed.unresolved = result.unresolvedSighting;
 	return observed;
 };
 
@@ -227,6 +263,87 @@ describe("which folder a message the database already holds lives in", () => {
 		assert.equal(observed.owned, false);
 	});
 
+	/**
+	 * The reported bug (#1146): a message that legitimately sits in two real
+	 * folders — a Gmail user label, a Sieve `fileinto` beside a `keep`, a Sent
+	 * copy a list echoes back — was re-pointed out of the Inbox by whichever
+	 * folder the round enumerated last.
+	 */
+	it("leaves a labelled message in the folder that still holds it", async () => {
+		const observed = await sync(
+			mailboxAt("Receipts"),
+			storedIn(INBOX),
+			threadRowIn(INBOX),
+			[],
+			"held",
+		);
+
+		assert.deepEqual(observed.repointedTo, []);
+		assert.deepEqual(observed.threadUpdates, []);
+		assert.equal(observed.owned, false);
+	});
+
+	/**
+	 * A decided sighting is decided for good: the message is in two folders and
+	 * asking again next round would get the same answer. Only an UNDECIDED one
+	 * comes back, so a settled decline must not hold the watermark.
+	 */
+	it("finishes with a labelled message rather than holding the watermark for it", async () => {
+		const observed = await sync(
+			mailboxAt("Receipts"),
+			storedIn(INBOX),
+			threadRowIn(INBOX),
+			[],
+			"held",
+		);
+
+		assert.equal(observed.unresolved, false);
+	});
+
+	it("holds the watermark when the source folder could not be asked", async () => {
+		const observed = await sync(
+			JUNK,
+			storedIn(INBOX),
+			threadRowIn(INBOX),
+			[],
+			"none",
+		);
+
+		assert.deepEqual(observed.repointedTo, []);
+		assert.equal(observed.unresolved, true);
+	});
+
+	/**
+	 * The probe reads the row outside the transaction that acts on it, so a user
+	 * move landing in between leaves a verdict about a placement this row no
+	 * longer has. Spending it would re-point the row against an answer reached
+	 * for somewhere else.
+	 */
+	it("spends no verdict reached for a placement the row has since left", async () => {
+		const observed = await sync(
+			JUNK,
+			storedIn(INBOX),
+			threadRowIn(INBOX),
+			[],
+			"stale",
+		);
+
+		assert.deepEqual(observed.repointedTo, []);
+		assert.equal(observed.unresolved, true);
+	});
+
+	it("holds the watermark for nothing when the row is already ours", async () => {
+		const observed = await sync(
+			INBOX,
+			storedIn(INBOX),
+			threadRowIn(INBOX),
+			[],
+			"none",
+		);
+
+		assert.equal(observed.unresolved, false);
+	});
+
 	it("declines a sighting in a folder that copies every message", async () => {
 		const observed = await sync(ALL_MAIL, storedIn(INBOX));
 
@@ -300,95 +417,3 @@ describe("what a re-pointed message does to its sender's standing", () => {
 		assert.deepEqual(observed.reconciled, []);
 	});
 });
-
-describe("repointsOnSighting", () => {
-	it("refuses a Gmail virtual folder the server never flagged", () => {
-		assert.equal(
-			repointsOnSighting(mailboxAt("[Gmail]/All Mail"), storedIn(INBOX)),
-			false,
-		);
-	});
-
-	it("accepts a folder the user named after a virtual one", () => {
-		assert.equal(
-			repointsOnSighting(mailboxAt("Starred ideas"), storedIn(INBOX)),
-			true,
-		);
-	});
-
-	it("accepts an ordinary inbound row the sync path left pending", () => {
-		assert.equal(
-			repointsOnSighting(
-				JUNK,
-				storedIn(INBOX, { syncStatus: MessageSyncStatus.pending }),
-			),
-			true,
-		);
-	});
-
-	it("accepts a row a settled mutation marked synced", () => {
-		assert.equal(
-			repointsOnSighting(
-				JUNK,
-				storedIn(INBOX, { syncStatus: MessageSyncStatus.synced }),
-			),
-			true,
-		);
-	});
-
-	/**
-	 * The row `abandonDelete` hands back: reader refused the delete, put the
-	 * message back where the server still has it, and nothing else is coming
-	 * for the row. Reader shares its mailboxes, so the user moving that same
-	 * message in another client is ordinary — and before R3 the sighting was
-	 * refused on `syncStatus` alone and the row never followed the move.
-	 */
-	it("accepts a row whose delete was abandoned, so a move made elsewhere still lands", () => {
-		assert.equal(
-			repointsOnSighting(
-				JUNK,
-				storedIn(INBOX, {
-					status: MessageStatus.active,
-					syncStatus: MessageSyncStatus.abandoned,
-				}),
-			),
-			true,
-		);
-	});
-
-	it("accepts a row left `failed` by a transient attempt that has since settled", () => {
-		assert.equal(
-			repointsOnSighting(
-				JUNK,
-				storedIn(INBOX, { syncStatus: MessageSyncStatus.failed }),
-			),
-			true,
-		);
-	});
-
-	it("refuses a row whose own move is still in flight", () => {
-		assert.equal(
-			repointsOnSighting(
-				JUNK,
-				storedIn(INBOX, {
-					status: MessageStatus.moving,
-					syncStatus: MessageSyncStatus.pending,
-				}),
-			),
-			false,
-		);
-	});
-
-	it("refuses a row whose own delete is still in flight", () => {
-		assert.equal(
-			repointsOnSighting(
-				JUNK,
-				storedIn(INBOX, {
-					status: MessageStatus.deleting,
-					syncStatus: MessageSyncStatus.pending,
-				}),
-			),
-			false,
-		);
-	});
-});