@remit/mailbox-service 0.0.40 → 0.0.42

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.40",
+  "version": "0.0.42",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/body-sync-placement-flags.test.ts

@@ -69,7 +69,7 @@ const MAILBOXES = {
 };
 
 const buildHarness = (
-	message: { messageId: string; mailboxId: string },
+	message: { messageId: string; mailboxId: string; movedByRemit?: boolean },
 	flags: AddressItem["flags"],
 ): Harness => {
 	const moves: MoveCall[] = [];
@@ -79,11 +79,7 @@ const buildHarness = (
 	}> = [];
 
 	const messageService = {
-		get: async () => ({
-			messageId: message.messageId,
-			mailboxId: message.mailboxId,
-			uid: 1,
-		}),
+		get: async () => ({ ...message, uid: 1 }),
 		update: async (messageId: string, input: UpdateMessageInput) => {
 			messageUpdates.push({ messageId, input });
 		},
@@ -300,3 +296,56 @@ describe("Address.flags.autoArchive drives placement (issue #300)", () => {
 		]);
 	});
 });
+
+/**
+ * Issue #398: not every `leave` verdict means "nothing confident to say".
+ * `already-moved-by-remit` is the guard against re-deciding a settled
+ * placement, and mail sitting in Junk gets a `leave` because the demote branch
+ * only fires outside Junk. Auto-archive is a filing preference relative to the
+ * Inbox and must yield to both, so these drive `BodySyncService` end-to-end and
+ * assert on the absence of a move.
+ */
+describe("autoArchive respects a leave verdict that decided something (issue #398)", () => {
+	it("leaves a blocked autoArchive sender's message in Junk", async () => {
+		const harness = buildHarness(
+			{ messageId: "m-1", mailboxId: MAILBOXES.junk.mailboxId },
+			{
+				blocked: { value: true, setAt: 1_000 },
+				autoArchive: { value: true, setAt: 1_000 },
+			},
+		);
+
+		await readBody(harness.service, "Junk");
+
+		assert.deepEqual(harness.moves, []);
+	});
+
+	it("leaves an autoArchive sender's provider-junked message in Junk", async () => {
+		const harness = buildHarness(
+			{ messageId: "m-1", mailboxId: MAILBOXES.junk.mailboxId },
+			{ autoArchive: { value: true, setAt: 1_000 } },
+		);
+
+		// Provider-spam + dmarc-pass from an untrusted sender is an unsure
+		// `leave`: too weak to rescue, and auto-archive is not a second chance
+		// at leaving Junk.
+		await readBody(harness.service, "Junk", SPAM_FLAGGED_DMARC_PASS_EML);
+
+		assert.deepEqual(harness.moves, []);
+	});
+
+	it("leaves a message Remit already placed alone on a re-entrant pass", async () => {
+		const harness = buildHarness(
+			{
+				messageId: "m-1",
+				mailboxId: MAILBOXES.inbox.mailboxId,
+				movedByRemit: true,
+			},
+			{ autoArchive: { value: true, setAt: 1_000 } },
+		);
+
+		await readBody(harness.service);
+
+		assert.deepEqual(harness.moves, []);
+	});
+});

package/src/body-sync.ts

@@ -48,6 +48,7 @@ import {
 import {
 	classifyPlacement,
 	type FolderPlacement,
+	type PlacementVerdict,
 	resolveBlockedVsTrust,
 } from "./heuristics/classifyPlacement.js";
 import type { PlacementMoveService } from "./placement-move.js";
@@ -199,6 +200,26 @@ const hasDecidedCategory = (
 const hasDecidedPlacement = (placementDecidedAt: number | undefined): boolean =>
 	placementDecidedAt !== undefined;
 
+/**
+ * Issue #398: `flags.autoArchive` is a filing preference relative to the Inbox.
+ * A `leave` verdict reaches it for two unrelated reasons and only one of them
+ * means "nothing confident to say":
+ *
+ * - `already-moved-by-remit` is {@link classifyPlacement}'s guard against
+ *   re-deciding a settled placement, and auto-archive is a re-decision. The
+ *   re-entrant paths `hasDecidedPlacement` names reach it on messages Remit
+ *   itself placed, or that a user moved back afterwards.
+ * - A message already in Junk gets an unsure `leave` whatever the sender's
+ *   `blocked` flag says, since the demote branch only fires outside Junk.
+ *   Moving mail out of Junk is a rescue, which this file reserves for a
+ *   confident signal plus sender trust.
+ */
+const autoArchiveApplies = (
+	verdict: PlacementVerdict,
+	placement: FolderPlacement,
+): boolean =>
+	placement !== "junk" && !verdict.reasons.includes("already-moved-by-remit");
+
 export const toParsedBody = (parsed: ParsedMail): ParsedBody => ({
 	text: parsed.text ?? null,
 	html: typeof parsed.html === "string" ? parsed.html : null,
@@ -1459,12 +1480,14 @@ export class BodySyncService {
 		// junk/inbox verdict computed above. Still marked decided (issue #383):
 		// "leave" is itself a verdict, not "not yet evaluated".
 		if (verdict.action === "leave") {
-			const outcome = await this.resolveAutoArchive(
-				mailboxSpecialUseService,
-				message,
-				accountId,
-				signals.autoArchive,
-			);
+			const outcome = autoArchiveApplies(verdict, placement)
+				? await this.resolveAutoArchive(
+						mailboxSpecialUseService,
+						message,
+						accountId,
+						signals.autoArchive,
+					)
+				: {};
 			return { ...outcome, placementDecidedAt: Date.now() };
 		}
 
@@ -1526,7 +1549,8 @@ export class BodySyncService {
 	 * straight to Archive, skipping Inbox. Only reached from
 	 * {@link computePlacement} when {@link classifyPlacement} had no confident
 	 * junk/inbox verdict of its own — `blocked`/DKIM/DMARC always take priority
-	 * over this filing preference.
+	 * over this filing preference — and only for the `leave` verdicts
+	 * {@link autoArchiveApplies} admits.
 	 *
 	 * No {@link MessagePlacementVerdict} is recorded: `PlacementAction` (the
 	 * audit enum) has only `MoveToInbox`/`MoveToJunk` — issue #300 is scoped to

package/src/list-id-backfill.test.ts

@@ -24,6 +24,7 @@ import type { StorageService } from "@remit/storage-service";
 import {
 	backfillListIds,
 	type ListIdBackfillCheckpoint,
+	type ListIdBackfillCheckpointStore,
 	type ListIdBackfillProgress,
 } from "./list-id-backfill.js";
 
@@ -99,6 +100,22 @@ const message = (overrides: Partial<MessageItem>): MessageItem =>
 		...overrides,
 	}) as unknown as MessageItem;
 
+const asAccount = (accountConfigId: string): AccountConfigItem =>
+	({ accountConfigId }) as unknown as AccountConfigItem;
+
+const inMemoryCheckpointStore = (): ListIdBackfillCheckpointStore => {
+	let checkpoint: ListIdBackfillCheckpoint | undefined;
+	return {
+		load: async () => checkpoint,
+		save: async (next) => {
+			checkpoint = next;
+		},
+		clear: async () => {
+			checkpoint = undefined;
+		},
+	};
+};
+
 interface Harness {
 	accountConfigService: Pick<IAccountConfigRepository, "listAll">;
 	threadMessageService: Pick<
@@ -351,6 +368,37 @@ describe("backfillListIds", () => {
 		assert.equal(harness.updates.length, 2);
 	});
 
+	it("scans accounts in account-id order, whatever order listAll returns", async () => {
+		const accountIds = ["acc-1", "acc-2", "acc-3"];
+		const rows = accountIds.map((accountConfigId) =>
+			row({
+				threadMessageId: `tm-${accountConfigId}`,
+				messageId: `m-${accountConfigId}`,
+				accountConfigId,
+			}),
+		);
+		const harness = buildHarness({
+			accounts: ["acc-3", "acc-1", "acc-2"].map(asAccount),
+			rows,
+			messages: rows.map((r) =>
+				message({
+					messageId: r.messageId,
+					bodyStorageKey: `s3://${r.messageId}`,
+				}),
+			),
+		});
+		const progress: ListIdBackfillProgress[] = [];
+
+		await backfillListIds(harness, {
+			onProgress: (p) => progress.push({ ...p }),
+		});
+
+		assert.deepEqual(
+			progress.map((p) => p.accountConfigId),
+			accountIds,
+		);
+	});
+
 	it("checkpoints after each page and clears it on completion", async () => {
 		const rows = Array.from({ length: 3 }, (_, i) =>
 			row({ threadMessageId: `tm-${i}`, messageId: `m-${i}` }),
@@ -380,6 +428,7 @@ describe("backfillListIds", () => {
 		});
 
 		assert.equal(saved.length, 2);
+		assert.equal(saved[0].accountConfigId, "acc-1");
 		assert.equal(saved[0].continuationToken, "2");
 		assert.equal(saved[1].continuationToken, undefined);
 		assert.equal(cleared, true);
@@ -400,7 +449,10 @@ describe("backfillListIds", () => {
 		const result = await backfillListIds(harness, {
 			batchSize: 2,
 			checkpointStore: {
-				load: async () => ({ accountIndex: 0, continuationToken: "2" }),
+				load: async () => ({
+					accountConfigId: "acc-1",
+					continuationToken: "2",
+				}),
 				save: async () => {},
 				clear: async () => {},
 			},
@@ -413,4 +465,109 @@ describe("backfillListIds", () => {
 			["tm-2"],
 		);
 	});
+
+	it("resumes the account it was working through after an earlier account is removed", async () => {
+		const accountIds = ["acc-1", "acc-2", "acc-3"];
+		const rows = [
+			row({
+				threadMessageId: "tm-1",
+				messageId: "m-1",
+				accountConfigId: "acc-1",
+			}),
+			row({
+				threadMessageId: "tm-2a",
+				messageId: "m-2a",
+				accountConfigId: "acc-2",
+			}),
+			row({
+				threadMessageId: "tm-2b",
+				messageId: "m-2b",
+				accountConfigId: "acc-2",
+			}),
+			row({
+				threadMessageId: "tm-3",
+				messageId: "m-3",
+				accountConfigId: "acc-3",
+			}),
+		];
+		const messages = rows.map((r) =>
+			message({
+				messageId: r.messageId,
+				bodyStorageKey: `s3://${r.messageId}`,
+			}),
+		);
+
+		const store = inMemoryCheckpointStore();
+
+		const interrupted = buildHarness({
+			accounts: accountIds.map(asAccount),
+			rows,
+			messages,
+			pageSize: 1,
+		});
+		const listByAccount = interrupted.threadMessageService.listByAccount;
+		interrupted.threadMessageService.listByAccount = async (
+			accountConfigId,
+			opts,
+		) => {
+			if (accountConfigId === "acc-2" && opts?.continuationToken) {
+				throw new Error("interrupted");
+			}
+			return listByAccount(accountConfigId, opts);
+		};
+
+		await assert.rejects(
+			backfillListIds(interrupted, { batchSize: 1, checkpointStore: store }),
+			/interrupted/,
+		);
+
+		const resumed = buildHarness({
+			accounts: ["acc-2", "acc-3"].map(asAccount),
+			rows,
+			messages,
+			pageSize: 1,
+		});
+
+		const result = await backfillListIds(resumed, {
+			batchSize: 1,
+			checkpointStore: store,
+		});
+
+		assert.deepEqual(
+			resumed.updates.map((u) => u.threadMessageId),
+			["tm-2b", "tm-3"],
+		);
+		assert.equal(result.backfilled, 2);
+	});
+
+	it("restarts the pass when the checkpointed account no longer exists", async () => {
+		const rows = Array.from({ length: 3 }, (_, i) =>
+			row({ threadMessageId: `tm-${i}`, messageId: `m-${i}` }),
+		);
+		const messages = rows.map((r) =>
+			message({
+				messageId: r.messageId,
+				bodyStorageKey: `s3://${r.messageId}`,
+			}),
+		);
+		const harness = buildHarness({ rows, messages, pageSize: 2 });
+
+		const result = await backfillListIds(harness, {
+			batchSize: 2,
+			checkpointStore: {
+				load: async () => ({
+					accountConfigId: "acc-removed",
+					continuationToken: "2",
+				}),
+				save: async () => {},
+				clear: async () => {},
+			},
+		});
+
+		assert.equal(result.scanned, 3);
+		assert.deepEqual(
+			harness.updates.map((u) => u.threadMessageId),
+			["tm-0", "tm-1", "tm-2"],
+		);
+	});
 });

package/src/list-id-backfill.ts

@@ -11,14 +11,14 @@ import { extractListId } from "./filters/list-id.js";
 const DEFAULT_BATCH_SIZE = 200;
 
 /**
- * Where the full-corpus pass left off: the index into the account list (in
- * `listAll()` order) it was working through, and the page cursor within that
- * account. Resuming skips every account before it entirely and re-opens the
- * in-flight one at its saved cursor, rather than re-scanning the whole corpus
- * from the top after an interruption.
+ * Where the full-corpus pass left off: the identity of the account it was
+ * working through, and the page cursor within that account. Resuming looks the
+ * account up by id and re-opens it at its saved cursor, rather than re-scanning
+ * the whole corpus from the top after an interruption. A checkpoint naming no
+ * configured account restarts the pass.
  */
 export interface ListIdBackfillCheckpoint {
-	accountIndex: number;
+	accountConfigId: string;
 	continuationToken?: string;
 }
 
@@ -138,7 +138,8 @@ const deriveAndApplyListId = async (
  * sync path will populate `listId` for it once the body lands.
  *
  * Chunked by `listByAccount`'s existing keyset pagination, one account at a
- * time in `listAll()` order. A failure reading or parsing one message's
+ * time in account-id order, so an interrupted run and its resume walk the same
+ * sequence. A failure reading or parsing one message's
  * stored body is contained to that message — logged, counted, and the pass
  * continues — the same containment `BodySyncService`'s classification
  * backfill uses for the same reason: one unreadable object must not strand
@@ -151,9 +152,29 @@ export const backfillListIds = async (
 	const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
 	const { logger, checkpointStore } = options;
 
-	const accounts = await deps.accountConfigService.listAll();
+	const accounts = [...(await deps.accountConfigService.listAll())].sort(
+		(left, right) => left.accountConfigId.localeCompare(right.accountConfigId),
+	);
 	const startingCheckpoint = await checkpointStore?.load();
-	const startIndex = startingCheckpoint?.accountIndex ?? 0;
+	const checkpointedIndex = startingCheckpoint
+		? accounts.findIndex(
+				(account) =>
+					account.accountConfigId === startingCheckpoint.accountConfigId,
+			)
+		: -1;
+
+	if (startingCheckpoint && checkpointedIndex === -1) {
+		logger?.info(
+			{ accountConfigId: startingCheckpoint.accountConfigId },
+			"ListId backfill checkpoint names no configured account; restarting the pass",
+		);
+	}
+
+	const startIndex = checkpointedIndex === -1 ? 0 : checkpointedIndex;
+	const startContinuationToken =
+		checkpointedIndex === -1
+			? undefined
+			: startingCheckpoint?.continuationToken;
 
 	const totals = emptyTotals();
 	const failedThreadMessageIds: string[] = [];
@@ -165,9 +186,7 @@ export const backfillListIds = async (
 	) {
 		const account = accounts[accountIndex];
 		let continuationToken: string | undefined =
-			accountIndex === startIndex
-				? startingCheckpoint?.continuationToken
-				: undefined;
+			accountIndex === startIndex ? startContinuationToken : undefined;
 
 		do {
 			const page = await deps.threadMessageService.listByAccount(
@@ -231,7 +250,7 @@ export const backfillListIds = async (
 
 			continuationToken = page.continuationToken;
 			await checkpointStore?.save({
-				accountIndex,
+				accountConfigId: account.accountConfigId,
 				continuationToken,
 			});