@remit/mailbox-service 0.0.41 → 0.0.43

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.41",
+  "version": "0.0.43",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/body-sync-auto-read-writeonce.test.ts

@@ -0,0 +1,219 @@
+/**
+ * Issue #499: `Address.flags.unsubscribed` auto-mark-read is a first-
+ * classification decision, like `Message.category` (#355) and the placement
+ * verdict (#383). Re-deriving it on the two shipped re-entrant paths —
+ * `fetchAndGetBody`'s `NoSuchKey` fallback and `syncBodies(..., force: true)`
+ * — undid a user who had deliberately marked such a message unread.
+ *
+ * The re-entrant fixtures below keep `flags.unsubscribed` set, so a regression
+ * that drops the guard shows up as a real `FlagQueueService.markAsRead`
+ * round-trip against a message the user owns, not as a fixture that happens to
+ * disagree with the flag.
+ */
+
+import assert from "node:assert/strict";
+import { Readable } from "node:stream";
+import { describe, it } from "node:test";
+import type {
+	AddressItem,
+	IAddressRepository,
+	IEnvelopeRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+	MessageItem,
+	UpdateMessageInput,
+} from "@remit/data-ports";
+import type { StorageService } from "@remit/storage-service";
+import { BodySyncService } from "./body-sync.js";
+import type { FlagQueueService } from "./flag-queue.js";
+import type { IImapConnection } from "./types.js";
+
+const UNSUBSCRIBED_SENDER = "newsletter@example.com";
+
+const PLAIN_EML = Buffer.from(
+	[
+		`From: Newsletter <${UNSUBSCRIBED_SENDER}>`,
+		"To: me@example.com",
+		"Subject: This week",
+		"List-Unsubscribe: <mailto:stop@example.com>",
+		"Content-Type: text/plain",
+		"",
+		"body",
+	].join("\r\n"),
+);
+
+interface MarkReadCall {
+	accountConfigId: string;
+	messageId: string;
+	accountId: string;
+}
+
+interface Harness {
+	service: BodySyncService;
+	markReadCalls: MarkReadCall[];
+}
+
+const buildHarness = (
+	message: Partial<MessageItem> & Pick<MessageItem, "messageId">,
+	retrieve: () => Promise<Buffer>,
+): Harness => {
+	const markReadCalls: MarkReadCall[] = [];
+
+	const messageRow = {
+		uid: 1,
+		mailboxId: "mb-inbox",
+		...message,
+	} as unknown as MessageItem;
+
+	const messageService = {
+		get: async () => messageRow,
+		update: async (_messageId: string, input: UpdateMessageInput) => {
+			Object.assign(messageRow, input);
+		},
+	} as unknown as IMessageRepository;
+
+	const threadMessageService = {
+		findAllByMessageId: async () => [
+			{
+				threadMessageId: "tm-1",
+				messageId: message.messageId,
+				mailboxId: messageRow.mailboxId,
+				sentDate: 1,
+				isRead: false,
+				isDeleted: false,
+				hasStars: false,
+				hasAttachment: false,
+			},
+		],
+		update: async () => {},
+	} as unknown as IThreadMessageRepository;
+
+	const storageService = {
+		retrieve,
+		storeMessageBody: async () => ({ uri: `s3://bodies/${message.messageId}` }),
+		storeMessageBodyStream: async () => ({
+			uri: `s3://bodies/${message.messageId}`,
+		}),
+		storeParsedBody: async () => {},
+		listBodyParts: async () => [],
+	} as unknown as StorageService;
+
+	const addressService = {
+		getAddress: async () =>
+			({
+				flags: { unsubscribed: { value: true, setAt: 1 } },
+			}) as unknown as AddressItem,
+		incrementInboundCount: async () => {},
+	} as unknown as IAddressRepository;
+
+	const envelopeService = {
+		listBodyParts: async () => [],
+	} as unknown as IEnvelopeRepository;
+
+	const flagQueueService = {
+		markAsRead: async (
+			accountConfigId: string,
+			messageId: string,
+			accountId: string,
+		) => {
+			markReadCalls.push({ accountConfigId, messageId, accountId });
+		},
+	} as unknown as FlagQueueService;
+
+	const service = new BodySyncService(
+		messageService,
+		storageService,
+		threadMessageService,
+		addressService,
+		envelopeService,
+		{ info: () => {}, error: () => {} },
+		undefined,
+		undefined,
+		undefined,
+		{ flagQueueService },
+	);
+
+	return { service, markReadCalls };
+};
+
+const noSuchKeyError = () =>
+	Object.assign(new Error("missing"), { name: "NoSuchKey" });
+
+const bodyConnection = () =>
+	({
+		openBox: async () => {},
+		fetchMessageBody: async () => PLAIN_EML,
+		async *fetchMessageBodies(uids: number[]) {
+			for (const uid of uids) {
+				yield { uid, source: Readable.from([PLAIN_EML]) };
+			}
+		},
+	}) as unknown as IImapConnection;
+
+describe("unsubscribed auto-mark-read is decided once (issue #499)", () => {
+	it("marks a message read on the pass that first classifies it", async () => {
+		const harness = buildHarness({ messageId: "m-1" }, async () => {
+			throw new Error("no body stored yet; must not retrieve");
+		});
+
+		await harness.service.fetchAndGetBody(
+			"m-1",
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			async () => bodyConnection(),
+		);
+
+		assert.deepEqual(harness.markReadCalls, [
+			{ accountConfigId: "cfg-1", messageId: "m-1", accountId: "acc-1" },
+		]);
+	});
+
+	it("leaves a manually-unread message alone through the NoSuchKey IMAP re-fetch", async () => {
+		const harness = buildHarness(
+			{ messageId: "m-1", bodyStorageKey: "s3://bodies/m-1" },
+			async () => {
+				throw noSuchKeyError();
+			},
+		);
+
+		await harness.service.fetchAndGetBody(
+			"m-1",
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			async () => bodyConnection(),
+		);
+
+		assert.deepEqual(
+			harness.markReadCalls,
+			[],
+			"a re-entrant pass must not re-apply auto-read over the user's unread",
+		);
+	});
+
+	it("leaves a manually-unread message alone when syncBodies re-fetches with force", async () => {
+		const harness = buildHarness(
+			{ messageId: "m-1", bodyStorageKey: "s3://bodies/m-1" },
+			async () => {
+				throw new Error("force path must not retrieve from storage");
+			},
+		);
+
+		const result = await harness.service.syncBodies(
+			["m-1"],
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			async () => bodyConnection(),
+			true,
+		);
+
+		assert.deepEqual(result.syncedMessageIds, ["m-1"]);
+		assert.deepEqual(
+			harness.markReadCalls,
+			[],
+			"a forced re-sync must not re-apply auto-read over the user's unread",
+		);
+	});
+});

package/src/body-sync.ts

@@ -200,6 +200,18 @@ const hasDecidedCategory = (
 const hasDecidedPlacement = (placementDecidedAt: number | undefined): boolean =>
 	placementDecidedAt !== undefined;
 
+/**
+ * Issue #499: whether a completed body-sync pass has already classified this
+ * message. `bodyStorageKey` is written last, once every derivation of that pass
+ * has run, so its presence means the pass that first classified the message
+ * finished. Reads it exactly as `syncBodies`' own skip guard does. The same two
+ * re-entrant paths `hasDecidedCategory` and `hasDecidedPlacement` guard
+ * (`fetchAndGetBody`'s `NoSuchKey` fallback, `syncBodies(..., force: true)`) are
+ * the ones that reach the derivations again with it already set.
+ */
+const hasClassifiedBody = (bodyStorageKey: string | undefined): boolean =>
+	Boolean(bodyStorageKey);
+
 /**
  * Issue #398: `flags.autoArchive` is a filing preference relative to the Inbox.
  * A `leave` verdict reaches it for two unrelated reasons and only one of them
@@ -869,6 +881,12 @@ export class BodySyncService {
 			});
 		}
 
+		// Read once for the two write-once guards below — the auto-read decision
+		// and the category carry-forward. Taken here rather than earlier because
+		// nothing between this point and the Message update writes either field;
+		// the moves above touch `mailboxId` only.
+		const existingMessage = await this.messageService.get(messageId);
+
 		// `flags.unsubscribed` (issue #302, RFC 039 Decision 3): auto-mark-read,
 		// reusing the same FlagQueueService.markAsRead round-trip a manual
 		// mark-as-read goes through — idempotent on a retry (flipFlag no-ops when
@@ -879,6 +897,7 @@ export class BodySyncService {
 			accountId,
 			accountConfigId,
 			parsed,
+			existingMessage.bodyStorageKey,
 		);
 
 		const moved = Boolean(resolved.move || filterMoved);
@@ -907,7 +926,6 @@ export class BodySyncService {
 		// re-entrant paths for placement: `computePlacement` already declined to
 		// recompute a verdict once this field is set, so it is only ever present
 		// here on the pass that first decided it.
-		const existingMessage = await this.messageService.get(messageId);
 		const finalCategory = hasDecidedCategory(existingMessage.category)
 			? existingMessage.category
 			: classification.category;
@@ -1284,14 +1302,22 @@ export class BodySyncService {
 	 * separate expiry mechanism, and no caching of the decision beyond this
 	 * per-message `Address` read. A no-op when body sync was built without an
 	 * {@link UnsubscribeConfig} or the message carries no `From` address.
+	 *
+	 * Write-once per message (issue #499), like the `category` and placement
+	 * derivations alongside it: read state is applied on the pass that first
+	 * classifies a message and never re-decided. A user who marks such a message
+	 * unread afterwards owns it — a re-entrant pass over an already-classified
+	 * body would otherwise silently undo that.
 	 */
 	private async applyUnsubscribedAutoRead(
 		messageId: string,
 		accountId: string,
 		accountConfigId: string,
 		parsed: ParsedMail,
+		storedBodyKey: string | undefined,
 	): Promise<void> {
 		if (!this.unsubscribeConfig) return;
+		if (hasClassifiedBody(storedBodyKey)) return;
 
 		const fromEmail = extractPrimaryFromEmail(parsed);
 		if (!fromEmail) return;

package/src/list-id-backfill.test.ts

@@ -24,6 +24,7 @@ import type { StorageService } from "@remit/storage-service";
 import {
 	backfillListIds,
 	type ListIdBackfillCheckpoint,
+	type ListIdBackfillCheckpointStore,
 	type ListIdBackfillProgress,
 } from "./list-id-backfill.js";
 
@@ -99,6 +100,22 @@ const message = (overrides: Partial<MessageItem>): MessageItem =>
 		...overrides,
 	}) as unknown as MessageItem;
 
+const asAccount = (accountConfigId: string): AccountConfigItem =>
+	({ accountConfigId }) as unknown as AccountConfigItem;
+
+const inMemoryCheckpointStore = (): ListIdBackfillCheckpointStore => {
+	let checkpoint: ListIdBackfillCheckpoint | undefined;
+	return {
+		load: async () => checkpoint,
+		save: async (next) => {
+			checkpoint = next;
+		},
+		clear: async () => {
+			checkpoint = undefined;
+		},
+	};
+};
+
 interface Harness {
 	accountConfigService: Pick<IAccountConfigRepository, "listAll">;
 	threadMessageService: Pick<
@@ -351,6 +368,37 @@ describe("backfillListIds", () => {
 		assert.equal(harness.updates.length, 2);
 	});
 
+	it("scans accounts in account-id order, whatever order listAll returns", async () => {
+		const accountIds = ["acc-1", "acc-2", "acc-3"];
+		const rows = accountIds.map((accountConfigId) =>
+			row({
+				threadMessageId: `tm-${accountConfigId}`,
+				messageId: `m-${accountConfigId}`,
+				accountConfigId,
+			}),
+		);
+		const harness = buildHarness({
+			accounts: ["acc-3", "acc-1", "acc-2"].map(asAccount),
+			rows,
+			messages: rows.map((r) =>
+				message({
+					messageId: r.messageId,
+					bodyStorageKey: `s3://${r.messageId}`,
+				}),
+			),
+		});
+		const progress: ListIdBackfillProgress[] = [];
+
+		await backfillListIds(harness, {
+			onProgress: (p) => progress.push({ ...p }),
+		});
+
+		assert.deepEqual(
+			progress.map((p) => p.accountConfigId),
+			accountIds,
+		);
+	});
+
 	it("checkpoints after each page and clears it on completion", async () => {
 		const rows = Array.from({ length: 3 }, (_, i) =>
 			row({ threadMessageId: `tm-${i}`, messageId: `m-${i}` }),
@@ -380,6 +428,7 @@ describe("backfillListIds", () => {
 		});
 
 		assert.equal(saved.length, 2);
+		assert.equal(saved[0].accountConfigId, "acc-1");
 		assert.equal(saved[0].continuationToken, "2");
 		assert.equal(saved[1].continuationToken, undefined);
 		assert.equal(cleared, true);
@@ -400,7 +449,10 @@ describe("backfillListIds", () => {
 		const result = await backfillListIds(harness, {
 			batchSize: 2,
 			checkpointStore: {
-				load: async () => ({ accountIndex: 0, continuationToken: "2" }),
+				load: async () => ({
+					accountConfigId: "acc-1",
+					continuationToken: "2",
+				}),
 				save: async () => {},
 				clear: async () => {},
 			},
@@ -413,4 +465,109 @@ describe("backfillListIds", () => {
 			["tm-2"],
 		);
 	});
+
+	it("resumes the account it was working through after an earlier account is removed", async () => {
+		const accountIds = ["acc-1", "acc-2", "acc-3"];
+		const rows = [
+			row({
+				threadMessageId: "tm-1",
+				messageId: "m-1",
+				accountConfigId: "acc-1",
+			}),
+			row({
+				threadMessageId: "tm-2a",
+				messageId: "m-2a",
+				accountConfigId: "acc-2",
+			}),
+			row({
+				threadMessageId: "tm-2b",
+				messageId: "m-2b",
+				accountConfigId: "acc-2",
+			}),
+			row({
+				threadMessageId: "tm-3",
+				messageId: "m-3",
+				accountConfigId: "acc-3",
+			}),
+		];
+		const messages = rows.map((r) =>
+			message({
+				messageId: r.messageId,
+				bodyStorageKey: `s3://${r.messageId}`,
+			}),
+		);
+
+		const store = inMemoryCheckpointStore();
+
+		const interrupted = buildHarness({
+			accounts: accountIds.map(asAccount),
+			rows,
+			messages,
+			pageSize: 1,
+		});
+		const listByAccount = interrupted.threadMessageService.listByAccount;
+		interrupted.threadMessageService.listByAccount = async (
+			accountConfigId,
+			opts,
+		) => {
+			if (accountConfigId === "acc-2" && opts?.continuationToken) {
+				throw new Error("interrupted");
+			}
+			return listByAccount(accountConfigId, opts);
+		};
+
+		await assert.rejects(
+			backfillListIds(interrupted, { batchSize: 1, checkpointStore: store }),
+			/interrupted/,
+		);
+
+		const resumed = buildHarness({
+			accounts: ["acc-2", "acc-3"].map(asAccount),
+			rows,
+			messages,
+			pageSize: 1,
+		});
+
+		const result = await backfillListIds(resumed, {
+			batchSize: 1,
+			checkpointStore: store,
+		});
+
+		assert.deepEqual(
+			resumed.updates.map((u) => u.threadMessageId),
+			["tm-2b", "tm-3"],
+		);
+		assert.equal(result.backfilled, 2);
+	});
+
+	it("restarts the pass when the checkpointed account no longer exists", async () => {
+		const rows = Array.from({ length: 3 }, (_, i) =>
+			row({ threadMessageId: `tm-${i}`, messageId: `m-${i}` }),
+		);
+		const messages = rows.map((r) =>
+			message({
+				messageId: r.messageId,
+				bodyStorageKey: `s3://${r.messageId}`,
+			}),
+		);
+		const harness = buildHarness({ rows, messages, pageSize: 2 });
+
+		const result = await backfillListIds(harness, {
+			batchSize: 2,
+			checkpointStore: {
+				load: async () => ({
+					accountConfigId: "acc-removed",
+					continuationToken: "2",
+				}),
+				save: async () => {},
+				clear: async () => {},
+			},
+		});
+
+		assert.equal(result.scanned, 3);
+		assert.deepEqual(
+			harness.updates.map((u) => u.threadMessageId),
+			["tm-0", "tm-1", "tm-2"],
+		);
+	});
 });

package/src/list-id-backfill.ts

@@ -11,14 +11,14 @@ import { extractListId } from "./filters/list-id.js";
 const DEFAULT_BATCH_SIZE = 200;
 
 /**
- * Where the full-corpus pass left off: the index into the account list (in
- * `listAll()` order) it was working through, and the page cursor within that
- * account. Resuming skips every account before it entirely and re-opens the
- * in-flight one at its saved cursor, rather than re-scanning the whole corpus
- * from the top after an interruption.
+ * Where the full-corpus pass left off: the identity of the account it was
+ * working through, and the page cursor within that account. Resuming looks the
+ * account up by id and re-opens it at its saved cursor, rather than re-scanning
+ * the whole corpus from the top after an interruption. A checkpoint naming no
+ * configured account restarts the pass.
  */
 export interface ListIdBackfillCheckpoint {
-	accountIndex: number;
+	accountConfigId: string;
 	continuationToken?: string;
 }
 
@@ -138,7 +138,8 @@ const deriveAndApplyListId = async (
  * sync path will populate `listId` for it once the body lands.
  *
  * Chunked by `listByAccount`'s existing keyset pagination, one account at a
- * time in `listAll()` order. A failure reading or parsing one message's
+ * time in account-id order, so an interrupted run and its resume walk the same
+ * sequence. A failure reading or parsing one message's
  * stored body is contained to that message — logged, counted, and the pass
  * continues — the same containment `BodySyncService`'s classification
  * backfill uses for the same reason: one unreadable object must not strand
@@ -151,9 +152,29 @@ export const backfillListIds = async (
 	const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
 	const { logger, checkpointStore } = options;
 
-	const accounts = await deps.accountConfigService.listAll();
+	const accounts = [...(await deps.accountConfigService.listAll())].sort(
+		(left, right) => left.accountConfigId.localeCompare(right.accountConfigId),
+	);
 	const startingCheckpoint = await checkpointStore?.load();
-	const startIndex = startingCheckpoint?.accountIndex ?? 0;
+	const checkpointedIndex = startingCheckpoint
+		? accounts.findIndex(
+				(account) =>
+					account.accountConfigId === startingCheckpoint.accountConfigId,
+			)
+		: -1;
+
+	if (startingCheckpoint && checkpointedIndex === -1) {
+		logger?.info(
+			{ accountConfigId: startingCheckpoint.accountConfigId },
+			"ListId backfill checkpoint names no configured account; restarting the pass",
+		);
+	}
+
+	const startIndex = checkpointedIndex === -1 ? 0 : checkpointedIndex;
+	const startContinuationToken =
+		checkpointedIndex === -1
+			? undefined
+			: startingCheckpoint?.continuationToken;
 
 	const totals = emptyTotals();
 	const failedThreadMessageIds: string[] = [];
@@ -165,9 +186,7 @@ export const backfillListIds = async (
 	) {
 		const account = accounts[accountIndex];
 		let continuationToken: string | undefined =
-			accountIndex === startIndex
-				? startingCheckpoint?.continuationToken
-				: undefined;
+			accountIndex === startIndex ? startContinuationToken : undefined;
 
 		do {
 			const page = await deps.threadMessageService.listByAccount(
@@ -231,7 +250,7 @@ export const backfillListIds = async (
 
 			continuationToken = page.continuationToken;
 			await checkpointStore?.save({
-				accountIndex,
+				accountConfigId: account.accountConfigId,
 				continuationToken,
 			});