@remit/mailbox-service 0.0.29 → 0.0.30

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.29",
+  "version": "0.0.30",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/index.ts

@@ -122,6 +122,17 @@ export {
 	createImapFlowConnectionWithCredentials,
 	ImapFlowConnection,
 } from "./imapflow-connection.js";
+export {
+	backfillListIds,
+	type ListIdBackfillCheckpoint,
+	type ListIdBackfillCheckpointStore,
+	type ListIdBackfillDeps,
+	type ListIdBackfillLogger,
+	type ListIdBackfillOptions,
+	type ListIdBackfillProgress,
+	type ListIdBackfillResult,
+	type ListIdBackfillTotals,
+} from "./list-id-backfill.js";
 export {
 	guardConnectionCursor,
 	guardMailboxCursor,

package/src/list-id-backfill.test.ts

@@ -0,0 +1,416 @@
+/**
+ * ListId is only written at body-sync time, going forward (issue #263). A row
+ * synced before header extraction shipped keeps `listId` unset forever, so a
+ * `ListId` filter clause silently under-matches the back catalogue.
+ *
+ * These tests pin the one-time, resumable pass that closes that gap: it reads
+ * each candidate's already-stored raw source (never IMAP), extracts
+ * `List-Id`, and writes only that field.
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	AccountConfigItem,
+	IAccountConfigRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+	MessageItem,
+	ResultList,
+	ThreadMessageItem,
+	UpdateThreadMessageInput,
+} from "@remit/data-ports";
+import type { StorageService } from "@remit/storage-service";
+import {
+	backfillListIds,
+	type ListIdBackfillCheckpoint,
+	type ListIdBackfillProgress,
+} from "./list-id-backfill.js";
+
+const LIST_EML = Buffer.from(
+	[
+		"From: Weekly News <news@example.com>",
+		"To: me@example.com",
+		"Subject: This week",
+		"List-Id: Weekly News <weekly.news.example.com>",
+		"Content-Type: text/plain",
+		"",
+		"news",
+	].join("\r\n"),
+);
+
+const PLAIN_EML = Buffer.from(
+	[
+		"From: Alice <alice@example.com>",
+		"To: me@example.com",
+		"Subject: Hi",
+		"Content-Type: text/plain",
+		"",
+		"hi",
+	].join("\r\n"),
+);
+
+const MALFORMED_LIST_ID_EML = Buffer.from(
+	[
+		"From: Weird <weird@example.com>",
+		"To: me@example.com",
+		"Subject: Odd header",
+		"List-Id:",
+		"Content-Type: text/plain",
+		"",
+		"body",
+	].join("\r\n"),
+);
+
+const row = (overrides: Partial<ThreadMessageItem>): ThreadMessageItem =>
+	({
+		threadMessageId: "tm-1",
+		accountConfigId: "acc-1",
+		threadId: "thread-1",
+		messageId: "m-1",
+		mailboxId: "mb-1",
+		uid: 1,
+		referenceOrder: 0,
+		internalDate: 1,
+		sentDate: 1,
+		isRead: false,
+		hasAttachment: false,
+		star: "none",
+		hasStars: false,
+		isDeleted: false,
+		category: "uncategorized",
+		createdAt: 1,
+		updatedAt: 1,
+		...overrides,
+	}) as unknown as ThreadMessageItem;
+
+const message = (overrides: Partial<MessageItem>): MessageItem =>
+	({
+		messageId: "m-1",
+		mailboxId: "mb-1",
+		uid: 1,
+		status: "active",
+		syncStatus: "synced",
+		category: "uncategorized",
+		hasListUnsubscribe: false,
+		movedByRemit: false,
+		createdAt: 1,
+		updatedAt: 1,
+		...overrides,
+	}) as unknown as MessageItem;
+
+interface Harness {
+	accountConfigService: Pick<IAccountConfigRepository, "listAll">;
+	threadMessageService: Pick<
+		IThreadMessageRepository,
+		"listByAccount" | "update"
+	>;
+	messageService: Pick<IMessageRepository, "get">;
+	storageService: Pick<StorageService, "retrieve">;
+	updates: Array<{ threadMessageId: string; input: UpdateThreadMessageInput }>;
+	retrieved: string[];
+}
+
+const buildHarness = (options: {
+	accounts?: AccountConfigItem[];
+	rows: ThreadMessageItem[];
+	messages: MessageItem[];
+	retrieve?: (key: string) => Promise<Buffer>;
+	pageSize?: number;
+}): Harness => {
+	const accounts = options.accounts ?? [
+		{ accountConfigId: "acc-1" } as unknown as AccountConfigItem,
+	];
+	const messagesById = new Map(options.messages.map((m) => [m.messageId, m]));
+	const updates: Array<{
+		threadMessageId: string;
+		input: UpdateThreadMessageInput;
+	}> = [];
+	const retrieved: string[] = [];
+	const pageSize = options.pageSize ?? 200;
+
+	const accountConfigService: Pick<IAccountConfigRepository, "listAll"> = {
+		listAll: async () => accounts,
+	};
+
+	const threadMessageService: Pick<
+		IThreadMessageRepository,
+		"listByAccount" | "update"
+	> = {
+		listByAccount: async (
+			accountConfigId: string,
+			opts?: { limit?: number; continuationToken?: string },
+		): Promise<ResultList<ThreadMessageItem>> => {
+			const scoped = options.rows.filter(
+				(r) => r.accountConfigId === accountConfigId,
+			);
+			const start = opts?.continuationToken
+				? Number(opts.continuationToken)
+				: 0;
+			const limit = opts?.limit ?? pageSize;
+			const page = scoped.slice(start, start + limit);
+			const nextStart = start + page.length;
+			return {
+				items: page,
+				continuationToken:
+					nextStart < scoped.length ? String(nextStart) : undefined,
+			};
+		},
+		update: async (
+			_accountConfigId: string,
+			threadMessageId: string,
+			input: UpdateThreadMessageInput,
+		) => {
+			updates.push({ threadMessageId, input });
+			return row({ threadMessageId, ...input });
+		},
+	};
+
+	const messageService: Pick<IMessageRepository, "get"> = {
+		get: (async (messageIds: string | string[]) => {
+			if (Array.isArray(messageIds)) {
+				return messageIds
+					.map((id) => messagesById.get(id))
+					.filter((m): m is MessageItem => m !== undefined);
+			}
+			const found = messagesById.get(messageIds);
+			if (!found) throw new Error(`no fixture for ${messageIds}`);
+			return found;
+		}) as IMessageRepository["get"],
+	};
+
+	const storageService: Pick<StorageService, "retrieve"> = {
+		retrieve: async (key: string) => {
+			retrieved.push(key);
+			return options.retrieve ? options.retrieve(key) : LIST_EML;
+		},
+	};
+
+	return {
+		accountConfigService,
+		threadMessageService,
+		messageService,
+		storageService,
+		updates,
+		retrieved,
+	};
+};
+
+describe("backfillListIds", () => {
+	it("writes the extracted List-Id for a candidate row", async () => {
+		const harness = buildHarness({
+			rows: [row({ threadMessageId: "tm-1", messageId: "m-1" })],
+			messages: [message({ messageId: "m-1", bodyStorageKey: "s3://m-1" })],
+		});
+
+		const result = await backfillListIds(harness);
+
+		assert.equal(result.backfilled, 1);
+		assert.equal(result.failed, 0);
+		assert.equal(harness.updates.length, 1);
+		assert.equal(harness.updates[0].input.listId, "weekly.news.example.com");
+	});
+
+	it("leaves listId empty, without error, when the message has no List-Id header", async () => {
+		const harness = buildHarness({
+			rows: [row({ threadMessageId: "tm-1", messageId: "m-1" })],
+			messages: [message({ messageId: "m-1", bodyStorageKey: "s3://m-1" })],
+			retrieve: async () => PLAIN_EML,
+		});
+
+		const result = await backfillListIds(harness);
+
+		assert.equal(result.noListId, 1);
+		assert.equal(result.failed, 0);
+		assert.deepEqual(harness.updates, []);
+	});
+
+	it("does not error on a malformed List-Id header", async () => {
+		const harness = buildHarness({
+			rows: [row({ threadMessageId: "tm-1", messageId: "m-1" })],
+			messages: [message({ messageId: "m-1", bodyStorageKey: "s3://m-1" })],
+			retrieve: async () => MALFORMED_LIST_ID_EML,
+		});
+
+		const result = await backfillListIds(harness);
+
+		assert.equal(result.failed, 0);
+		assert.equal(result.noListId, 1);
+		assert.deepEqual(harness.updates, []);
+	});
+
+	it("skips a row whose listId is already set, without reading storage", async () => {
+		const harness = buildHarness({
+			rows: [
+				row({
+					threadMessageId: "tm-1",
+					messageId: "m-1",
+					listId: "already.set",
+				}),
+			],
+			messages: [message({ messageId: "m-1", bodyStorageKey: "s3://m-1" })],
+		});
+
+		const result = await backfillListIds(harness);
+
+		assert.equal(result.alreadySet, 1);
+		assert.equal(result.backfilled, 0);
+		assert.deepEqual(harness.retrieved, []);
+		assert.deepEqual(harness.updates, []);
+	});
+
+	it("skips a candidate whose body was never synced, without reading storage", async () => {
+		const harness = buildHarness({
+			rows: [row({ threadMessageId: "tm-1", messageId: "m-1" })],
+			messages: [message({ messageId: "m-1", bodyStorageKey: undefined })],
+		});
+
+		const result = await backfillListIds(harness);
+
+		assert.equal(result.skippedNoBody, 1);
+		assert.deepEqual(harness.retrieved, []);
+		assert.deepEqual(harness.updates, []);
+	});
+
+	it("contains a storage failure to the one message and keeps going", async () => {
+		const harness = buildHarness({
+			rows: [
+				row({ threadMessageId: "tm-bad", messageId: "m-bad" }),
+				row({ threadMessageId: "tm-good", messageId: "m-good" }),
+			],
+			messages: [
+				message({ messageId: "m-bad", bodyStorageKey: "s3://m-bad" }),
+				message({ messageId: "m-good", bodyStorageKey: "s3://m-good" }),
+			],
+			retrieve: async (key) => {
+				if (key === "s3://m-bad") throw new Error("AccessDenied");
+				return LIST_EML;
+			},
+		});
+
+		const result = await backfillListIds(harness);
+
+		assert.equal(result.failed, 1);
+		assert.deepEqual(result.failedThreadMessageIds, ["tm-bad"]);
+		assert.equal(result.backfilled, 1);
+		assert.equal(harness.updates.length, 1);
+		assert.equal(harness.updates[0].threadMessageId, "tm-good");
+	});
+
+	it("reports progress as pages are processed", async () => {
+		const rows = Array.from({ length: 3 }, (_, i) =>
+			row({ threadMessageId: `tm-${i}`, messageId: `m-${i}` }),
+		);
+		const messages = rows.map((r) =>
+			message({
+				messageId: r.messageId,
+				bodyStorageKey: `s3://${r.messageId}`,
+			}),
+		);
+		const harness = buildHarness({ rows, messages, pageSize: 2 });
+		const progress: ListIdBackfillProgress[] = [];
+
+		const result = await backfillListIds(harness, {
+			batchSize: 2,
+			onProgress: (p) => progress.push({ ...p }),
+		});
+
+		assert.equal(result.backfilled, 3);
+		assert.equal(progress.length, 2);
+		assert.equal(progress[0].scanned, 2);
+		assert.equal(progress[1].scanned, 3);
+	});
+
+	it("scans every account returned by listAll", async () => {
+		const harness = buildHarness({
+			accounts: [
+				{ accountConfigId: "acc-1" } as unknown as AccountConfigItem,
+				{ accountConfigId: "acc-2" } as unknown as AccountConfigItem,
+			],
+			rows: [
+				row({
+					threadMessageId: "tm-1",
+					messageId: "m-1",
+					accountConfigId: "acc-1",
+				}),
+				row({
+					threadMessageId: "tm-2",
+					messageId: "m-2",
+					accountConfigId: "acc-2",
+				}),
+			],
+			messages: [
+				message({ messageId: "m-1", bodyStorageKey: "s3://m-1" }),
+				message({ messageId: "m-2", bodyStorageKey: "s3://m-2" }),
+			],
+		});
+
+		const result = await backfillListIds(harness);
+
+		assert.equal(result.backfilled, 2);
+		assert.equal(harness.updates.length, 2);
+	});
+
+	it("checkpoints after each page and clears it on completion", async () => {
+		const rows = Array.from({ length: 3 }, (_, i) =>
+			row({ threadMessageId: `tm-${i}`, messageId: `m-${i}` }),
+		);
+		const messages = rows.map((r) =>
+			message({
+				messageId: r.messageId,
+				bodyStorageKey: `s3://${r.messageId}`,
+			}),
+		);
+		const harness = buildHarness({ rows, messages, pageSize: 2 });
+
+		const saved: ListIdBackfillCheckpoint[] = [];
+		let cleared = false;
+
+		await backfillListIds(harness, {
+			batchSize: 2,
+			checkpointStore: {
+				load: async () => undefined,
+				save: async (checkpoint) => {
+					saved.push(checkpoint);
+				},
+				clear: async () => {
+					cleared = true;
+				},
+			},
+		});
+
+		assert.equal(saved.length, 2);
+		assert.equal(saved[0].continuationToken, "2");
+		assert.equal(saved[1].continuationToken, undefined);
+		assert.equal(cleared, true);
+	});
+
+	it("resumes from a saved checkpoint instead of rescanning from the start", async () => {
+		const rows = Array.from({ length: 3 }, (_, i) =>
+			row({ threadMessageId: `tm-${i}`, messageId: `m-${i}` }),
+		);
+		const messages = rows.map((r) =>
+			message({
+				messageId: r.messageId,
+				bodyStorageKey: `s3://${r.messageId}`,
+			}),
+		);
+		const harness = buildHarness({ rows, messages, pageSize: 2 });
+
+		const result = await backfillListIds(harness, {
+			batchSize: 2,
+			checkpointStore: {
+				load: async () => ({ accountIndex: 0, continuationToken: "2" }),
+				save: async () => {},
+				clear: async () => {},
+			},
+		});
+
+		assert.equal(result.scanned, 1);
+		assert.equal(result.backfilled, 1);
+		assert.deepEqual(
+			harness.updates.map((u) => u.threadMessageId),
+			["tm-2"],
+		);
+	});
+});

package/src/list-id-backfill.ts

@@ -0,0 +1,252 @@
+import type {
+	IAccountConfigRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+	ThreadMessageItem,
+} from "@remit/data-ports";
+import type { StorageService } from "@remit/storage-service";
+import { parseMessageBody } from "./body-parse.js";
+import { extractListId } from "./filters/list-id.js";
+
+const DEFAULT_BATCH_SIZE = 200;
+
+/**
+ * Where the full-corpus pass left off: the index into the account list (in
+ * `listAll()` order) it was working through, and the page cursor within that
+ * account. Resuming skips every account before it entirely and re-opens the
+ * in-flight one at its saved cursor, rather than re-scanning the whole corpus
+ * from the top after an interruption.
+ */
+export interface ListIdBackfillCheckpoint {
+	accountIndex: number;
+	continuationToken?: string;
+}
+
+/**
+ * Persistence for {@link ListIdBackfillCheckpoint}, injected so the pass stays
+ * testable with an in-memory fake; the real entrypoint backs it with a file.
+ * `clear()` runs once the whole corpus has been scanned, so a later run starts
+ * fresh rather than reading a stale finished-run checkpoint.
+ */
+export interface ListIdBackfillCheckpointStore {
+	load(): Promise<ListIdBackfillCheckpoint | undefined>;
+	save(checkpoint: ListIdBackfillCheckpoint): Promise<void>;
+	clear(): Promise<void>;
+}
+
+export interface ListIdBackfillDeps {
+	accountConfigService: Pick<IAccountConfigRepository, "listAll">;
+	threadMessageService: Pick<
+		IThreadMessageRepository,
+		"listByAccount" | "update"
+	>;
+	messageService: Pick<IMessageRepository, "get">;
+	storageService: Pick<StorageService, "retrieve">;
+}
+
+export interface ListIdBackfillLogger {
+	info(obj: Record<string, unknown>, msg: string): void;
+	error?(obj: Record<string, unknown>, msg: string): void;
+}
+
+export interface ListIdBackfillTotals {
+	scanned: number;
+	alreadySet: number;
+	skippedNoBody: number;
+	backfilled: number;
+	noListId: number;
+	failed: number;
+}
+
+export interface ListIdBackfillResult extends ListIdBackfillTotals {
+	failedThreadMessageIds: string[];
+}
+
+export interface ListIdBackfillProgress extends ListIdBackfillTotals {
+	accountConfigId: string;
+}
+
+export interface ListIdBackfillOptions {
+	/** Rows fetched per `listByAccount` page. */
+	batchSize?: number;
+	checkpointStore?: ListIdBackfillCheckpointStore;
+	logger?: ListIdBackfillLogger;
+	/** Called once per page, after that page's rows are settled. */
+	onProgress?: (progress: ListIdBackfillProgress) => void;
+}
+
+const emptyTotals = (): ListIdBackfillTotals => ({
+	scanned: 0,
+	alreadySet: 0,
+	skippedNoBody: 0,
+	backfilled: 0,
+	noListId: 0,
+	failed: 0,
+});
+
+/**
+ * Read the stored raw source, derive `List-Id`, and — only when one is
+ * present — write it. Kept as its own function (rather than inline in a
+ * try/catch) so the caller can contain a failure with `.then(fulfilled,
+ * rejected)` instead of a block catch, matching how `BodySyncService`
+ * contains a per-message backfill failure.
+ */
+const deriveAndApplyListId = async (
+	deps: ListIdBackfillDeps,
+	accountConfigId: string,
+	row: ThreadMessageItem,
+	bodyStorageKey: string,
+): Promise<string> => {
+	const body = await deps.storageService.retrieve(bodyStorageKey);
+	const parsed = await parseMessageBody(body);
+	const listId = extractListId(parsed);
+
+	if (listId) {
+		await deps.threadMessageService.update(
+			accountConfigId,
+			row.threadMessageId,
+			{ listId },
+			{
+				composites: {
+					sentDate: row.sentDate,
+					mailboxId: row.mailboxId,
+					isRead: row.isRead,
+					isDeleted: row.isDeleted,
+					hasStars: row.hasStars,
+					hasAttachment: row.hasAttachment,
+				},
+			},
+		);
+	}
+
+	return listId;
+};
+
+/**
+ * One-time, resumable pass that derives `ThreadMessage.listId` for rows synced
+ * before header extraction shipped (issue #263). Read-only against the stored
+ * raw source: it never opens IMAP and never touches anything but the single
+ * `listId` field.
+ *
+ * A row is a candidate when `listId` is `undefined` — written before the
+ * column existed, or written since by a path that found no `List-Id` header
+ * (body-sync only sets the field when the header is present, so "no header"
+ * and "not yet backfilled" look the same on the row; re-checking an
+ * already-correct empty row is wasted work, never a wrong answer). A
+ * candidate whose Message has no `bodyStorageKey` yet is left alone — its
+ * body was never synced, so there is nothing local to read, and the ordinary
+ * sync path will populate `listId` for it once the body lands.
+ *
+ * Chunked by `listByAccount`'s existing keyset pagination, one account at a
+ * time in `listAll()` order. A failure reading or parsing one message's
+ * stored body is contained to that message — logged, counted, and the pass
+ * continues — the same containment `BodySyncService`'s classification
+ * backfill uses for the same reason: one unreadable object must not strand
+ * the rest of the corpus.
+ */
+export const backfillListIds = async (
+	deps: ListIdBackfillDeps,
+	options: ListIdBackfillOptions = {},
+): Promise<ListIdBackfillResult> => {
+	const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
+	const { logger, checkpointStore } = options;
+
+	const accounts = await deps.accountConfigService.listAll();
+	const startingCheckpoint = await checkpointStore?.load();
+	const startIndex = startingCheckpoint?.accountIndex ?? 0;
+
+	const totals = emptyTotals();
+	const failedThreadMessageIds: string[] = [];
+
+	for (
+		let accountIndex = startIndex;
+		accountIndex < accounts.length;
+		accountIndex++
+	) {
+		const account = accounts[accountIndex];
+		let continuationToken: string | undefined =
+			accountIndex === startIndex
+				? startingCheckpoint?.continuationToken
+				: undefined;
+
+		do {
+			const page = await deps.threadMessageService.listByAccount(
+				account.accountConfigId,
+				{ limit: batchSize, continuationToken },
+			);
+
+			totals.scanned += page.items.length;
+			const candidates = page.items.filter((row) => row.listId === undefined);
+			totals.alreadySet += page.items.length - candidates.length;
+
+			if (candidates.length > 0) {
+				const messages = await deps.messageService.get(
+					candidates.map((row) => row.messageId),
+				);
+				const messageByMessageId = new Map(
+					messages.map((message) => [message.messageId, message]),
+				);
+
+				for (const row of candidates) {
+					const message = messageByMessageId.get(row.messageId);
+					if (!message?.bodyStorageKey) {
+						totals.skippedNoBody++;
+						continue;
+					}
+
+					const outcome = await deriveAndApplyListId(
+						deps,
+						account.accountConfigId,
+						row,
+						message.bodyStorageKey,
+					).then(
+						(listId) => ({ error: null, listId }) as const,
+						(error: unknown) => ({ error, listId: null }) as const,
+					);
+
+					if (outcome.error !== null) {
+						totals.failed++;
+						failedThreadMessageIds.push(row.threadMessageId);
+						logger?.error?.(
+							{
+								threadMessageId: row.threadMessageId,
+								messageId: row.messageId,
+								error:
+									outcome.error instanceof Error
+										? outcome.error.message
+										: String(outcome.error),
+							},
+							"ListId backfill failed for a message; leaving it for a later pass",
+						);
+						continue;
+					}
+
+					if (outcome.listId) {
+						totals.backfilled++;
+					} else {
+						totals.noListId++;
+					}
+				}
+			}
+
+			continuationToken = page.continuationToken;
+			await checkpointStore?.save({
+				accountIndex,
+				continuationToken,
+			});
+
+			logger?.info(
+				{ accountConfigId: account.accountConfigId, ...totals },
+				"ListId backfill progress",
+			);
+			options.onProgress?.({
+				accountConfigId: account.accountConfigId,
+				...totals,
+			});
+		} while (continuationToken);
+	}
+
+	await checkpointStore?.clear();
+
+	return { ...totals, failedThreadMessageIds };
+};