@remit/mailbox-service 0.0.111 → 0.0.113

package/src/list-id-backfill.ts

@@ -1,274 +0,0 @@
-import type {
-	IAccountConfigRepository,
-	IMessageRepository,
-	IThreadMessageRepository,
-	ThreadMessageItem,
-} from "@remit/data-ports";
-import type { StorageService } from "@remit/storage-service";
-import { parseMessageBody } from "./body-parse.js";
-import { extractListId } from "./filters/list-id.js";
-
-const DEFAULT_BATCH_SIZE = 200;
-
-/**
- * Where the full-corpus pass left off: the identity of the account it was
- * working through, and the page cursor within that account. Resuming looks the
- * account up by id and re-opens it at its saved cursor, rather than re-scanning
- * the whole corpus from the top after an interruption. A checkpoint naming no
- * configured account restarts the pass.
- */
-export interface ListIdBackfillCheckpoint {
-	accountConfigId: string;
-	continuationToken?: string;
-}
-
-/**
- * Persistence for {@link ListIdBackfillCheckpoint}, injected so the pass stays
- * testable with an in-memory fake; the real entrypoint backs it with a file.
- * `clear()` runs once the whole corpus has been scanned, so a later run starts
- * fresh rather than reading a stale finished-run checkpoint.
- */
-export interface ListIdBackfillCheckpointStore {
-	load(): Promise<ListIdBackfillCheckpoint | undefined>;
-	save(checkpoint: ListIdBackfillCheckpoint): Promise<void>;
-	clear(): Promise<void>;
-}
-
-export interface ListIdBackfillDeps {
-	accountConfigService: Pick<IAccountConfigRepository, "listAll">;
-	threadMessageService: Pick<
-		IThreadMessageRepository,
-		"listByAccount" | "update"
-	>;
-	messageService: Pick<IMessageRepository, "get">;
-	storageService: Pick<StorageService, "retrieve">;
-}
-
-export interface ListIdBackfillLogger {
-	info(obj: Record<string, unknown>, msg: string): void;
-	error?(obj: Record<string, unknown>, msg: string): void;
-}
-
-export interface ListIdBackfillTotals {
-	scanned: number;
-	alreadySet: number;
-	skippedNoBody: number;
-	backfilled: number;
-	noListId: number;
-	failed: number;
-}
-
-export interface ListIdBackfillResult extends ListIdBackfillTotals {
-	failedThreadMessageIds: string[];
-}
-
-export interface ListIdBackfillProgress extends ListIdBackfillTotals {
-	accountConfigId: string;
-}
-
-export interface ListIdBackfillOptions {
-	/** Rows fetched per `listByAccount` page. */
-	batchSize?: number;
-	checkpointStore?: ListIdBackfillCheckpointStore;
-	logger?: ListIdBackfillLogger;
-	/** Called once per page, after that page's rows are settled. */
-	onProgress?: (progress: ListIdBackfillProgress) => void;
-}
-
-const emptyTotals = (): ListIdBackfillTotals => ({
-	scanned: 0,
-	alreadySet: 0,
-	skippedNoBody: 0,
-	backfilled: 0,
-	noListId: 0,
-	failed: 0,
-});
-
-/**
- * Read the stored raw source, derive `List-Id`, and — only when one is
- * present — write it. Kept as its own function (rather than inline in a
- * try/catch) so the caller can contain a failure with `.then(fulfilled,
- * rejected)` instead of a block catch, which keeps the `try` around the one
- * call whose failure this contains.
- */
-const deriveAndApplyListId = async (
-	deps: ListIdBackfillDeps,
-	accountConfigId: string,
-	row: ThreadMessageItem,
-	bodyStorageKey: string,
-): Promise<string> => {
-	const body = await deps.storageService.retrieve(bodyStorageKey);
-	const parsed = await parseMessageBody(body);
-	const listId = extractListId(parsed);
-
-	if (listId) {
-		await deps.threadMessageService.update(
-			accountConfigId,
-			row.threadMessageId,
-			{ listId },
-			{
-				composites: {
-					sentDate: row.sentDate,
-					mailboxId: row.mailboxId,
-					isRead: row.isRead,
-					isDeleted: row.isDeleted,
-					hasStars: row.hasStars,
-					hasAttachment: row.hasAttachment,
-				},
-			},
-		);
-	}
-
-	return listId;
-};
-
-/**
- * One-time, resumable pass that derives `ThreadMessage.listId` for rows synced
- * before header extraction shipped (issue #263). Read-only against the stored
- * raw source: it never opens IMAP and never touches anything but the single
- * `listId` field.
- *
- * A row is a candidate when `listId` is `undefined` — written before the
- * column existed, or written since by a path that found no `List-Id` header
- * (body-sync only sets the field when the header is present, so "no header"
- * and "not yet backfilled" look the same on the row; re-checking an
- * already-correct empty row is wasted work, never a wrong answer). A
- * candidate whose Message has no `bodyStorageKey` yet is left alone — its
- * body was never synced, so there is nothing local to read, and the ordinary
- * sync path will populate `listId` for it once the body lands.
- *
- * Chunked by `listByAccount`'s existing keyset pagination, one account at a
- * time in account-id order, so an interrupted run and its resume walk the same
- * sequence. A failure reading or parsing one message's
- * stored body is contained to that message — logged, counted, and the pass
- * continues: one unreadable object must not strand the rest of the corpus.
- *
- * The shape a later pass over `Message.classificationState` should take
- * (issue #331): a one-time, resumable, storage-only sweep that selects its
- * cohort by an explicit field, so it can be run once, deliberately, after a
- * classifier release — rather than a re-derivation folded into every sync pass.
- */
-export const backfillListIds = async (
-	deps: ListIdBackfillDeps,
-	options: ListIdBackfillOptions = {},
-): Promise<ListIdBackfillResult> => {
-	const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
-	const { logger, checkpointStore } = options;
-
-	const accounts = [...(await deps.accountConfigService.listAll())].sort(
-		(left, right) => left.accountConfigId.localeCompare(right.accountConfigId),
-	);
-	const startingCheckpoint = await checkpointStore?.load();
-	const checkpointedIndex = startingCheckpoint
-		? accounts.findIndex(
-				(account) =>
-					account.accountConfigId === startingCheckpoint.accountConfigId,
-			)
-		: -1;
-
-	if (startingCheckpoint && checkpointedIndex === -1) {
-		logger?.info(
-			{ accountConfigId: startingCheckpoint.accountConfigId },
-			"ListId backfill checkpoint names no configured account; restarting the pass",
-		);
-	}
-
-	const startIndex = checkpointedIndex === -1 ? 0 : checkpointedIndex;
-	const startContinuationToken =
-		checkpointedIndex === -1
-			? undefined
-			: startingCheckpoint?.continuationToken;
-
-	const totals = emptyTotals();
-	const failedThreadMessageIds: string[] = [];
-
-	for (
-		let accountIndex = startIndex;
-		accountIndex < accounts.length;
-		accountIndex++
-	) {
-		const account = accounts[accountIndex];
-		let continuationToken: string | undefined =
-			accountIndex === startIndex ? startContinuationToken : undefined;
-
-		do {
-			const page = await deps.threadMessageService.listByAccount(
-				account.accountConfigId,
-				{ limit: batchSize, continuationToken },
-			);
-
-			totals.scanned += page.items.length;
-			const candidates = page.items.filter((row) => row.listId === undefined);
-			totals.alreadySet += page.items.length - candidates.length;
-
-			if (candidates.length > 0) {
-				const messages = await deps.messageService.get(
-					candidates.map((row) => row.messageId),
-				);
-				const messageByMessageId = new Map(
-					messages.map((message) => [message.messageId, message]),
-				);
-
-				for (const row of candidates) {
-					const message = messageByMessageId.get(row.messageId);
-					if (!message?.bodyStorageKey) {
-						totals.skippedNoBody++;
-						continue;
-					}
-
-					const outcome = await deriveAndApplyListId(
-						deps,
-						account.accountConfigId,
-						row,
-						message.bodyStorageKey,
-					).then(
-						(listId) => ({ error: null, listId }) as const,
-						(error: unknown) => ({ error, listId: null }) as const,
-					);
-
-					if (outcome.error !== null) {
-						totals.failed++;
-						failedThreadMessageIds.push(row.threadMessageId);
-						logger?.error?.(
-							{
-								threadMessageId: row.threadMessageId,
-								messageId: row.messageId,
-								error:
-									outcome.error instanceof Error
-										? outcome.error.message
-										: String(outcome.error),
-							},
-							"ListId backfill failed for a message; leaving it for a later pass",
-						);
-						continue;
-					}
-
-					if (outcome.listId) {
-						totals.backfilled++;
-					} else {
-						totals.noListId++;
-					}
-				}
-			}
-
-			continuationToken = page.continuationToken;
-			await checkpointStore?.save({
-				accountConfigId: account.accountConfigId,
-				continuationToken,
-			});
-
-			logger?.info(
-				{ accountConfigId: account.accountConfigId, ...totals },
-				"ListId backfill progress",
-			);
-			options.onProgress?.({
-				accountConfigId: account.accountConfigId,
-				...totals,
-			});
-		} while (continuationToken);
-	}
-
-	await checkpointStore?.clear();
-
-	return { ...totals, failedThreadMessageIds };
-};

package/src/mailbox-cursor-rebuild.test.ts

@@ -1,171 +0,0 @@
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import {
-	type CursorRebuildRow,
-	type CursorRebuildSnapshot,
-	matchCursorRebuild,
-} from "./mailbox-cursor-rebuild.js";
-
-const row = (over: Partial<CursorRebuildRow>): CursorRebuildRow => ({
-	messageId: "msg-1",
-	messageIdHeader: "<a@example.com>",
-	internalDate: 1_700_000_000_000,
-	uid: 10,
-	...over,
-});
-
-const snapshot = (
-	over: Partial<CursorRebuildSnapshot>,
-): CursorRebuildSnapshot => ({
-	uid: 10,
-	messageId: "<a@example.com>",
-	internalDate: 1_700_000_000_000,
-	...over,
-});
-
-describe("matchCursorRebuild", () => {
-	it("matches by Message-ID and rewrites the UID mapping when the UID changed", () => {
-		const existingRows = [row({ messageId: "msg-1", uid: 10 })];
-		const serverSnapshots = [snapshot({ uid: 55 })];
-
-		const result = matchCursorRebuild(serverSnapshots, existingRows);
-
-		assert.deepEqual(result.matched, [
-			{ messageId: "msg-1", oldUid: 10, newUid: 55 },
-		]);
-		assert.deepEqual(result.newUids, []);
-		assert.deepEqual(result.staleMessageIds, []);
-	});
-
-	it("does not emit a rewrite when the matched UID is unchanged (frugal — no-op write)", () => {
-		const existingRows = [row({ messageId: "msg-1", uid: 10 })];
-		const serverSnapshots = [snapshot({ uid: 10 })];
-
-		const result = matchCursorRebuild(serverSnapshots, existingRows);
-
-		assert.deepEqual(result.matched, []);
-		assert.deepEqual(result.staleMessageIds, []);
-	});
-
-	it("classifies a server message with no matching row as new — normal new-message sync", () => {
-		const existingRows: CursorRebuildRow[] = [];
-		const serverSnapshots = [
-			snapshot({ uid: 99, messageId: "<never-seen@example.com>" }),
-		];
-
-		const result = matchCursorRebuild(serverSnapshots, existingRows);
-
-		assert.deepEqual(result.matched, []);
-		assert.deepEqual(result.newUids, [99]);
-		assert.deepEqual(result.staleMessageIds, []);
-	});
-
-	it("classifies a row with no matching server message as stale — expunged, reconcile", () => {
-		const existingRows = [
-			row({ messageId: "msg-gone", messageIdHeader: "<gone@example.com>" }),
-		];
-		const serverSnapshots: CursorRebuildSnapshot[] = [];
-
-		const result = matchCursorRebuild(serverSnapshots, existingRows);
-
-		assert.deepEqual(result.matched, []);
-		assert.deepEqual(result.newUids, []);
-		assert.deepEqual(result.staleMessageIds, ["msg-gone"]);
-	});
-
-	it("handles a mixed batch: one match, one new, one stale", () => {
-		const existingRows = [
-			row({
-				messageId: "msg-keep",
-				messageIdHeader: "<keep@example.com>",
-				uid: 5,
-			}),
-			row({
-				messageId: "msg-gone",
-				messageIdHeader: "<gone@example.com>",
-				uid: 6,
-			}),
-		];
-		const serverSnapshots = [
-			snapshot({ uid: 105, messageId: "<keep@example.com>" }),
-			snapshot({ uid: 200, messageId: "<new@example.com>" }),
-		];
-
-		const result = matchCursorRebuild(serverSnapshots, existingRows);
-
-		assert.deepEqual(result.matched, [
-			{ messageId: "msg-keep", oldUid: 5, newUid: 105 },
-		]);
-		assert.deepEqual(result.newUids, [200]);
-		assert.deepEqual(result.staleMessageIds, ["msg-gone"]);
-	});
-
-	it("falls back to an exact INTERNALDATE match for headerless messages on both sides", () => {
-		const existingRows = [
-			row({
-				messageId: "msg-headerless",
-				messageIdHeader: "",
-				internalDate: 42,
-				uid: 3,
-			}),
-		];
-		const serverSnapshots = [
-			snapshot({ uid: 77, messageId: "", internalDate: 42 }),
-		];
-
-		const result = matchCursorRebuild(serverSnapshots, existingRows);
-
-		assert.deepEqual(result.matched, [
-			{ messageId: "msg-headerless", oldUid: 3, newUid: 77 },
-		]);
-		assert.deepEqual(result.staleMessageIds, []);
-	});
-
-	it("picks the closest INTERNALDATE among multiple rows sharing one Message-ID (legitimate resends)", () => {
-		const existingRows = [
-			row({ messageId: "msg-early", internalDate: 1000, uid: 1 }),
-			row({ messageId: "msg-late", internalDate: 5000, uid: 2 }),
-		];
-		const serverSnapshots = [snapshot({ uid: 50, internalDate: 4900 })];
-
-		const result = matchCursorRebuild(serverSnapshots, existingRows);
-
-		assert.deepEqual(result.matched, [
-			{ messageId: "msg-late", oldUid: 2, newUid: 50 },
-		]);
-		assert.deepEqual(result.staleMessageIds, ["msg-early"]);
-	});
-
-	it("carries the row's threadMessage ref through to the match, so the caller can rewrite ThreadMessage.uid alongside Message.uid", () => {
-		const threadMessage = {
-			accountConfigId: "acfg-1",
-			threadMessageId: "tm-1",
-			sentDate: 1_700_000_000_000,
-			mailboxId: "mbx-1",
-			isRead: true,
-			isDeleted: false,
-			hasStars: false,
-			hasAttachment: false,
-		};
-		const existingRows = [row({ messageId: "msg-1", uid: 10, threadMessage })];
-		const serverSnapshots = [snapshot({ uid: 99 })];
-
-		const result = matchCursorRebuild(serverSnapshots, existingRows);
-
-		assert.deepEqual(result.matched, [
-			{ messageId: "msg-1", oldUid: 10, newUid: 99, threadMessage },
-		]);
-	});
-
-	it("omits threadMessage from the match when the row carries none (test fixtures that don't need it)", () => {
-		const existingRows = [row({ messageId: "msg-1", uid: 10 })];
-		const serverSnapshots = [snapshot({ uid: 99 })];
-
-		const result = matchCursorRebuild(serverSnapshots, existingRows);
-
-		assert.deepEqual(result.matched, [
-			{ messageId: "msg-1", oldUid: 10, newUid: 99 },
-		]);
-		assert.equal("threadMessage" in result.matched[0], false);
-	});
-});

package/src/mailbox-cursor-rebuild.ts

@@ -1,146 +0,0 @@
-/**
- * Pure matching logic for the UIDVALIDITY cursor rebuild (issue #1272).
- *
- * Message identity is the Message-ID header; UIDs are cursor positions on
- * the server's axis. When UIDVALIDITY changes, the axis is gone but the
- * data isn't — this reconciles a fresh envelope-level snapshot (UID +
- * Message-ID + INTERNALDATE, no body fetch) against the rows already stored
- * for the mailbox, by identity rather than by position.
- */
-
-/** One server message from the cheap envelope-only rebuild FETCH. */
-export interface CursorRebuildSnapshot {
-	uid: number;
-	/** Raw RFC 822 Message-ID header; empty string when absent/unparseable. */
-	messageId: string;
-	/** INTERNALDATE in epoch milliseconds. */
-	internalDate: number;
-}
-
-/**
- * ThreadMessage identity + every composite-key attribute ElectroDB needs to
- * run a conditional update against the row (see `buildThreadMessageMoveUpdate`
- * in `message-move.ts` for the same pattern on a normal move). Carried
- * alongside a {@link CursorRebuildRow} purely so a match can rewrite
- * ThreadMessage.uid together with Message.uid — matching itself never reads
- * these fields.
- */
-export interface CursorRebuildThreadMessageRef {
-	accountConfigId: string;
-	threadMessageId: string;
-	sentDate: number;
-	mailboxId: string;
-	isRead: boolean;
-	isDeleted: boolean;
-	hasStars: boolean;
-	hasAttachment: boolean;
-}
-
-/** One existing row for the mailbox, as needed to match it against a snapshot. */
-export interface CursorRebuildRow {
-	/** Our internal messageId (the Message row's primary key). */
-	messageId: string;
-	/** Raw RFC 822 Message-ID header the row was created with. */
-	messageIdHeader: string;
-	/** INTERNALDATE in epoch milliseconds. */
-	internalDate: number;
-	/** The row's currently-stored UID (on the old, invalid axis). */
-	uid: number;
-	/** Present when the caller wants a match to also rewrite ThreadMessage.uid. */
-	threadMessage?: CursorRebuildThreadMessageRef;
-}
-
-export interface CursorRebuildMatch {
-	messageId: string;
-	oldUid: number;
-	newUid: number;
-	/** Carried through from the matched row so the caller can rewrite
-	 * ThreadMessage.uid alongside Message.uid without a second lookup. */
-	threadMessage?: CursorRebuildThreadMessageRef;
-}
-
-export interface CursorRebuildMatchResult {
-	/** Rows whose Message-ID matched a server message under a different UID — rewrite in place. */
-	matched: CursorRebuildMatch[];
-	/** Server UIDs with no matching row — run through normal new-message sync. */
-	newUids: number[];
-	/** Rows with no matching server message — expunged; reconcile (#1283). */
-	staleMessageIds: string[];
-}
-
-const pickClosestByInternalDate = (
-	snapshot: CursorRebuildSnapshot,
-	candidates: CursorRebuildRow[],
-	consumed: Set<string>,
-): CursorRebuildRow | undefined => {
-	const available = candidates.filter((row) => !consumed.has(row.messageId));
-	if (available.length === 0) return undefined;
-	if (available.length === 1) return available[0];
-	return available.reduce((best, row) =>
-		Math.abs(row.internalDate - snapshot.internalDate) <
-		Math.abs(best.internalDate - snapshot.internalDate)
-			? row
-			: best,
-	);
-};
-
-/**
- * Match a fresh server snapshot against the mailbox's existing rows by
- * Message-ID, falling back to an exact INTERNALDATE match for the (rare)
- * messages with no usable Message-ID header on either side — mirrors the
- * same headerless fallback `message-sync.ts` already uses for threading.
- *
- * Pure and side-effect free: the caller is responsible for acting on the
- * three outcomes (rewrite / new-message sync / reconcile).
- */
-export const matchCursorRebuild = (
-	serverSnapshots: CursorRebuildSnapshot[],
-	existingRows: CursorRebuildRow[],
-): CursorRebuildMatchResult => {
-	const rowsByHeader = new Map<string, CursorRebuildRow[]>();
-	const rowsByInternalDateOnly = new Map<number, CursorRebuildRow[]>();
-
-	for (const row of existingRows) {
-		if (row.messageIdHeader) {
-			const list = rowsByHeader.get(row.messageIdHeader) ?? [];
-			list.push(row);
-			rowsByHeader.set(row.messageIdHeader, list);
-		} else {
-			const list = rowsByInternalDateOnly.get(row.internalDate) ?? [];
-			list.push(row);
-			rowsByInternalDateOnly.set(row.internalDate, list);
-		}
-	}
-
-	const consumed = new Set<string>();
-	const matched: CursorRebuildMatch[] = [];
-	const newUids: number[] = [];
-
-	for (const snapshot of serverSnapshots) {
-		const candidates = snapshot.messageId
-			? (rowsByHeader.get(snapshot.messageId) ?? [])
-			: (rowsByInternalDateOnly.get(snapshot.internalDate) ?? []);
-		const row = pickClosestByInternalDate(snapshot, candidates, consumed);
-
-		if (!row) {
-			newUids.push(snapshot.uid);
-			continue;
-		}
-
-		consumed.add(row.messageId);
-		if (row.uid !== snapshot.uid) {
-			matched.push({
-				messageId: row.messageId,
-				oldUid: row.uid,
-				newUid: snapshot.uid,
-				...(row.threadMessage ? { threadMessage: row.threadMessage } : {}),
-			});
-		}
-	}
-
-	const staleMessageIds = existingRows
-		.filter((row) => !consumed.has(row.messageId))
-		.map((row) => row.messageId);
-
-	return { matched, newUids, staleMessageIds };
-};