@remit/mailbox-service 0.0.111 → 0.0.113

package/src/classification-backfill.ts

@@ -1,363 +0,0 @@
-import type {
-	IAccountConfigRepository,
-	IAccountRepository,
-	IAddressRepository,
-	IMessageRepository,
-	IThreadMessageRepository,
-	ThreadMessageItem,
-} from "@remit/data-ports";
-import { MessageClassificationState } from "@remit/domain-enums";
-import type { StorageService } from "@remit/storage-service";
-import { parseMessageBody } from "./body-parse.js";
-import { extractSnippet } from "./body-sync.js";
-import { classifyParsedMessage } from "./classify-message.js";
-import { extractListId } from "./filters/list-id.js";
-import {
-	type FolderPlacementLookup,
-	resolveAccountFolderPlacements,
-} from "./folder-placement.js";
-import type { FolderPlacement } from "./heuristics/classifyPlacement.js";
-import {
-	denormalizeMessageCategory,
-	hasDecidedCategory,
-} from "./message-category.js";
-
-const DEFAULT_BATCH_SIZE = 200;
-
-/**
- * Where the full-corpus pass left off: the identity of the account it was
- * working through, and the page cursor within that account. Resuming looks the
- * account up by id and re-opens it at its saved cursor, rather than re-scanning
- * the whole corpus from the top after an interruption. A checkpoint naming no
- * configured account restarts the pass.
- */
-export interface ClassificationBackfillCheckpoint {
-	accountConfigId: string;
-	continuationToken?: string;
-}
-
-/**
- * Persistence for {@link ClassificationBackfillCheckpoint}, injected so the
- * pass stays testable with an in-memory fake; the real entrypoint backs it
- * with a file. `clear()` runs once the whole corpus has been scanned, so a
- * later run starts fresh rather than reading a stale finished-run checkpoint.
- */
-export interface ClassificationBackfillCheckpointStore {
-	load(): Promise<ClassificationBackfillCheckpoint | undefined>;
-	save(checkpoint: ClassificationBackfillCheckpoint): Promise<void>;
-	clear(): Promise<void>;
-}
-
-export interface ClassificationBackfillDeps {
-	accountConfigService: Pick<IAccountConfigRepository, "listAll">;
-	accountService: Pick<IAccountRepository, "listAllByAccountConfig">;
-	mailboxSpecialUseService: FolderPlacementLookup;
-	/** Reads the sender's `Address.flags.category` override (issue #299). */
-	addressService: Pick<IAddressRepository, "getAddress">;
-	threadMessageService: Pick<
-		IThreadMessageRepository,
-		"listByAccount" | "findAllByMessageId" | "update"
-	>;
-	messageService: Pick<IMessageRepository, "get" | "update">;
-	storageService: Pick<StorageService, "retrieve">;
-}
-
-export interface ClassificationBackfillLogger {
-	info(obj: Record<string, unknown>, msg: string): void;
-	error?(obj: Record<string, unknown>, msg: string): void;
-}
-
-export interface ClassificationBackfillTotals {
-	scanned: number;
-	alreadyExamined: number;
-	alreadyCategorized: number;
-	skippedNoBody: number;
-	backfilled: number;
-	failed: number;
-}
-
-export interface ClassificationBackfillResult
-	extends ClassificationBackfillTotals {
-	failedThreadMessageIds: string[];
-}
-
-export interface ClassificationBackfillProgress
-	extends ClassificationBackfillTotals {
-	accountConfigId: string;
-}
-
-export interface ClassificationBackfillOptions {
-	/** Rows fetched per `listByAccount` page. */
-	batchSize?: number;
-	checkpointStore?: ClassificationBackfillCheckpointStore;
-	logger?: ClassificationBackfillLogger;
-	/** Called once per page, after that page's rows are settled. */
-	onProgress?: (progress: ClassificationBackfillProgress) => void;
-}
-
-const emptyTotals = (): ClassificationBackfillTotals => ({
-	scanned: 0,
-	alreadyExamined: 0,
-	alreadyCategorized: 0,
-	skippedNoBody: 0,
-	backfilled: 0,
-	failed: 0,
-});
-
-/**
- * Read the stored raw source, classify it once, and record that the
- * classifier ran. Kept as its own function (rather than inline in a try/catch)
- * so the caller can contain a failure with `.then(fulfilled, rejected)`
- * instead of a block catch, which keeps the `try` around the one call whose
- * failure this contains.
- *
- * The ThreadMessage rows are denormalized BEFORE the Message update, for the
- * same reason the deleted in-sync backfill did it that way (issue #320): the
- * signal this pass selects by — `classificationState` — is written in the
- * Message update, so a failure between the two writes leaves both undone and
- * the rerun redoes them. Writing the Message first would strand the
- * denormalized rows at `uncategorized` forever — the row reads as examined and
- * the retry returns early.
- */
-const classifyAndApply = async (
-	deps: ClassificationBackfillDeps,
-	accountConfigId: string,
-	row: ThreadMessageItem,
-	bodyStorageKey: string,
-	placement: FolderPlacement,
-): Promise<void> => {
-	const body = await deps.storageService.retrieve(bodyStorageKey);
-	const parsed = await parseMessageBody(body);
-	const classification = await classifyParsedMessage(
-		deps.addressService,
-		accountConfigId,
-		parsed,
-		placement,
-	);
-
-	// The same denormalized fields the body-store path writes (see
-	// `applyPostStoreSteps`), not just the category: the snippet and `List-Id`
-	// are derived from the same bytes already in hand, and the rows this pass
-	// reaches never got them — the pass that stored the body is the pass that
-	// classifies, and it never ran for this cohort.
-	await denormalizeMessageCategory(
-		{ threadMessageService: deps.threadMessageService },
-		accountConfigId,
-		row.messageId,
-		{
-			category: classification.category,
-			snippet: extractSnippet(parsed),
-			listId: extractListId(parsed),
-		},
-	);
-
-	await deps.messageService.update(row.messageId, {
-		...classification,
-		classificationState: MessageClassificationState.Examined,
-	});
-};
-
-/**
- * One-time, resumable pass that classifies the stored-body cohort the
- * classifier has never examined (issue #1197): rows whose bodies were synced
- * before the classifier reached them, which carry the `NotExamined` sentinel
- * on `Message.classificationState` and so stay `uncategorized` in the list —
- * the field is what selects the cohort, and nothing on the sync path selects
- * it. Read-only against the stored raw source: it never opens IMAP, and it
- * has no placement or filter side effects — the same bounds the in-sync
- * backfill #1173 removed had. Index-time moves and filter actions are
- * decisions that already ran (or were declined) when the body first landed;
- * re-running them would move mail the user has since filed by hand.
- *
- * A row is a candidate when its Message carries `NotExamined` — the named
- * pending state, never inferred from `category` or from `bodyStorageKey` —
- * and has a `bodyStorageKey`; a candidate without a stored body has nothing
- * local to read, and the ordinary sync path will classify it once the body
- * lands. A candidate whose category is already decided is marked `Examined`
- * without re-deriving anything: those rows were classified by a pass that
- * predates the field, and RFC 034 Decision 3.1's write-once category must not
- * be recomputed (#355). The classification itself is
- * {@link classifyParsedMessage} — the same rule table, and the same sender
- * override, the body-store path classifies with.
- *
- * Chunked by `listByAccount`'s existing keyset pagination, one account at a
- * time in account-id order, so an interrupted run and its resume walk the
- * same sequence. A failure reading, parsing, or classifying one message is
- * contained to that message — logged, counted, and the pass continues: one
- * unreadable object must not strand the rest of the corpus.
- */
-export const backfillClassifications = async (
-	deps: ClassificationBackfillDeps,
-	options: ClassificationBackfillOptions = {},
-): Promise<ClassificationBackfillResult> => {
-	const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
-	const { logger, checkpointStore } = options;
-
-	const accounts = [...(await deps.accountConfigService.listAll())].sort(
-		(left, right) => left.accountConfigId.localeCompare(right.accountConfigId),
-	);
-	const startingCheckpoint = await checkpointStore?.load();
-	const checkpointedIndex = startingCheckpoint
-		? accounts.findIndex(
-				(account) =>
-					account.accountConfigId === startingCheckpoint.accountConfigId,
-			)
-		: -1;
-
-	if (startingCheckpoint && checkpointedIndex === -1) {
-		logger?.info(
-			{ accountConfigId: startingCheckpoint.accountConfigId },
-			"Classification backfill checkpoint names no configured account; restarting the pass",
-		);
-	}
-
-	const startIndex = checkpointedIndex === -1 ? 0 : checkpointedIndex;
-	const startContinuationToken =
-		checkpointedIndex === -1
-			? undefined
-			: startingCheckpoint?.continuationToken;
-
-	const totals = emptyTotals();
-	const failedThreadMessageIds: string[] = [];
-	// Messages settled this run. More than one ThreadMessage row can index one
-	// Message, and the candidate fact lives on the Message, so without this a
-	// second row would re-read the same body and re-classify it within the
-	// same page — idempotent in outcome, wasteful in storage reads.
-	const examinedMessageIds = new Set<string>();
-
-	for (
-		let accountIndex = startIndex;
-		accountIndex < accounts.length;
-		accountIndex++
-	) {
-		const account = accounts[accountIndex];
-		let continuationToken: string | undefined =
-			accountIndex === startIndex ? startContinuationToken : undefined;
-		const mailAccounts = await deps.accountService.listAllByAccountConfig(
-			account.accountConfigId,
-		);
-		const placements = await resolveAccountFolderPlacements(
-			deps.mailboxSpecialUseService,
-			mailAccounts.map((mailAccount) => mailAccount.accountId),
-		);
-
-		do {
-			const page = await deps.threadMessageService.listByAccount(
-				account.accountConfigId,
-				{ limit: batchSize, continuationToken },
-			);
-
-			totals.scanned += page.items.length;
-
-			if (page.items.length > 0) {
-				const messages = await deps.messageService.get(
-					page.items.map((row) => row.messageId),
-				);
-				const messageByMessageId = new Map(
-					messages.map((message) => [message.messageId, message]),
-				);
-
-				for (const row of page.items) {
-					const message = messageByMessageId.get(row.messageId);
-					if (!message?.bodyStorageKey) {
-						totals.skippedNoBody++;
-						continue;
-					}
-
-					if (
-						message.classificationState !==
-							MessageClassificationState.NotExamined ||
-						examinedMessageIds.has(row.messageId)
-					) {
-						totals.alreadyExamined++;
-						continue;
-					}
-
-					// Classified by a pass that predates the field: record that it
-					// ran, and touch nothing else — the write-once category is not
-					// recomputed (#355), and the derived fields that pass wrote are
-					// already on the row.
-					if (hasDecidedCategory(message.category)) {
-						const outcome = await deps.messageService
-							.update(row.messageId, {
-								classificationState: MessageClassificationState.Examined,
-							})
-							.then(
-								() => null,
-								(error: unknown) => error,
-							);
-
-						if (outcome !== null) {
-							totals.failed++;
-							failedThreadMessageIds.push(row.threadMessageId);
-							logger?.error?.(
-								{
-									threadMessageId: row.threadMessageId,
-									messageId: row.messageId,
-									error:
-										outcome instanceof Error
-											? outcome.message
-											: String(outcome),
-								},
-								"Classification backfill failed to record an already-categorized message; leaving it for a later pass",
-							);
-							continue;
-						}
-
-						examinedMessageIds.add(row.messageId);
-						totals.alreadyCategorized++;
-						continue;
-					}
-
-					const outcome = await classifyAndApply(
-						deps,
-						account.accountConfigId,
-						row,
-						message.bodyStorageKey,
-						placements.get(row.mailboxId) ?? "other",
-					).then(
-						() => null,
-						(error: unknown) => error,
-					);
-
-					if (outcome !== null) {
-						totals.failed++;
-						failedThreadMessageIds.push(row.threadMessageId);
-						logger?.error?.(
-							{
-								threadMessageId: row.threadMessageId,
-								messageId: row.messageId,
-								error:
-									outcome instanceof Error ? outcome.message : String(outcome),
-							},
-							"Classification backfill failed for a message; leaving it for a later pass",
-						);
-						continue;
-					}
-
-					examinedMessageIds.add(row.messageId);
-					totals.backfilled++;
-				}
-			}
-
-			continuationToken = page.continuationToken;
-			await checkpointStore?.save({
-				accountConfigId: account.accountConfigId,
-				continuationToken,
-			});
-
-			logger?.info(
-				{ accountConfigId: account.accountConfigId, ...totals },
-				"Classification backfill progress",
-			);
-			options.onProgress?.({
-				accountConfigId: account.accountConfigId,
-				...totals,
-			});
-		} while (continuationToken);
-	}
-
-	await checkpointStore?.clear();
-
-	return { ...totals, failedThreadMessageIds };
-};

package/src/classify-message.test.ts

@@ -1,205 +0,0 @@
-/**
- * Tests for `classifyParsedMessage` (issue #1197): the one classification
- * implementation shared by the body-store path and the classification
- * backfill, so the two can never classify the same bytes differently. Pins
- * the `Address.flags.category` override contract (issue #299, RFC 039
- * Decision 3) on its own, rather than only through body-sync's public
- * surface: the override wins outright, a genuinely-absent Address is "no
- * override", and any other read failure propagates rather than silently
- * classifying by headers when the user asked for something else.
- */
-
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import type { AddressItem, IAddressRepository } from "@remit/data-ports";
-import { NotFoundError } from "@remit/data-ports/errors";
-import { DisplayNameCorrespondence } from "@remit/domain-enums";
-import { parseMessageBody } from "./body-parse.js";
-import {
-	classifyParsedMessage,
-	extractPrimaryFromEmail,
-} from "./classify-message.js";
-
-const PLAIN_EML = Buffer.from(
-	[
-		"From: Alice <alice@example.com>",
-		"To: me@example.com",
-		"Subject: Hi",
-		"Content-Type: text/plain",
-		"",
-		"hi",
-	].join("\r\n"),
-);
-
-const LIST_EML = Buffer.from(
-	[
-		"From: Alice <alice@example.com>",
-		"To: me@example.com",
-		"Subject: Newsletter",
-		"List-Id: <news.example.com>",
-		"List-Unsubscribe: <https://example.com/unsub>",
-		"Content-Type: text/plain",
-		"",
-		"this week in news",
-	].join("\r\n"),
-);
-
-const IMPERSONATION_EML = Buffer.from(
-	[
-		"From: InfoMedics <billing@serviceupdatebank.atlassian.net>",
-		"To: me@example.com",
-		"Subject: Openstaande factuur",
-		"DKIM-Signature: v=1; a=rsa-sha256; d=atlassian.net; s=sel; b=xxx",
-		"Content-Type: text/html",
-		"",
-		'<a href="https://betaal-vordering.example.org/pay">Betaal uw factuur</a>',
-	].join("\r\n"),
-);
-
-const addressService = (
-	behavior: (accountConfigId: string, addressId: string) => unknown,
-): Pick<IAddressRepository, "getAddress"> => ({
-	getAddress: (async (accountConfigId: string, addressId: string) =>
-		behavior(
-			accountConfigId,
-			addressId,
-		)) as unknown as IAddressRepository["getAddress"],
-});
-
-const withFlags =
-	(flags: AddressItem["flags"]): (() => AddressItem) =>
-	() =>
-		({ flags }) as AddressItem;
-
-describe("classifyParsedMessage", () => {
-	it("answers from the header rule table when the sender has no override", async () => {
-		const parsed = await parseMessageBody(LIST_EML);
-
-		const classification = await classifyParsedMessage(
-			addressService(async () => {
-				throw new NotFoundError("no Address row");
-			}),
-			"acc-1",
-			parsed,
-			"inbox",
-		);
-
-		assert.equal(classification.category, "newsletter");
-		assert.equal(classification.hasListUnsubscribe, true);
-	});
-
-	it("substitutes the sender's flags.category override outright", async () => {
-		const parsed = await parseMessageBody(PLAIN_EML);
-
-		const classification = await classifyParsedMessage(
-			addressService(withFlags({ category: { value: "marketing", setAt: 1 } })),
-			"acc-1",
-			parsed,
-			"inbox",
-		);
-
-		// The header table alone would answer `personal`; the override wins
-		// without blending (RFC 039 Decision 3).
-		assert.equal(classification.category, "marketing");
-	});
-
-	it("propagates an Address read failure rather than classifying by headers", async () => {
-		const parsed = await parseMessageBody(PLAIN_EML);
-
-		await assert.rejects(
-			classifyParsedMessage(
-				addressService(async () => {
-					throw new Error("throttled");
-				}),
-				"acc-1",
-				parsed,
-				"inbox",
-			),
-			/throttled/,
-		);
-	});
-
-	it("reads no Address at all when the message has no From address", async () => {
-		const parsed = await parseMessageBody(
-			Buffer.from(
-				[
-					"To: me@example.com",
-					"Subject: Hi",
-					"Content-Type: text/plain",
-					"",
-					"hi",
-				].join("\r\n"),
-			),
-		);
-		let reads = 0;
-
-		const classification = await classifyParsedMessage(
-			addressService(async () => {
-				reads++;
-				throw new NotFoundError("no Address row");
-			}),
-			"acc-1",
-			parsed,
-			"inbox",
-		);
-
-		assert.equal(reads, 0);
-		assert.equal(classification.category, "personal");
-	});
-});
-
-describe("classifyParsedMessage sender mismatch", () => {
-	const noAddress = addressService(async () => {
-		throw new NotFoundError("no Address row");
-	});
-
-	it("compares the sender of mail in Junk the provider rated clean", async () => {
-		const parsed = await parseMessageBody(IMPERSONATION_EML);
-
-		const classification = await classifyParsedMessage(
-			noAddress,
-			"acc-1",
-			parsed,
-			"junk",
-		);
-
-		assert.equal(
-			classification.authenticity?.displayNameCorrespondence,
-			DisplayNameCorrespondence.Unrelated,
-		);
-		assert.deepEqual(classification.authenticity?.offDomainLinkDomains, [
-			"example.org",
-		]);
-	});
-
-	it("compares nothing for the same mail outside Junk without a provider verdict", async () => {
-		const parsed = await parseMessageBody(IMPERSONATION_EML);
-
-		const classification = await classifyParsedMessage(
-			noAddress,
-			"acc-1",
-			parsed,
-			"inbox",
-		);
-
-		assert.equal(
-			classification.authenticity?.displayNameCorrespondence,
-			undefined,
-		);
-		assert.equal(classification.authenticity?.offDomainLinkDomains, undefined);
-	});
-});
-
-describe("extractPrimaryFromEmail", () => {
-	it("reads the first From address", async () => {
-		const parsed = await parseMessageBody(PLAIN_EML);
-		assert.equal(extractPrimaryFromEmail(parsed), "alice@example.com");
-	});
-
-	it("answers null when there is no From address to read", async () => {
-		const parsed = await parseMessageBody(
-			Buffer.from(["To: me@example.com", "", "hi"].join("\r\n")),
-		);
-		assert.equal(extractPrimaryFromEmail(parsed), null);
-	});
-});

package/src/classify-message.ts

@@ -1,116 +0,0 @@
-import type {
-	IAddressRepository,
-	ThreadMessageItem,
-	UpdateMessageInput,
-} from "@remit/data-ports";
-import { NotFoundError } from "@remit/data-ports/errors";
-import { deriveAddressId } from "@remit/data-ports/id";
-import type { ParsedMail } from "mailparser";
-import {
-	classifyByHeaders,
-	extractAuthenticity,
-	extractAuthResult,
-	extractHasListUnsubscribe,
-	extractProviderSpam,
-} from "./heuristics/classifyByHeaders.js";
-import type { FolderPlacement } from "./heuristics/classifyPlacement.js";
-import { extractSenderMismatch } from "./heuristics/senderMismatch.js";
-
-type ThreadMessageCategory = ThreadMessageItem["category"];
-
-/**
- * The From address the classifier reads, when there is one to read. Shared by
- * every path that classifies or counts a sender (body-sync's classification,
- * inbound counter and placement signals, and the classification backfill) so
- * they all read the same address off the same bytes.
- */
-export const extractPrimaryFromEmail = (parsed: ParsedMail): string | null => {
-	const from = parsed.from;
-	if (!from || !from.value || from.value.length === 0) return null;
-	const address = from.value[0]?.address;
-	if (!address) return null;
-	return address.toLowerCase();
-};
-
-/**
- * The one-sided half of `Address.flags.category` (issue #299, RFC 039
- * Decision 3): a real value here overrides `classifyByHeaders` outright.
- * Only a genuinely-absent Address is "no override" — any other failure
- * (throttle, infra) propagates, because this feeds the write-once
- * `Message.category` at the moment it is decided, where a masked infra
- * failure would silently classify by headers when the user asked for
- * something else.
- */
-const resolveCategoryOverride = async (
-	addressService: Pick<IAddressRepository, "getAddress">,
-	accountConfigId: string,
-	fromEmail: string,
-): Promise<ThreadMessageCategory | undefined> => {
-	try {
-		const addressId = deriveAddressId(accountConfigId, fromEmail);
-		const address = await addressService.getAddress(accountConfigId, addressId);
-		return address.flags?.category?.value;
-	} catch (err) {
-		if (!(err instanceof NotFoundError)) throw err;
-		return undefined;
-	}
-};
-
-/**
- * Header classification, with the sender's `Address.flags.category` override
- * (issue #299, RFC 039 Decision 3) substituted for the header-derived
- * category when one is set. Returns the subset of the Message update that
- * carries the derived fields; the caller folds it into a single UpdateItem
- * alongside `bodyStorageKey` (body-sync) or `classificationState` (the
- * classification backfill, issue #1197). Optional signals are omitted when
- * absent so we never overwrite an existing value with `undefined`.
- *
- * The override wins outright rather than blending with the heuristic — RFC
- * 039 Decision 3 treats a direct reclassification as final, the same as
- * `flags.blocked`/`vip` already override placement. Every caller gates on
- * `hasDecidedCategory` before this result reaches a write, so a message that
- * already carries a real category is never re-touched regardless of what
- * this returns.
- */
-export const classifyParsedMessage = async (
-	addressService: Pick<IAddressRepository, "getAddress">,
-	accountConfigId: string,
-	parsed: ParsedMail,
-	placement: FolderPlacement,
-): Promise<UpdateMessageInput & { category: ThreadMessageCategory }> => {
-	const headerCategory = classifyByHeaders(parsed);
-	const authenticity = extractAuthenticity(parsed);
-	const authResult = extractAuthResult(parsed);
-	const providerSpam = extractProviderSpam(parsed);
-	const hasListUnsubscribe = extractHasListUnsubscribe(parsed);
-
-	// A passing SPF/DKIM/DMARC check proves the sending domain, not the
-	// identity the message claims — on a shared-tenant host the verified
-	// subdomain belongs to whoever signed up. These two comparisons say
-	// whether the claim holds, and run only over mail the provider already
-	// called spam or that sits in the Junk folder.
-	const senderMismatch =
-		authenticity === null
-			? {}
-			: extractSenderMismatch(parsed, {
-					fromDomain: authenticity.fromDomain,
-					providerSpamClassified: providerSpam?.classified === true,
-					placement,
-					bulkSender: hasListUnsubscribe,
-				});
-
-	const fromEmail = extractPrimaryFromEmail(parsed);
-	const categoryOverride = fromEmail
-		? await resolveCategoryOverride(addressService, accountConfigId, fromEmail)
-		: undefined;
-
-	return {
-		category: categoryOverride ?? headerCategory,
-		hasListUnsubscribe,
-		...(authenticity !== null
-			? { authenticity: { ...authenticity, ...senderMismatch } }
-			: {}),
-		...(authResult !== null ? { authResult } : {}),
-		...(providerSpam !== null ? { providerSpam } : {}),
-	};
-};