@remit/mailbox-service 0.0.24 → 0.0.26

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.24",
+  "version": "0.0.26",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -39,6 +39,7 @@
     "mailparser": "^3.9.14",
     "nodemailer": "^9.0.3",
     "p-map": "^7.0.4",
+    "tldts": "^7.4.9",
     "@remit/data-ports": "*",
     "@remit/domain-enums": "*",
     "@remit/mail-oauth-service": "*",

package/src/body-sync-backfill.test.ts

@@ -72,15 +72,17 @@ const buildHarness = (
 	} as unknown as IMessageRepository;
 
 	const threadMessageService = {
-		getByMessageId: async () => ({
-			threadMessageId: "tm-1",
-			sentDate: 1,
-			mailboxId: "mb-1",
-			isRead: false,
-			isDeleted: false,
-			hasStars: false,
-			hasAttachment: false,
-		}),
+		findAllByMessageId: async () => [
+			{
+				threadMessageId: "tm-1",
+				sentDate: 1,
+				mailboxId: "mb-1",
+				isRead: false,
+				isDeleted: false,
+				hasStars: false,
+				hasAttachment: false,
+			},
+		],
 		update: async (
 			_accountConfigId: string,
 			_threadMessageId: string,

package/src/body-sync-category-denormalize.test.ts

@@ -0,0 +1,341 @@
+/**
+ * `thread_message.category` is the copy the list read path is about to filter
+ * on, and its write-path defects only appear under conditions a happy-path test
+ * never creates (issue #320).
+ *
+ * 1. The retro classification path wrote the Message before the ThreadMessage
+ *    while its skip guard keyed off `message.category`. A failure between the
+ *    two left the guard satisfied and the denormalized row stranded at
+ *    `uncategorized` forever, because the requeued retry returned early.
+ * 2. `denormalizeCategory` resolved one arbitrary row per message through an
+ *    unordered `.limit(1)`. More than one row per messageId is schema-legal —
+ *    reachable through thread-root drift, not through a second mailbox, whose ids
+ *    collapse to one row — and every row a message has must end up correct.
+ * 3. It also wrote only `category` on the retro path, so a copied message, which
+ *    reaches no other denormalizing path, never got a `listId` at all.
+ *
+ * None of these show up on a single row in a single successful pass.
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	IAddressRepository,
+	IEnvelopeRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+	MessageItem,
+	ThreadMessageItem,
+	UpdateMessageInput,
+	UpdateThreadMessageInput,
+} from "@remit/data-ports";
+import { MessageCategory } from "@remit/domain-enums";
+import type { StorageService } from "@remit/storage-service";
+import { BodySyncService } from "./body-sync.js";
+import type { IImapConnection } from "./types.js";
+
+const LINKEDIN_EML = Buffer.from(
+	[
+		"From: LinkedIn <messages-noreply@linkedin.com>",
+		"To: me@example.com",
+		"Subject: You have a new invitation",
+		"List-Id: LinkedIn Invitations <invitations.linkedin.com>",
+		"List-Unsubscribe: <https://www.linkedin.com/e/unsub>",
+		"Content-Type: text/plain",
+		"",
+		"invitation body text",
+	].join("\r\n"),
+);
+
+type Row = Pick<
+	ThreadMessageItem,
+	| "threadMessageId"
+	| "messageId"
+	| "accountConfigId"
+	| "mailboxId"
+	| "sentDate"
+	| "isRead"
+	| "isDeleted"
+	| "hasStars"
+	| "hasAttachment"
+	| "category"
+> &
+	Partial<Pick<ThreadMessageItem, "snippet" | "listId">>;
+
+interface ThreadUpdate {
+	threadMessageId: string;
+	input: UpdateThreadMessageInput;
+	composites: Record<string, unknown> | undefined;
+}
+
+interface Harness {
+	service: BodySyncService;
+	message: MessageItem;
+	rows: Row[];
+	messageUpdates: Array<{ messageId: string; input: UpdateMessageInput }>;
+	threadUpdates: ThreadUpdate[];
+	writeOrder: string[];
+}
+
+const buildRow = (
+	overrides: Partial<Row> & Pick<Row, "threadMessageId">,
+): Row =>
+	({
+		messageId: "m-1",
+		accountConfigId: "cfg-1",
+		mailboxId: "mb-inbox",
+		sentDate: 1,
+		isRead: false,
+		isDeleted: false,
+		hasStars: false,
+		hasAttachment: false,
+		category: MessageCategory.uncategorized,
+		...overrides,
+	}) as Row;
+
+/**
+ * The Message and ThreadMessage fixtures are mutated by their writes, so the
+ * skip guard sees the state a requeued delivery would. A harness that only
+ * records the write cannot show a stranded row: the strand is the guard reading
+ * a value that outlived a failed pass.
+ */
+const buildHarness = (
+	rows: Row[],
+	options: {
+		failThreadUpdates?: number;
+		bodyStored?: boolean;
+	} = {},
+): Harness => {
+	const messageUpdates: Array<{
+		messageId: string;
+		input: UpdateMessageInput;
+	}> = [];
+	const threadUpdates: ThreadUpdate[] = [];
+	const writeOrder: string[] = [];
+	let threadUpdateAttempts = 0;
+
+	const message = {
+		messageId: "m-1",
+		mailboxId: "mb-inbox",
+		uid: 1,
+		category: MessageCategory.uncategorized,
+		...(options.bodyStored === false
+			? {}
+			: { bodyStorageKey: "s3://bodies/m-1" }),
+	} as unknown as MessageItem;
+
+	const messageService = {
+		get: async () => message,
+		update: async (messageId: string, input: UpdateMessageInput) => {
+			writeOrder.push("message");
+			messageUpdates.push({ messageId, input });
+			Object.assign(message, input);
+		},
+	} as unknown as IMessageRepository;
+
+	const threadMessageService = {
+		findAllByMessageId: async () => rows,
+		// Load-bearing, not a leftover: the current code never calls it, but
+		// reverting body-sync.ts to reproduce these failures does. Deleting it
+		// makes the fail-before run throw instead of demonstrating the defect.
+		getByMessageId: async () => rows[0],
+		update: async (
+			_accountConfigId: string,
+			threadMessageId: string,
+			input: UpdateThreadMessageInput,
+			updateOptions?: { composites?: Record<string, unknown> },
+		) => {
+			threadUpdateAttempts++;
+			if (threadUpdateAttempts <= (options.failThreadUpdates ?? 0)) {
+				throw new Error("thread_message write failed");
+			}
+			writeOrder.push("thread");
+			threadUpdates.push({
+				threadMessageId,
+				input,
+				composites: updateOptions?.composites,
+			});
+			const row = rows.find((r) => r.threadMessageId === threadMessageId);
+			if (row) Object.assign(row, input);
+		},
+	} as unknown as IThreadMessageRepository;
+
+	const storageService = {
+		retrieve: async () => LINKEDIN_EML,
+		storeMessageBody: async () => ({ uri: "s3://bodies/m-1" }),
+		storeParsedBody: async () => {},
+		listBodyParts: async () => [],
+	} as unknown as StorageService;
+
+	const service = new BodySyncService(
+		messageService,
+		storageService,
+		threadMessageService,
+		{ incrementInboundCount: async () => {} } as unknown as IAddressRepository,
+		{ listBodyParts: async () => [] } as unknown as IEnvelopeRepository,
+		{ info: () => {}, error: () => {} },
+	);
+
+	return {
+		service,
+		message,
+		rows,
+		messageUpdates,
+		threadUpdates,
+		writeOrder,
+	};
+};
+
+const backfill = (harness: Harness) =>
+	harness.service.syncBodies(["m-1"], "acc-1", "cfg-1", "INBOX", async () => {
+		throw new Error("the backfill must not open IMAP");
+	});
+
+const syncFromImap = (harness: Harness) =>
+	harness.service.fetchAndGetBody(
+		"m-1",
+		"acc-1",
+		"cfg-1",
+		"INBOX",
+		async () =>
+			({
+				openBox: async () => {},
+				fetchMessageBody: async () => LINKEDIN_EML,
+			}) as unknown as IImapConnection,
+	);
+
+describe("the classification backfill survives an interrupted write", () => {
+	it("leaves the Message unclassified when the ThreadMessage write fails, so the retry runs again", async () => {
+		const harness = buildHarness([buildRow({ threadMessageId: "tm-1" })], {
+			failThreadUpdates: 1,
+		});
+
+		const failed = await backfill(harness);
+
+		assert.deepEqual(failed.failedMessageIds, ["m-1"]);
+		assert.equal(harness.message.category, MessageCategory.uncategorized);
+		assert.deepEqual(harness.messageUpdates, []);
+
+		const retried = await backfill(harness);
+
+		assert.deepEqual(retried.failedMessageIds, []);
+		assert.equal(harness.message.category, MessageCategory.social);
+		assert.equal(harness.rows[0].category, MessageCategory.social);
+	});
+
+	it("writes the Message only after the denormalized row is durable", async () => {
+		const harness = buildHarness([buildRow({ threadMessageId: "tm-1" })]);
+
+		await backfill(harness);
+
+		assert.deepEqual(harness.writeOrder, ["thread", "message"]);
+	});
+});
+
+describe("denormalization reaches every row a message has", () => {
+	it("carries the backfilled category onto all of a message's rows", async () => {
+		const harness = buildHarness([
+			buildRow({ threadMessageId: "tm-inbox", mailboxId: "mb-inbox" }),
+			buildRow({
+				threadMessageId: "tm-archive",
+				mailboxId: "mb-archive",
+				isRead: true,
+			}),
+		]);
+
+		await backfill(harness);
+
+		assert.deepEqual(
+			harness.rows.map((r) => r.category),
+			[MessageCategory.social, MessageCategory.social],
+		);
+	});
+
+	it("builds the composite block from each row rather than reusing the first", async () => {
+		const harness = buildHarness([
+			buildRow({ threadMessageId: "tm-inbox", mailboxId: "mb-inbox" }),
+			buildRow({
+				threadMessageId: "tm-archive",
+				mailboxId: "mb-archive",
+				isRead: true,
+			}),
+		]);
+
+		await backfill(harness);
+
+		assert.deepEqual(
+			harness.threadUpdates.map((u) => ({
+				threadMessageId: u.threadMessageId,
+				mailboxId: u.composites?.mailboxId,
+				isRead: u.composites?.isRead,
+			})),
+			[
+				{ threadMessageId: "tm-inbox", mailboxId: "mb-inbox", isRead: false },
+				{
+					threadMessageId: "tm-archive",
+					mailboxId: "mb-archive",
+					isRead: true,
+				},
+			],
+		);
+	});
+
+	it("fans the snippet and the list id out to every row on the sync path", async () => {
+		// A snippet and a List-Id belong to the message, not to the mailbox a row
+		// happens to name, so every row carries them.
+		const harness = buildHarness(
+			[
+				buildRow({ threadMessageId: "tm-inbox", mailboxId: "mb-inbox" }),
+				buildRow({ threadMessageId: "tm-archive", mailboxId: "mb-archive" }),
+			],
+			{ bodyStored: false },
+		);
+
+		await syncFromImap(harness);
+
+		for (const row of harness.rows) {
+			assert.equal(row.category, MessageCategory.social);
+			assert.equal(row.listId, "invitations.linkedin.com");
+			assert.equal(row.snippet, "invitation body text");
+		}
+	});
+
+	it("writes nothing for a row that already carries every value", async () => {
+		const harness = buildHarness([
+			buildRow({
+				threadMessageId: "tm-inbox",
+				mailboxId: "mb-inbox",
+				category: MessageCategory.social,
+				snippet: "invitation body text",
+				listId: "invitations.linkedin.com",
+			}),
+			buildRow({ threadMessageId: "tm-archive", mailboxId: "mb-archive" }),
+		]);
+
+		await backfill(harness);
+
+		assert.deepEqual(
+			harness.threadUpdates.map((u) => u.threadMessageId),
+			["tm-archive"],
+		);
+	});
+});
+
+describe("the classification backfill writes the whole denormalized set", () => {
+	// A copied message inherits its source's stored body AND its decided
+	// category, so the full body-store path skips it and this path's guard
+	// returns early — and nothing but these two paths ever writes `listId`. The
+	// backfill passing only `category` left every copy's `list_id` NULL for good.
+	it("carries the list id and the snippet, not just the category", async () => {
+		const harness = buildHarness([buildRow({ threadMessageId: "tm-1" })]);
+
+		await backfill(harness);
+
+		assert.deepEqual(harness.threadUpdates.length, 1);
+		assert.deepEqual(harness.threadUpdates[0].input, {
+			category: MessageCategory.social,
+			snippet: "invitation body text",
+			listId: "invitations.linkedin.com",
+		});
+	});
+});

package/src/body-sync-filter.test.ts

@@ -102,15 +102,17 @@ const buildHarness = (
 	} as unknown as IMessageRepository;
 
 	const threadMessageService = {
-		getByMessageId: async () => ({
-			threadMessageId: "tm-1",
-			sentDate: 1,
-			mailboxId: message.mailboxId,
-			isRead: false,
-			isDeleted: false,
-			hasStars: false,
-			hasAttachment: false,
-		}),
+		findAllByMessageId: async () => [
+			{
+				threadMessageId: "tm-1",
+				sentDate: 1,
+				mailboxId: message.mailboxId,
+				isRead: false,
+				isDeleted: false,
+				hasStars: false,
+				hasAttachment: false,
+			},
+		],
 		update: async () => {},
 	} as unknown as IThreadMessageRepository;
 

package/src/body-sync-quarantine.test.ts

@@ -78,15 +78,17 @@ const buildHarness = (options: {
 		messageService,
 		storageService,
 		{
-			getByMessageId: async () => ({
-				threadMessageId: "tm-1",
-				sentDate: 1,
-				mailboxId: "mbx-1",
-				isRead: false,
-				isDeleted: false,
-				hasStars: false,
-				hasAttachment: false,
-			}),
+			findAllByMessageId: async () => [
+				{
+					threadMessageId: "tm-1",
+					sentDate: 1,
+					mailboxId: "mbx-1",
+					isRead: false,
+					isDeleted: false,
+					hasStars: false,
+					hasAttachment: false,
+				},
+			],
 			update: async () => {},
 		} as unknown as IThreadMessageRepository,
 		{} as unknown as IAddressRepository,

package/src/body-sync.ts

@@ -30,6 +30,7 @@ import { type ParsedMail, simpleParser } from "mailparser";
 import pMap from "p-map";
 import { BodyParseError, parseMessageBody } from "./body-parse.js";
 import { mapBodyPartsToContent } from "./body-part-mapper.js";
+import { extractListId } from "./filters/list-id.js";
 import type { FilterMessage } from "./filters/match.js";
 import {
 	type FilterConfig,
@@ -134,8 +135,40 @@ const toFilterMessage = (parsed: ParsedMail): FilterMessage => ({
 	fromName: parsed.from?.value?.[0]?.name ?? "",
 	subject: parsed.subject ?? "",
 	text: parsed.text ?? "",
+	listId: extractListId(parsed),
 });
 
+const SNIPPET_LENGTH = 256;
+
+/**
+ * The snippet the list row shows, from whichever body part carries text. Shared
+ * by both paths that denormalize onto the ThreadMessage so they cannot derive it
+ * differently.
+ */
+const extractSnippet = (parsed: ParsedMail): string =>
+	extractSnippetFromEmail(
+		parsed.text,
+		typeof parsed.html === "string" ? parsed.html : undefined,
+		SNIPPET_LENGTH,
+	);
+
+/**
+ * A row is skipped only when every field the denormalization would write
+ * already matches. `snippet` and `listId` are absent from the update when the
+ * message has neither, and an absent field is not a mismatch.
+ */
+const alreadyDenormalized = (
+	row: ThreadMessageItem,
+	update: {
+		category: ThreadMessageCategory;
+		snippet?: string;
+		listId?: string;
+	},
+): boolean =>
+	row.category === update.category &&
+	(update.snippet === undefined || row.snippet === update.snippet) &&
+	(update.listId === undefined || row.listId === update.listId);
+
 export const toParsedBody = (parsed: ParsedMail): ParsedBody => ({
 	text: parsed.text ?? null,
 	html: typeof parsed.html === "string" ? parsed.html : null,
@@ -940,12 +973,26 @@ export class BodySyncService {
 		const parsed = await parseMessageBody(body);
 		const classification = this.classifyMessage(parsed);
 
-		await this.messageService.update(message.messageId, classification);
+		// Same order as {@link applyPostStoreSteps}, for the same reason: the
+		// signal the skip guard reads is written last. `message.category` is that
+		// signal here, so a failure between the two writes leaves both undone and
+		// the requeued retry redoes both. Writing the Message first strands the
+		// denormalized row at `uncategorized` forever — the guard is satisfied and
+		// the retry returns early (issue #320).
+		// The same three denormalized fields `updateSnippets` writes on the
+		// full body-store path, not just the category. A copied message inherits
+		// `bodyStorageKey` and a decided category from its source, so it reaches
+		// neither that path nor this one's classification — but nothing else ever
+		// writes `listId`, so leaving it out here made a copy's `list_id`
+		// permanently NULL. Both are derived from the same bytes already in hand.
 		await this.denormalizeCategory(
 			accountConfigId,
 			message.messageId,
 			classification.category,
+			extractSnippet(parsed),
+			extractListId(parsed),
 		);
+		await this.messageService.update(message.messageId, classification);
 
 		this.log.info(
 			{ messageId: message.messageId, category: classification.category },
@@ -1423,10 +1470,11 @@ export class BodySyncService {
 	}
 
 	/**
-	 * Extract snippet and header category from the body and denormalize both
-	 * onto the ThreadMessage. `category` mirrors the Message: created as
-	 * `uncategorized` at metadata-sync and set to the classified value here, so
-	 * the list/search read path carries it without a per-row Message fetch.
+	 * Extract snippet, header category and the normalized `List-Id` from the body
+	 * and denormalize them onto the ThreadMessage. `category` mirrors the Message:
+	 * created as `uncategorized` at metadata-sync and set to the classified value
+	 * here; `listId` is written so the back-apply corpus projection can match a
+	 * `ListId` clause vector-free, off the same row the list/search path reads.
 	 * Returns the parsed mail so callers can reuse it (e.g., to write the
 	 * parsed-body cache) without paying for mailparser twice.
 	 */
@@ -1442,20 +1490,16 @@ export class BodySyncService {
 		// two apart and quarantine only this one (issue #72).
 		const parsed = await parseMessageBody(body);
 
-		// Extract snippet from text or HTML content
-		const snippet = extractSnippetFromEmail(
-			parsed.text,
-			typeof parsed.html === "string" ? parsed.html : undefined,
-			256,
-		);
-
+		const snippet = extractSnippet(parsed);
 		const category = classifyByHeaders(parsed);
+		const listId = extractListId(parsed);
 
 		await this.denormalizeCategory(
 			accountConfigId,
 			messageId,
 			category,
 			snippet,
+			listId,
 		);
 
 		this.log.debug?.(
@@ -1467,41 +1511,71 @@ export class BodySyncService {
 	}
 
 	/**
-	 * Write the denormalized `category` (and optionally the snippet) onto the
-	 * message's ThreadMessage row — the copy the list/search read path serves
-	 * without a per-row Message fetch.
+	 * Write the denormalized `category` (and optionally the snippet and the
+	 * normalized `List-Id`) onto EVERY ThreadMessage row the message has — the
+	 * copy the list/search read path serves without a per-row Message fetch.
+	 *
+	 * More than one row per messageId is schema-legal but not normally produced,
+	 * and this iterates for the same reason `message-move.ts` does (see the model
+	 * stated at its `deleteThreadMessagesForMessage`): the key permits it and
+	 * nothing enforces otherwise. It is NOT the second mailbox a message appears
+	 * in — `deriveMessageId` and `deriveThreadMessageId` are both
+	 * mailbox-independent, so INBOX and Archive resolve to one row, and a copy
+	 * gets its own messageId. The reachable case is thread-root drift: the same
+	 * message re-saved under different `References`, which mints a second
+	 * threadId and so a second row. Iterating is therefore hardening against a
+	 * legal state, not a repair for one the sync path manufactures, which is why
+	 * the tree's other single-row `messageId` lookups are correct as they stand.
+	 * `flag-queue.ts` iterates the same list.
 	 *
-	 * The ThreadMessage is looked up by messageId (GSI), so it does not depend
-	 * on the RFC822 Message-ID header — a headerless message still gets
-	 * denormalized, matching the unconditional Message.category write. The full
-	 * composite set is passed so that a future key-attribute addition touching
-	 * the lsi3/lsi4/lsi5/gsi2 sort keys keeps the index rows consistent.
+	 * Rows are looked up by messageId, so this does not depend on the RFC822
+	 * Message-ID header — a headerless message still gets denormalized, matching
+	 * the unconditional Message.category write. The composite set is built per
+	 * row, never reused: `mailboxId` and `isRead` can differ between two rows for
+	 * one message, and it is passed at all so that a future key-attribute
+	 * addition touching the lsi3/lsi4/lsi5/gsi2 sort keys keeps the index rows
+	 * consistent.
 	 */
 	private async denormalizeCategory(
 		accountConfigId: string,
 		messageId: string,
 		category: ThreadMessageCategory,
 		snippet?: string,
+		listId?: string,
 	): Promise<void> {
-		const threadMessage = await this.threadMessageService.getByMessageId(
+		const rows = await this.threadMessageService.findAllByMessageId(
 			accountConfigId,
 			messageId,
 		);
+		if (rows.length === 0) {
+			throw new NotFoundError(
+				`ThreadMessage not found for message ${messageId}`,
+			);
+		}
 
-		await this.threadMessageService.update(
-			accountConfigId,
-			threadMessage.threadMessageId,
-			{ category, ...(snippet ? { snippet } : {}) },
-			{
-				composites: {
-					sentDate: threadMessage.sentDate,
-					mailboxId: threadMessage.mailboxId,
-					isRead: threadMessage.isRead,
-					isDeleted: threadMessage.isDeleted,
-					hasStars: threadMessage.hasStars,
-					hasAttachment: threadMessage.hasAttachment,
+		const update = {
+			category,
+			...(snippet ? { snippet } : {}),
+			...(listId ? { listId } : {}),
+		};
+
+		for (const row of rows) {
+			if (alreadyDenormalized(row, update)) continue;
+			await this.threadMessageService.update(
+				accountConfigId,
+				row.threadMessageId,
+				update,
+				{
+					composites: {
+						sentDate: row.sentDate,
+						mailboxId: row.mailboxId,
+						isRead: row.isRead,
+						isDeleted: row.isDeleted,
+						hasStars: row.hasStars,
+						hasAttachment: row.hasAttachment,
+					},
 				},
-			},
-		);
+			);
+		}
 	}
 }

package/src/filters/list-id.test.ts

@@ -0,0 +1,50 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { simpleParser } from "mailparser";
+import { extractListId, normalizeListId } from "./list-id.js";
+
+const parse = async (lines: string[]) =>
+	simpleParser(Buffer.from(lines.join("\r\n")));
+
+describe("normalizeListId", () => {
+	it("extracts the bracketed identifier and folds case", () => {
+		assert.equal(
+			normalizeListId("Weekly News <Weekly.News.Example.COM>"),
+			"weekly.news.example.com",
+		);
+	});
+
+	it("keeps a bare value and folds case", () => {
+		assert.equal(
+			normalizeListId("  Weekly.News.Example.COM  "),
+			"weekly.news.example.com",
+		);
+	});
+
+	it("normalizes an empty value to the empty string", () => {
+		assert.equal(normalizeListId("   "), "");
+	});
+});
+
+describe("extractListId", () => {
+	it("reads and normalizes a List-Id header", async () => {
+		const parsed = await parse([
+			"From: list@example.com",
+			"Subject: hi",
+			"List-Id: Weekly News <weekly.news.example.com>",
+			"",
+			"body",
+		]);
+		assert.equal(extractListId(parsed), "weekly.news.example.com");
+	});
+
+	it("returns the empty string when there is no List-Id header", async () => {
+		const parsed = await parse([
+			"From: alice@example.com",
+			"Subject: hi",
+			"",
+			"body",
+		]);
+		assert.equal(extractListId(parsed), "");
+	});
+});

package/src/filters/list-id.ts

@@ -0,0 +1,29 @@
+import type { ParsedMail } from "mailparser";
+
+/**
+ * Canonical form of a `List-Id` value for exact comparison: the bracketed
+ * identifier when the header carries the RFC 2919 `Name <list.id>` shape,
+ * otherwise the whole value, trimmed and case-folded. Both the stored copy and
+ * a `ListId` clause pass through this, so `<weekly.news.example.com>` and
+ * `weekly.news.example.com` are one list. An empty input normalizes to `""`.
+ */
+export const normalizeListId = (value: string): string => {
+	const trimmed = value.trim();
+	const bracketed = trimmed.match(/<([^>]+)>/);
+	return (bracketed ? bracketed[1] : trimmed).trim().toLowerCase();
+};
+
+/**
+ * The normalized `List-Id` header value of a parsed message, or `""` when the
+ * message carries no `List-Id`. Read from the raw header line so the exact
+ * value survives regardless of how the parser structures the header.
+ */
+export const extractListId = (parsed: ParsedMail): string => {
+	const line = parsed.headerLines.find(
+		(header) => header.key.toLowerCase() === "list-id",
+	);
+	if (!line) return "";
+	const colon = line.line.indexOf(":");
+	if (colon < 0) return "";
+	return normalizeListId(line.line.slice(colon + 1));
+};

package/src/filters/match.test.ts

@@ -18,6 +18,7 @@ const message = (overrides: Partial<FilterMessage> = {}): FilterMessage => ({
 	fromName: "Alice Example",
 	subject: "Q3 invoice attached",
 	text: "Please find the invoice for the quarter attached.",
+	listId: "",
 	...overrides,
 });
 
@@ -69,6 +70,93 @@ describe("clauseMatches", () => {
 			false,
 		);
 	});
+
+	it("matches ListId exactly, never as a substring", () => {
+		const msg = message({ listId: "weekly.news.example.com" });
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.ListId, "weekly.news.example.com"),
+				msg,
+			),
+			true,
+		);
+		assert.equal(
+			clauseMatches(clause(FilterClauseField.ListId, "news.example.com"), msg),
+			false,
+		);
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.ListId, "weekly.news.example.com.other"),
+				msg,
+			),
+			false,
+		);
+	});
+
+	it("normalizes ListId brackets and case on both sides", () => {
+		const msg = message({ listId: "weekly.news.example.com" });
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.ListId, "<Weekly.News.Example.COM>"),
+				msg,
+			),
+			true,
+		);
+	});
+
+	it("never matches ListId on a message with no List-Id", () => {
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.ListId, "weekly.news.example.com"),
+				message({ listId: "" }),
+			),
+			false,
+		);
+	});
+
+	it("matches FromDomain on the registrable domain, including subdomains", () => {
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.FromDomain, "github.com"),
+				message({ from: "notifications@github.com" }),
+			),
+			true,
+		);
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.FromDomain, "github.com"),
+				message({ from: "notifications@sub.github.com" }),
+			),
+			true,
+		);
+	});
+
+	it("never matches FromDomain on a look-alike subdomain (public-suffix aware)", () => {
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.FromDomain, "github.com"),
+				message({ from: "attacker@github.com.evil.example" }),
+			),
+			false,
+		);
+	});
+
+	it("matches FromDomain across multi-level public suffixes", () => {
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.FromDomain, "example.co.uk"),
+				message({ from: "hr@mail.example.co.uk" }),
+			),
+			true,
+		);
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.FromDomain, "example.co.uk"),
+				message({ from: "hr@example.co.uk.evil.example" }),
+			),
+			false,
+		);
+	});
 });
 
 describe("literalClausesMatch", () => {

package/src/filters/match.ts

@@ -1,5 +1,7 @@
 import type { FilterItem } from "@remit/data-ports";
 import { FilterClauseField, FilterMatchOperator } from "@remit/domain-enums";
+import { getDomain } from "tldts";
+import { normalizeListId } from "./list-id.js";
 
 type FilterClause = FilterItem["literalClauses"][number];
 
@@ -36,15 +38,32 @@ export interface FilterMessage {
 	fromName: string;
 	subject: string;
 	text: string;
+	/** Normalized `List-Id` header value (see `normalizeListId`); `""` when absent. */
+	listId: string;
 }
 
 const includesFold = (haystack: string, needle: string): boolean =>
 	haystack.toLowerCase().includes(needle.toLowerCase());
 
+const hostOf = (address: string): string => {
+	const at = address.lastIndexOf("@");
+	return at >= 0 ? address.slice(at + 1) : address;
+};
+
+/**
+ * The registrable, public-suffix-aware domain of an email address or host, or
+ * `null` when none resolves. `getDomain` folds case and applies the ICANN
+ * suffix list, so `github.com.evil.example` yields `evil.example`, never
+ * `github.com` — a `FromDomain` clause cannot be spoofed by a crafted subdomain.
+ */
+const registrableDomain = (addressOrHost: string): string | null =>
+	getDomain(hostOf(addressOrHost.trim()));
+
 /**
  * Whether one literal clause matches the message. From matches against the
  * sender address and display name; Subject against the subject; HasWords against
- * subject or body. An empty clause value never matches.
+ * subject or body; ListId against the exact normalized `List-Id`; FromDomain
+ * against the sender's registrable domain. An empty clause value never matches.
  */
 export const clauseMatches = (
 	clause: FilterClause,
@@ -59,6 +78,15 @@ export const clauseMatches = (
 			return includesFold(msg.subject, value);
 		case FilterClauseField.HasWords:
 			return includesFold(msg.subject, value) || includesFold(msg.text, value);
+		case FilterClauseField.ListId: {
+			const target = normalizeListId(value);
+			return target !== "" && normalizeListId(msg.listId) === target;
+		}
+		case FilterClauseField.FromDomain: {
+			const target = registrableDomain(value);
+			if (target === null) return false;
+			return registrableDomain(msg.from) === target;
+		}
 		default:
 			return false;
 	}

package/src/filters/pipeline.test.ts

@@ -74,6 +74,7 @@ describe("FilterPipeline — anchorless From/Or filter at index time", () => {
 			fromName: "npm",
 			subject: "A new version of left-pad is available",
 			text: "body",
+			listId: "",
 		});
 
 		assert.deepEqual(decision.move, {
@@ -92,6 +93,7 @@ describe("FilterPipeline — anchorless From/Or filter at index time", () => {
 			fromName: "Stripe",
 			subject: "Your receipt",
 			text: "body",
+			listId: "",
 		});
 
 		assert.equal(decision.move, undefined);

package/src/index.ts

@@ -59,6 +59,7 @@ export {
 	testImapConnection,
 	testSmtpConnection,
 } from "./connection-test.js";
+export { extractListId, normalizeListId } from "./filters/list-id.js";
 export {
 	buildMatchText,
 	clauseMatches,

package/src/message-sync-category.test.ts

@@ -0,0 +1,132 @@
+/**
+ * A ThreadMessage row created for a message that is already classified used to
+ * start at `uncategorized` and stay there: `denormalizeCategory` runs at
+ * body-sync, and body-sync skips a message whose body is already stored (issue
+ * #320). A second mailbox does not produce such a row — the ids are
+ * mailbox-independent — but thread-root drift does: the same message re-saved
+ * under different `References` mints a second threadId, so a new row for a
+ * Message whose category is already decided.
+ *
+ * The row now carries the category of the Message the same save just wrote.
+ * These pin the create input, which is where the threading can go wrong; the
+ * stored round-trip belongs to #322's conformance suite. The negative case is
+ * here because parameter threading is exactly the change that starts writing the
+ * wrong value when a caller's variable is undefined — and `uncategorized` is the
+ * declared pending state, never to be folded into `personal` (issue #45).
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	CreateMessageInput,
+	CreateThreadMessageInput,
+	IAddressRepository,
+	IEnvelopeRepository,
+	IMailboxRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+	MessageItem,
+	ThreadMessageItem,
+} from "@remit/data-ports";
+import { MessageCategory } from "@remit/domain-enums";
+import type { ManagedConnectionFactory } from "./connection-factory.js";
+import { MessageSyncService } from "./message-sync.js";
+import type { ImapMessage } from "./types.js";
+
+const envelope = {
+	date: new Date(0).toISOString(),
+	messageId: "<root@example.com>",
+	subject: "Subject",
+	from: [{ name: "Sender", mailbox: "sender", host: "example.com" }],
+	sender: [],
+	replyTo: [],
+	to: [],
+	cc: [],
+	bcc: [],
+	inReplyTo: "",
+};
+
+const imapMessage = {
+	uid: 42,
+	seq: 1,
+	size: 100,
+	internalDate: new Date(0),
+	flags: [],
+	envelope,
+} as unknown as ImapMessage;
+
+const stub = <T>(): T => ({}) as T;
+
+/**
+ * Drive the real save path so the value the ThreadMessage gets is the one the
+ * Message write returned, not one a test handed to a private method.
+ */
+const saveIntoMailbox = async (
+	storedCategory: MessageItem["category"],
+): Promise<CreateThreadMessageInput> => {
+	const inputs: CreateThreadMessageInput[] = [];
+
+	const messageService = {
+		upsertWithStatus: async (input: CreateMessageInput) => ({
+			item: {
+				...input,
+				category: storedCategory,
+			} as unknown as MessageItem,
+			created: false,
+		}),
+	} as unknown as IMessageRepository;
+
+	const threadMessageService = {
+		create: async (input: CreateThreadMessageInput) => {
+			inputs.push(input);
+			return input as unknown as ThreadMessageItem;
+		},
+	} as unknown as IThreadMessageRepository;
+
+	const envelopeService = {
+		upsertEnvelope: async () => {},
+		upsertBodyParts: async () => {},
+	} as unknown as IEnvelopeRepository;
+
+	const addressService = {
+		upsertAddress: async () => {},
+		upsertEnvelopeAddress: async () => {},
+	} as unknown as IAddressRepository;
+
+	const service = new MessageSyncService(
+		stub<ManagedConnectionFactory>(),
+		stub<IMailboxRepository>(),
+		messageService,
+		envelopeService,
+		addressService,
+		threadMessageService,
+	);
+
+	await (
+		service as unknown as {
+			saveMessage: (
+				mailboxId: string,
+				accountId: string,
+				accountConfigId: string,
+				msg: ImapMessage,
+			) => Promise<unknown>;
+		}
+	).saveMessage("mbx-1", "acct-1", "cfg-1", imapMessage);
+
+	assert.equal(inputs.length, 1);
+	const [input] = inputs;
+	assert.ok(input);
+	return input;
+};
+
+describe("a created ThreadMessage carries the Message's category", () => {
+	it("gives a row for an already-classified message that message's category", async () => {
+		const input = await saveIntoMailbox(MessageCategory.newsletter);
+		assert.equal(input.category, MessageCategory.newsletter);
+	});
+
+	it("leaves a row for a new, unclassified message at uncategorized", async () => {
+		const input = await saveIntoMailbox(MessageCategory.uncategorized);
+		assert.equal(input.category, MessageCategory.uncategorized);
+	});
+});

package/src/message-sync-flags.test.ts

@@ -9,7 +9,7 @@ import type {
 	IThreadMessageRepository,
 	ThreadMessageItem,
 } from "@remit/data-ports";
-import { MessageSystemFlag } from "@remit/domain-enums";
+import { MessageCategory, MessageSystemFlag } from "@remit/domain-enums";
 import type { ManagedConnectionFactory } from "./connection-factory.js";
 import { MessageSyncService } from "./message-sync.js";
 import type { ImapEnvelope } from "./types.js";
@@ -72,6 +72,7 @@ type CreateThreadForMessage = (
 	sentDate: number,
 	envelope: ImapEnvelope,
 	flags: string[],
+	category: ThreadMessageItem["category"],
 	references?: string[],
 	hasAttachment?: boolean,
 ) => Promise<void>;
@@ -103,6 +104,7 @@ const createThreadWithFlags = async (
 		now,
 		envelope,
 		flags,
+		MessageCategory.uncategorized,
 	);
 
 	assert.equal(inputs.length, 1);

package/src/message-sync.ts

@@ -1317,6 +1317,15 @@ export class MessageSyncService {
 				sentDate,
 				envelope,
 				msg.flags,
+				// The category of the Message this same save just wrote. A new
+				// message is `uncategorized` until body-sync classifies it; a row
+				// created for a Message that is ALREADY classified takes the decided
+				// value rather than sitting stale forever behind a body-sync pass that
+				// will skip it. That happens on thread-root drift — the same message
+				// re-saved under different `References` mints a second threadId and so
+				// a second row — not on a second mailbox, which resolves to the row
+				// that already exists (issue #320).
+				item.category,
 				msg.references,
 				hasAttachment,
 			);
@@ -1400,6 +1409,10 @@ export class MessageSyncService {
 	 * 3. Fall back to Message-ID (this message is a thread root)
 	 *
 	 * This ensures proper threading even when messages arrive out of order.
+	 *
+	 * `category` is the value the caller already holds from the Message it just
+	 * wrote, so the row starts out agreeing with it rather than defaulting to
+	 * `uncategorized` behind an already-classified message.
 	 */
 	private async createThreadForMessage(
 		threadMessageService: IThreadMessageRepository,
@@ -1412,6 +1425,7 @@ export class MessageSyncService {
 		sentDate: number,
 		envelope: ImapEnvelope,
 		flags: string[],
+		category: ThreadMessageItem["category"],
 		references?: string[],
 		hasAttachment = false,
 	): Promise<void> {
@@ -1484,6 +1498,7 @@ export class MessageSyncService {
 				hasAttachment,
 				hasStars,
 				star: hasStars ? StarColor.Yellow : StarColor.None,
+				category,
 			})
 			.catch((error: unknown) => {
 				// Ignore conflict errors (idempotent create)