@remit/mailbox-service 0.0.23 → 0.0.25

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.23",
+  "version": "0.0.25",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -39,6 +39,7 @@
     "mailparser": "^3.9.14",
     "nodemailer": "^9.0.3",
     "p-map": "^7.0.4",
+    "tldts": "^7.4.9",
     "@remit/data-ports": "*",
     "@remit/domain-enums": "*",
     "@remit/mail-oauth-service": "*",

package/src/body-sync.ts

@@ -30,6 +30,7 @@ import { type ParsedMail, simpleParser } from "mailparser";
 import pMap from "p-map";
 import { BodyParseError, parseMessageBody } from "./body-parse.js";
 import { mapBodyPartsToContent } from "./body-part-mapper.js";
+import { extractListId } from "./filters/list-id.js";
 import type { FilterMessage } from "./filters/match.js";
 import {
 	type FilterConfig,
@@ -134,6 +135,7 @@ const toFilterMessage = (parsed: ParsedMail): FilterMessage => ({
 	fromName: parsed.from?.value?.[0]?.name ?? "",
 	subject: parsed.subject ?? "",
 	text: parsed.text ?? "",
+	listId: extractListId(parsed),
 });
 
 export const toParsedBody = (parsed: ParsedMail): ParsedBody => ({
@@ -1423,10 +1425,11 @@ export class BodySyncService {
 	}
 
 	/**
-	 * Extract snippet and header category from the body and denormalize both
-	 * onto the ThreadMessage. `category` mirrors the Message: created as
-	 * `uncategorized` at metadata-sync and set to the classified value here, so
-	 * the list/search read path carries it without a per-row Message fetch.
+	 * Extract snippet, header category and the normalized `List-Id` from the body
+	 * and denormalize them onto the ThreadMessage. `category` mirrors the Message:
+	 * created as `uncategorized` at metadata-sync and set to the classified value
+	 * here; `listId` is written so the back-apply corpus projection can match a
+	 * `ListId` clause vector-free, off the same row the list/search path reads.
 	 * Returns the parsed mail so callers can reuse it (e.g., to write the
 	 * parsed-body cache) without paying for mailparser twice.
 	 */
@@ -1450,12 +1453,14 @@ export class BodySyncService {
 		);
 
 		const category = classifyByHeaders(parsed);
+		const listId = extractListId(parsed);
 
 		await this.denormalizeCategory(
 			accountConfigId,
 			messageId,
 			category,
 			snippet,
+			listId,
 		);
 
 		this.log.debug?.(
@@ -1482,6 +1487,7 @@ export class BodySyncService {
 		messageId: string,
 		category: ThreadMessageCategory,
 		snippet?: string,
+		listId?: string,
 	): Promise<void> {
 		const threadMessage = await this.threadMessageService.getByMessageId(
 			accountConfigId,
@@ -1491,7 +1497,11 @@ export class BodySyncService {
 		await this.threadMessageService.update(
 			accountConfigId,
 			threadMessage.threadMessageId,
-			{ category, ...(snippet ? { snippet } : {}) },
+			{
+				category,
+				...(snippet ? { snippet } : {}),
+				...(listId ? { listId } : {}),
+			},
 			{
 				composites: {
 					sentDate: threadMessage.sentDate,

package/src/filters/list-id.test.ts

@@ -0,0 +1,50 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { simpleParser } from "mailparser";
+import { extractListId, normalizeListId } from "./list-id.js";
+
+const parse = async (lines: string[]) =>
+	simpleParser(Buffer.from(lines.join("\r\n")));
+
+describe("normalizeListId", () => {
+	it("extracts the bracketed identifier and folds case", () => {
+		assert.equal(
+			normalizeListId("Weekly News <Weekly.News.Example.COM>"),
+			"weekly.news.example.com",
+		);
+	});
+
+	it("keeps a bare value and folds case", () => {
+		assert.equal(
+			normalizeListId("  Weekly.News.Example.COM  "),
+			"weekly.news.example.com",
+		);
+	});
+
+	it("normalizes an empty value to the empty string", () => {
+		assert.equal(normalizeListId("   "), "");
+	});
+});
+
+describe("extractListId", () => {
+	it("reads and normalizes a List-Id header", async () => {
+		const parsed = await parse([
+			"From: list@example.com",
+			"Subject: hi",
+			"List-Id: Weekly News <weekly.news.example.com>",
+			"",
+			"body",
+		]);
+		assert.equal(extractListId(parsed), "weekly.news.example.com");
+	});
+
+	it("returns the empty string when there is no List-Id header", async () => {
+		const parsed = await parse([
+			"From: alice@example.com",
+			"Subject: hi",
+			"",
+			"body",
+		]);
+		assert.equal(extractListId(parsed), "");
+	});
+});

package/src/filters/list-id.ts

@@ -0,0 +1,29 @@
+import type { ParsedMail } from "mailparser";
+
+/**
+ * Canonical form of a `List-Id` value for exact comparison: the bracketed
+ * identifier when the header carries the RFC 2919 `Name <list.id>` shape,
+ * otherwise the whole value, trimmed and case-folded. Both the stored copy and
+ * a `ListId` clause pass through this, so `<weekly.news.example.com>` and
+ * `weekly.news.example.com` are one list. An empty input normalizes to `""`.
+ */
+export const normalizeListId = (value: string): string => {
+	const trimmed = value.trim();
+	const bracketed = trimmed.match(/<([^>]+)>/);
+	return (bracketed ? bracketed[1] : trimmed).trim().toLowerCase();
+};
+
+/**
+ * The normalized `List-Id` header value of a parsed message, or `""` when the
+ * message carries no `List-Id`. Read from the raw header line so the exact
+ * value survives regardless of how the parser structures the header.
+ */
+export const extractListId = (parsed: ParsedMail): string => {
+	const line = parsed.headerLines.find(
+		(header) => header.key.toLowerCase() === "list-id",
+	);
+	if (!line) return "";
+	const colon = line.line.indexOf(":");
+	if (colon < 0) return "";
+	return normalizeListId(line.line.slice(colon + 1));
+};

package/src/filters/match.test.ts

@@ -18,6 +18,7 @@ const message = (overrides: Partial<FilterMessage> = {}): FilterMessage => ({
 	fromName: "Alice Example",
 	subject: "Q3 invoice attached",
 	text: "Please find the invoice for the quarter attached.",
+	listId: "",
 	...overrides,
 });
 
@@ -69,6 +70,93 @@ describe("clauseMatches", () => {
 			false,
 		);
 	});
+
+	it("matches ListId exactly, never as a substring", () => {
+		const msg = message({ listId: "weekly.news.example.com" });
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.ListId, "weekly.news.example.com"),
+				msg,
+			),
+			true,
+		);
+		assert.equal(
+			clauseMatches(clause(FilterClauseField.ListId, "news.example.com"), msg),
+			false,
+		);
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.ListId, "weekly.news.example.com.other"),
+				msg,
+			),
+			false,
+		);
+	});
+
+	it("normalizes ListId brackets and case on both sides", () => {
+		const msg = message({ listId: "weekly.news.example.com" });
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.ListId, "<Weekly.News.Example.COM>"),
+				msg,
+			),
+			true,
+		);
+	});
+
+	it("never matches ListId on a message with no List-Id", () => {
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.ListId, "weekly.news.example.com"),
+				message({ listId: "" }),
+			),
+			false,
+		);
+	});
+
+	it("matches FromDomain on the registrable domain, including subdomains", () => {
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.FromDomain, "github.com"),
+				message({ from: "notifications@github.com" }),
+			),
+			true,
+		);
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.FromDomain, "github.com"),
+				message({ from: "notifications@sub.github.com" }),
+			),
+			true,
+		);
+	});
+
+	it("never matches FromDomain on a look-alike subdomain (public-suffix aware)", () => {
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.FromDomain, "github.com"),
+				message({ from: "attacker@github.com.evil.example" }),
+			),
+			false,
+		);
+	});
+
+	it("matches FromDomain across multi-level public suffixes", () => {
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.FromDomain, "example.co.uk"),
+				message({ from: "hr@mail.example.co.uk" }),
+			),
+			true,
+		);
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.FromDomain, "example.co.uk"),
+				message({ from: "hr@example.co.uk.evil.example" }),
+			),
+			false,
+		);
+	});
 });
 
 describe("literalClausesMatch", () => {

package/src/filters/match.ts

@@ -1,5 +1,7 @@
 import type { FilterItem } from "@remit/data-ports";
 import { FilterClauseField, FilterMatchOperator } from "@remit/domain-enums";
+import { getDomain } from "tldts";
+import { normalizeListId } from "./list-id.js";
 
 type FilterClause = FilterItem["literalClauses"][number];
 
@@ -36,15 +38,32 @@ export interface FilterMessage {
 	fromName: string;
 	subject: string;
 	text: string;
+	/** Normalized `List-Id` header value (see `normalizeListId`); `""` when absent. */
+	listId: string;
 }
 
 const includesFold = (haystack: string, needle: string): boolean =>
 	haystack.toLowerCase().includes(needle.toLowerCase());
 
+const hostOf = (address: string): string => {
+	const at = address.lastIndexOf("@");
+	return at >= 0 ? address.slice(at + 1) : address;
+};
+
+/**
+ * The registrable, public-suffix-aware domain of an email address or host, or
+ * `null` when none resolves. `getDomain` folds case and applies the ICANN
+ * suffix list, so `github.com.evil.example` yields `evil.example`, never
+ * `github.com` — a `FromDomain` clause cannot be spoofed by a crafted subdomain.
+ */
+const registrableDomain = (addressOrHost: string): string | null =>
+	getDomain(hostOf(addressOrHost.trim()));
+
 /**
  * Whether one literal clause matches the message. From matches against the
  * sender address and display name; Subject against the subject; HasWords against
- * subject or body. An empty clause value never matches.
+ * subject or body; ListId against the exact normalized `List-Id`; FromDomain
+ * against the sender's registrable domain. An empty clause value never matches.
  */
 export const clauseMatches = (
 	clause: FilterClause,
@@ -59,6 +78,15 @@ export const clauseMatches = (
 			return includesFold(msg.subject, value);
 		case FilterClauseField.HasWords:
 			return includesFold(msg.subject, value) || includesFold(msg.text, value);
+		case FilterClauseField.ListId: {
+			const target = normalizeListId(value);
+			return target !== "" && normalizeListId(msg.listId) === target;
+		}
+		case FilterClauseField.FromDomain: {
+			const target = registrableDomain(value);
+			if (target === null) return false;
+			return registrableDomain(msg.from) === target;
+		}
 		default:
 			return false;
 	}

package/src/filters/pipeline.test.ts

@@ -0,0 +1,103 @@
+/**
+ * The index-time filter pass over an anchorless literal filter — the shape the
+ * organize widen creates on a deployment without the vector pipeline: `From`
+ * clauses combined with `Or`, `hasAnchor: false`. This is what keeps a standing
+ * "move all mail from these senders" filter working on future mail with no
+ * vectors, so it is pinned directly: a matching sender resolves the move, a
+ * non-matching one resolves nothing, and the embedder is never touched.
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	FilterItem,
+	IFilterAnchorRepository,
+	IFilterRepository,
+	IMessageLabelRepository,
+} from "@remit/data-ports";
+import {
+	FilterClauseField,
+	FilterMatchOperator,
+	FilterState,
+} from "@remit/domain-enums";
+import type { PlacementMoveService } from "../placement-move.js";
+import { NO_ACTION } from "./match.js";
+import { type FilterConfig, FilterPipeline } from "./pipeline.js";
+
+const senderFilter = (destinationMailboxId: string): FilterItem =>
+	({
+		filterId: "flt-senders",
+		accountConfigId: "cfg-1",
+		name: "npm barrage",
+		scope: "Standing",
+		state: FilterState.Active,
+		hasAnchor: false,
+		ruleChangedAt: 1,
+		matchOperator: FilterMatchOperator.Or,
+		literalClauses: [
+			{ field: FilterClauseField.From, value: "npm@github.com" },
+			{ field: FilterClauseField.From, value: "notifications@github.com" },
+		],
+		actionLabelId: NO_ACTION,
+		actionMailboxId: destinationMailboxId,
+		createdAt: 1,
+		updatedAt: 1,
+	}) as unknown as FilterItem;
+
+const buildPipeline = (filter: FilterItem) => {
+	const state = { embedCalls: 0 };
+	const config: FilterConfig = {
+		filterService: {
+			listByAccountAndState: async () => [filter],
+			refreshExpiry: async (f: FilterItem) => f,
+		} as unknown as IFilterRepository,
+		filterAnchorService: {
+			get: async () => undefined,
+		} as unknown as IFilterAnchorRepository,
+		messageLabelService: {} as unknown as IMessageLabelRepository,
+		placementMoveService: {} as unknown as PlacementMoveService,
+		embedder: {
+			embed: async () => {
+				state.embedCalls += 1;
+				return [];
+			},
+		},
+	};
+	return { pipeline: new FilterPipeline(config, { info: () => {} }), state };
+};
+
+describe("FilterPipeline — anchorless From/Or filter at index time", () => {
+	it("resolves the move for a matching sender and never embeds", async () => {
+		const { pipeline, state } = buildPipeline(senderFilter("mbx-archive"));
+		const decision = await pipeline.evaluate("cfg-1", "m-1", {
+			from: "npm@github.com",
+			fromName: "npm",
+			subject: "A new version of left-pad is available",
+			text: "body",
+			listId: "",
+		});
+
+		assert.deepEqual(decision.move, {
+			destinationMailboxId: "mbx-archive",
+			filterId: "flt-senders",
+		});
+		// A literal-only filter is decided by its clauses alone — the embedder is
+		// never constructed or called.
+		assert.equal(state.embedCalls, 0);
+	});
+
+	it("resolves nothing for a sender none of the clauses match", async () => {
+		const { pipeline, state } = buildPipeline(senderFilter("mbx-archive"));
+		const decision = await pipeline.evaluate("cfg-1", "m-2", {
+			from: "hello@stripe.com",
+			fromName: "Stripe",
+			subject: "Your receipt",
+			text: "body",
+			listId: "",
+		});
+
+		assert.equal(decision.move, undefined);
+		assert.deepEqual(decision.labels, []);
+		assert.equal(state.embedCalls, 0);
+	});
+});

package/src/index.ts

@@ -59,6 +59,7 @@ export {
 	testImapConnection,
 	testSmtpConnection,
 } from "./connection-test.js";
+export { extractListId, normalizeListId } from "./filters/list-id.js";
 export {
 	buildMatchText,
 	clauseMatches,