@remit/mailbox-service 0.0.43 → 0.0.44

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.43",
+  "version": "0.0.44",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/body-sync.ts

@@ -51,6 +51,7 @@ import {
 	type PlacementVerdict,
 	resolveBlockedVsTrust,
 } from "./heuristics/classifyPlacement.js";
+import { extractSenderMismatch } from "./heuristics/senderMismatch.js";
 import type { PlacementMoveService } from "./placement-move.js";
 import { type QuarantineService, shapeFromMessageData } from "./quarantine.js";
 import { extractSnippetFromEmail } from "./snippet.js";
@@ -1159,6 +1160,20 @@ export class BodySyncService {
 		const providerSpam = extractProviderSpam(parsed);
 		const hasListUnsubscribe = extractHasListUnsubscribe(parsed);
 
+		// A passing SPF/DKIM/DMARC check proves the sending domain, not the
+		// identity the message claims — on a shared-tenant host the verified
+		// subdomain belongs to whoever signed up. These two comparisons say
+		// whether the claim holds, and run only over mail the provider already
+		// called spam.
+		const senderMismatch =
+			authenticity === null
+				? {}
+				: extractSenderMismatch(parsed, {
+						fromDomain: authenticity.fromDomain,
+						spamClassified: providerSpam?.classified === true,
+						bulkSender: hasListUnsubscribe,
+					});
+
 		const fromEmail = extractPrimaryFromEmail(parsed);
 		const categoryOverride = fromEmail
 			? await this.resolveCategoryOverride(accountConfigId, fromEmail)
@@ -1167,7 +1182,9 @@ export class BodySyncService {
 		return {
 			category: categoryOverride ?? headerCategory,
 			hasListUnsubscribe,
-			...(authenticity !== null ? { authenticity } : {}),
+			...(authenticity !== null
+				? { authenticity: { ...authenticity, ...senderMismatch } }
+				: {}),
 			...(authResult !== null ? { authResult } : {}),
 			...(providerSpam !== null ? { providerSpam } : {}),
 		};

package/src/heuristics/senderMismatch.test.ts

@@ -0,0 +1,213 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { DisplayNameCorrespondence } from "@remit/domain-enums";
+import { simpleParser } from "mailparser";
+import {
+	classifyDisplayNameCorrespondence,
+	extractOffDomainLinkDomains,
+	extractSenderMismatch,
+} from "./senderMismatch.js";
+
+const parse = async (lines: string[]) =>
+	simpleParser(Buffer.from(lines.join("\r\n")));
+
+describe("classifyDisplayNameCorrespondence", () => {
+	it("corresponds when the name is a label of the From domain", () => {
+		assert.equal(
+			classifyDisplayNameCorrespondence("GitHub", "notifications.github.com"),
+			DisplayNameCorrespondence.Corresponds,
+		);
+	});
+
+	it("corresponds when a word of a decorated name is a label of the From domain", () => {
+		assert.equal(
+			classifyDisplayNameCorrespondence(
+				"GitHub Actions",
+				"notifications.github.com",
+			),
+			DisplayNameCorrespondence.Corresponds,
+		);
+	});
+
+	it("corresponds across a multi-part public suffix", () => {
+		assert.equal(
+			classifyDisplayNameCorrespondence("Sainsbury's", "mail.sainsburys.co.uk"),
+			DisplayNameCorrespondence.Corresponds,
+		);
+	});
+
+	it("is unrelated when the name appears nowhere in the From domain", () => {
+		assert.equal(
+			classifyDisplayNameCorrespondence(
+				"InfoMedics",
+				"serviceupdatebank.atlassian.net",
+			),
+			DisplayNameCorrespondence.Unrelated,
+		);
+	});
+
+	it("is a lookalike when a digit stands in for a letter of the domain", () => {
+		assert.equal(
+			classifyDisplayNameCorrespondence("InfoMedics", "1nfomedics.nl"),
+			DisplayNameCorrespondence.Lookalike,
+		);
+	});
+
+	it("is a lookalike for a short brand one character off", () => {
+		assert.equal(
+			classifyDisplayNameCorrespondence("PayPal", "paypa1.com"),
+			DisplayNameCorrespondence.Lookalike,
+		);
+	});
+
+	it("claims nothing when the display name is empty", () => {
+		assert.equal(
+			classifyDisplayNameCorrespondence("", "example.com"),
+			DisplayNameCorrespondence.NoClaim,
+		);
+		assert.equal(
+			classifyDisplayNameCorrespondence(undefined, "example.com"),
+			DisplayNameCorrespondence.NoClaim,
+		);
+	});
+
+	it("claims nothing when the display name is the address itself", () => {
+		assert.equal(
+			classifyDisplayNameCorrespondence(
+				"billing@serviceupdatebank.atlassian.net",
+				"serviceupdatebank.atlassian.net",
+			),
+			DisplayNameCorrespondence.NoClaim,
+		);
+	});
+
+	it("does not read the public suffix as a match for a brand containing it", () => {
+		assert.equal(
+			classifyDisplayNameCorrespondence("Netflix", "mailer.example.net"),
+			DisplayNameCorrespondence.Unrelated,
+		);
+	});
+});
+
+describe("extractOffDomainLinkDomains", () => {
+	it("names only the registrable domains that leave the sender's own", async () => {
+		const parsed = await parse([
+			"From: Jira <jira@serviceupdatebank.atlassian.net>",
+			"To: bob@example.com",
+			"Subject: Vordering",
+			"Content-Type: text/html",
+			"",
+			'<a href="https://serviceupdatebank.atlassian.net/browse/X">ticket</a>',
+			'<a href="https://betaal-vordering.example.org/pay">betaal nu</a>',
+			'<a href="https://cdn.betaal-vordering.example.org/logo.png">logo</a>',
+		]);
+		assert.deepEqual(
+			extractOffDomainLinkDomains(parsed, "serviceupdatebank.atlassian.net"),
+			["example.org"],
+		);
+	});
+
+	it("compares public-suffix-aware, so a co.uk sibling is not off-domain", async () => {
+		const parsed = await parse([
+			"From: Shop <shop@mail.example.co.uk>",
+			"To: bob@example.com",
+			"Subject: order",
+			"Content-Type: text/html",
+			"",
+			'<a href="https://www.example.co.uk/orders">orders</a>',
+		]);
+		assert.deepEqual(
+			extractOffDomainLinkDomains(parsed, "mail.example.co.uk"),
+			[],
+		);
+	});
+
+	it("ignores mailto, anchors and relative hrefs", async () => {
+		const parsed = await parse([
+			"From: Shop <shop@example.com>",
+			"To: bob@example.com",
+			"Subject: order",
+			"Content-Type: text/html",
+			"",
+			'<a href="mailto:help@elsewhere.example">mail us</a>',
+			'<a href="#top">top</a>',
+			'<a href="/orders">orders</a>',
+			'<a href="tel:+31201234567">call</a>',
+		]);
+		assert.deepEqual(extractOffDomainLinkDomains(parsed, "example.com"), []);
+	});
+
+	it("reads bare URLs out of a plain-text body", async () => {
+		const parsed = await parse([
+			"From: Shop <shop@example.com>",
+			"To: bob@example.com",
+			"Subject: order",
+			"",
+			"Betaal hier: https://betaal.elsewhere.example/pay?id=1",
+		]);
+		assert.deepEqual(extractOffDomainLinkDomains(parsed, "example.com"), [
+			"elsewhere.example",
+		]);
+	});
+});
+
+describe("extractSenderMismatch", () => {
+	const infoMedicsPhish = [
+		"From: InfoMedics <jira@serviceupdatebank.atlassian.net>",
+		"To: bob@example.com",
+		"Subject: Vordering",
+		"Authentication-Results: mx.example.com; dmarc=pass; spf=pass; dkim=pass",
+		"X-HalOne-Spam-Probability: 1",
+		"DKIM-Signature: v=1; a=rsa-sha256; d=custmx.one.com; s=sel; b=xxx",
+		"Content-Type: text/html",
+		"",
+		'<a href="https://betaal-vordering.example.org/pay">Betaal uw factuur</a>',
+	];
+
+	it("flags the display name and the links on a spam-classified message", async () => {
+		const parsed = await parse(infoMedicsPhish);
+		assert.deepEqual(
+			extractSenderMismatch(parsed, {
+				fromDomain: "serviceupdatebank.atlassian.net",
+				spamClassified: true,
+				bulkSender: false,
+			}),
+			{
+				displayNameCorrespondence: DisplayNameCorrespondence.Unrelated,
+				offDomainLinkDomains: ["example.org"],
+			},
+		);
+	});
+
+	it("compares nothing when the provider did not call the message spam", async () => {
+		const parsed = await parse(infoMedicsPhish);
+		assert.deepEqual(
+			extractSenderMismatch(parsed, {
+				fromDomain: "serviceupdatebank.atlassian.net",
+				spamClassified: false,
+				bulkSender: false,
+			}),
+			{},
+		);
+	});
+
+	it("leaves the display name uncompared for a bulk sender", async () => {
+		const parsed = await parse([
+			"From: Dutch Cycling Weekly <bounce-9f2@mailer.esp.example>",
+			"To: bob@example.com",
+			"Subject: This week in cycling",
+			"List-Unsubscribe: <https://mailer.esp.example/u/9f2>",
+			"X-HalOne-Spam-Probability: 1",
+			"Content-Type: text/html",
+			"",
+			'<a href="https://dutchcyclingweekly.example.org/issue/12">read</a>',
+		]);
+		const signals = extractSenderMismatch(parsed, {
+			fromDomain: "mailer.esp.example",
+			spamClassified: true,
+			bulkSender: true,
+		});
+		assert.equal(signals.displayNameCorrespondence, undefined);
+		assert.deepEqual(signals.offDomainLinkDomains, ["example.org"]);
+	});
+});

package/src/heuristics/senderMismatch.ts

@@ -0,0 +1,213 @@
+import { DisplayNameCorrespondence } from "@remit/domain-enums";
+import type { ParsedMail } from "mailparser";
+import { getDomain, parse as parseHost } from "tldts";
+
+type CorrespondenceValue =
+	(typeof DisplayNameCorrespondence)[keyof typeof DisplayNameCorrespondence];
+
+/**
+ * The two signals that survive a passing SPF/DKIM/DMARC check: a display name
+ * that belongs to nobody at the sending domain, and body links that leave it.
+ *
+ * Both are deliberately aggressive, and both are computed only for mail the
+ * provider's own filter already called spam (see {@link extractSenderMismatch}).
+ * On a shared-tenant host — a free Atlassian, Salesforce or Zendesk instance —
+ * the verified subdomain is chosen by whoever signed up, so a passing signature
+ * proves the domain and nothing about the identity the message claims.
+ */
+export interface SenderMismatchSignals {
+	displayNameCorrespondence?: CorrespondenceValue;
+	offDomainLinkDomains?: string[];
+}
+
+export interface SenderMismatchContext {
+	fromDomain: string;
+	spamClassified: boolean;
+	/**
+	 * List-Unsubscribe present. Bulk mail routinely shows a brand name over an
+	 * ESP's sending domain, so the display-name comparison says nothing there and
+	 * is not made.
+	 */
+	bulkSender: boolean;
+}
+
+const MAX_LINK_DOMAINS = 10;
+
+const normalize = (value: string): string =>
+	value.toLowerCase().replace(/[^a-z0-9]/g, "");
+
+/**
+ * The parts of a host a display name could honestly correspond to: the
+ * registrable name without its public suffix, every subdomain label, and the
+ * registrable domain run together. The suffix itself is dropped — `net` and
+ * `com` sit inside half the brand names in existence.
+ */
+const correspondenceCandidates = (host: string): string[] => {
+	const parsed = parseHost(host);
+	const labels = (parsed.subdomain ?? "").split(".");
+	if (parsed.domainWithoutSuffix !== null) {
+		labels.push(parsed.domainWithoutSuffix);
+	} else {
+		labels.push(...host.split(".").slice(0, -1));
+	}
+	if (parsed.domain !== null) labels.push(parsed.domain);
+	return labels.map(normalize).filter((label) => label.length >= 3);
+};
+
+const words = (displayName: string): string[] =>
+	displayName
+		.toLowerCase()
+		.split(/[^a-z0-9]+/)
+		.filter((word) => word.length >= 3);
+
+const editDistance = (a: string, b: string): number => {
+	let previous = Array.from({ length: b.length + 1 }, (_, i) => i);
+	for (let i = 1; i <= a.length; i++) {
+		const current = [i];
+		for (let j = 1; j <= b.length; j++) {
+			current[j] = Math.min(
+				previous[j] + 1,
+				current[j - 1] + 1,
+				previous[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1),
+			);
+		}
+		previous = current;
+	}
+	return previous[b.length];
+};
+
+/**
+ * The distance a name of this length may be from a domain label and still be
+ * read as an imitation of it. Tight on purpose: an unrelated brand name and an
+ * ESP's domain are always far apart, so distance alone must never decide.
+ */
+const lookalikeThreshold = (length: number): number => {
+	if (length < 5) return 0;
+	if (length < 9) return 1;
+	return 2;
+};
+
+/**
+ * Whether the From display name corresponds to the From domain.
+ *
+ * Containment decides: the normalised name, or any word of it, appearing inside
+ * the registrable domain or one of the domain's labels. `GitHub` sits inside
+ * `notifications.github.com`; `InfoMedics` sits nowhere inside
+ * `serviceupdatebank.atlassian.net`.
+ *
+ * A bounded edit distance is the secondary test, and only reaches names that
+ * nearly match a label — `InfoMedics` against `1nfomedics.nl`. It cannot promote
+ * an unrelated name on its own.
+ */
+export const classifyDisplayNameCorrespondence = (
+	displayName: string | undefined,
+	fromDomain: string,
+): CorrespondenceValue => {
+	const raw = (displayName ?? "").trim();
+	if (raw === "" || raw.includes("@")) {
+		return DisplayNameCorrespondence.NoClaim;
+	}
+
+	const name = normalize(raw);
+	if (name.length < 3) return DisplayNameCorrespondence.NoClaim;
+
+	const candidates = correspondenceCandidates(fromDomain);
+	if (candidates.length === 0) return DisplayNameCorrespondence.NoClaim;
+
+	const terms = [name, ...words(raw)];
+	for (const term of terms) {
+		for (const candidate of candidates) {
+			if (candidate.includes(term) || term.includes(candidate)) {
+				return DisplayNameCorrespondence.Corresponds;
+			}
+		}
+	}
+
+	for (const candidate of candidates) {
+		const threshold = lookalikeThreshold(
+			Math.min(name.length, candidate.length),
+		);
+		if (threshold === 0) continue;
+		if (Math.abs(name.length - candidate.length) > threshold) continue;
+		if (editDistance(name, candidate) <= threshold) {
+			return DisplayNameCorrespondence.Lookalike;
+		}
+	}
+
+	return DisplayNameCorrespondence.Unrelated;
+};
+
+const hrefsFromHtml = (html: string): string[] => {
+	const out: string[] = [];
+	const pattern = /href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
+	let match = pattern.exec(html);
+	while (match !== null) {
+		out.push(match[1] ?? match[2] ?? match[3] ?? "");
+		match = pattern.exec(html);
+	}
+	return out;
+};
+
+const urlsFromText = (text: string): string[] =>
+	text.match(/https?:\/\/[^\s<>"')\]]+/gi) ?? [];
+
+/**
+ * Registrable domains the body's links point at, minus the sender's own.
+ *
+ * `mailto:`, `tel:`, in-page anchors and relative hrefs carry no destination
+ * domain and are skipped; anything else is resolved public-suffix-aware through
+ * tldts, so `co.uk` and `atlassian.net` both survive the comparison a naive dot
+ * split gets wrong.
+ */
+export const extractOffDomainLinkDomains = (
+	parsed: ParsedMail,
+	fromDomain: string,
+): string[] => {
+	const senderDomain = getDomain(fromDomain);
+	const html = typeof parsed.html === "string" ? parsed.html : "";
+	const text = typeof parsed.text === "string" ? parsed.text : "";
+
+	const seen = new Set<string>();
+	const out: string[] = [];
+	for (const href of [...hrefsFromHtml(html), ...urlsFromText(text)]) {
+		const value = href.trim();
+		if (value === "" || !/^https?:\/\//i.test(value)) continue;
+		const domain = getDomain(value);
+		if (domain === null || domain === senderDomain) continue;
+		if (seen.has(domain)) continue;
+		seen.add(domain);
+		out.push(domain);
+		if (out.length === MAX_LINK_DOMAINS) break;
+	}
+	return out;
+};
+
+/**
+ * Both signals, gated on the provider's own spam verdict.
+ *
+ * The gate is what lets the checks be this aggressive: ordinary mail never
+ * reaches them, so a brand name over an ESP domain or a newsletter full of
+ * tracking links can never be flagged. Returns an empty object when the gate
+ * does not open — the fields stay absent, meaning "not compared".
+ */
+export const extractSenderMismatch = (
+	parsed: ParsedMail,
+	context: SenderMismatchContext,
+): SenderMismatchSignals => {
+	if (!context.spamClassified) return {};
+
+	const offDomainLinkDomains = extractOffDomainLinkDomains(
+		parsed,
+		context.fromDomain,
+	);
+
+	if (context.bulkSender) return { offDomainLinkDomains };
+
+	return {
+		displayNameCorrespondence: classifyDisplayNameCorrespondence(
+			parsed.from?.value?.[0]?.name,
+			context.fromDomain,
+		),
+		offDomainLinkDomains,
+	};
+};

package/src/index.ts

@@ -114,6 +114,13 @@ export {
 	type PlacementAction,
 	type PlacementVerdict,
 } from "./heuristics/classifyPlacement.js";
+export {
+	classifyDisplayNameCorrespondence,
+	extractOffDomainLinkDomains,
+	extractSenderMismatch,
+	type SenderMismatchContext,
+	type SenderMismatchSignals,
+} from "./heuristics/senderMismatch.js";
 export { SOCIAL_DOMAINS } from "./heuristics/socialDomains.js";
 export { TRANSACTIONAL_DOMAINS } from "./heuristics/transactionalDomains.js";
 // IMAP connection (ImapFlow-based)