@remit/mailbox-service 0.0.99 → 0.0.101

package/src/heuristics/classifyByHeaders.test.ts

@@ -1,7 +1,12 @@
 import assert from "node:assert/strict";
 import { describe, it } from "node:test";
-import { MessageCategory } from "@remit/domain-enums";
-import { simpleParser } from "mailparser";
+import type { MessageItem } from "@remit/data-ports";
+import {
+	MessageCategory,
+	SenderOverride,
+	SenderTrust,
+} from "@remit/domain-enums";
+import { type ParsedMail, simpleParser } from "mailparser";
 import {
 	classifyByHeaders,
 	extractAuthenticity,
@@ -9,6 +14,7 @@ import {
 	extractHasListUnsubscribe,
 	extractProviderSpam,
 } from "./classifyByHeaders.js";
+import { classifyPlacement } from "./classifyPlacement.js";
 
 const buildEml = (lines: string[]): Buffer => Buffer.from(lines.join("\r\n"));
 
@@ -576,3 +582,308 @@ describe("extractHasListUnsubscribe", () => {
 		assert.equal(extractHasListUnsubscribe(parsed), true);
 	});
 });
+
+const messageFrom = (parsed: ParsedMail): MessageItem => ({
+	messageId: "m-1",
+	mailboxId: "mb-1",
+	uid: 1,
+	sequenceNumber: 1,
+	rfc822Size: 1,
+	internalDate: 1,
+	envelopeId: "e-1",
+	rootBodyPartId: "bp-1",
+	status: "active",
+	syncStatus: "synced",
+	abandonedMutation: "none",
+	category: "uncategorized",
+	classificationState: "NotExamined",
+	authenticityVerdict: "NotEvaluated",
+	hasListUnsubscribe: false,
+	movedByRemit: false,
+	createdAt: 1,
+	updatedAt: 1,
+	providerSpam: extractProviderSpam(parsed) ?? undefined,
+	authResult: extractAuthResult(parsed) ?? undefined,
+	authenticity: extractAuthenticity(parsed) ?? undefined,
+});
+
+const inboxVerdict = (parsed: ParsedMail) =>
+	classifyPlacement(
+		messageFrom(parsed),
+		"inbox",
+		SenderTrust.Unknown,
+		SenderOverride.None,
+	);
+
+const withBody = (lines: string[]): string[] => [...lines, "", "body"];
+
+describe("Authentication-Results adversarial findings (#657)", () => {
+	it("keeps the aligned dkim result when a comment carries a semicolon", async () => {
+		const parsed = await parse(
+			withBody([
+				"From: Alice <alice@acme.example>",
+				"Authentication-Results: gw.example; dkim=fail (1024-bit key; unprotected) header.d=acme.example; dkim=pass header.d=relay.example; dmarc=fail (p=REJECT) header.from=acme.example",
+				"X-Spam-Status: No, score=0.1",
+			]),
+		);
+		assert.deepEqual(extractAuthResult(parsed), {
+			dmarc: "Fail",
+			spf: undefined,
+			dkim: "Fail",
+		});
+	});
+
+	it("checks alignment even when an unrelated dkim=pass is present", async () => {
+		const header = (extra: string) =>
+			parse(
+				withBody([
+					"From: Alice <alice@acme.example>",
+					`Authentication-Results: gw.corp.example; ${extra}dkim=fail (body hash did not verify) header.d=acme.example; dmarc=fail (p=REJECT) header.from=acme.example`,
+					"X-Spam-Status: No, score=0.1",
+				]),
+			);
+		const without = await header("");
+		const withEsp = await header("dkim=pass header.d=esp-mailer.example; ");
+		assert.deepEqual(extractAuthResult(withEsp), extractAuthResult(without));
+		assert.deepEqual(inboxVerdict(withEsp), inboxVerdict(without));
+		assert.equal(inboxVerdict(withEsp).action, "leave");
+	});
+
+	it("never junks list mail whose relay stripped the original signature", async () => {
+		const parsed = await parse(
+			withBody([
+				"From: Alice <alice@acme.example>",
+				"To: list@googlegroups.example",
+				"Authentication-Results: mx.google.example; dkim=pass header.d=googlegroups.example; spf=pass smtp.mailfrom=googlegroups.example; dmarc=fail (p=REJECT) header.from=acme.example",
+				"List-Id: <list.googlegroups.example>",
+				"X-Spam-Status: No, score=0.2",
+			]),
+		);
+		assert.equal(extractAuthenticity(parsed), null);
+		assert.equal(inboxVerdict(parsed).action, "leave");
+	});
+
+	it("never lets a prepended header override the raw signing domain", async () => {
+		const parsed = await parse(
+			withBody([
+				"Authentication-Results: anything-i-like; dkim=pass header.d=bank.example",
+				"From: Security <security@bank.example>",
+				"To: victim@example.com",
+				"Subject: Verify your account",
+				"DKIM-Signature: v=1; a=rsa-sha256; d=phish-host.example; s=sel; b=xxx",
+				"X-Spam-Status: No, score=0.1",
+			]),
+		);
+		assert.deepEqual(extractAuthenticity(parsed), {
+			fromDomain: "bank.example",
+			dkimDomain: "phish-host.example",
+			dkimMismatch: true,
+		});
+	});
+
+	it("never lets a prepended header hide the receiving MTA's dmarc=fail", async () => {
+		const lines = (order: "forged-first" | "real-first") => {
+			const forged =
+				"Authentication-Results: whatever; dkim=pass header.d=evil-mimic.example; dmarc=pass header.from=evil-mimic.example";
+			const real = "Authentication-Results: mx.example.com; dmarc=fail";
+			return withBody([
+				...(order === "forged-first" ? [forged, real] : [real, forged]),
+				"From: Support <support@evil-mimic.example>",
+				"To: me@example.com",
+				"Subject: Verify your account",
+				"DKIM-Signature: v=1; a=rsa-sha256; d=relay.example.net; s=sel; b=xxx",
+				"X-Spam-Status: No, score=0.1",
+			]);
+		};
+		for (const order of ["forged-first", "real-first"] as const) {
+			const parsed = await parse(lines(order));
+			assert.equal(extractAuthResult(parsed)?.dmarc, "Fail", order);
+			assert.equal(inboxVerdict(parsed).action, "move-to-junk", order);
+		}
+	});
+
+	it("never lets a sender-added aligned dkim=pass override the receiver's verdict", async () => {
+		const receivers = [
+			"Authentication-Results: mx.me.example; dkim=fail header.d=phish.example; dmarc=fail header.from=bank.example",
+			"Authentication-Results: mx.me.example; dkim=none; dmarc=fail header.from=bank.example",
+		];
+		const forged = "Authentication-Results: x; dkim=pass header.d=bank.example";
+		for (const [receiver, expected] of [
+			[receivers[0], "Fail"],
+			[receivers[1], "None"],
+		] as const) {
+			for (const headers of [
+				[receiver, forged],
+				[forged, receiver],
+			]) {
+				const parsed = await parse(
+					withBody([...headers, "From: B <s@bank.example>"]),
+				);
+				assert.equal(
+					extractAuthResult(parsed)?.dkim,
+					expected,
+					headers.join(" | "),
+				);
+			}
+		}
+	});
+
+	it("never reads dkim=fail out of another method's comment", async () => {
+		const parsed = await parse(
+			withBody([
+				"From: Alice <alice@acme.example>",
+				"Authentication-Results: mx.example.net; arc=fail (i=1 spf=fail; dkim=fail",
+				"  header.d=arcsigner.example; dmarc=fail); spf=pass smtp.mailfrom=acme.example;",
+				"  dmarc=pass header.from=acme.example",
+			]),
+		);
+		assert.deepEqual(extractAuthResult(parsed), {
+			dmarc: "Pass",
+			spf: "Pass",
+			dkim: undefined,
+		});
+	});
+
+	it("reads dmarc from its own resinfo, not from an arc comment", async () => {
+		const parsed = await parse(
+			withBody([
+				"From: Alice <alice@acme.example>",
+				"Authentication-Results: gw.example; arc=pass (i=1 dmarc=fail fromdomain=acme.example); dmarc=pass header.from=acme.example",
+			]),
+		);
+		assert.equal(extractAuthResult(parsed)?.dmarc, "Pass");
+	});
+
+	describe("grammar edge cases", () => {
+		const dkimFor = async (value: string) =>
+			extractAuthResult(
+				await parse(
+					withBody([
+						"From: Alice <alice@acme.example>",
+						`Authentication-Results: ${value}`,
+					]),
+				),
+			)?.dkim;
+
+		it("aligns a signing domain written with a trailing root dot", async () => {
+			assert.equal(
+				await dkimFor(
+					"gw.example; dkim=fail header.d=esp.example; dkim=pass header.d=acme.example.",
+				),
+				"Pass",
+			);
+		});
+
+		it("never aligns a header.i without an @", async () => {
+			assert.equal(
+				await dkimFor(
+					"gw.example; dkim=fail header.d=esp.example; dkim=pass header.i=acme.example",
+				),
+				"Fail",
+			);
+		});
+
+		it("never aligns a result whose header.i falls outside header.d", async () => {
+			assert.equal(
+				await dkimFor(
+					"gw.example; dkim=fail header.d=esp.example; dkim=pass header.d=acme.example header.i=@evil.example",
+				),
+				"Fail",
+			);
+		});
+
+		it("aligns through header.i when header.d is absent", async () => {
+			assert.equal(
+				await dkimFor(
+					"gw.example; dkim=fail header.d=esp.example; dkim=pass header.i=alice@mail.acme.example",
+				),
+				"Pass",
+			);
+		});
+
+		it("reads upper-case tokens", async () => {
+			const parsed = await parse(
+				withBody([
+					"From: Alice <alice@acme.example>",
+					"Authentication-Results: GW.EXAMPLE; DKIM=FAIL HEADER.D=EVIL.EXAMPLE; DMARC=FAIL",
+				]),
+			);
+			assert.deepEqual(extractAuthResult(parsed), {
+				dmarc: "Fail",
+				spf: undefined,
+				dkim: "Fail",
+			});
+		});
+
+		it("reads a folded header whose continuation starts with dkim=", async () => {
+			const parsed = await parse(
+				withBody([
+					"From: Alice <alice@acme.example>",
+					"Authentication-Results: mx.example.com;",
+					"\tdkim=pass header.d=acme.example;",
+					"\tdmarc=fail header.from=acme.example",
+				]),
+			);
+			assert.deepEqual(extractAuthResult(parsed), {
+				dmarc: "Fail",
+				spf: undefined,
+				dkim: "Pass",
+			});
+		});
+
+		it("prefers the aligned result wherever it is listed", async () => {
+			assert.equal(
+				await dkimFor(
+					"gw.example; dkim=fail header.d=esp-mailer.example; dkim=pass header.d=acme.example; dmarc=fail",
+				),
+				"Pass",
+			);
+		});
+
+		it("reads dkim=fail with no domain", async () => {
+			assert.equal(await dkimFor("gw.example; dkim=fail; dmarc=fail"), "Fail");
+		});
+
+		it("never reads an unrecognised result as a pass", async () => {
+			assert.equal(
+				await dkimFor(
+					"gw.example; dkim=pass header.d=acme.example; dkim=permerror header.d=acme.example",
+				),
+				undefined,
+			);
+		});
+	});
+
+	describe("fail closed", () => {
+		it("yields no result for an empty header", async () => {
+			const parsed = await parse(
+				withBody([
+					"From: Alice <alice@acme.example>",
+					"Authentication-Results:    ",
+				]),
+			);
+			assert.equal(extractAuthResult(parsed), null);
+		});
+
+		it("yields no result for a header that does not parse", async () => {
+			const parsed = await parse(
+				withBody([
+					"From: Alice <alice@acme.example>",
+					"Authentication-Results: mx.example; dmarc=pass (unterminated",
+				]),
+			);
+			assert.equal(extractAuthResult(parsed), null);
+		});
+
+		it("ignores an unparseable header beside one that parses", async () => {
+			const parsed = await parse(
+				withBody([
+					"Authentication-Results: forged; dmarc=pass )",
+					"From: Alice <alice@acme.example>",
+					"Authentication-Results: mx.example.com; dmarc=fail header.from=acme.example",
+				]),
+			);
+			assert.equal(extractAuthResult(parsed)?.dmarc, "Fail");
+		});
+	});
+});

package/src/heuristics/classifyByHeaders.ts

@@ -6,6 +6,11 @@ import type {
 	ParsedMail,
 	StructuredHeader,
 } from "mailparser";
+import {
+	type AuthenticationResult,
+	type AuthenticationResults,
+	parseAuthenticationResults,
+} from "./authenticationResults.js";
 import { hasMachineHeader, isMachineLocalPart } from "./machineSenders.js";
 import { SOCIAL_DOMAINS } from "./socialDomains.js";
 import { TRANSACTIONAL_DOMAINS } from "./transactionalDomains.js";
@@ -139,41 +144,104 @@ export const extractAuthenticity = (
 	};
 };
 
-const extractVerdict = (
-	text: string,
-	mechanism: string,
+const RESULTS_BY_FAVOURABILITY = [
+	"fail",
+	"softfail",
+	"neutral",
+	"none",
+	"unrecognised",
+	"pass",
+] as const;
+
+type RankedResult = (typeof RESULTS_BY_FAVOURABILITY)[number];
+
+const VERDICT_BY_RESULT: ReadonlyMap<RankedResult, AuthVerdictValue> = new Map([
+	["fail", AuthResultVerdict.Fail],
+	["softfail", AuthResultVerdict.Softfail],
+	["neutral", AuthResultVerdict.Neutral],
+	["none", AuthResultVerdict.None],
+	["pass", AuthResultVerdict.Pass],
+]);
+
+const isRankedResult = (result: string): result is RankedResult =>
+	(RESULTS_BY_FAVOURABILITY as readonly string[]).includes(result);
+
+const rankOf = (result: string): number =>
+	RESULTS_BY_FAVOURABILITY.indexOf(
+		isRankedResult(result) ? result : "unrecognised",
+	);
+
+const leastFavourableVerdict = (
+	results: AuthenticationResult[],
 ): AuthVerdictValue | undefined => {
-	const match = text.match(new RegExp(`${mechanism}=(\\w+)`, "i"));
-	if (!match) return undefined;
-	const raw = match[1].toLowerCase();
-	const map: Record<string, AuthVerdictValue> = {
-		pass: AuthResultVerdict.Pass,
-		fail: AuthResultVerdict.Fail,
-		none: AuthResultVerdict.None,
-		neutral: AuthResultVerdict.Neutral,
-		softfail: AuthResultVerdict.Softfail,
-	};
-	return map[raw];
+	if (results.length === 0) return undefined;
+	const worst = Math.min(...results.map((r) => rankOf(r.result)));
+	return VERDICT_BY_RESULT.get(RESULTS_BY_FAVOURABILITY[worst]);
+};
+
+const normaliseDomain = (value: string | undefined): string | null => {
+	if (value === undefined) return null;
+	const domain = value.trim().toLowerCase().replace(/\.$/, "");
+	return domain.length > 0 ? domain : null;
+};
+
+const identityDomain = (value: string | undefined): string | null => {
+	if (value === undefined) return null;
+	const at = value.lastIndexOf("@");
+	if (at < 0) return null;
+	return normaliseDomain(value.slice(at + 1));
+};
+
+const dkimSigningDomain = (result: AuthenticationResult): string | null => {
+	const signing = normaliseDomain(result.properties.get("header.d"));
+	const hasIdentity = result.properties.has("header.i");
+	const identity = identityDomain(result.properties.get("header.i"));
+	if (hasIdentity && identity === null) return null;
+	if (signing === null) return identity;
+	if (identity === null) return signing;
+	const identityWithinSigning =
+		identity === signing || identity.endsWith(`.${signing}`);
+	return identityWithinSigning ? signing : null;
+};
+
+const dkimResultsForFrom = (
+	results: AuthenticationResult[],
+	fromDomain: string | null,
+): AuthenticationResult[] => {
+	if (!fromDomain) return results;
+	const aligned = results.filter((r) => {
+		const domain = dkimSigningDomain(r);
+		return domain !== null && domainsAligned(domain, fromDomain);
+	});
+	return aligned.length > 0 ? aligned : results;
 };
 
-/**
- * Extract provider authentication-results verdict from the Authentication-Results header.
- * Returns null when the header is absent.
- */
 export const extractAuthResult = (
 	parsed: ParsedMail,
 ): MessageAuthResult | null => {
-	const line = parsed.headerLines.find(
-		(l) => l.key.toLowerCase() === "authentication-results",
-	);
-	if (!line) return null;
+	const headers = parsed.headerLines
+		.filter((l) => l.key.toLowerCase() === "authentication-results")
+		.map((l) => parseAuthenticationResults(stripHeaderName(l.line)))
+		.filter((h): h is AuthenticationResults => h !== null);
+	if (headers.length === 0) return null;
 
-	const text = stripHeaderName(line.line);
-	const dmarc = extractVerdict(text, "dmarc");
-	const spf = extractVerdict(text, "spf");
-	const dkim = extractVerdict(text, "dkim");
+	const fromDomain = getFromDomain(parsed);
+	const byMethod = (
+		results: AuthenticationResult[],
+		method: string,
+	): AuthenticationResult[] => results.filter((r) => r.method === method);
+	const acrossHeaders = (method: string): AuthenticationResult[] =>
+		headers.flatMap((h) => byMethod(h.results, method));
 
-	return { dmarc, spf, dkim };
+	return {
+		dmarc: leastFavourableVerdict(acrossHeaders("dmarc")),
+		spf: leastFavourableVerdict(acrossHeaders("spf")),
+		dkim: leastFavourableVerdict(
+			headers.flatMap((h) =>
+				dkimResultsForFrom(byMethod(h.results, "dkim"), fromDomain),
+			),
+		),
+	};
 };
 
 /**

package/src/index.ts

@@ -28,7 +28,7 @@ export {
 	type BodySyncLogger,
 	BodySyncService,
 	type ConnectionGetter,
-	extractPrimaryFromEmail,
+	extractSnippet,
 	type FetchBodyResult,
 	type PlacementConfig,
 	type QuarantineConfig,
@@ -48,6 +48,21 @@ export {
 	type ResolveExhaustedBodySyncResult,
 	resolveExhaustedBodySyncFailures,
 } from "./body-sync-terminal.js";
+export {
+	backfillClassifications,
+	type ClassificationBackfillCheckpoint,
+	type ClassificationBackfillCheckpointStore,
+	type ClassificationBackfillDeps,
+	type ClassificationBackfillLogger,
+	type ClassificationBackfillOptions,
+	type ClassificationBackfillProgress,
+	type ClassificationBackfillResult,
+	type ClassificationBackfillTotals,
+} from "./classification-backfill.js";
+export {
+	classifyParsedMessage,
+	extractPrimaryFromEmail,
+} from "./classify-message.js";
 // Connection factory
 export {
 	createConnection,

package/src/message-category.ts

@@ -3,9 +3,27 @@ import type {
 	ThreadMessageItem,
 } from "@remit/data-ports";
 import { NotFoundError } from "@remit/data-ports/errors";
+import { MessageCategory } from "@remit/domain-enums";
 
 export type MessageCategoryValue = ThreadMessageItem["category"];
 
+/**
+ * RFC 034 Decision 3.1: `Message.category` is written once and never mutated
+ * after — RFC 030's message-list GSI sort key depends on it never churning.
+ * "Already decided" is any real category, so a message that carries one is
+ * never re-categorized by a re-entrant pass.
+ *
+ * `uncategorized` fails this test whether or not the classifier has run, which
+ * is deliberate: this asks what the row holds, not what was done to it. Whether
+ * the classifier has run is {@link Message.classificationState}, and it is the
+ * skip guard in `BodySyncService.syncBodies` — not this one — that keeps a
+ * declined message from being examined twice.
+ */
+export const hasDecidedCategory = (
+	category: MessageCategoryValue | undefined,
+): boolean =>
+	category !== undefined && category !== MessageCategory.uncategorized;
+
 export interface MessageCategoryDenormalizeDeps {
 	threadMessageService: Pick<
 		IThreadMessageRepository,