@remit/mailbox-service 0.0.111 → 0.0.113

package/src/heuristics/classifyByHeaders.ts

@@ -1,467 +0,0 @@
-import { AuthResultVerdict, MessageCategory } from "@remit/domain-enums";
-import type {
-	Attachment,
-	HeaderLines,
-	Headers,
-	ParsedMail,
-	StructuredHeader,
-} from "mailparser";
-import {
-	type AuthenticationResult,
-	type AuthenticationResults,
-	parseAuthenticationResults,
-} from "./authenticationResults.js";
-import { hasMachineHeader, isMachineLocalPart } from "./machineSenders.js";
-import { SOCIAL_DOMAINS } from "./socialDomains.js";
-import { TRANSACTIONAL_DOMAINS } from "./transactionalDomains.js";
-
-type Category = (typeof MessageCategory)[keyof typeof MessageCategory];
-type AuthVerdictValue =
-	(typeof AuthResultVerdict)[keyof typeof AuthResultVerdict];
-
-export interface MessageAuthResult {
-	dmarc?: AuthVerdictValue;
-	spf?: AuthVerdictValue;
-	dkim?: AuthVerdictValue;
-}
-
-export interface MessageProviderSpam {
-	classified: boolean;
-	score?: string;
-	source?: string;
-}
-
-/**
- * Structured sender-authenticity signal extracted from DKIM headers.
- * Persisted alongside `category` during body-sync so the intelligence
- * sidebar (#425) can render phishing verdicts without re-parsing.
- *
- * Present only when a DKIM-Signature header exists; absent otherwise
- * (absence means "no signal" — no comparison was possible).
- */
-export interface MessageAuthenticity {
-	/** Domain of the From header address */
-	fromDomain: string;
-	/** DKIM signing domain (d=) that was compared, when a signature is present */
-	dkimDomain?: string;
-	/** True when none of the DKIM signing domains aligns with the From domain */
-	dkimMismatch: boolean;
-}
-
-/**
- * Header-only classification. Pure function. First match wins. Falls through
- * to `personal` so misclassification stays in the safest bucket.
- *
- * Rules are ordered most-specific-signal first. Two properties drive the
- * order, and both were violated by the original table (issue #45):
- *
- * - A signal that identifies WHO sent the mail (calendar part, allow-listed
- *   domain) outranks a signal that only says the mail was sent in bulk.
- * - `Precedence` and `Auto-Submitted` say "a machine sent this", which is true
- *   of nearly every newsletter, marketing blast, and platform notification.
- *   Ranking them first collapsed `newsletter`, `marketing`, `social` and
- *   `transactional` into `automated`, leaving those buckets empty.
- *
- * 1. `Content-Type: text/calendar` part anywhere → `transactional`
- * 2. From-domain in TRANSACTIONAL_DOMAINS → `transactional`
- * 3. From-domain in SOCIAL_DOMAINS → `social`
- * 4. `List-Unsubscribe` AND `List-Id` → `newsletter`
- * 5. `List-Unsubscribe` only → `marketing`
- * 6. `Auto-Submitted: auto-generated|auto-replied` → `automated`
- * 7. `Precedence: bulk|list|junk` → `automated`
- * 8. Machine sender (no-reply local-part, `Feedback-ID`,
- *    `X-Auto-Response-Suppress`) → `automated`
- * 9. DKIM `d=` differs from From domain → `automated`
- * 10. fallback → `personal`
- */
-export const classifyByHeaders = (parsed: ParsedMail): Category => {
-	const headers = parsed.headers;
-	const lines = parsed.headerLines;
-
-	if (hasCalendarPart(parsed.attachments)) return MessageCategory.transactional;
-
-	const fromDomain = getFromDomain(parsed);
-
-	if (fromDomain && domainMatches(fromDomain, TRANSACTIONAL_DOMAINS)) {
-		return MessageCategory.transactional;
-	}
-
-	if (fromDomain && domainMatches(fromDomain, SOCIAL_DOMAINS)) {
-		return MessageCategory.social;
-	}
-
-	const hasListUnsubscribe = hasHeaderLine(lines, "list-unsubscribe");
-	const hasListId = hasHeaderLine(lines, "list-id");
-
-	if (hasListUnsubscribe && hasListId) return MessageCategory.newsletter;
-	if (hasListUnsubscribe) return MessageCategory.marketing;
-
-	if (matchesAutoSubmitted(headers)) return MessageCategory.automated;
-	if (matchesPrecedence(headers)) return MessageCategory.automated;
-	if (isMachineSender(parsed, lines)) return MessageCategory.automated;
-
-	if (
-		fromDomain &&
-		pickAlignedOrFirstMismatch(extractDkimDomains(headers, lines), fromDomain)
-			.mismatch
-	) {
-		return MessageCategory.automated;
-	}
-
-	return MessageCategory.personal;
-};
-
-/**
- * Extract the structured authenticity signal from parsed headers.
- *
- * Returns a `MessageAuthenticity` when at least one DKIM-Signature header
- * is present (a comparison was possible). Returns `null` when there are no
- * DKIM-Signature headers so callers can omit the field entirely — absence
- * means "no signal", not "mismatch: false".
- *
- * The alignment rule exactly mirrors the category heuristic (rule 7) so the
- * structured `dkimMismatch` boolean and the `automated` category can never
- * disagree.
- */
-export const extractAuthenticity = (
-	parsed: ParsedMail,
-): MessageAuthenticity | null => {
-	const headers = parsed.headers;
-	const lines = parsed.headerLines;
-	const fromDomain = getFromDomain(parsed);
-
-	if (!fromDomain) return null;
-
-	const dkimDomains = extractDkimDomains(headers, lines);
-	if (dkimDomains.length === 0) return null;
-
-	const picked = pickAlignedOrFirstMismatch(dkimDomains, fromDomain);
-
-	return {
-		fromDomain,
-		dkimDomain: picked.domain ?? undefined,
-		dkimMismatch: picked.mismatch,
-	};
-};
-
-const RESULTS_BY_FAVOURABILITY = [
-	"fail",
-	"softfail",
-	"neutral",
-	"none",
-	"unrecognised",
-	"pass",
-] as const;
-
-type RankedResult = (typeof RESULTS_BY_FAVOURABILITY)[number];
-
-const VERDICT_BY_RESULT: ReadonlyMap<RankedResult, AuthVerdictValue> = new Map([
-	["fail", AuthResultVerdict.Fail],
-	["softfail", AuthResultVerdict.Softfail],
-	["neutral", AuthResultVerdict.Neutral],
-	["none", AuthResultVerdict.None],
-	["pass", AuthResultVerdict.Pass],
-]);
-
-const isRankedResult = (result: string): result is RankedResult =>
-	(RESULTS_BY_FAVOURABILITY as readonly string[]).includes(result);
-
-const rankOf = (result: string): number =>
-	RESULTS_BY_FAVOURABILITY.indexOf(
-		isRankedResult(result) ? result : "unrecognised",
-	);
-
-const leastFavourableVerdict = (
-	results: AuthenticationResult[],
-): AuthVerdictValue | undefined => {
-	if (results.length === 0) return undefined;
-	const worst = Math.min(...results.map((r) => rankOf(r.result)));
-	return VERDICT_BY_RESULT.get(RESULTS_BY_FAVOURABILITY[worst]);
-};
-
-const normaliseDomain = (value: string | undefined): string | null => {
-	if (value === undefined) return null;
-	const domain = value.trim().toLowerCase().replace(/\.$/, "");
-	return domain.length > 0 ? domain : null;
-};
-
-const identityDomain = (value: string | undefined): string | null => {
-	if (value === undefined) return null;
-	const at = value.lastIndexOf("@");
-	if (at < 0) return null;
-	return normaliseDomain(value.slice(at + 1));
-};
-
-const dkimSigningDomain = (result: AuthenticationResult): string | null => {
-	const signing = normaliseDomain(result.properties.get("header.d"));
-	const hasIdentity = result.properties.has("header.i");
-	const identity = identityDomain(result.properties.get("header.i"));
-	if (hasIdentity && identity === null) return null;
-	if (signing === null) return identity;
-	if (identity === null) return signing;
-	const identityWithinSigning =
-		identity === signing || identity.endsWith(`.${signing}`);
-	return identityWithinSigning ? signing : null;
-};
-
-const dkimResultsForFrom = (
-	results: AuthenticationResult[],
-	fromDomain: string | null,
-): AuthenticationResult[] => {
-	if (!fromDomain) return results;
-	const aligned = results.filter((r) => {
-		const domain = dkimSigningDomain(r);
-		return domain !== null && domainsAligned(domain, fromDomain);
-	});
-	return aligned.length > 0 ? aligned : results;
-};
-
-export const extractAuthResult = (
-	parsed: ParsedMail,
-): MessageAuthResult | null => {
-	const headers = parsed.headerLines
-		.filter((l) => l.key.toLowerCase() === "authentication-results")
-		.map((l) => parseAuthenticationResults(stripHeaderName(l.line)))
-		.filter((h): h is AuthenticationResults => h !== null);
-	if (headers.length === 0) return null;
-
-	const fromDomain = getFromDomain(parsed);
-	const byMethod = (
-		results: AuthenticationResult[],
-		method: string,
-	): AuthenticationResult[] => results.filter((r) => r.method === method);
-	const acrossHeaders = (method: string): AuthenticationResult[] =>
-		headers.flatMap((h) => byMethod(h.results, method));
-
-	return {
-		dmarc: leastFavourableVerdict(acrossHeaders("dmarc")),
-		spf: leastFavourableVerdict(acrossHeaders("spf")),
-		dkim: leastFavourableVerdict(
-			headers.flatMap((h) =>
-				dkimResultsForFrom(byMethod(h.results, "dkim"), fromDomain),
-			),
-		),
-	};
-};
-
-/**
- * Extract provider spam-filter signal from X-SpamExperts-Class, X-Spam-Status,
- * or X-HalOne-Spam-Probability headers. Returns null when none are present.
- */
-export const extractProviderSpam = (
-	parsed: ParsedMail,
-): MessageProviderSpam | null => {
-	const lines = parsed.headerLines;
-
-	const spamExperts = lines.find(
-		(l) => l.key.toLowerCase() === "x-spamexperts-class",
-	);
-	if (spamExperts) {
-		const value = stripHeaderName(spamExperts.line).trim().toLowerCase();
-		return {
-			classified: value === "spam",
-			source: "x-spamexperts-class",
-		};
-	}
-
-	const spamStatus = lines.find((l) => l.key.toLowerCase() === "x-spam-status");
-	if (spamStatus) {
-		const text = stripHeaderName(spamStatus.line).trim();
-		const classified = text.toLowerCase().startsWith("yes");
-		const scoreMatch = text.match(/score=([\d.+-]+)/i);
-		return {
-			classified,
-			score: scoreMatch ? scoreMatch[1] : undefined,
-			source: "x-spam-status",
-		};
-	}
-
-	const halOne = lines.find(
-		(l) => l.key.toLowerCase() === "x-halone-spam-probability",
-	);
-	if (halOne) {
-		const value = stripHeaderName(halOne.line).trim();
-		return {
-			classified: parseFloat(value) > 0.5,
-			score: value,
-			source: "x-halone-spam-probability",
-		};
-	}
-
-	return null;
-};
-
-/**
- * Returns true when a List-Unsubscribe header is present, indicating a
- * bulk/mailing-list sender.
- */
-export const extractHasListUnsubscribe = (parsed: ParsedMail): boolean =>
-	parsed.headerLines.some((l) => l.key.toLowerCase() === "list-unsubscribe");
-
-const matchesAutoSubmitted = (headers: Headers): boolean => {
-	const value = readStringHeader(headers, "auto-submitted");
-	if (!value) return false;
-	const normalized = value.split(";")[0].trim().toLowerCase();
-	return normalized === "auto-generated" || normalized === "auto-replied";
-};
-
-const matchesPrecedence = (headers: Headers): boolean => {
-	const value = readStringHeader(headers, "precedence");
-	if (!value) return false;
-	const normalized = value.trim().toLowerCase();
-	return (
-		normalized === "bulk" || normalized === "list" || normalized === "junk"
-	);
-};
-
-const hasCalendarPart = (attachments: Attachment[] | undefined): boolean => {
-	if (!attachments) return false;
-	for (const att of attachments) {
-		if ((att.contentType ?? "").toLowerCase().startsWith("text/calendar")) {
-			return true;
-		}
-	}
-	return false;
-};
-
-/**
- * Whether the sender is a machine that does not read replies — a no-reply
- * local-part, or a header only bulk/notification infrastructure sets. This is
- * what routes platform notifications (npm, CI, password resets) to `automated`
- * instead of the `personal` fallback; they carry no list or bulk headers.
- */
-const isMachineSender = (parsed: ParsedMail, lines: HeaderLines): boolean => {
-	if (hasMachineHeader(lines.map((line) => line.key))) return true;
-	const localPart = getFromLocalPart(parsed);
-	return localPart !== null && isMachineLocalPart(localPart);
-};
-
-const getFromLocalPart = (parsed: ParsedMail): string | null => {
-	const address = parsed.from?.value?.[0]?.address;
-	if (!address) return null;
-	const at = address.lastIndexOf("@");
-	if (at <= 0) return null;
-	return address.slice(0, at);
-};
-
-const getFromDomain = (parsed: ParsedMail): string | null => {
-	const from = parsed.from;
-	if (!from || !from.value || from.value.length === 0) return null;
-	const address = from.value[0]?.address;
-	if (!address) return null;
-	const at = address.lastIndexOf("@");
-	if (at < 0 || at === address.length - 1) return null;
-	return address.slice(at + 1).toLowerCase();
-};
-
-const domainMatches = (
-	domain: string,
-	allowList: readonly string[],
-): boolean => {
-	for (const entry of allowList) {
-		if (domain === entry || domain.endsWith(`.${entry}`)) return true;
-	}
-	return false;
-};
-
-/**
- * Whether a signing domain aligns with the From domain: equal, or one is a
- * subdomain of the other (parent/child). A legitimate re-mailer signing under
- * a subdomain is not suspicious, so either direction counts as aligned.
- */
-const domainsAligned = (signingDomain: string, fromDomain: string): boolean =>
-	signingDomain === fromDomain ||
-	fromDomain.endsWith(`.${signingDomain}`) ||
-	signingDomain.endsWith(`.${fromDomain}`);
-
-/**
- * Pick the domain to report out of a list of candidate signing domains: the
- * first one aligned with the From domain, so a legitimate signature is never
- * shadowed by an earlier unrelated one. When none align, the first is
- * reported as the mismatching evidence — the UI can then show "signed by
- * relay.example.net, claims example.com". Shared by the category heuristic
- * (rule 9, which reads only `.mismatch`) and the authenticity extractor
- * (which also reads `.domain`), so the two can never disagree.
- */
-const pickAlignedOrFirstMismatch = (
-	domains: string[],
-	fromDomain: string,
-): { mismatch: boolean; domain: string | null } => {
-	let firstMismatching: string | null = null;
-	for (const d of domains) {
-		if (domainsAligned(d, fromDomain)) return { mismatch: false, domain: d };
-		if (!firstMismatching) firstMismatching = d;
-	}
-	return { mismatch: firstMismatching !== null, domain: firstMismatching };
-};
-
-const extractDkimDomains = (headers: Headers, lines: HeaderLines): string[] => {
-	const domains: string[] = [];
-
-	const raw = headers.get("dkim-signature");
-	const structured = collectStructured(raw);
-	for (const s of structured) {
-		const d = s.params?.d;
-		if (typeof d === "string") domains.push(d.trim().toLowerCase());
-	}
-
-	for (const line of lines) {
-		if (line.key.toLowerCase() !== "dkim-signature") continue;
-		const value = stripHeaderName(line.line);
-		const match = value.match(/(?:^|;)\s*d\s*=\s*([^;\s]+)/i);
-		if (match) {
-			const candidate = match[1].trim().toLowerCase();
-			if (!domains.includes(candidate)) domains.push(candidate);
-		}
-	}
-	return domains;
-};
-
-const collectStructured = (value: unknown): StructuredHeader[] => {
-	if (!value) return [];
-	if (Array.isArray(value)) {
-		const out: StructuredHeader[] = [];
-		for (const v of value) {
-			if (isStructuredHeader(v)) out.push(v);
-		}
-		return out;
-	}
-	if (isStructuredHeader(value)) return [value];
-	return [];
-};
-
-const isStructuredHeader = (v: unknown): v is StructuredHeader => {
-	if (!v || typeof v !== "object") return false;
-	return (
-		"value" in v &&
-		typeof (v as { value: unknown }).value === "string" &&
-		"params" in v &&
-		typeof (v as { params: unknown }).params === "object"
-	);
-};
-
-const stripHeaderName = (line: string): string => {
-	const idx = line.indexOf(":");
-	return idx >= 0 ? line.slice(idx + 1).trim() : line;
-};
-
-const hasHeaderLine = (lines: HeaderLines, name: string): boolean => {
-	const lc = name.toLowerCase();
-	for (const line of lines) {
-		if (line.key.toLowerCase() === lc) return true;
-	}
-	return false;
-};
-
-const readStringHeader = (
-	headers: Headers,
-	key: string,
-): string | undefined => {
-	const value = headers.get(key);
-	if (typeof value === "string") return value;
-	if (Array.isArray(value)) {
-		const first = value[0];
-		if (typeof first === "string") return first;
-	}
-	return undefined;
-};