@remit/mailbox-service 0.0.111 → 0.0.113

package/src/heuristics/senderMismatch.ts

@@ -1,259 +0,0 @@
-import { DisplayNameCorrespondence } from "@remit/domain-enums";
-import type { ParsedMail } from "mailparser";
-import { getDomain, parse as parseHost } from "tldts";
-import type { FolderPlacement } from "./classifyPlacement.js";
-
-type CorrespondenceValue =
-	(typeof DisplayNameCorrespondence)[keyof typeof DisplayNameCorrespondence];
-
-/**
- * The two signals that survive a passing SPF/DKIM/DMARC check: a display name
- * that belongs to nobody at the sending domain, and body links that leave it.
- *
- * Both are deliberately aggressive, and both are computed only for mail the
- * provider's own filter already called spam or that sits in the Junk folder
- * (see {@link extractSenderMismatch}).
- * On a shared-tenant host — a free Atlassian, Salesforce or Zendesk instance —
- * the verified subdomain is chosen by whoever signed up, so a passing signature
- * proves the domain and nothing about the identity the message claims.
- */
-export interface SenderMismatchSignals {
-	displayNameCorrespondence?: CorrespondenceValue;
-	offDomainLinkDomains?: string[];
-}
-
-export interface SenderMismatchContext {
-	fromDomain: string;
-	providerSpamClassified: boolean;
-	placement: FolderPlacement;
-	/**
-	 * List-Unsubscribe present. Bulk mail routinely shows a brand name over an
-	 * ESP's sending domain, so the display-name comparison says nothing there and
-	 * is not made.
-	 */
-	bulkSender: boolean;
-}
-
-const MAX_LINK_DOMAINS = 10;
-
-const normalize = (value: string): string =>
-	value.toLowerCase().replace(/[^a-z0-9]/g, "");
-
-/**
- * The parts of a host a display name could honestly correspond to: the
- * registrable name without its public suffix, every subdomain label, and the
- * registrable domain run together. The suffix itself is dropped — `net` and
- * `com` sit inside half the brand names in existence.
- */
-const correspondenceCandidates = (host: string): string[] => {
-	const parsed = parseHost(host);
-	const labels = (parsed.subdomain ?? "").split(".");
-	if (parsed.domainWithoutSuffix !== null) {
-		labels.push(parsed.domainWithoutSuffix);
-	} else {
-		labels.push(...host.split(".").slice(0, -1));
-	}
-	if (parsed.domain !== null) labels.push(parsed.domain);
-	return labels.map(normalize).filter((label) => label.length >= 3);
-};
-
-const words = (displayName: string): string[] =>
-	displayName
-		.toLowerCase()
-		.split(/[^a-z0-9]+/)
-		.filter((word) => word.length >= 3);
-
-const editDistance = (a: string, b: string): number => {
-	let previous = Array.from({ length: b.length + 1 }, (_, i) => i);
-	for (let i = 1; i <= a.length; i++) {
-		const current = [i];
-		for (let j = 1; j <= b.length; j++) {
-			current[j] = Math.min(
-				previous[j] + 1,
-				current[j - 1] + 1,
-				previous[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1),
-			);
-		}
-		previous = current;
-	}
-	return previous[b.length];
-};
-
-const ADDRESS_SHAPE = /[^\s<>@,;:"]+@([a-z0-9-]+(?:\.[a-z0-9-]+)+)/i;
-
-const registrableDomain = (host: string): string =>
-	getDomain(host) ?? host.toLowerCase();
-
-/**
- * The registrable domain of the address a display name spells out, or
- * `undefined` when it holds an `@` without an address behind it — `Bob @ Acme`
- * asserts no address and is left to the ordinary comparison.
- */
-const claimedAddressDomain = (displayName: string): string | undefined => {
-	const host = ADDRESS_SHAPE.exec(displayName)?.[1];
-	if (host === undefined) return undefined;
-	return registrableDomain(host);
-};
-
-/**
- * The distance a name of this length may be from a domain label and still be
- * read as an imitation of it. Tight on purpose: an unrelated brand name and an
- * ESP's domain are always far apart, so distance alone must never decide.
- */
-const lookalikeThreshold = (length: number): number => {
-	if (length < 5) return 0;
-	if (length < 9) return 1;
-	return 2;
-};
-
-/**
- * Whether the From display name corresponds to the From domain.
- *
- * Containment decides: the normalised name, or any word of it, containing an
- * entire domain candidate. `GitHub` contains the label `github` from
- * `notifications.github.com`; `InfoMedics` contains no label of
- * `serviceupdatebank.atlassian.net`.
- *
- * Only that direction counts — a domain candidate containing the (shorter)
- * name does not. `ING Fraudedesk` is not a match for `secure-ing-verify.tk`
- * just because the three-letter word "ing" sits inside "secureingverify":
- * that is a coincidental substring of a longer label the domain owner chose,
- * not a domain that names the brand. The direction this drops is exactly the
- * one a short, valuable brand name is deliberately embedded into a longer,
- * unrelated-looking domain to exploit — the live Dutch-bank shape this was
- * fixed against (`ING`). A real short brand over its own domain (`ING` /
- * `ing.nl`) still matches: name and label are then equal, and equality
- * satisfies containment in either direction.
- *
- * A bounded edit distance is the secondary test, and only reaches names that
- * nearly match a label — `InfoMedics` against `1nfomedics.nl`. It cannot promote
- * an unrelated name on its own.
- *
- * A name that spells out an address is decided before any of that. Over the
- * sending domain it claims nothing — `billing@` shown over
- * `serviceupdatebank.atlassian.net` names the same party the envelope does.
- * Over any other registrable domain it is the strongest non-correspondence
- * signal there is: the message asserts, in the one field the recipient reads,
- * that it comes from an address that cannot have sent it.
- */
-export const classifyDisplayNameCorrespondence = (
-	displayName: string | undefined,
-	fromDomain: string,
-): CorrespondenceValue => {
-	const raw = (displayName ?? "").trim();
-	if (raw === "") return DisplayNameCorrespondence.NoClaim;
-
-	const claimed = claimedAddressDomain(raw);
-	if (claimed !== undefined) {
-		if (claimed === registrableDomain(fromDomain)) {
-			return DisplayNameCorrespondence.NoClaim;
-		}
-		return DisplayNameCorrespondence.ForeignAddress;
-	}
-
-	const name = normalize(raw);
-	if (name.length < 3) return DisplayNameCorrespondence.NoClaim;
-
-	const candidates = correspondenceCandidates(fromDomain);
-	if (candidates.length === 0) return DisplayNameCorrespondence.NoClaim;
-
-	const terms = [name, ...words(raw)];
-	for (const term of terms) {
-		for (const candidate of candidates) {
-			if (term.includes(candidate)) {
-				return DisplayNameCorrespondence.Corresponds;
-			}
-		}
-	}
-
-	for (const candidate of candidates) {
-		const threshold = lookalikeThreshold(
-			Math.min(name.length, candidate.length),
-		);
-		if (threshold === 0) continue;
-		if (Math.abs(name.length - candidate.length) > threshold) continue;
-		if (editDistance(name, candidate) <= threshold) {
-			return DisplayNameCorrespondence.Lookalike;
-		}
-	}
-
-	return DisplayNameCorrespondence.Unrelated;
-};
-
-const hrefsFromHtml = (html: string): string[] => {
-	const out: string[] = [];
-	const pattern = /href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
-	let match = pattern.exec(html);
-	while (match !== null) {
-		out.push(match[1] ?? match[2] ?? match[3] ?? "");
-		match = pattern.exec(html);
-	}
-	return out;
-};
-
-const urlsFromText = (text: string): string[] =>
-	text.match(/https?:\/\/[^\s<>"')\]]+/gi) ?? [];
-
-/**
- * Registrable domains the body's links point at, minus the sender's own.
- *
- * `mailto:`, `tel:`, in-page anchors and relative hrefs carry no destination
- * domain and are skipped; anything else is resolved public-suffix-aware through
- * tldts, so `co.uk` and `atlassian.net` both survive the comparison a naive dot
- * split gets wrong.
- */
-export const extractOffDomainLinkDomains = (
-	parsed: ParsedMail,
-	fromDomain: string,
-): string[] => {
-	const senderDomain = getDomain(fromDomain);
-	const html = typeof parsed.html === "string" ? parsed.html : "";
-	const text = typeof parsed.text === "string" ? parsed.text : "";
-
-	const seen = new Set<string>();
-	const out: string[] = [];
-	for (const href of [...hrefsFromHtml(html), ...urlsFromText(text)]) {
-		const value = href.trim();
-		if (value === "" || !/^https?:\/\//i.test(value)) continue;
-		const domain = getDomain(value);
-		if (domain === null || domain === senderDomain) continue;
-		if (seen.has(domain)) continue;
-		seen.add(domain);
-		out.push(domain);
-		if (out.length === MAX_LINK_DOMAINS) break;
-	}
-	return out;
-};
-
-/**
- * Both signals, gated on a spam verdict: the provider's own, or the message
- * sitting in the Junk folder.
- *
- * The gate is what lets the checks be this aggressive: ordinary mail never
- * reaches them, so a brand name over an ESP domain or a newsletter full of
- * tracking links can never be flagged. Returns an empty object when the gate
- * does not open — the fields stay absent, meaning "not compared".
- */
-export const extractSenderMismatch = (
-	parsed: ParsedMail,
-	context: SenderMismatchContext,
-): SenderMismatchSignals => {
-	if (!context.providerSpamClassified && context.placement !== "junk") {
-		return {};
-	}
-
-	const offDomainLinkDomains = extractOffDomainLinkDomains(
-		parsed,
-		context.fromDomain,
-	);
-
-	if (context.bulkSender) return { offDomainLinkDomains };
-
-	return {
-		displayNameCorrespondence: classifyDisplayNameCorrespondence(
-			parsed.from?.value?.[0]?.name,
-			context.fromDomain,
-		),
-		offDomainLinkDomains,
-	};
-};

package/src/imapflow-connect-retry.test.ts

@@ -1,134 +0,0 @@
-/**
- * Unit tests for ImapFlowConnection.connect() retry semantics.
- *
- * Verifies the retry policy without touching a real IMAP server:
- * - Network errors (e.g. ECONNREFUSED) are retried with backoff before the
- *   classified MailConnectionError("network") is finally thrown.
- * - Auth errors are thrown immediately and NEVER retried.
- *
- * The private `attemptConnect` and `sleep` instance methods are stubbed so the
- * test exercises the loop logic directly and runs instantly.
- */
-
-import assert from "node:assert/strict";
-import { describe, it, mock } from "node:test";
-import { ImapFlowConnection } from "./imapflow-connection.js";
-import { MailConnectionError } from "./types.js";
-
-const baseConfig = {
-	host: "localhost",
-	port: 143,
-	user: "alice@example.com",
-	credentials: { kind: "password" as const, password: "s3cr3t" },
-	tls: false,
-};
-
-type WithPrivates = {
-	attemptConnect: () => Promise<void>;
-	sleep: (ms: number) => Promise<void>;
-};
-
-const makeErrnoError = (code: string): Error => {
-	const err = new Error(`connect ${code}`);
-	return Object.assign(err, { code });
-};
-
-describe("ImapFlowConnection.connect() retry policy", () => {
-	it("retries network errors (ECONNREFUSED) 3x with backoff, then throws classified network error", async () => {
-		const connection = new ImapFlowConnection(baseConfig);
-		const priv = connection as unknown as WithPrivates;
-
-		// Always fail with a network-level error.
-		const attempt = mock.method(priv, "attemptConnect", async () => {
-			throw makeErrnoError("ECONNREFUSED");
-		});
-		// Stub sleep so the backoff does not actually delay the test.
-		mock.method(priv, "sleep", async () => {});
-
-		await assert.rejects(
-			() => connection.connect(),
-			(error: unknown) => {
-				assert.ok(
-					error instanceof MailConnectionError,
-					"final error should be a MailConnectionError",
-				);
-				assert.equal(error.kind, "network");
-				// The host:port endpoint must appear in the message so a DNS /
-				// connect failure is self-describing in the logs (issue #835).
-				assert.match(error.message, /localhost:143/);
-				return true;
-			},
-		);
-
-		// 3 attempts total (the retry loop runs maxRetries times).
-		assert.equal(attempt.mock.callCount(), 3);
-	});
-
-	it("succeeds on a later attempt when a transient network error clears", async () => {
-		const connection = new ImapFlowConnection(baseConfig);
-		const priv = connection as unknown as WithPrivates;
-
-		let calls = 0;
-		const attempt = mock.method(priv, "attemptConnect", async () => {
-			calls += 1;
-			if (calls < 2) {
-				throw makeErrnoError("ECONNRESET");
-			}
-			// Second attempt succeeds.
-		});
-		mock.method(priv, "sleep", async () => {});
-
-		await connection.connect();
-
-		assert.equal(attempt.mock.callCount(), 2);
-	});
-
-	it("does NOT retry auth errors — throws immediately on attempt 1", async () => {
-		const connection = new ImapFlowConnection(baseConfig);
-		const priv = connection as unknown as WithPrivates;
-
-		const attempt = mock.method(priv, "attemptConnect", async () => {
-			throw new Error("Invalid credentials");
-		});
-		const sleep = mock.method(priv, "sleep", async () => {});
-
-		await assert.rejects(
-			() => connection.connect(),
-			(error: unknown) => {
-				assert.ok(error instanceof MailConnectionError);
-				assert.equal(error.kind, "auth");
-				return true;
-			},
-		);
-
-		// Only one attempt, no backoff sleep.
-		assert.equal(attempt.mock.callCount(), 1);
-		assert.equal(sleep.mock.callCount(), 0);
-	});
-
-	it("carries the server's own refusal, which is not always a re-auth", async () => {
-		const connection = new ImapFlowConnection(baseConfig);
-		const priv = connection as unknown as WithPrivates;
-
-		// imapflow's shape for a tagged NO: the message is its own, the server's
-		// words are on `responseText`. Signing in again fixes nothing here — the
-		// tenant has IMAP switched off — so the text has to reach the user.
-		mock.method(priv, "attemptConnect", async () => {
-			throw Object.assign(new Error("Command failed"), {
-				authenticationFailed: true,
-				responseText:
-					"LOGIN failed. User is authenticated but not connected. SmtpClientAuthentication is disabled for the Tenant.",
-			});
-		});
-
-		await assert.rejects(
-			() => connection.connect(),
-			(error: unknown) => {
-				assert.ok(error instanceof MailConnectionError);
-				assert.equal(error.kind, "auth");
-				assert.match(error.message, /SmtpClientAuthentication is disabled/);
-				return true;
-			},
-		);
-	});
-});