@remit/mailbox-service 0.0.111 → 0.0.113

package/src/filters/match.test.ts

@@ -1,429 +0,0 @@
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import type { FilterItem } from "@remit/data-ports";
-import { FilterClauseField, FilterMatchOperator } from "@remit/domain-enums";
-import {
-	buildMatchText,
-	clauseMatches,
-	cosineSimilarity,
-	type FilterMessage,
-	literalClausesMatch,
-	literalClauseTerms,
-	selectMoveWinner,
-} from "./match.js";
-
-type FilterClause = FilterItem["literalClauses"][number];
-
-const message = (overrides: Partial<FilterMessage> = {}): FilterMessage => ({
-	from: "alice@acme.example",
-	fromName: "Alice Example",
-	subject: "Q3 invoice attached",
-	text: "Please find the invoice for the quarter attached.",
-	listId: "",
-	...overrides,
-});
-
-const clause = (field: FilterClause["field"], value: string): FilterClause => ({
-	field,
-	value,
-});
-
-describe("clauseMatches", () => {
-	it("matches From against the sender address, case-insensitively", () => {
-		assert.equal(
-			clauseMatches(clause(FilterClauseField.From, "ACME.example"), message()),
-			true,
-		);
-	});
-
-	it("matches From against the sender display name", () => {
-		assert.equal(
-			clauseMatches(clause(FilterClauseField.From, "alice example"), message()),
-			true,
-		);
-	});
-
-	it("matches Subject as a substring", () => {
-		assert.equal(
-			clauseMatches(clause(FilterClauseField.Subject, "invoice"), message()),
-			true,
-		);
-		assert.equal(
-			clauseMatches(clause(FilterClauseField.Subject, "receipt"), message()),
-			false,
-		);
-	});
-
-	it("matches HasWords against subject or body", () => {
-		assert.equal(
-			clauseMatches(clause(FilterClauseField.HasWords, "quarter"), message()),
-			true,
-		);
-		assert.equal(
-			clauseMatches(clause(FilterClauseField.HasWords, "Q3"), message()),
-			true,
-		);
-	});
-
-	it("matches HasWords on the sender display name alone, as search does", () => {
-		const msg = message({
-			from: "booking@mail.example",
-			fromName: "Austrian Airlines Booking",
-			subject: "Your trip to Vienna",
-			text: "Check in opens 24 hours before departure.",
-		});
-		assert.equal(
-			clauseMatches(
-				clause(FilterClauseField.HasWords, "Austrian airlines"),
-				msg,
-			),
-			true,
-		);
-	});
-
-	it("matches HasWords regardless of word order and case", () => {
-		assert.equal(
-			clauseMatches(
-				clause(FilterClauseField.HasWords, "ALICE invoice"),
-				message(),
-			),
-			true,
-		);
-	});
-
-	it("does not match HasWords when one word is missing", () => {
-		assert.equal(
-			clauseMatches(
-				clause(FilterClauseField.HasWords, "invoice receipt"),
-				message(),
-			),
-			false,
-		);
-	});
-
-	it("never matches an empty clause value", () => {
-		assert.equal(
-			clauseMatches(clause(FilterClauseField.HasWords, "   "), message()),
-			false,
-		);
-	});
-
-	it("matches ListId exactly, never as a substring", () => {
-		const msg = message({ listId: "weekly.news.example.com" });
-		assert.equal(
-			clauseMatches(
-				clause(FilterClauseField.ListId, "weekly.news.example.com"),
-				msg,
-			),
-			true,
-		);
-		assert.equal(
-			clauseMatches(clause(FilterClauseField.ListId, "news.example.com"), msg),
-			false,
-		);
-		assert.equal(
-			clauseMatches(
-				clause(FilterClauseField.ListId, "weekly.news.example.com.other"),
-				msg,
-			),
-			false,
-		);
-	});
-
-	it("normalizes ListId brackets and case on both sides", () => {
-		const msg = message({ listId: "weekly.news.example.com" });
-		assert.equal(
-			clauseMatches(
-				clause(FilterClauseField.ListId, "<Weekly.News.Example.COM>"),
-				msg,
-			),
-			true,
-		);
-	});
-
-	it("never matches ListId on a message with no List-Id", () => {
-		assert.equal(
-			clauseMatches(
-				clause(FilterClauseField.ListId, "weekly.news.example.com"),
-				message({ listId: "" }),
-			),
-			false,
-		);
-	});
-
-	it("matches FromDomain on the registrable domain, including subdomains", () => {
-		assert.equal(
-			clauseMatches(
-				clause(FilterClauseField.FromDomain, "github.com"),
-				message({ from: "notifications@github.com" }),
-			),
-			true,
-		);
-		assert.equal(
-			clauseMatches(
-				clause(FilterClauseField.FromDomain, "github.com"),
-				message({ from: "notifications@sub.github.com" }),
-			),
-			true,
-		);
-	});
-
-	it("never matches FromDomain on a look-alike subdomain (public-suffix aware)", () => {
-		assert.equal(
-			clauseMatches(
-				clause(FilterClauseField.FromDomain, "github.com"),
-				message({ from: "attacker@github.com.evil.example" }),
-			),
-			false,
-		);
-	});
-
-	it("matches FromDomain across multi-level public suffixes", () => {
-		assert.equal(
-			clauseMatches(
-				clause(FilterClauseField.FromDomain, "example.co.uk"),
-				message({ from: "hr@mail.example.co.uk" }),
-			),
-			true,
-		);
-		assert.equal(
-			clauseMatches(
-				clause(FilterClauseField.FromDomain, "example.co.uk"),
-				message({ from: "hr@example.co.uk.evil.example" }),
-			),
-			false,
-		);
-	});
-});
-
-describe("literalClausesMatch", () => {
-	it("passes vacuously with no clauses (a purely-semantic filter)", () => {
-		assert.equal(
-			literalClausesMatch([], FilterMatchOperator.And, message()),
-			true,
-		);
-	});
-
-	it("And requires every clause to match", () => {
-		const clauses = [
-			clause(FilterClauseField.From, "acme.example"),
-			clause(FilterClauseField.Subject, "invoice"),
-		];
-		assert.equal(
-			literalClausesMatch(clauses, FilterMatchOperator.And, message()),
-			true,
-		);
-		assert.equal(
-			literalClausesMatch(
-				[...clauses, clause(FilterClauseField.Subject, "nope")],
-				FilterMatchOperator.And,
-				message(),
-			),
-			false,
-		);
-	});
-
-	it("Or requires only one clause to match", () => {
-		const clauses = [
-			clause(FilterClauseField.Subject, "nope"),
-			clause(FilterClauseField.Subject, "invoice"),
-		];
-		assert.equal(
-			literalClausesMatch(clauses, FilterMatchOperator.Or, message()),
-			true,
-		);
-		assert.equal(
-			literalClausesMatch(
-				[clause(FilterClauseField.Subject, "nope")],
-				FilterMatchOperator.Or,
-				message(),
-			),
-			false,
-		);
-	});
-});
-
-describe("cosineSimilarity", () => {
-	it("scores identical vectors as 1", () => {
-		assert.equal(cosineSimilarity([1, 2, 3], [1, 2, 3]), 1);
-	});
-
-	it("scores orthogonal vectors as 0", () => {
-		assert.equal(cosineSimilarity([1, 0], [0, 1]), 0);
-	});
-
-	it("scores a zero vector as 0 rather than dividing by zero", () => {
-		assert.equal(cosineSimilarity([0, 0], [1, 1]), 0);
-	});
-
-	it("throws on a dimension mismatch instead of scoring incomparable vectors", () => {
-		assert.throws(
-			() => cosineSimilarity([1, 2], [1, 2, 3]),
-			/dimension mismatch/,
-		);
-	});
-});
-
-describe("selectMoveWinner", () => {
-	const filter = (
-		filterId: string,
-		actionChangedAt: number,
-		ruleChangedAt = actionChangedAt,
-	): FilterItem => ({ filterId, actionChangedAt, ruleChangedAt }) as FilterItem;
-
-	it("returns undefined with no candidates", () => {
-		assert.equal(selectMoveWinner([]), undefined);
-	});
-
-	it("picks the most-recently action-changed filter", () => {
-		const winner = selectMoveWinner([
-			filter("a", 100),
-			filter("b", 300),
-			filter("c", 200),
-		]);
-		assert.equal(winner?.filterId, "b");
-	});
-
-	it("tie-breaks on filterId when actionChangedAt is identical", () => {
-		const winner = selectMoveWinner([
-			filter("a", 100),
-			filter("c", 100),
-			filter("b", 100),
-		]);
-		assert.equal(winner?.filterId, "c");
-	});
-
-	it("ignores a ruleChangedAt-only bump — extending scope/expiry alone must not promote a filter to move-winner (reader #384)", () => {
-		// filter A's predicate/action last changed at 10:00 (actionChangedAt).
-		// filter B was created at 09:00 and never had its predicate/action
-		// touched since, but a user extended its expiry at 11:00 — bumping only
-		// its ruleChangedAt (RFC 034 Decision 3.2 / #294), not what it matches or
-		// does. B must not out-rank A.
-		const filterA = filter("filter-a", 10_00);
-		const filterB = filter("filter-b", 9_00, 11_00);
-
-		const winner = selectMoveWinner([filterA, filterB]);
-		assert.equal(
-			winner?.filterId,
-			"filter-a",
-			"A still wins: its actionChangedAt (10:00) beats B's (09:00), even though B's ruleChangedAt (11:00) is later",
-		);
-	});
-});
-
-describe("buildMatchText", () => {
-	it("joins subject and body and bounds the length", () => {
-		const text = buildMatchText(message({ subject: "hello", text: "world" }));
-		assert.equal(text, "hello\nworld");
-	});
-
-	it("caps at 512 characters", () => {
-		const text = buildMatchText(
-			message({ subject: "", text: "x".repeat(1000) }),
-		);
-		assert.equal(text.length, 512);
-	});
-});
-
-describe("literalClauseTerms (reader #459)", () => {
-	it("narrows From and Subject onto the columns that carry them", () => {
-		assert.deepEqual(
-			literalClauseTerms(
-				[
-					clause(FilterClauseField.From, " alice@acme.example "),
-					clause(FilterClauseField.Subject, "invoice"),
-				],
-				FilterMatchOperator.And,
-			),
-			{
-				terms: [
-					{ field: "sender", contains: "alice@acme.example" },
-					{ field: "subject", contains: "invoice" },
-				],
-				operator: "and",
-			},
-		);
-	});
-
-	it("narrows FromDomain to the registrable domain, which the sender always contains", () => {
-		const narrowing = literalClauseTerms(
-			[clause(FilterClauseField.FromDomain, "billing@mail.acme.example")],
-			FilterMatchOperator.And,
-		);
-
-		assert.deepEqual(narrowing?.terms, [
-			{ field: "sender", contains: "acme.example" },
-		]);
-		assert.equal(
-			clauseMatches(
-				clause(FilterClauseField.FromDomain, "billing@mail.acme.example"),
-				message(),
-			),
-			true,
-		);
-	});
-
-	it("narrows ListId to its canonical form", () => {
-		assert.deepEqual(
-			literalClauseTerms(
-				[clause(FilterClauseField.ListId, "News <Weekly.News.Example.com>")],
-				FilterMatchOperator.And,
-			)?.terms,
-			[{ field: "listId", contains: "weekly.news.example.com" }],
-		);
-	});
-
-	it("returns null for clauses nothing can satisfy, so no query is worth running", () => {
-		assert.equal(
-			literalClauseTerms(
-				[clause(FilterClauseField.From, "   ")],
-				FilterMatchOperator.And,
-			),
-			null,
-		);
-		assert.equal(
-			literalClauseTerms(
-				[clause(FilterClauseField.FromDomain, "not a domain")],
-				FilterMatchOperator.Or,
-			),
-			null,
-		);
-	});
-
-	it("drops an unsatisfiable clause from an Or instead of losing the rest", () => {
-		assert.deepEqual(
-			literalClauseTerms(
-				[
-					clause(FilterClauseField.From, ""),
-					clause(FilterClauseField.Subject, "invoice"),
-				],
-				FilterMatchOperator.Or,
-			),
-			{ terms: [{ field: "subject", contains: "invoice" }], operator: "or" },
-		);
-	});
-
-	it("narrows nothing when an Or reaches a clause no column carries", () => {
-		assert.deepEqual(
-			literalClauseTerms(
-				[
-					clause(FilterClauseField.HasWords, "quarter"),
-					clause(FilterClauseField.Subject, "invoice"),
-				],
-				FilterMatchOperator.Or,
-			),
-			{ terms: [], operator: "or" },
-		);
-	});
-
-	it("narrows nothing for no clauses, matching the vacuous pass of literalClausesMatch", () => {
-		assert.deepEqual(literalClauseTerms([], FilterMatchOperator.And), {
-			terms: [],
-			operator: "and",
-		});
-		assert.equal(
-			literalClausesMatch([], FilterMatchOperator.And, message()),
-			true,
-		);
-	});
-});

package/src/filters/match.ts

@@ -1,274 +0,0 @@
-import {
-	FILTER_NO_ACTION,
-	type FilterItem,
-	type ThreadMessageFieldTerm,
-} from "@remit/data-ports";
-import { FilterClauseField, FilterMatchOperator } from "@remit/domain-enums";
-import { getDomain } from "tldts";
-import { normalizeListId } from "./list-id.js";
-
-type FilterClause = FilterItem["literalClauses"][number];
-
-/**
- * The `"None"` sentinel a filter's action fields carry when that action is
- * absent (RFC 034 Decision 3.1) — `actionLabelId`/`actionMailboxId` are never
- * empty/optional strings, so a missing action is this exact value, never `""`.
- */
-export const NO_ACTION = FILTER_NO_ACTION;
-
-/**
- * Default cosine cut-off for a semantic-anchor match (RFC 031 "the semantic
- * anchor evaluates as a kNN threshold against the new message's embedding").
- * Tunable per pipeline via {@link FilterConfig.similarityThreshold}.
- */
-export const DEFAULT_SEMANTIC_MATCH_THRESHOLD = 0.75;
-
-/**
- * Bound on the text embedded for a semantic match — the same 512-char budget
- * `buildTextPreview` applies to chunk vectors and `FilterAnchor.anchorSourceText`
- * persists, so the candidate side of the comparison is derived on the same
- * footing as the anchor side.
- */
-const MATCH_TEXT_LIMIT = 512;
-
-/**
- * The message fields a filter evaluates against — the literal-clause targets
- * (from / subject / body) plus the text embedded for a semantic anchor. A plain
- * value object so the matcher stays independent of the `ParsedMail` shape and is
- * trivially constructed in a test.
- */
-export interface FilterMessage {
-	from: string;
-	fromName: string;
-	subject: string;
-	text: string;
-	/** Normalized `List-Id` header value (see `normalizeListId`); `""` when absent. */
-	listId: string;
-}
-
-const includesFold = (haystack: string, needle: string): boolean =>
-	haystack.toLowerCase().includes(needle.toLowerCase());
-
-/**
- * Whether every whitespace-separated word of `value` appears somewhere in the
- * sender address, sender name, subject or body — the shape free-text search
- * matches with (`buildSearchConditions`), so a filter made from a search keeps
- * the mail that search showed.
- */
-const hasEveryWord = (value: string, msg: FilterMessage): boolean => {
-	const fields = [msg.from, msg.fromName, msg.subject, msg.text];
-	return value
-		.split(/\s+/)
-		.filter(Boolean)
-		.every((word) => fields.some((field) => includesFold(field, word)));
-};
-
-const hostOf = (address: string): string => {
-	const at = address.lastIndexOf("@");
-	return at >= 0 ? address.slice(at + 1) : address;
-};
-
-/**
- * The registrable, public-suffix-aware domain of an email address or host, or
- * `null` when none resolves. `getDomain` folds case and applies the ICANN
- * suffix list, so `github.com.evil.example` yields `evil.example`, never
- * `github.com` — a `FromDomain` clause cannot be spoofed by a crafted subdomain.
- */
-const registrableDomain = (addressOrHost: string): string | null =>
-	getDomain(hostOf(addressOrHost.trim()));
-
-/**
- * Whether one literal clause matches the message. From matches against the
- * sender address and display name; Subject against the subject; HasWords
- * requires every word somewhere in sender, subject or body; ListId against the exact normalized `List-Id`; FromDomain
- * against the sender's registrable domain. An empty clause value never matches.
- */
-export const clauseMatches = (
-	clause: FilterClause,
-	msg: FilterMessage,
-): boolean => {
-	const value = clause.value.trim();
-	if (value === "") return false;
-	switch (clause.field) {
-		case FilterClauseField.From:
-			return includesFold(msg.from, value) || includesFold(msg.fromName, value);
-		case FilterClauseField.Subject:
-			return includesFold(msg.subject, value);
-		case FilterClauseField.HasWords:
-			return hasEveryWord(value, msg);
-		case FilterClauseField.ListId: {
-			const target = normalizeListId(value);
-			return target !== "" && normalizeListId(msg.listId) === target;
-		}
-		case FilterClauseField.FromDomain: {
-			const target = registrableDomain(value);
-			if (target === null) return false;
-			return registrableDomain(msg.from) === target;
-		}
-		default:
-			return false;
-	}
-};
-
-/**
- * Whether the literal clauses match under their combine operator. Empty clauses
- * carry no literal constraint and pass vacuously — a purely-semantic filter (no
- * clauses, only an anchor) is gated by its anchor alone, not by this.
- */
-export const literalClausesMatch = (
-	clauses: readonly FilterClause[],
-	operator: FilterItem["matchOperator"],
-	msg: FilterMessage,
-): boolean => {
-	if (clauses.length === 0) return true;
-	return operator === FilterMatchOperator.Or
-		? clauses.some((clause) => clauseMatches(clause, msg))
-		: clauses.every((clause) => clauseMatches(clause, msg));
-};
-
-/**
- * What a store can be asked for on behalf of one clause: a column term, or the
- * two answers a term cannot express — a clause nothing can satisfy, and a
- * clause selecting on something no ThreadMessage column carries.
- */
-type ClauseNarrowing =
-	| { kind: "Term"; term: ThreadMessageFieldTerm }
-	| { kind: "NeverMatches" }
-	| { kind: "Unnarrowable" };
-
-const clauseNarrowing = (clause: FilterClause): ClauseNarrowing => {
-	const value = clause.value.trim();
-	if (value === "") return { kind: "NeverMatches" };
-	switch (clause.field) {
-		case FilterClauseField.From:
-			return { kind: "Term", term: { field: "sender", contains: value } };
-		case FilterClauseField.Subject:
-			return { kind: "Term", term: { field: "subject", contains: value } };
-		case FilterClauseField.ListId: {
-			const target = normalizeListId(value);
-			if (target === "") return { kind: "NeverMatches" };
-			return { kind: "Term", term: { field: "listId", contains: target } };
-		}
-		case FilterClauseField.FromDomain: {
-			const target = registrableDomain(value);
-			if (target === null) return { kind: "NeverMatches" };
-			return { kind: "Term", term: { field: "sender", contains: target } };
-		}
-		default:
-			return { kind: "Unnarrowable" };
-	}
-};
-
-/**
- * The terms to query with, and how to combine them. Empty terms narrow nothing
- * — the caller reads the whole corpus and refines it.
- */
-export interface LiteralClauseNarrowing {
-	terms: ThreadMessageFieldTerm[];
-	operator: "and" | "or";
-}
-
-/**
- * The store-side half of {@link literalClausesMatch}: the column terms a query
- * can evaluate on behalf of these clauses, or `null` when they can match
- * nothing at all and there is no query worth running.
- *
- * A term is deliberately WIDER than the clause it stands for — a `FromDomain`
- * clause narrows to the registrable domain appearing anywhere in the sender,
- * so `github.com.evil.example` still comes back — and a `HasWords` clause
- * cannot be narrowed at all, since no row column carries the body. The rows a
- * term selects are candidates; `literalClausesMatch` still decides. Pushing the
- * terms down is what makes the caller's bound a bound on RESULTS: filtering a
- * date-ordered page instead answers "matches among the newest N messages"
- * (#459).
- */
-export const literalClauseTerms = (
-	clauses: readonly FilterClause[],
-	operator: FilterItem["matchOperator"],
-): LiteralClauseNarrowing | null => {
-	const combine = operator === FilterMatchOperator.Or ? "or" : "and";
-	if (clauses.length === 0) return { terms: [], operator: combine };
-	const narrowings = clauses.map(clauseNarrowing);
-	const terms = narrowings.flatMap((narrowing) =>
-		narrowing.kind === "Term" ? [narrowing.term] : [],
-	);
-	if (combine === "and") {
-		if (narrowings.some((narrowing) => narrowing.kind === "NeverMatches")) {
-			return null;
-		}
-		return { terms, operator: "and" };
-	}
-	if (narrowings.every((narrowing) => narrowing.kind === "NeverMatches")) {
-		return null;
-	}
-	if (narrowings.some((narrowing) => narrowing.kind === "Unnarrowable")) {
-		return { terms: [], operator: "or" };
-	}
-	return { terms, operator: "or" };
-};
-
-/**
- * Cosine similarity of two equal-length vectors. Throws on a dimension mismatch
- * rather than silently scoring incomparable vectors — a mismatch means a stale
- * anchor embedded under a different model (`anchorEmbeddingId`), which is a
- * migration concern, not a match to guess at.
- */
-export const cosineSimilarity = (
-	a: readonly number[],
-	b: readonly number[],
-): number => {
-	if (a.length !== b.length) {
-		throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
-	}
-	let dot = 0;
-	let normA = 0;
-	let normB = 0;
-	for (let i = 0; i < a.length; i++) {
-		dot += a[i] * b[i];
-		normA += a[i] * a[i];
-		normB += b[i] * b[i];
-	}
-	if (normA === 0 || normB === 0) return 0;
-	return dot / (Math.sqrt(normA) * Math.sqrt(normB));
-};
-
-/**
- * The move a message ends in when several filters matched: the filter whose
- * predicate or action was most recently *changed* wins (RFC 034 Decision 3.2),
- * tie-broken on `filterId` for the unreachable identical-timestamp case.
- * `actionChangedAt` — not `ruleChangedAt` — is the signal: `ruleChangedAt` also
- * bumps on a scope/expiry-only edit (reader #266), which changes a filter's
- * lifecycle, not what it matches or does, and must not reorder exclusive-move
- * precedence (reader #384). Nor is it `updatedAt`, so a cosmetic rename never
- * flips an exclusive move either.
- */
-export const selectMoveWinner = (
-	candidates: readonly FilterItem[],
-): FilterItem | undefined => {
-	let winner: FilterItem | undefined;
-	for (const candidate of candidates) {
-		if (!winner) {
-			winner = candidate;
-			continue;
-		}
-		if (candidate.actionChangedAt > winner.actionChangedAt) {
-			winner = candidate;
-			continue;
-		}
-		if (
-			candidate.actionChangedAt === winner.actionChangedAt &&
-			candidate.filterId > winner.filterId
-		) {
-			winner = candidate;
-		}
-	}
-	return winner;
-};
-
-/**
- * The candidate text embedded once for a semantic match — subject then body,
- * bounded to the anchor's `anchorSourceText` budget so both sides of the cosine
- * comparison are derived from comparable inputs.
- */
-export const buildMatchText = (msg: FilterMessage): string =>
-	`${msg.subject}\n${msg.text}`.slice(0, MATCH_TEXT_LIMIT);