@remit/mailbox-service 0.0.112 → 0.0.113

package/src/filters/match.ts

@@ -1,225 +0,0 @@
-import {
-	FILTER_NO_ACTION,
-	type FilterItem,
-	type ThreadMessageFieldTerm,
-} from "@remit/data-ports";
-import { FilterClauseField, FilterMatchOperator } from "@remit/domain-enums";
-import { getDomain } from "tldts";
-import { normalizeListId } from "./list-id.js";
-
-type FilterClause = FilterItem["literalClauses"][number];
-
-/**
- * The `"None"` sentinel a filter's action fields carry when that action is
- * absent (RFC 034 Decision 3.1) — `actionLabelId`/`actionMailboxId` are never
- * empty/optional strings, so a missing action is this exact value, never `""`.
- */
-export const NO_ACTION = FILTER_NO_ACTION;
-
-/**
- * The message fields a filter evaluates against — the literal-clause targets
- * (from / subject / body / list). A plain value object so the matcher stays independent of the `ParsedMail` shape and is
- * trivially constructed in a test.
- */
-export interface FilterMessage {
-	from: string;
-	fromName: string;
-	subject: string;
-	text: string;
-	/** Normalized `List-Id` header value (see `normalizeListId`); `""` when absent. */
-	listId: string;
-}
-
-const includesFold = (haystack: string, needle: string): boolean =>
-	haystack.toLowerCase().includes(needle.toLowerCase());
-
-/**
- * Whether every whitespace-separated word of `value` appears somewhere in the
- * sender address, sender name, subject or body — the shape free-text search
- * matches with (`buildSearchConditions`), so a filter made from a search keeps
- * the mail that search showed.
- */
-const hasEveryWord = (value: string, msg: FilterMessage): boolean => {
-	const fields = [msg.from, msg.fromName, msg.subject, msg.text];
-	return value
-		.split(/\s+/)
-		.filter(Boolean)
-		.every((word) => fields.some((field) => includesFold(field, word)));
-};
-
-const hostOf = (address: string): string => {
-	const at = address.lastIndexOf("@");
-	return at >= 0 ? address.slice(at + 1) : address;
-};
-
-/**
- * The registrable, public-suffix-aware domain of an email address or host, or
- * `null` when none resolves. `getDomain` folds case and applies the ICANN
- * suffix list, so `github.com.evil.example` yields `evil.example`, never
- * `github.com` — a `FromDomain` clause cannot be spoofed by a crafted subdomain.
- */
-const registrableDomain = (addressOrHost: string): string | null =>
-	getDomain(hostOf(addressOrHost.trim()));
-
-/**
- * Whether one literal clause matches the message. From matches against the
- * sender address and display name; Subject against the subject; HasWords
- * requires every word somewhere in sender, subject or body; ListId against the exact normalized `List-Id`; FromDomain
- * against the sender's registrable domain. An empty clause value never matches.
- */
-export const clauseMatches = (
-	clause: FilterClause,
-	msg: FilterMessage,
-): boolean => {
-	const value = clause.value.trim();
-	if (value === "") return false;
-	switch (clause.field) {
-		case FilterClauseField.From:
-			return includesFold(msg.from, value) || includesFold(msg.fromName, value);
-		case FilterClauseField.Subject:
-			return includesFold(msg.subject, value);
-		case FilterClauseField.HasWords:
-			return hasEveryWord(value, msg);
-		case FilterClauseField.ListId: {
-			const target = normalizeListId(value);
-			return target !== "" && normalizeListId(msg.listId) === target;
-		}
-		case FilterClauseField.FromDomain: {
-			const target = registrableDomain(value);
-			if (target === null) return false;
-			return registrableDomain(msg.from) === target;
-		}
-		default:
-			return false;
-	}
-};
-
-/**
- * Whether the literal clauses match under their combine operator. Empty clauses
- * carry no literal constraint and pass vacuously; a caller that needs a
- * predicate checks for clauses itself.
- */
-export const literalClausesMatch = (
-	clauses: readonly FilterClause[],
-	operator: FilterItem["matchOperator"],
-	msg: FilterMessage,
-): boolean => {
-	if (clauses.length === 0) return true;
-	return operator === FilterMatchOperator.Or
-		? clauses.some((clause) => clauseMatches(clause, msg))
-		: clauses.every((clause) => clauseMatches(clause, msg));
-};
-
-/**
- * What a store can be asked for on behalf of one clause: a column term, or the
- * two answers a term cannot express — a clause nothing can satisfy, and a
- * clause selecting on something no ThreadMessage column carries.
- */
-type ClauseNarrowing =
-	| { kind: "Term"; term: ThreadMessageFieldTerm }
-	| { kind: "NeverMatches" }
-	| { kind: "Unnarrowable" };
-
-const clauseNarrowing = (clause: FilterClause): ClauseNarrowing => {
-	const value = clause.value.trim();
-	if (value === "") return { kind: "NeverMatches" };
-	switch (clause.field) {
-		case FilterClauseField.From:
-			return { kind: "Term", term: { field: "sender", contains: value } };
-		case FilterClauseField.Subject:
-			return { kind: "Term", term: { field: "subject", contains: value } };
-		case FilterClauseField.ListId: {
-			const target = normalizeListId(value);
-			if (target === "") return { kind: "NeverMatches" };
-			return { kind: "Term", term: { field: "listId", contains: target } };
-		}
-		case FilterClauseField.FromDomain: {
-			const target = registrableDomain(value);
-			if (target === null) return { kind: "NeverMatches" };
-			return { kind: "Term", term: { field: "sender", contains: target } };
-		}
-		default:
-			return { kind: "Unnarrowable" };
-	}
-};
-
-/**
- * The terms to query with, and how to combine them. Empty terms narrow nothing
- * — the caller reads the whole corpus and refines it.
- */
-export interface LiteralClauseNarrowing {
-	terms: ThreadMessageFieldTerm[];
-	operator: "and" | "or";
-}
-
-/**
- * The store-side half of {@link literalClausesMatch}: the column terms a query
- * can evaluate on behalf of these clauses, or `null` when they can match
- * nothing at all and there is no query worth running.
- *
- * A term is deliberately WIDER than the clause it stands for — a `FromDomain`
- * clause narrows to the registrable domain appearing anywhere in the sender,
- * so `github.com.evil.example` still comes back — and a `HasWords` clause
- * cannot be narrowed at all, since no row column carries the body. The rows a
- * term selects are candidates; `literalClausesMatch` still decides. Pushing the
- * terms down is what makes the caller's bound a bound on RESULTS: filtering a
- * date-ordered page instead answers "matches among the newest N messages"
- * (#459).
- */
-export const literalClauseTerms = (
-	clauses: readonly FilterClause[],
-	operator: FilterItem["matchOperator"],
-): LiteralClauseNarrowing | null => {
-	const combine = operator === FilterMatchOperator.Or ? "or" : "and";
-	if (clauses.length === 0) return { terms: [], operator: combine };
-	const narrowings = clauses.map(clauseNarrowing);
-	const terms = narrowings.flatMap((narrowing) =>
-		narrowing.kind === "Term" ? [narrowing.term] : [],
-	);
-	if (combine === "and") {
-		if (narrowings.some((narrowing) => narrowing.kind === "NeverMatches")) {
-			return null;
-		}
-		return { terms, operator: "and" };
-	}
-	if (narrowings.every((narrowing) => narrowing.kind === "NeverMatches")) {
-		return null;
-	}
-	if (narrowings.some((narrowing) => narrowing.kind === "Unnarrowable")) {
-		return { terms: [], operator: "or" };
-	}
-	return { terms, operator: "or" };
-};
-
-/**
- * The move a message ends in when several filters matched: the filter whose
- * predicate or action was most recently *changed* wins (RFC 034 Decision 3.2),
- * tie-broken on `filterId` for the unreachable identical-timestamp case.
- * `actionChangedAt` — not `ruleChangedAt` — is the signal: `ruleChangedAt` also
- * bumps on a scope/expiry-only edit (reader #266), which changes a filter's
- * lifecycle, not what it matches or does, and must not reorder exclusive-move
- * precedence (reader #384). Nor is it `updatedAt`, so a cosmetic rename never
- * flips an exclusive move either.
- */
-export const selectMoveWinner = (
-	candidates: readonly FilterItem[],
-): FilterItem | undefined => {
-	let winner: FilterItem | undefined;
-	for (const candidate of candidates) {
-		if (!winner) {
-			winner = candidate;
-			continue;
-		}
-		if (candidate.actionChangedAt > winner.actionChangedAt) {
-			winner = candidate;
-			continue;
-		}
-		if (
-			candidate.actionChangedAt === winner.actionChangedAt &&
-			candidate.filterId > winner.filterId
-		) {
-			winner = candidate;
-		}
-	}
-	return winner;
-};

package/src/filters/pipeline.test.ts

@@ -1,199 +0,0 @@
-import assert from "node:assert/strict";
-import { afterEach, beforeEach, describe, it, mock } from "node:test";
-import type { AccountItem, CreateFilterInput } from "@remit/data-ports";
-import {
-	AccountAuthType,
-	ConnectionState,
-	FilterClauseField,
-	FilterMatchOperator,
-	FilterScope,
-} from "@remit/domain-enums";
-import { PlacementMoveService } from "../placement-move.js";
-import {
-	openStore,
-	type Store,
-	seedAccount,
-	seedMailbox,
-} from "../test-helpers/test-store.js";
-import type { FilterMessage } from "./match.js";
-import { FilterPipeline } from "./pipeline.js";
-
-const ACCOUNT = "cfg-1";
-
-let store: Store;
-let account: AccountItem;
-let archive: string;
-let projects: string;
-
-beforeEach(async () => {
-	store = await openStore();
-	account = await seedAccount(store, { accountConfigId: ACCOUNT });
-	archive = (await seedMailbox(store, account, "Archive")).mailboxId;
-	projects = (await seedMailbox(store, account, "Projects")).mailboxId;
-});
-
-afterEach(async () => {
-	mock.timers.reset();
-	await store.close();
-});
-
-const fromGithub: FilterMessage = {
-	from: "npm@github.com",
-	fromName: "npm",
-	subject: "A new version of left-pad is available",
-	text: "body",
-	listId: "",
-};
-
-const receipt: FilterMessage = {
-	from: "billing@stripe.com",
-	fromName: "Stripe",
-	subject: "Your receipt",
-	text: "Thanks for your payment",
-	listId: "",
-};
-
-const senders = (actionMailboxId: string) =>
-	store.filter.create({
-		accountConfigId: ACCOUNT,
-		name: "rule",
-		scope: FilterScope.Standing,
-		matchOperator: FilterMatchOperator.Or,
-		literalClauses: [
-			{ field: FilterClauseField.From, value: "npm@github.com" },
-			{ field: FilterClauseField.From, value: "notifications@github.com" },
-		],
-		actionMailboxId,
-	});
-
-const pipeline = () =>
-	new FilterPipeline(
-		{
-			filterService: store.filter,
-			messageLabelService: store.messageLabel,
-			mailboxService: store.mailbox,
-			placementMoveService: new PlacementMoveService({
-				messageService: store.message,
-				threadMessageService: store.threadMessage,
-				markerService: store.placementMove,
-				addressService: store.address,
-				mailboxSpecialUseService: store.mailboxSpecialUse,
-				mailboxService: store.mailbox,
-				sqsQueueUrl: "https://sqs.test/message-mgmt",
-			}),
-		},
-		{ info: () => {}, error: () => {} },
-	);
-
-describe("FilterPipeline over literal filters", () => {
-	it("resolves the move for a matching sender", async () => {
-		const filter = await senders(archive);
-
-		const decision = await pipeline().evaluate(
-			ACCOUNT,
-			account.accountId,
-			"m-1",
-			fromGithub,
-		);
-
-		assert.deepEqual(decision.move, {
-			destinationMailboxId: archive,
-			filterId: filter.filterId,
-		});
-	});
-
-	it("resolves nothing for a sender no clause names", async () => {
-		await senders(archive);
-
-		const decision = await pipeline().evaluate(
-			ACCOUNT,
-			account.accountId,
-			"m-2",
-			{
-				...fromGithub,
-				from: "hello@stripe.com",
-			},
-		);
-
-		assert.equal(decision.move, undefined);
-		assert.deepEqual(decision.labels, []);
-	});
-
-	it("gives a contested message to the more recently changed action", async () => {
-		mock.timers.enable({ apis: ["Date"], now: 1_000_000 });
-		await senders(archive);
-		mock.timers.setTime(9_000_000);
-		const rival = await senders(projects);
-
-		const decision = await pipeline().evaluate(
-			ACCOUNT,
-			account.accountId,
-			"m-3",
-			fromGithub,
-		);
-
-		assert.deepEqual(decision.move, {
-			destinationMailboxId: projects,
-			filterId: rival.filterId,
-		});
-	});
-});
-
-describe("FilterPipeline scopes a filter to its folder's account", () => {
-	it("leaves another account's mail alone, as if the filter had not matched", async () => {
-		const sibling = await store.account.create({
-			accountConfigId: ACCOUNT,
-			username: "me-too@example.com",
-			email: "me-too@example.com",
-			authType: AccountAuthType.Password,
-			passwordHash: '{"ciphertext":"x"}',
-			imapHost: "imap.example.com",
-			imapPort: 993,
-			imapTls: true,
-			imapStartTls: false,
-			isActive: true,
-			connectionState: ConnectionState.Authenticated,
-		});
-		const travel = await seedMailbox(store, sibling, "Travel");
-		const filter = await senders(travel.mailboxId);
-
-		const mine = await pipeline().evaluate(
-			ACCOUNT,
-			account.accountId,
-			"m-1",
-			fromGithub,
-		);
-		const theirs = await pipeline().evaluate(
-			ACCOUNT,
-			sibling.accountId,
-			"m-2",
-			fromGithub,
-		);
-
-		assert.deepEqual(mine, { labels: [] });
-		assert.deepEqual(theirs.move, {
-			destinationMailboxId: travel.mailboxId,
-			filterId: filter.filterId,
-		});
-		assert.equal(
-			(await store.filter.get(ACCOUNT, filter.filterId)).state,
-			"Active",
-		);
-	});
-
-	it("turns the filter off when no account holds its folder any more", async () => {
-		const filter = await senders("mbx-gone");
-
-		const decision = await pipeline().evaluate(
-			ACCOUNT,
-			account.accountId,
-			"m-1",
-			fromGithub,
-		);
-
-		assert.deepEqual(decision, { labels: [] });
-		const stored = await store.filter.get(ACCOUNT, filter.filterId);
-		assert.equal(stored.state, "Disabled");
-		assert.equal(stored.disabledReason, "FolderMissing");
-	});
-});

package/src/filters/pipeline.ts

@@ -1,170 +0,0 @@
-import { inspect } from "node:util";
-import type {
-	FilterItem,
-	IFilterRepository,
-	IMailboxRepository,
-	IMessageLabelRepository,
-} from "@remit/data-ports";
-import { FilterDisabledReason, FilterState } from "@remit/domain-enums";
-import type { PlacementMoveService } from "../placement-move.js";
-import {
-	type FilterMessage,
-	literalClausesMatch,
-	NO_ACTION,
-	selectMoveWinner,
-} from "./match.js";
-
-export interface FilterLogger {
-	info(obj: Record<string, unknown>, msg: string): void;
-	debug?(obj: Record<string, unknown>, msg: string): void;
-	warn?(obj: Record<string, unknown>, msg: string): void;
-	error?(obj: Record<string, unknown>, msg: string): void;
-}
-
-/**
- * Dependencies for the index-time filter pass (RFC 034). Optional on
- * {@link BodySyncService} the same way {@link PlacementConfig} is — a deployment
- * without it simply runs no filter matching.
- */
-export interface FilterConfig {
-	filterService: IFilterRepository;
-	messageLabelService: IMessageLabelRepository;
-	/** Names the account a filter's folder belongs to. */
-	mailboxService: Pick<IMailboxRepository, "resolveAccountId">;
-	placementMoveService: PlacementMoveService;
-}
-
-/**
- * What the matched filters resolve to for one message: every matching filter's
- * label (additive — RFC 034 Decision 3.1), and at most one move (exclusive — the
- * most-recently-changed matching filter, Decision 3.2).
- */
-export interface FilterDecision {
-	labels: Array<{ labelId: string; filterId: string }>;
-	move?: { destinationMailboxId: string; filterId: string };
-}
-
-const EMPTY_DECISION: FilterDecision = { labels: [] };
-
-/**
- * Evaluates a synced message against the account's active filters and resolves
- * the actions to apply (RFC 034). Reads only — listing filters and refreshing
- * lazy expiry; the caller applies the
- * resulting {@link FilterDecision}.
- *
- * `evaluate` isolates every failure in this read/decision phase from body sync,
- * matching the placement-resolution precedent (#1246): a schema-drifted query
- * must never fail the surrounding message store, since the body is
- * already durable by the time this runs. The apply of the decision — label
- * writes and the move enqueue — is the caller's responsibility and is *not*
- * swallowed there, so a genuine DDB/SQS write failure still propagates and
- * requeues the message.
- */
-export class FilterPipeline {
-	constructor(
-		private readonly config: FilterConfig,
-		private readonly log: FilterLogger,
-	) {}
-
-	async evaluate(
-		accountConfigId: string,
-		accountId: string,
-		messageId: string,
-		msg: FilterMessage,
-	): Promise<FilterDecision> {
-		return this.match(accountConfigId, accountId, msg).catch(
-			(error: unknown) => {
-				this.log.error?.(
-					{
-						alert: "body_sync_filter_match_failed",
-						messageId,
-						accountConfigId,
-						errorName: (error as { name?: string })?.name,
-						error: inspect(error),
-					},
-					"Filter matching failed; body already stored, continuing without filter actions (best-effort, non-fatal)",
-				);
-				return EMPTY_DECISION;
-			},
-		);
-	}
-
-	/**
-	 * A filter sits on the account config and sees every account's mail, but its
-	 * folder is one account's: mail of another account is not this filter's to
-	 * act on, exactly as if it had not matched. A folder no account holds any
-	 * more turns the filter off, with the reason the filter list shows.
-	 */
-	private async appliesTo(
-		accountConfigId: string,
-		accountId: string,
-		filter: FilterItem,
-	): Promise<boolean> {
-		if (filter.actionMailboxId === NO_ACTION) return true;
-		const owner = await this.config.mailboxService.resolveAccountId(
-			filter.actionMailboxId,
-		);
-		if (owner !== null) return owner === accountId;
-		await this.config.filterService.update(accountConfigId, filter.filterId, {
-			state: FilterState.Disabled,
-			disabledReason: FilterDisabledReason.FolderMissing,
-		});
-		this.log.warn?.(
-			{ filterId: filter.filterId, accountConfigId },
-			"Filter turned off: the folder it moves mail into no longer exists",
-		);
-		return false;
-	}
-
-	private async match(
-		accountConfigId: string,
-		accountId: string,
-		msg: FilterMessage,
-	): Promise<FilterDecision> {
-		const active = await this.config.filterService.listByAccountAndState(
-			accountConfigId,
-			FilterState.Active,
-		);
-		if (active.length === 0) return EMPTY_DECISION;
-
-		const matched: FilterItem[] = [];
-		for (const filter of active) {
-			// Lazy expiry (RFC 034 Decision 1.2): reading a Temporary filter past
-			// its expiresAt patches it to Expired on this read and drops it from
-			// evaluation, independent of whether the TTL delete has run.
-			const usable = await this.config.filterService.refreshExpiry(filter);
-			if (usable.state !== FilterState.Active) continue;
-			if (!(await this.appliesTo(accountConfigId, accountId, usable))) continue;
-			if (filterMatches(usable, msg)) matched.push(usable);
-		}
-		if (matched.length === 0) return EMPTY_DECISION;
-
-		const labels: FilterDecision["labels"] = [];
-		const seenLabels = new Set<string>();
-		for (const filter of matched) {
-			if (filter.actionLabelId === NO_ACTION) continue;
-			if (seenLabels.has(filter.actionLabelId)) continue;
-			seenLabels.add(filter.actionLabelId);
-			labels.push({ labelId: filter.actionLabelId, filterId: filter.filterId });
-		}
-
-		const winner = selectMoveWinner(
-			matched.filter((filter) => filter.actionMailboxId !== NO_ACTION),
-		);
-
-		return {
-			labels,
-			...(winner
-				? {
-						move: {
-							destinationMailboxId: winner.actionMailboxId,
-							filterId: winner.filterId,
-						},
-					}
-				: {}),
-		};
-	}
-}
-
-const filterMatches = (filter: FilterItem, msg: FilterMessage): boolean =>
-	literalClausesMatch(filter.literalClauses, filter.matchOperator, msg);

package/src/flag-push.test.ts

@@ -1,112 +0,0 @@
-import assert from "node:assert/strict";
-import { afterEach, beforeEach, describe, it } from "node:test";
-import { MessageSystemFlag } from "@remit/domain-enums";
-import { FlagPushService } from "./flag-push.js";
-import {
-	captureQueue,
-	openStore,
-	type QueueCapture,
-	type Store,
-} from "./test-helpers/test-store.js";
-
-const target = {
-	accountId: "acc-1",
-	accountConfigId: "cfg-1",
-	messageId: "msg-1",
-	mailboxId: "mbx-1",
-	flagName: MessageSystemFlag.Seen,
-	operation: "add" as const,
-};
-
-let store: Store;
-let capture: QueueCapture;
-let errors: Array<Record<string, unknown>>;
-
-beforeEach(async () => {
-	store = await openStore();
-	capture = captureQueue();
-	errors = [];
-});
-
-afterEach(async () => {
-	capture.restore();
-	await store.close();
-});
-
-const queueAnswers = (answer: () => Promise<unknown>) => capture.answer(answer);
-
-const flips = (queueUrl = "http://localhost:9324/000/remit-flags") =>
-	new FlagPushService({
-		markerService: store.flagPush,
-		sqsQueueUrl: queueUrl,
-		logger: {
-			info: () => {},
-			error: (fields: Record<string, unknown>) => errors.push(fields),
-		},
-	});
-
-const marker = () => store.flagPush.find(target.messageId, target.flagName);
-
-describe("FlagPushService.flip", () => {
-	it("records the owed push and marks it queued once the hint is sent", async () => {
-		queueAnswers(async () => ({}));
-
-		await flips().flip(target);
-
-		const recorded = await marker();
-		assert.equal(recorded?.operation, "add");
-		assert.equal(recorded?.mailboxId, target.mailboxId);
-		assert.equal(recorded?.state, "queued");
-	});
-
-	it("names the message and flag in the hint, never the operation or a uid", async () => {
-		queueAnswers(async () => ({}));
-
-		await flips().flip(target);
-
-		const event = JSON.parse(capture.inputs()[0].MessageBody ?? "{}");
-		assert.equal(event.type, "FLAG_PUSH");
-		assert.equal(event.messageId, target.messageId);
-		assert.equal(event.flagName, target.flagName);
-		assert.equal("operation" in event, false);
-		assert.equal("uid" in event, false);
-	});
-
-	it("groups a FIFO hint by account and leaves a standard queue ungrouped", async () => {
-		queueAnswers(async () => ({}));
-
-		await flips("http://localhost:9324/000/remit-flags.fifo").flip(target);
-		await flips("http://localhost:9324/000/remit-flags").flip(target);
-
-		assert.equal(capture.inputs()[0].MessageGroupId, target.accountId);
-		assert.equal(capture.inputs()[1].MessageGroupId, undefined);
-	});
-
-	it("keeps the push owed and alarms, without rejecting, when the queue is down", async () => {
-		queueAnswers(async () => {
-			throw Object.assign(new Error("connect ECONNREFUSED"), {
-				code: "ECONNREFUSED",
-			});
-		});
-
-		await flips().flip(target);
-
-		assert.equal((await marker())?.state, "pending");
-		const alarm = errors.find(
-			(entry) => entry.alert === "flag_push_hint_enqueue_failed",
-		);
-		assert.equal(alarm?.errorCode, "ECONNREFUSED");
-	});
-
-	it("raises no alarm when the worker drained the push before it was marked queued", async () => {
-		queueAnswers(async () => {
-			await store.flagPush.delete(target.messageId, target.flagName);
-			return {};
-		});
-
-		await flips().flip(target);
-
-		assert.equal(await marker(), null);
-		assert.deepEqual(errors, []);
-	});
-});