@remit/mailbox-service 0.0.1

package/src/filters/match.ts

@@ -0,0 +1,143 @@
+import type { FilterItem } from "@remit/data-ports";
+import { FilterClauseField, FilterMatchOperator } from "@remit/domain-enums";
+
+type FilterClause = FilterItem["literalClauses"][number];
+
+/**
+ * The `"None"` sentinel a filter's action fields carry when that action is
+ * absent (RFC 034 Decision 3.1) — `actionLabelId`/`actionMailboxId` are never
+ * empty/optional strings, so a missing action is this exact value, never `""`.
+ */
+export const NO_ACTION = "None";
+
+/**
+ * Default cosine cut-off for a semantic-anchor match (RFC 031 "the semantic
+ * anchor evaluates as a kNN threshold against the new message's embedding").
+ * Tunable per pipeline via {@link FilterConfig.similarityThreshold}.
+ */
+export const DEFAULT_SEMANTIC_MATCH_THRESHOLD = 0.75;
+
+/**
+ * Bound on the text embedded for a semantic match — the same 512-char budget
+ * `buildTextPreview` applies to chunk vectors and `FilterAnchor.anchorSourceText`
+ * persists, so the candidate side of the comparison is derived on the same
+ * footing as the anchor side.
+ */
+const MATCH_TEXT_LIMIT = 512;
+
+/**
+ * The message fields a filter evaluates against — the literal-clause targets
+ * (from / subject / body) plus the text embedded for a semantic anchor. A plain
+ * value object so the matcher stays independent of the `ParsedMail` shape and is
+ * trivially constructed in a test.
+ */
+export interface FilterMessage {
+	from: string;
+	fromName: string;
+	subject: string;
+	text: string;
+}
+
+const includesFold = (haystack: string, needle: string): boolean =>
+	haystack.toLowerCase().includes(needle.toLowerCase());
+
+/**
+ * Whether one literal clause matches the message. From matches against the
+ * sender address and display name; Subject against the subject; HasWords against
+ * subject or body. An empty clause value never matches.
+ */
+export const clauseMatches = (
+	clause: FilterClause,
+	msg: FilterMessage,
+): boolean => {
+	const value = clause.value.trim();
+	if (value === "") return false;
+	switch (clause.field) {
+		case FilterClauseField.From:
+			return includesFold(msg.from, value) || includesFold(msg.fromName, value);
+		case FilterClauseField.Subject:
+			return includesFold(msg.subject, value);
+		case FilterClauseField.HasWords:
+			return includesFold(msg.subject, value) || includesFold(msg.text, value);
+		default:
+			return false;
+	}
+};
+
+/**
+ * Whether the literal clauses match under their combine operator. Empty clauses
+ * carry no literal constraint and pass vacuously — a purely-semantic filter (no
+ * clauses, only an anchor) is gated by its anchor alone, not by this.
+ */
+export const literalClausesMatch = (
+	clauses: readonly FilterClause[],
+	operator: FilterItem["matchOperator"],
+	msg: FilterMessage,
+): boolean => {
+	if (clauses.length === 0) return true;
+	return operator === FilterMatchOperator.Or
+		? clauses.some((clause) => clauseMatches(clause, msg))
+		: clauses.every((clause) => clauseMatches(clause, msg));
+};
+
+/**
+ * Cosine similarity of two equal-length vectors. Throws on a dimension mismatch
+ * rather than silently scoring incomparable vectors — a mismatch means a stale
+ * anchor embedded under a different model (`anchorEmbeddingId`), which is a
+ * migration concern, not a match to guess at.
+ */
+export const cosineSimilarity = (
+	a: readonly number[],
+	b: readonly number[],
+): number => {
+	if (a.length !== b.length) {
+		throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+	}
+	let dot = 0;
+	let normA = 0;
+	let normB = 0;
+	for (let i = 0; i < a.length; i++) {
+		dot += a[i] * b[i];
+		normA += a[i] * a[i];
+		normB += b[i] * b[i];
+	}
+	if (normA === 0 || normB === 0) return 0;
+	return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+};
+
+/**
+ * The move a message ends in when several filters matched: the most-recently
+ * *changed* filter wins (RFC 034 Decision 3.2), tie-broken on `filterId` for the
+ * unreachable identical-timestamp case. `ruleChangedAt` — not `updatedAt` — is
+ * the signal, so a cosmetic rename never flips an exclusive move.
+ */
+export const selectMoveWinner = (
+	candidates: readonly FilterItem[],
+): FilterItem | undefined => {
+	let winner: FilterItem | undefined;
+	for (const candidate of candidates) {
+		if (!winner) {
+			winner = candidate;
+			continue;
+		}
+		if (candidate.ruleChangedAt > winner.ruleChangedAt) {
+			winner = candidate;
+			continue;
+		}
+		if (
+			candidate.ruleChangedAt === winner.ruleChangedAt &&
+			candidate.filterId > winner.filterId
+		) {
+			winner = candidate;
+		}
+	}
+	return winner;
+};
+
+/**
+ * The candidate text embedded once for a semantic match — subject then body,
+ * bounded to the anchor's `anchorSourceText` budget so both sides of the cosine
+ * comparison are derived from comparable inputs.
+ */
+export const buildMatchText = (msg: FilterMessage): string =>
+	`${msg.subject}\n${msg.text}`.slice(0, MATCH_TEXT_LIMIT);

package/src/filters/pipeline.ts

@@ -0,0 +1,243 @@
+import { inspect } from "node:util";
+import type {
+	FilterItem,
+	IFilterAnchorRepository,
+	IFilterRepository,
+	IMessageLabelRepository,
+} from "@remit/data-ports";
+import { FilterState } from "@remit/domain-enums";
+import type { PlacementMoveService } from "../placement-move.js";
+import {
+	buildMatchText,
+	cosineSimilarity,
+	DEFAULT_SEMANTIC_MATCH_THRESHOLD,
+	type FilterMessage,
+	literalClausesMatch,
+	NO_ACTION,
+	selectMoveWinner,
+} from "./match.js";
+
+/**
+ * Turns the candidate message's text into a single message-level vector to
+ * compare against a filter's persisted `anchorEmbedding`. The anchor side is
+ * never embedded here — it is read from `FilterAnchor` as a fixed fact (RFC 034
+ * Decision 2.1/2.3); only the incoming message is embedded, and only once per
+ * message, and only when a semantic filter is actually in play.
+ */
+export interface MessageEmbedder {
+	embed(text: string): Promise<number[]>;
+}
+
+export interface FilterLogger {
+	info(obj: Record<string, unknown>, msg: string): void;
+	debug?(obj: Record<string, unknown>, msg: string): void;
+	warn?(obj: Record<string, unknown>, msg: string): void;
+	error?(obj: Record<string, unknown>, msg: string): void;
+}
+
+/**
+ * Dependencies for the index-time filter pass (RFC 034). Optional on
+ * {@link BodySyncService} the same way {@link PlacementConfig} is — a deployment
+ * without it simply runs no filter matching.
+ *
+ * `embedder` is optional on purpose: semantic-anchor filters cannot be evaluated
+ * without it, so they are skipped when it is absent, but literal-clause filters
+ * always run. The `FilterAnchor` write path (a separate ticket) and a wired
+ * embedder are what light the semantic half up.
+ */
+export interface FilterConfig {
+	filterService: IFilterRepository;
+	filterAnchorService: IFilterAnchorRepository;
+	messageLabelService: IMessageLabelRepository;
+	placementMoveService: PlacementMoveService;
+	embedder?: MessageEmbedder;
+	similarityThreshold?: number;
+}
+
+/**
+ * What the matched filters resolve to for one message: every matching filter's
+ * label (additive — RFC 034 Decision 3.1), and at most one move (exclusive — the
+ * most-recently-changed matching filter, Decision 3.2).
+ */
+export interface FilterDecision {
+	labels: Array<{ labelId: string; filterId: string }>;
+	move?: { destinationMailboxId: string; filterId: string };
+}
+
+const EMPTY_DECISION: FilterDecision = { labels: [] };
+
+/**
+ * Evaluates a synced message against the account's active filters and resolves
+ * the actions to apply (RFC 034). Reads only — listing filters, refreshing lazy
+ * expiry, reading anchors, and embedding the message; the caller applies the
+ * resulting {@link FilterDecision}.
+ *
+ * `evaluate` isolates every failure in this read/decision phase from body sync,
+ * matching the placement-resolution precedent (#1246): a schema-drifted query or
+ * a bad anchor must never fail the surrounding message store, since the body is
+ * already durable by the time this runs. The apply of the decision — label
+ * writes and the move enqueue — is the caller's responsibility and is *not*
+ * swallowed there, so a genuine DDB/SQS write failure still propagates and
+ * requeues the message.
+ */
+export class FilterPipeline {
+	constructor(
+		private readonly config: FilterConfig,
+		private readonly log: FilterLogger,
+	) {}
+
+	async evaluate(
+		accountConfigId: string,
+		messageId: string,
+		msg: FilterMessage,
+	): Promise<FilterDecision> {
+		return this.match(accountConfigId, msg).catch((error: unknown) => {
+			this.log.error?.(
+				{
+					alert: "body_sync_filter_match_failed",
+					messageId,
+					accountConfigId,
+					errorName: (error as { name?: string })?.name,
+					error: inspect(error),
+				},
+				"Filter matching failed; body already stored, continuing without filter actions (best-effort, non-fatal)",
+			);
+			return EMPTY_DECISION;
+		});
+	}
+
+	private async match(
+		accountConfigId: string,
+		msg: FilterMessage,
+	): Promise<FilterDecision> {
+		const active = await this.config.filterService.listByAccountAndState(
+			accountConfigId,
+			FilterState.Active,
+		);
+		if (active.length === 0) return EMPTY_DECISION;
+
+		// The message is embedded lazily and at most once: `undefined` means not
+		// yet computed, `null` means computed-but-unavailable (no embedder). The
+		// embed only ever fires when a filter survives the literal pre-filter and
+		// carries an anchor — a purely-literal account never pays for it.
+		let messageEmbedding: number[] | null | undefined;
+		const embed = async (): Promise<number[] | null> => {
+			if (messageEmbedding !== undefined) return messageEmbedding;
+			const embedder = this.config.embedder;
+			if (!embedder) {
+				messageEmbedding = null;
+				return null;
+			}
+			messageEmbedding = await embedder.embed(buildMatchText(msg));
+			return messageEmbedding;
+		};
+
+		const matched: FilterItem[] = [];
+		for (const filter of active) {
+			// Lazy expiry (RFC 034 Decision 1.2): reading a Temporary filter past
+			// its expiresAt patches it to Expired on this read and drops it from
+			// evaluation, independent of whether the TTL delete has run.
+			const usable = await this.config.filterService.refreshExpiry(filter);
+			if (usable.state !== FilterState.Active) continue;
+			// A bad or stale anchor vector (a dimension mismatch under a changed
+			// embedding model) throws from cosineSimilarity. Isolate it to this
+			// filter: it is skipped and loudly logged, and every other filter on
+			// the message still evaluates and applies. This sits inside the
+			// evaluate-level catch — a whole-phase failure still degrades to no
+			// filter actions — but stops one poisoned anchor from doing so.
+			const isMatch = await this.filterMatches(
+				accountConfigId,
+				usable,
+				msg,
+				embed,
+			).catch((error: unknown) => {
+				this.log.error?.(
+					{
+						alert: "filter_anchor_match_failed",
+						filterId: usable.filterId,
+						accountConfigId,
+						errorName: (error as { name?: string })?.name,
+						error: inspect(error),
+					},
+					"Filter anchor comparison failed; skipping this filter, other filters still evaluate (bad/stale anchor vector, non-fatal)",
+				);
+				return "skip" as const;
+			});
+			if (isMatch === "skip") continue;
+			if (isMatch) matched.push(usable);
+		}
+		if (matched.length === 0) return EMPTY_DECISION;
+
+		const labels: FilterDecision["labels"] = [];
+		const seenLabels = new Set<string>();
+		for (const filter of matched) {
+			if (filter.actionLabelId === NO_ACTION) continue;
+			if (seenLabels.has(filter.actionLabelId)) continue;
+			seenLabels.add(filter.actionLabelId);
+			labels.push({ labelId: filter.actionLabelId, filterId: filter.filterId });
+		}
+
+		const winner = selectMoveWinner(
+			matched.filter((filter) => filter.actionMailboxId !== NO_ACTION),
+		);
+
+		return {
+			labels,
+			...(winner
+				? {
+						move: {
+							destinationMailboxId: winner.actionMailboxId,
+							filterId: winner.filterId,
+						},
+					}
+				: {}),
+		};
+	}
+
+	private async filterMatches(
+		accountConfigId: string,
+		filter: FilterItem,
+		msg: FilterMessage,
+		embed: () => Promise<number[] | null>,
+	): Promise<boolean> {
+		if (
+			!literalClausesMatch(filter.literalClauses, filter.matchOperator, msg)
+		) {
+			return false;
+		}
+
+		if (!filter.hasAnchor) {
+			// A filter with neither clauses nor an anchor has no predicate and
+			// matches nothing; a literal-only filter matched iff it had clauses.
+			return filter.literalClauses.length > 0;
+		}
+
+		// One deterministic point read per semantic candidate that survived the
+		// literal pre-filter — never a scan, never the anchor message itself
+		// (RFC 034 Decision 2.3).
+		const anchor = await this.config.filterAnchorService.get(
+			accountConfigId,
+			filter.filterId,
+		);
+		if (!anchor) {
+			this.log.warn?.(
+				{ filterId: filter.filterId, accountConfigId },
+				"Filter marked hasAnchor but no FilterAnchor row found; skipping semantic match",
+			);
+			return false;
+		}
+
+		const vector = await embed();
+		if (!vector) {
+			this.log.debug?.(
+				{ filterId: filter.filterId },
+				"No embedder configured; skipping semantic filter",
+			);
+			return false;
+		}
+
+		const threshold =
+			this.config.similarityThreshold ?? DEFAULT_SEMANTIC_MATCH_THRESHOLD;
+		return cosineSimilarity(vector, anchor.anchorEmbedding) >= threshold;
+	}
+}

package/src/flag-push-terminal.test.ts

@@ -0,0 +1,156 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	IMessageRepository,
+	IThreadMessageRepository,
+} from "@remit/data-ports";
+import type { FlagPushLogger } from "./flag-push.js";
+import {
+	type ResolveExhaustedFlagPushDeps,
+	resolveExhaustedFlagPushFailure,
+} from "./flag-push-terminal.js";
+import type { IImapConnection } from "./types.js";
+
+interface LogEntry {
+	obj: Record<string, unknown>;
+	msg: string;
+}
+
+const buildLogger = (): {
+	log: FlagPushLogger;
+	infos: LogEntry[];
+	errors: LogEntry[];
+} => {
+	const infos: LogEntry[] = [];
+	const errors: LogEntry[] = [];
+	return {
+		log: {
+			info: (obj, msg) => infos.push({ obj, msg }),
+			error: (obj, msg) => errors.push({ obj, msg }),
+		},
+		infos,
+		errors,
+	};
+};
+
+const buildConnection = (present: Set<number>): IImapConnection =>
+	({
+		openBox: async () => ({}) as never,
+		fetchMessages: async (uids: number[]) =>
+			uids
+				.filter((uid) => present.has(uid))
+				.map((uid) => ({ uid }) as unknown as never),
+	}) as unknown as IImapConnection;
+
+describe("resolveExhaustedFlagPushFailure — the two terminal outcomes (mirrors #1289/#1270 for flag pushes)", () => {
+	it("RECONCILED (expected): the message is gone from its mailbox — marker dropped, stale row reconciled, no alarm", async () => {
+		const deletedMessages: string[] = [];
+		const deletedThreadMessages: Array<{
+			accountConfigId: string;
+			threadMessageId: string;
+		}> = [];
+		const markerDeletes: Array<{ messageId: string; flagName: string }> = [];
+		const { log, infos, errors } = buildLogger();
+
+		const deps: ResolveExhaustedFlagPushDeps = {
+			markerService: {
+				delete: async (messageId: string, flagName: string) => {
+					markerDeletes.push({ messageId, flagName });
+				},
+			},
+			messageService: {
+				delete: async (id: string) => {
+					deletedMessages.push(id);
+				},
+			} as unknown as Pick<IMessageRepository, "delete">,
+			threadMessageService: {
+				findAllByMessageId: async (
+					accountConfigId: string,
+					messageId: string,
+				) => [{ accountConfigId, threadMessageId: `tm-${messageId}` }],
+				deleteMany: async (
+					keys: Array<{ accountConfigId: string; threadMessageId: string }>,
+				) => {
+					deletedThreadMessages.push(...keys);
+				},
+			} as unknown as Pick<
+				IThreadMessageRepository,
+				"findAllByMessageId" | "deleteMany"
+			>,
+			log,
+		};
+
+		const result = await resolveExhaustedFlagPushFailure(deps, {
+			accountId: "acc-1",
+			accountConfigId: "cfg-1",
+			messageId: "msg-gone",
+			flagName: "\\Seen",
+			uid: 101,
+			mailboxPath: "INBOX",
+			getConnection: async () => buildConnection(new Set()),
+		});
+
+		assert.equal(result.outcome, "reconciled");
+		assert.deepEqual(markerDeletes, [
+			{ messageId: "msg-gone", flagName: "\\Seen" },
+		]);
+		assert.deepEqual(deletedMessages, ["msg-gone"]);
+		assert.equal(deletedThreadMessages.length, 1);
+
+		assert.equal(errors.length, 0, "reconciled outcome must not log an alert");
+		const metricLog = infos.find(
+			(entry) => entry.obj.metric === "flag_push_stale_row_reconciled",
+		);
+		assert.ok(metricLog, "expected a routine reconciliation metric log");
+	});
+
+	it("BROKEN (should never happen): the message still exists — marker left in place, alert logged", async () => {
+		const markerDeletes: Array<{ messageId: string; flagName: string }> = [];
+		const { log, errors } = buildLogger();
+
+		const deps: ResolveExhaustedFlagPushDeps = {
+			markerService: {
+				delete: async (messageId: string, flagName: string) => {
+					markerDeletes.push({ messageId, flagName });
+				},
+			},
+			messageService: {
+				delete: async () => {
+					throw new Error("must not be called — message still exists");
+				},
+			} as unknown as Pick<IMessageRepository, "delete">,
+			threadMessageService: {
+				findAllByMessageId: async () => {
+					throw new Error("must not be called — message still exists");
+				},
+				deleteMany: async () => {
+					throw new Error("must not be called — message still exists");
+				},
+			} as unknown as Pick<
+				IThreadMessageRepository,
+				"findAllByMessageId" | "deleteMany"
+			>,
+			log,
+		};
+
+		const result = await resolveExhaustedFlagPushFailure(deps, {
+			accountId: "acc-1",
+			accountConfigId: "cfg-1",
+			messageId: "msg-still-here",
+			flagName: "\\Flagged",
+			uid: 202,
+			mailboxPath: "INBOX",
+			getConnection: async () => buildConnection(new Set([202])),
+		});
+
+		assert.equal(result.outcome, "broken");
+		assert.deepEqual(
+			markerDeletes,
+			[],
+			"marker must be left in place — resync never reverts a pending flag while it is",
+		);
+
+		const alerted = errors.find((e) => e.obj.alert === "flag_push_failed");
+		assert.ok(alerted, "expected an alertable flag_push_failed log");
+	});
+});

package/src/flag-push-terminal.ts

@@ -0,0 +1,106 @@
+import type { FlagPushLogger } from "./flag-push.js";
+import {
+	reconcileStaleMessage,
+	type StaleMessageReconcileDeps,
+} from "./stale-message-reconcile.js";
+import type { IImapConnection } from "./types.js";
+
+export interface ResolveExhaustedFlagPushDeps
+	extends StaleMessageReconcileDeps {
+	markerService: { delete(messageId: string, flagName: string): Promise<void> };
+	log: FlagPushLogger;
+}
+
+export interface ResolveExhaustedFlagPushInput {
+	accountId: string;
+	accountConfigId: string;
+	messageId: string;
+	flagName: string;
+	uid: number;
+	mailboxPath: string;
+	getConnection: () => Promise<IImapConnection>;
+}
+
+export type FlagPushTerminalOutcome = "reconciled" | "broken";
+
+export interface ResolveExhaustedFlagPushResult {
+	outcome: FlagPushTerminalOutcome;
+}
+
+/**
+ * Resolve a FLAG_PUSH failure that has exhausted the delivering queue's
+ * redelivery budget into exactly one of two terminal outcomes, mirroring
+ * `resolveExhaustedPlacementMoveFailure` (#1289) and
+ * `resolveExhaustedBodySyncFailures` (#1270) for the same failure taxonomy
+ * (epic #1281 invariant 3) — no third, softer outcome.
+ *
+ * 1. RECONCILED (expected) — the message no longer exists at its mailbox on
+ *    IMAP. Per invariant 2, an external delete supersedes the marker
+ *    entirely: the marker is dropped and the stale Message/ThreadMessage rows
+ *    are deleted via {@link reconcileStaleMessage}. Metric only, no alarm —
+ *    routine.
+ * 2. BROKEN — the message still exists, but the flag push keeps failing.
+ *    Broken code or a broken account, not a transient blip. The marker is
+ *    left in place (not cleared) — while pending, resync never reverts the
+ *    local flag (invariant 3 of this issue's own spec), so leaving it does
+ *    not risk correctness, and dropping it here would discard the only
+ *    record that IMAP still owes this push. Logged with an `alert`-shaped
+ *    entry for an operator alarm; never re-thrown (terminal — the caller acks
+ *    either way, since retrying a stale or permanently-broken push can never
+ *    succeed).
+ */
+export const resolveExhaustedFlagPushFailure = async (
+	deps: ResolveExhaustedFlagPushDeps,
+	input: ResolveExhaustedFlagPushInput,
+): Promise<ResolveExhaustedFlagPushResult> => {
+	const {
+		accountId,
+		accountConfigId,
+		messageId,
+		flagName,
+		uid,
+		mailboxPath,
+		getConnection,
+	} = input;
+
+	const connection = await getConnection();
+	await connection.openBox(mailboxPath, true);
+	const found = await connection.fetchMessages([uid]);
+
+	if (found.length === 0) {
+		await deps.markerService.delete(messageId, flagName);
+		const { threadMessagesDeleted } = await reconcileStaleMessage(
+			deps,
+			accountConfigId,
+			messageId,
+		);
+		deps.log.info(
+			{
+				metric: "flag_push_stale_row_reconciled",
+				accountId,
+				accountConfigId,
+				messageId,
+				flagName,
+				uid,
+				mailboxPath,
+				threadMessagesDeleted,
+			},
+			"Message no longer at its flag-push mailbox after retry exhaustion (external delete); marker dropped, stale row reconciled",
+		);
+		return { outcome: "reconciled" };
+	}
+
+	deps.log.error(
+		{
+			alert: "flag_push_failed",
+			accountId,
+			accountConfigId,
+			messageId,
+			flagName,
+			uid,
+			mailboxPath,
+		},
+		"Flag push could not be pushed to IMAP after retry exhaustion; message still exists — marker left pending for operator investigation",
+	);
+	return { outcome: "broken" };
+};