@remit/mailbox-service 0.0.111 → 0.0.113

package/src/filters/message-embedder.test.ts

@@ -1,39 +0,0 @@
-import assert from "node:assert/strict";
-import { afterEach, describe, it } from "node:test";
-import {
-	_resetMessageEmbedderForTest,
-	getMessageEmbedder,
-} from "./message-embedder.js";
-
-describe("getMessageEmbedder", () => {
-	afterEach(() => {
-		_resetMessageEmbedderForTest();
-	});
-
-	it("adapts the batch embedding service into a single-text message embedder", async () => {
-		const vector = await getMessageEmbedder().embed("invoice from stripe");
-
-		assert.ok(Array.isArray(vector));
-		assert.ok(vector.length > 0);
-		assert.ok(vector.every((value) => typeof value === "number"));
-	});
-
-	it("embeds the same text deterministically", async () => {
-		const embedder = getMessageEmbedder();
-		const a = await embedder.embed("your monthly receipt");
-		const b = await embedder.embed("your monthly receipt");
-
-		assert.deepEqual(a, b);
-	});
-
-	it("memoizes the embedder across calls", () => {
-		assert.strictEqual(getMessageEmbedder(), getMessageEmbedder());
-	});
-
-	it("rebuilds after a reset", () => {
-		const first = getMessageEmbedder();
-		_resetMessageEmbedderForTest();
-
-		assert.notStrictEqual(getMessageEmbedder(), first);
-	});
-});

package/src/filters/message-embedder.ts

@@ -1,33 +0,0 @@
-import { buildEmbeddingServiceFromEnv } from "@remit/search-service/from-env";
-import type { MessageEmbedder } from "./pipeline.js";
-
-/**
- * Adapt the batch {@link EmbeddingService} the rest of the stack composes from
- * env into the single-text {@link MessageEmbedder} the filter pipeline needs: one
- * incoming message becomes one message-level vector to compare against a filter's
- * persisted anchor (RFC 034 Decision 2.1/2.3).
- *
- * The embedder is selected by the same `SEARCH_EMBEDDING_*` env the search-index
- * worker reads, so every filter pass embeds under the identical model the anchors
- * were built with. The instance is memoized across warm invocations so a local
- * Transformers.js model loads once per container rather than per event.
- */
-let cached: MessageEmbedder | undefined;
-
-export const getMessageEmbedder = (): MessageEmbedder => {
-	if (cached) return cached;
-	const service = buildEmbeddingServiceFromEnv();
-	cached = {
-		embed: async (text: string): Promise<number[]> => {
-			const [vector] = await service.embed([text]);
-			return vector;
-		},
-		embeddingId: service.embeddingId,
-	};
-	return cached;
-};
-
-/** Reset the singleton — test use only. */
-export const _resetMessageEmbedderForTest = (): void => {
-	cached = undefined;
-};

package/src/filters/pipeline.test.ts

@@ -1,330 +0,0 @@
-/**
- * The index-time filter pass over an anchorless literal filter — the shape the
- * organize widen creates on a deployment without the vector pipeline: `From`
- * clauses combined with `Or`, `hasAnchor: false`. This is what keeps a standing
- * "move all mail from these senders" filter working on future mail with no
- * vectors, so it is pinned directly: a matching sender resolves the move, a
- * non-matching one resolves nothing, and the embedder is never touched.
- */
-
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import type {
-	FilterItem,
-	IFilterAnchorRepository,
-	IFilterRepository,
-	IMessageLabelRepository,
-} from "@remit/data-ports";
-import {
-	FilterClauseField,
-	FilterMatchOperator,
-	FilterState,
-} from "@remit/domain-enums";
-import type { PlacementMoveService } from "../placement-move.js";
-import { NO_ACTION } from "./match.js";
-import { type FilterConfig, FilterPipeline } from "./pipeline.js";
-
-const senderFilter = (destinationMailboxId: string): FilterItem =>
-	({
-		filterId: "flt-senders",
-		accountConfigId: "cfg-1",
-		name: "npm barrage",
-		scope: "Standing",
-		state: FilterState.Active,
-		hasAnchor: false,
-		ruleChangedAt: 1,
-		matchOperator: FilterMatchOperator.Or,
-		literalClauses: [
-			{ field: FilterClauseField.From, value: "npm@github.com" },
-			{ field: FilterClauseField.From, value: "notifications@github.com" },
-		],
-		actionLabelId: NO_ACTION,
-		actionMailboxId: destinationMailboxId,
-		createdAt: 1,
-		updatedAt: 1,
-	}) as unknown as FilterItem;
-
-const buildPipeline = (...filters: FilterItem[]) => {
-	const state = { embedCalls: 0 };
-	const config: FilterConfig = {
-		filterService: {
-			listByAccountAndState: async () => filters,
-			refreshExpiry: async (f: FilterItem) => f,
-		} as unknown as IFilterRepository,
-		filterAnchorService: {
-			get: async () => undefined,
-		} as unknown as IFilterAnchorRepository,
-		messageLabelService: {} as unknown as IMessageLabelRepository,
-		placementMoveService: {} as unknown as PlacementMoveService,
-		embedder: {
-			embed: async () => {
-				state.embedCalls += 1;
-				return [];
-			},
-			embeddingId: "test-model@0",
-		},
-	};
-	return { pipeline: new FilterPipeline(config, { info: () => {} }), state };
-};
-
-describe("FilterPipeline — anchorless From/Or filter at index time", () => {
-	it("resolves the move for a matching sender and never embeds", async () => {
-		const { pipeline, state } = buildPipeline(senderFilter("mbx-archive"));
-		const decision = await pipeline.evaluate("cfg-1", "m-1", {
-			from: "npm@github.com",
-			fromName: "npm",
-			subject: "A new version of left-pad is available",
-			text: "body",
-			listId: "",
-		});
-
-		assert.deepEqual(decision.move, {
-			destinationMailboxId: "mbx-archive",
-			filterId: "flt-senders",
-		});
-		// A literal-only filter is decided by its clauses alone — the embedder is
-		// never constructed or called.
-		assert.equal(state.embedCalls, 0);
-	});
-
-	it("resolves nothing for a sender none of the clauses match", async () => {
-		const { pipeline, state } = buildPipeline(senderFilter("mbx-archive"));
-		const decision = await pipeline.evaluate("cfg-1", "m-2", {
-			from: "hello@stripe.com",
-			fromName: "Stripe",
-			subject: "Your receipt",
-			text: "body",
-			listId: "",
-		});
-
-		assert.equal(decision.move, undefined);
-		assert.deepEqual(decision.labels, []);
-		assert.equal(state.embedCalls, 0);
-	});
-});
-
-/**
- * Arbitration survives exactly here: two standing filters colliding over the
- * same message at the same trigger. A move is exclusive, so the pass resolves
- * one destination — the more-recently-changed action wins (reader #497 leaves
- * this untouched while removing arbitration from the user-initiated apply).
- */
-describe("FilterPipeline — two standing filters colliding on one message", () => {
-	const rivalFilter = (): FilterItem =>
-		({
-			...senderFilter("mbx-projects"),
-			filterId: "flt-rival",
-			name: "github to projects",
-			actionChangedAt: 2,
-		}) as unknown as FilterItem;
-
-	it("resolves a single destination from the more-recently-changed action", async () => {
-		const older = {
-			...senderFilter("mbx-archive"),
-			actionChangedAt: 1,
-		} as unknown as FilterItem;
-		const { pipeline } = buildPipeline(older, rivalFilter());
-		const decision = await pipeline.evaluate("cfg-1", "m-3", {
-			from: "npm@github.com",
-			fromName: "npm",
-			subject: "A new version of left-pad is available",
-			text: "body",
-			listId: "",
-		});
-
-		assert.deepEqual(decision.move, {
-			destinationMailboxId: "mbx-projects",
-			filterId: "flt-rival",
-		});
-	});
-});
-
-describe("FilterPipeline — anchor lazy re-embed on model drift (reader #295)", () => {
-	const anchoredFilter = (): FilterItem =>
-		({
-			filterId: "flt-semantic",
-			accountConfigId: "cfg-1",
-			name: "receipts",
-			scope: "Standing",
-			state: FilterState.Active,
-			hasAnchor: true,
-			ruleChangedAt: 1,
-			matchOperator: FilterMatchOperator.And,
-			literalClauses: [],
-			actionLabelId: "lbl-1",
-			actionMailboxId: NO_ACTION,
-			createdAt: 1,
-			updatedAt: 1,
-		}) as unknown as FilterItem;
-
-	const message = {
-		from: "billing@stripe.com",
-		fromName: "Stripe",
-		subject: "Your receipt",
-		text: "Thanks for your payment",
-		listId: "",
-	};
-
-	it("re-embeds a stale anchor and matches on the very next evaluation, persisting the refreshed row", async () => {
-		const putCalls: Array<Record<string, unknown>> = [];
-		let embedCalls = 0;
-		const config: FilterConfig = {
-			filterService: {
-				listByAccountAndState: async () => [anchoredFilter()],
-				refreshExpiry: async (f: FilterItem) => f,
-			} as unknown as IFilterRepository,
-			filterAnchorService: {
-				get: async () => ({
-					accountConfigId: "cfg-1",
-					filterId: "flt-semantic",
-					// Stale-model space — orthogonal to what the current model
-					// produces, so a raw comparison against it would not match.
-					anchorEmbedding: [0, 1, 0],
-					anchorEmbeddingId: "old-model@3",
-					anchorSourceText: "your receipt is ready",
-					anchorMessageId: "msg-anchor",
-				}),
-				put: async (input: Record<string, unknown>) => {
-					putCalls.push(input);
-					return { ...input, createdAt: 1, updatedAt: 2 };
-				},
-			} as unknown as IFilterAnchorRepository,
-			messageLabelService: {} as unknown as IMessageLabelRepository,
-			placementMoveService: {} as unknown as PlacementMoveService,
-			embedder: {
-				embed: async () => {
-					embedCalls += 1;
-					return [1, 0, 0];
-				},
-				embeddingId: "new-model@3",
-			},
-		};
-
-		const decision = await new FilterPipeline(config, {
-			info: () => {},
-		}).evaluate("cfg-1", "m-1", message);
-
-		assert.deepEqual(
-			decision.labels,
-			[{ labelId: "lbl-1", filterId: "flt-semantic" }],
-			"the anchor matches once re-embedded into the current model's space",
-		);
-		assert.equal(
-			putCalls.length,
-			1,
-			"the refreshed anchor is written back exactly once",
-		);
-		assert.deepEqual(putCalls[0], {
-			accountConfigId: "cfg-1",
-			filterId: "flt-semantic",
-			anchorEmbedding: [1, 0, 0],
-			anchorEmbeddingId: "new-model@3",
-			anchorSourceText: "your receipt is ready",
-			anchorMessageId: "msg-anchor",
-		});
-		assert.equal(
-			embedCalls,
-			2,
-			"one embed re-embeds the anchor's source text, one embeds the candidate message",
-		);
-	});
-
-	it("never re-embeds when the anchor's embeddingId is already current", async () => {
-		let putCalls = 0;
-		let embedCalls = 0;
-		const config: FilterConfig = {
-			filterService: {
-				listByAccountAndState: async () => [anchoredFilter()],
-				refreshExpiry: async (f: FilterItem) => f,
-			} as unknown as IFilterRepository,
-			filterAnchorService: {
-				get: async () => ({
-					anchorEmbedding: [1, 0, 0],
-					anchorEmbeddingId: "current-model@3",
-					anchorSourceText: "your receipt is ready",
-					anchorMessageId: "msg-anchor",
-				}),
-				put: async () => {
-					putCalls += 1;
-					throw new Error("must not be called");
-				},
-			} as unknown as IFilterAnchorRepository,
-			messageLabelService: {} as unknown as IMessageLabelRepository,
-			placementMoveService: {} as unknown as PlacementMoveService,
-			embedder: {
-				embed: async () => {
-					embedCalls += 1;
-					return [1, 0, 0];
-				},
-				embeddingId: "current-model@3",
-			},
-		};
-
-		const decision = await new FilterPipeline(config, {
-			info: () => {},
-		}).evaluate("cfg-1", "m-1", message);
-
-		assert.deepEqual(decision.labels, [
-			{ labelId: "lbl-1", filterId: "flt-semantic" },
-		]);
-		assert.equal(putCalls, 0, "a current anchor is never rewritten");
-		assert.equal(
-			embedCalls,
-			1,
-			"only the candidate message is embedded — no wasted re-embed call",
-		);
-	});
-
-	it("degrades a re-embed failure to skipping just this filter, not the whole pass", async () => {
-		const literalFilter: FilterItem = {
-			filterId: "flt-literal",
-			accountConfigId: "cfg-1",
-			name: "stripe receipts",
-			scope: "Standing",
-			state: FilterState.Active,
-			hasAnchor: false,
-			ruleChangedAt: 1,
-			matchOperator: FilterMatchOperator.Or,
-			literalClauses: [{ field: FilterClauseField.From, value: "stripe.com" }],
-			actionLabelId: "lbl-literal",
-			actionMailboxId: NO_ACTION,
-			createdAt: 1,
-			updatedAt: 1,
-		} as unknown as FilterItem;
-
-		const config: FilterConfig = {
-			filterService: {
-				listByAccountAndState: async () => [anchoredFilter(), literalFilter],
-				refreshExpiry: async (f: FilterItem) => f,
-			} as unknown as IFilterRepository,
-			filterAnchorService: {
-				get: async () => ({
-					anchorEmbedding: [0, 1, 0],
-					anchorEmbeddingId: "old-model@3",
-					anchorSourceText: "your receipt is ready",
-					anchorMessageId: "msg-anchor",
-				}),
-				put: async () => {
-					throw new Error("SQLITE_BUSY");
-				},
-			} as unknown as IFilterAnchorRepository,
-			messageLabelService: {} as unknown as IMessageLabelRepository,
-			placementMoveService: {} as unknown as PlacementMoveService,
-			embedder: {
-				embed: async () => [1, 0, 0],
-				embeddingId: "new-model@3",
-			},
-		};
-
-		const decision = await new FilterPipeline(config, {
-			info: () => {},
-			error: () => {},
-		}).evaluate("cfg-1", "m-1", message);
-
-		assert.deepEqual(
-			decision.labels,
-			[{ labelId: "lbl-literal", filterId: "flt-literal" }],
-			"the anchored filter's re-embed failure is isolated — the literal filter still applies",
-		);
-	});
-});

package/src/filters/pipeline.ts

@@ -1,284 +0,0 @@
-import { inspect } from "node:util";
-import type {
-	FilterItem,
-	IFilterAnchorRepository,
-	IFilterRepository,
-	IMessageLabelRepository,
-} from "@remit/data-ports";
-import { FilterState } from "@remit/domain-enums";
-import {
-	isEmbeddingCapabilityAbsence,
-	isEmbeddingCapabilityUnavailable,
-	recordEmbeddingCapabilityAbsence,
-} from "@remit/search-service/capability";
-import type { PlacementMoveService } from "../placement-move.js";
-import {
-	type AnchorEmbedder,
-	refreshAnchorForEmbedder,
-} from "./anchor-drift.js";
-import {
-	buildMatchText,
-	cosineSimilarity,
-	DEFAULT_SEMANTIC_MATCH_THRESHOLD,
-	type FilterMessage,
-	literalClausesMatch,
-	NO_ACTION,
-	selectMoveWinner,
-} from "./match.js";
-
-/**
- * Turns the candidate message's text into a single message-level vector to
- * compare against a filter's persisted `anchorEmbedding`. The anchor side is
- * embedded here only to repair a drifted anchor in place (see
- * {@link refreshAnchorForEmbedder}); the match itself reads `FilterAnchor` as
- * a fixed fact (RFC 034 Decision 2.1/2.3), and the incoming message is
- * embedded once per message, only when a semantic filter is actually in play.
- */
-export type MessageEmbedder = AnchorEmbedder;
-
-export interface FilterLogger {
-	info(obj: Record<string, unknown>, msg: string): void;
-	debug?(obj: Record<string, unknown>, msg: string): void;
-	warn?(obj: Record<string, unknown>, msg: string): void;
-	error?(obj: Record<string, unknown>, msg: string): void;
-}
-
-/**
- * Dependencies for the index-time filter pass (RFC 034). Optional on
- * {@link BodySyncService} the same way {@link PlacementConfig} is — a deployment
- * without it simply runs no filter matching.
- *
- * `embedder` is optional on purpose: semantic-anchor filters cannot be evaluated
- * without it, so they are skipped when it is absent, but literal-clause filters
- * always run. The `FilterAnchor` write path (a separate ticket) and a wired
- * embedder are what light the semantic half up.
- */
-export interface FilterConfig {
-	filterService: IFilterRepository;
-	filterAnchorService: IFilterAnchorRepository;
-	messageLabelService: IMessageLabelRepository;
-	placementMoveService: PlacementMoveService;
-	embedder?: MessageEmbedder;
-	similarityThreshold?: number;
-}
-
-/**
- * What the matched filters resolve to for one message: every matching filter's
- * label (additive — RFC 034 Decision 3.1), and at most one move (exclusive — the
- * most-recently-changed matching filter, Decision 3.2).
- */
-export interface FilterDecision {
-	labels: Array<{ labelId: string; filterId: string }>;
-	move?: { destinationMailboxId: string; filterId: string };
-}
-
-const EMPTY_DECISION: FilterDecision = { labels: [] };
-
-/**
- * Evaluates a synced message against the account's active filters and resolves
- * the actions to apply (RFC 034). Reads only — listing filters, refreshing lazy
- * expiry, reading anchors, and embedding the message; the caller applies the
- * resulting {@link FilterDecision}.
- *
- * `evaluate` isolates every failure in this read/decision phase from body sync,
- * matching the placement-resolution precedent (#1246): a schema-drifted query or
- * a bad anchor must never fail the surrounding message store, since the body is
- * already durable by the time this runs. The apply of the decision — label
- * writes and the move enqueue — is the caller's responsibility and is *not*
- * swallowed there, so a genuine DDB/SQS write failure still propagates and
- * requeues the message.
- */
-export class FilterPipeline {
-	constructor(
-		private readonly config: FilterConfig,
-		private readonly log: FilterLogger,
-	) {}
-
-	async evaluate(
-		accountConfigId: string,
-		messageId: string,
-		msg: FilterMessage,
-	): Promise<FilterDecision> {
-		return this.match(accountConfigId, msg).catch((error: unknown) => {
-			this.log.error?.(
-				{
-					alert: "body_sync_filter_match_failed",
-					messageId,
-					accountConfigId,
-					errorName: (error as { name?: string })?.name,
-					error: inspect(error),
-				},
-				"Filter matching failed; body already stored, continuing without filter actions (best-effort, non-fatal)",
-			);
-			return EMPTY_DECISION;
-		});
-	}
-
-	private async match(
-		accountConfigId: string,
-		msg: FilterMessage,
-	): Promise<FilterDecision> {
-		const active = await this.config.filterService.listByAccountAndState(
-			accountConfigId,
-			FilterState.Active,
-		);
-		if (active.length === 0) return EMPTY_DECISION;
-
-		// The message is embedded lazily and at most once: `undefined` means not
-		// yet computed, `null` means computed-but-unavailable (no embedder). The
-		// embed only ever fires when a filter survives the literal pre-filter and
-		// carries an anchor — a purely-literal account never pays for it.
-		let messageEmbedding: number[] | null | undefined;
-		const embed = async (): Promise<number[] | null> => {
-			if (messageEmbedding !== undefined) return messageEmbedding;
-			const embedder = this.config.embedder;
-			if (!embedder) {
-				messageEmbedding = null;
-				return null;
-			}
-			messageEmbedding = await embedder
-				.embed(buildMatchText(msg))
-				.catch((error: unknown) => {
-					// This deployment carries no embedding pipeline at all — the
-					// container images ship without `@huggingface/transformers`. That is
-					// the same absence `SEARCH_EMBEDDING_PROVIDER=off` states, so it takes
-					// the same designed skip rather than an error-level
-					// `filter_anchor_match_failed` per semantic filter per message. The
-					// absence is remembered process-wide, so this is the only embed that
-					// pays for it: `buildFilterConfig` wires no embedder afterwards.
-					if (!isEmbeddingCapabilityAbsence(error)) throw error;
-					if (!isEmbeddingCapabilityUnavailable()) {
-						this.log.warn?.(
-							{
-								accountConfigId,
-								error: inspect(error),
-							},
-							"No embedding pipeline in this deployment; semantic filters are skipped (literal filters still run)",
-						);
-					}
-					recordEmbeddingCapabilityAbsence(error);
-					return null;
-				});
-			return messageEmbedding;
-		};
-
-		const matched: FilterItem[] = [];
-		for (const filter of active) {
-			// Lazy expiry (RFC 034 Decision 1.2): reading a Temporary filter past
-			// its expiresAt patches it to Expired on this read and drops it from
-			// evaluation, independent of whether the TTL delete has run.
-			const usable = await this.config.filterService.refreshExpiry(filter);
-			if (usable.state !== FilterState.Active) continue;
-			// A bad or stale anchor vector (a dimension mismatch under a changed
-			// embedding model) throws from cosineSimilarity. Isolate it to this
-			// filter: it is skipped and loudly logged, and every other filter on
-			// the message still evaluates and applies. This sits inside the
-			// evaluate-level catch — a whole-phase failure still degrades to no
-			// filter actions — but stops one poisoned anchor from doing so.
-			const isMatch = await this.filterMatches(
-				accountConfigId,
-				usable,
-				msg,
-				embed,
-			).catch((error: unknown) => {
-				this.log.error?.(
-					{
-						alert: "filter_anchor_match_failed",
-						filterId: usable.filterId,
-						accountConfigId,
-						errorName: (error as { name?: string })?.name,
-						error: inspect(error),
-					},
-					"Filter anchor comparison failed; skipping this filter, other filters still evaluate (bad/stale anchor vector, non-fatal)",
-				);
-				return "skip" as const;
-			});
-			if (isMatch === "skip") continue;
-			if (isMatch) matched.push(usable);
-		}
-		if (matched.length === 0) return EMPTY_DECISION;
-
-		const labels: FilterDecision["labels"] = [];
-		const seenLabels = new Set<string>();
-		for (const filter of matched) {
-			if (filter.actionLabelId === NO_ACTION) continue;
-			if (seenLabels.has(filter.actionLabelId)) continue;
-			seenLabels.add(filter.actionLabelId);
-			labels.push({ labelId: filter.actionLabelId, filterId: filter.filterId });
-		}
-
-		const winner = selectMoveWinner(
-			matched.filter((filter) => filter.actionMailboxId !== NO_ACTION),
-		);
-
-		return {
-			labels,
-			...(winner
-				? {
-						move: {
-							destinationMailboxId: winner.actionMailboxId,
-							filterId: winner.filterId,
-						},
-					}
-				: {}),
-		};
-	}
-
-	private async filterMatches(
-		accountConfigId: string,
-		filter: FilterItem,
-		msg: FilterMessage,
-		embed: () => Promise<number[] | null>,
-	): Promise<boolean> {
-		if (
-			!literalClausesMatch(filter.literalClauses, filter.matchOperator, msg)
-		) {
-			return false;
-		}
-
-		if (!filter.hasAnchor) {
-			// A filter with neither clauses nor an anchor has no predicate and
-			// matches nothing; a literal-only filter matched iff it had clauses.
-			return filter.literalClauses.length > 0;
-		}
-
-		// One deterministic point read per semantic candidate that survived the
-		// literal pre-filter — never a scan, never the anchor message itself
-		// (RFC 034 Decision 2.3).
-		let anchor = await this.config.filterAnchorService.get(
-			accountConfigId,
-			filter.filterId,
-		);
-		if (!anchor) {
-			this.log.warn?.(
-				{ filterId: filter.filterId, accountConfigId },
-				"Filter marked hasAnchor but no FilterAnchor row found; skipping semantic match",
-			);
-			return false;
-		}
-
-		// A re-embed failure here propagates to the per-filter catch in
-		// `match()`, which logs and skips this filter for this one evaluation
-		// exactly as an unrecoverable stale anchor does today.
-		anchor = await refreshAnchorForEmbedder(
-			{
-				anchorRepository: this.config.filterAnchorService,
-				embedder: this.config.embedder,
-			},
-			anchor,
-		);
-
-		const vector = await embed();
-		if (!vector) {
-			this.log.debug?.(
-				{ filterId: filter.filterId },
-				"No embedder configured; skipping semantic filter",
-			);
-			return false;
-		}
-
-		const threshold =
-			this.config.similarityThreshold ?? DEFAULT_SEMANTIC_MATCH_THRESHOLD;
-		return cosineSimilarity(vector, anchor.anchorEmbedding) >= threshold;
-	}
-}