@remit/mailbox-service 0.0.90 → 0.0.91

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.90",
+  "version": "0.0.91",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -9,6 +9,10 @@
       "types": "./src/index.ts",
       "default": "./src/index.ts"
     },
+    "./filter-config": {
+      "types": "./src/filters/filter-config.ts",
+      "default": "./src/filters/filter-config.ts"
+    },
     "./body-sync-queue": {
       "types": "./src/body-sync-queue.ts",
       "default": "./src/body-sync-queue.ts"
@@ -42,6 +46,7 @@
     "@remit/data-ports": "*",
     "@remit/domain-enums": "*",
     "@remit/mail-oauth-service": "*",
+    "@remit/search-service": "*",
     "@remit/secrets-service": "*",
     "@remit/sqs-client": "*",
     "@remit/storage-service": "*",

package/src/filters/filter-config.test.ts

@@ -0,0 +1,229 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	FilterItem,
+	IFilterAnchorRepository,
+	IFilterRepository,
+	IMessageLabelRepository,
+} from "@remit/data-ports";
+import { FilterMatchOperator, FilterState } from "@remit/domain-enums";
+import { _resetEmbeddingCapabilityForTest } from "@remit/search-service/capability";
+import type { PlacementMoveService } from "../placement-move.js";
+import { buildFilterConfig, type FilterConfigDeps } from "./filter-config.js";
+import { type FilterMessage, NO_ACTION } from "./match.js";
+import { _resetMessageEmbedderForTest } from "./message-embedder.js";
+import { FilterPipeline, type MessageEmbedder } from "./pipeline.js";
+
+const anchorOnlyFilter = (destinationMailboxId: string): FilterItem =>
+	({
+		filterId: "flt-semantic",
+		accountConfigId: "cfg-1",
+		name: "receipts",
+		scope: "Standing",
+		state: FilterState.Active,
+		hasAnchor: true,
+		ruleChangedAt: 1,
+		matchOperator: FilterMatchOperator.Or,
+		literalClauses: [],
+		actionLabelId: NO_ACTION,
+		actionMailboxId: destinationMailboxId,
+		createdAt: 1,
+		updatedAt: 1,
+	}) as unknown as FilterItem;
+
+const CURRENT_MODEL = "test-model@3";
+
+const repos = (filter: FilterItem): FilterConfigDeps => ({
+	filterService: {
+		listByAccountAndState: async () => [filter],
+		refreshExpiry: async (f: FilterItem) => f,
+	} as unknown as IFilterRepository,
+	filterAnchorService: {
+		get: async () => ({
+			anchorEmbedding: [1, 0, 0],
+			anchorEmbeddingId: CURRENT_MODEL,
+		}),
+	} as unknown as IFilterAnchorRepository,
+	messageLabelService: {} as unknown as IMessageLabelRepository,
+	placementMoveService: {} as unknown as PlacementMoveService,
+});
+
+const message: FilterMessage = {
+	from: "billing@stripe.com",
+	fromName: "Stripe",
+	subject: "Your receipt",
+	text: "Thanks for your payment",
+	listId: "",
+};
+
+describe("buildFilterConfig", () => {
+	it("returns undefined without a placement mover — no move path, filters stay off", () => {
+		const deps = repos(anchorOnlyFilter("mbx-archive"));
+		deps.placementMoveService = undefined;
+
+		assert.equal(buildFilterConfig(deps), undefined);
+	});
+
+	it("wires an embedder into the config", () => {
+		const config = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
+
+		assert.ok(config);
+		assert.ok(config.embedder);
+	});
+
+	it("lights up a semantic anchor-only filter on the body-sync pass", async () => {
+		let embedCalls = 0;
+		const embedder: MessageEmbedder = {
+			embed: async () => {
+				embedCalls += 1;
+				return [1, 0, 0];
+			},
+			embeddingId: CURRENT_MODEL,
+		};
+		const config = buildFilterConfig(
+			repos(anchorOnlyFilter("mbx-archive")),
+			embedder,
+		);
+		assert.ok(config);
+
+		const decision = await new FilterPipeline(config, {
+			info: () => {},
+		}).evaluate("cfg-1", "m-1", message);
+
+		assert.deepEqual(decision.move, {
+			destinationMailboxId: "mbx-archive",
+			filterId: "flt-semantic",
+		});
+		assert.equal(embedCalls, 1);
+	});
+
+	it("does not match when the message embedding diverges from the anchor", async () => {
+		const embedder: MessageEmbedder = {
+			embed: async () => [0, 1, 0],
+			embeddingId: CURRENT_MODEL,
+		};
+		const config = buildFilterConfig(
+			repos(anchorOnlyFilter("mbx-archive")),
+			embedder,
+		);
+		assert.ok(config);
+
+		const decision = await new FilterPipeline(config, {
+			info: () => {},
+		}).evaluate("cfg-1", "m-2", message);
+
+		assert.equal(decision.move, undefined);
+	});
+});
+
+// An instance with semantic search off (issue #1068). `off` selects an embedder
+// that throws by design, and the throw lands in the pipeline's per-filter catch:
+// one error-level `filter_anchor_match_failed` for every semantic filter on
+// every message it syncs. The pipeline already has a designed skip for a
+// missing embedder, so the config passes none.
+describe("buildFilterConfig with semantic search off", () => {
+	const withProviderOff = async (body: () => Promise<void>): Promise<void> => {
+		const previous = process.env.SEARCH_EMBEDDING_PROVIDER;
+		process.env.SEARCH_EMBEDDING_PROVIDER = "off";
+		_resetMessageEmbedderForTest();
+		try {
+			await body();
+		} finally {
+			if (previous === undefined) {
+				delete process.env.SEARCH_EMBEDDING_PROVIDER;
+			} else {
+				process.env.SEARCH_EMBEDDING_PROVIDER = previous;
+			}
+			_resetMessageEmbedderForTest();
+		}
+	};
+
+	it("wires no embedder", async () => {
+		await withProviderOff(async () => {
+			const config = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
+
+			assert.ok(config);
+			assert.equal(config.embedder, undefined);
+		});
+	});
+
+	it("skips the semantic filter without logging an error", async () => {
+		await withProviderOff(async () => {
+			const config = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
+			assert.ok(config);
+
+			const errors: unknown[] = [];
+			const decision = await new FilterPipeline(config, {
+				info: () => {},
+				debug: () => {},
+				warn: () => {},
+				error: (fields) => errors.push(fields.alert),
+			}).evaluate("cfg-1", "m-off", message);
+
+			assert.deepEqual(decision.labels, []);
+			assert.equal(decision.move, undefined);
+			assert.deepEqual(errors, []);
+		});
+	});
+});
+
+// The self-host `local` profile asks for the Transformers.js embedder, but the
+// backend and imap-worker images ship without `@huggingface/transformers`
+// (npm-scripts/docker-bundle.mjs). The first embed throws ERR_MODULE_NOT_FOUND,
+// which is the same capability absence `off` states and must take the same
+// designed skip — not an error-level `filter_anchor_match_failed` per semantic
+// filter per message on a user-facing GET.
+describe("buildFilterConfig when this deployment carries no embedder", () => {
+	const moduleNotFound = (): Error =>
+		Object.assign(
+			new Error("Cannot find package '@huggingface/transformers'"),
+			{ code: "ERR_MODULE_NOT_FOUND" },
+		);
+
+	it("skips the semantic filter, logs no error, and stops wiring an embedder", async () => {
+		const previous = process.env.SEARCH_EMBEDDING_PROVIDER;
+		process.env.SEARCH_EMBEDDING_PROVIDER = "local";
+		_resetMessageEmbedderForTest();
+		_resetEmbeddingCapabilityForTest();
+
+		try {
+			const unavailable: MessageEmbedder = {
+				embed: async () => {
+					throw moduleNotFound();
+				},
+				embeddingId: CURRENT_MODEL,
+			};
+			const config = buildFilterConfig(
+				repos(anchorOnlyFilter("mbx-archive")),
+				unavailable,
+			);
+			assert.ok(config);
+
+			const errors: unknown[] = [];
+			const decision = await new FilterPipeline(config, {
+				info: () => {},
+				debug: () => {},
+				warn: () => {},
+				error: (fields) => errors.push(fields.alert),
+			}).evaluate("cfg-1", "m-local", message);
+
+			assert.deepEqual(decision.labels, []);
+			assert.equal(decision.move, undefined);
+			assert.deepEqual(errors, []);
+
+			// Probed once, not per filter or per message: every later config on this
+			// process wires no embedder at all, so nothing else attempts the embed.
+			const next = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
+			assert.ok(next);
+			assert.equal(next.embedder, undefined);
+		} finally {
+			if (previous === undefined) {
+				delete process.env.SEARCH_EMBEDDING_PROVIDER;
+			} else {
+				process.env.SEARCH_EMBEDDING_PROVIDER = previous;
+			}
+			_resetMessageEmbedderForTest();
+			_resetEmbeddingCapabilityForTest();
+		}
+	});
+});

package/src/filters/filter-config.ts

@@ -0,0 +1,69 @@
+import type {
+	IFilterAnchorRepository,
+	IFilterRepository,
+	IMessageLabelRepository,
+} from "@remit/data-ports";
+import { isEmbeddingCapabilityUnavailable } from "@remit/search-service/capability";
+import {
+	EMBEDDING_PROVIDER_OFF,
+	readEmbeddingProviderFromEnv,
+} from "@remit/search-service/from-env";
+import type { PlacementMoveService } from "../placement-move.js";
+import { getMessageEmbedder } from "./message-embedder.js";
+import type { FilterConfig, MessageEmbedder } from "./pipeline.js";
+
+export interface FilterConfigDeps {
+	filterService: IFilterRepository;
+	filterAnchorService: IFilterAnchorRepository;
+	messageLabelService: IMessageLabelRepository;
+	placementMoveService?: PlacementMoveService;
+}
+
+/**
+ * The env-selected embedder, or none on an instance that cannot embed.
+ *
+ * `off` embeds nothing by design, and its embedder says so by throwing
+ * (EmbeddingDisabledError). The pipeline has a designed skip for a *missing*
+ * embedder — semantic filters are passed over and logged at debug — but a
+ * present embedder that throws lands in the per-filter catch instead, which is
+ * an error-level `filter_anchor_match_failed` for every semantic filter on
+ * every synced message. Same outcome, a log nobody can read.
+ *
+ * `local` on the self-host profiles reaches the same place by a different road:
+ * the backend and imap-worker images ship without `@huggingface/transformers`,
+ * so the first embed throws ERR_MODULE_NOT_FOUND. The pipeline classifies that
+ * one throw through `@remit/search-service/capability` and remembers it, and
+ * from then on the config here wires no embedder either — one probe per process,
+ * not one per filter per message.
+ */
+const embedderFromEnv = (): MessageEmbedder | undefined =>
+	readEmbeddingProviderFromEnv() === EMBEDDING_PROVIDER_OFF ||
+	isEmbeddingCapabilityUnavailable()
+		? undefined
+		: getMessageEmbedder();
+
+/**
+ * Assemble the filter config a body-materializing pass runs (RFC 034), shared by
+ * the imap-worker's sync path and the backend's read-path backfill so both embed
+ * under the same model that produced the anchors — a semantic (anchor-only)
+ * filter is evaluated on either path instead of silently skipped.
+ *
+ * Absent the placement mover there is no move path, so filters stay off — a
+ * matched filter's actions reuse the same enqueue plumbing the placement mover
+ * owns.
+ */
+export const buildFilterConfig = (
+	deps: FilterConfigDeps,
+	embedder?: MessageEmbedder,
+): FilterConfig | undefined => {
+	const { placementMoveService } = deps;
+	if (!placementMoveService) return undefined;
+	const resolved = embedder ?? embedderFromEnv();
+	return {
+		filterService: deps.filterService,
+		filterAnchorService: deps.filterAnchorService,
+		messageLabelService: deps.messageLabelService,
+		placementMoveService,
+		...(resolved ? { embedder: resolved } : {}),
+	};
+};

package/src/filters/message-embedder.test.ts

@@ -0,0 +1,39 @@
+import assert from "node:assert/strict";
+import { afterEach, describe, it } from "node:test";
+import {
+	_resetMessageEmbedderForTest,
+	getMessageEmbedder,
+} from "./message-embedder.js";
+
+describe("getMessageEmbedder", () => {
+	afterEach(() => {
+		_resetMessageEmbedderForTest();
+	});
+
+	it("adapts the batch embedding service into a single-text message embedder", async () => {
+		const vector = await getMessageEmbedder().embed("invoice from stripe");
+
+		assert.ok(Array.isArray(vector));
+		assert.ok(vector.length > 0);
+		assert.ok(vector.every((value) => typeof value === "number"));
+	});
+
+	it("embeds the same text deterministically", async () => {
+		const embedder = getMessageEmbedder();
+		const a = await embedder.embed("your monthly receipt");
+		const b = await embedder.embed("your monthly receipt");
+
+		assert.deepEqual(a, b);
+	});
+
+	it("memoizes the embedder across calls", () => {
+		assert.strictEqual(getMessageEmbedder(), getMessageEmbedder());
+	});
+
+	it("rebuilds after a reset", () => {
+		const first = getMessageEmbedder();
+		_resetMessageEmbedderForTest();
+
+		assert.notStrictEqual(getMessageEmbedder(), first);
+	});
+});

package/src/filters/message-embedder.ts

@@ -0,0 +1,33 @@
+import { buildEmbeddingServiceFromEnv } from "@remit/search-service/from-env";
+import type { MessageEmbedder } from "./pipeline.js";
+
+/**
+ * Adapt the batch {@link EmbeddingService} the rest of the stack composes from
+ * env into the single-text {@link MessageEmbedder} the filter pipeline needs: one
+ * incoming message becomes one message-level vector to compare against a filter's
+ * persisted anchor (RFC 034 Decision 2.1/2.3).
+ *
+ * The embedder is selected by the same `SEARCH_EMBEDDING_*` env the search-index
+ * worker reads, so every filter pass embeds under the identical model the anchors
+ * were built with. The instance is memoized across warm invocations so a local
+ * Transformers.js model loads once per container rather than per event.
+ */
+let cached: MessageEmbedder | undefined;
+
+export const getMessageEmbedder = (): MessageEmbedder => {
+	if (cached) return cached;
+	const service = buildEmbeddingServiceFromEnv();
+	cached = {
+		embed: async (text: string): Promise<number[]> => {
+			const [vector] = await service.embed([text]);
+			return vector;
+		},
+		embeddingId: service.embeddingId,
+	};
+	return cached;
+};
+
+/** Reset the singleton — test use only. */
+export const _resetMessageEmbedderForTest = (): void => {
+	cached = undefined;
+};

package/src/filters/pipeline.ts

@@ -6,6 +6,11 @@ import type {
 	IMessageLabelRepository,
 } from "@remit/data-ports";
 import { FilterState } from "@remit/domain-enums";
+import {
+	isEmbeddingCapabilityAbsence,
+	isEmbeddingCapabilityUnavailable,
+	recordEmbeddingCapabilityAbsence,
+} from "@remit/search-service/capability";
 import type { PlacementMoveService } from "../placement-move.js";
 import {
 	type AnchorEmbedder,
@@ -131,7 +136,29 @@ export class FilterPipeline {
 				messageEmbedding = null;
 				return null;
 			}
-			messageEmbedding = await embedder.embed(buildMatchText(msg));
+			messageEmbedding = await embedder
+				.embed(buildMatchText(msg))
+				.catch((error: unknown) => {
+					// This deployment carries no embedding pipeline at all — the
+					// container images ship without `@huggingface/transformers`. That is
+					// the same absence `SEARCH_EMBEDDING_PROVIDER=off` states, so it takes
+					// the same designed skip rather than an error-level
+					// `filter_anchor_match_failed` per semantic filter per message. The
+					// absence is remembered process-wide, so this is the only embed that
+					// pays for it: `buildFilterConfig` wires no embedder afterwards.
+					if (!isEmbeddingCapabilityAbsence(error)) throw error;
+					if (!isEmbeddingCapabilityUnavailable()) {
+						this.log.warn?.(
+							{
+								accountConfigId,
+								error: inspect(error),
+							},
+							"No embedding pipeline in this deployment; semantic filters are skipped (literal filters still run)",
+						);
+					}
+					recordEmbeddingCapabilityAbsence(error);
+					return null;
+				});
 			return messageEmbedding;
 		};