@remit/imap-worker 0.0.22 → 0.0.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.22",
+  "version": "0.0.23",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -37,6 +37,7 @@
     "@remit/logger-lambda": "*",
     "@remit/mailbox-service": "*",
     "@remit/search-index-worker": "*",
+    "@remit/search-service": "*",
     "@remit/sqs-client": "*",
     "@remit/backend": "*",
     "@remit/domain-enums": "*",

package/src/filter-config.test.ts

@@ -0,0 +1,112 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	FilterItem,
+	IFilterAnchorRepository,
+	IFilterRepository,
+	IMessageLabelRepository,
+} from "@remit/data-ports";
+import { FilterMatchOperator, FilterState } from "@remit/domain-enums";
+import {
+	type FilterMessage,
+	FilterPipeline,
+	type MessageEmbedder,
+	NO_ACTION,
+	type PlacementMoveService,
+} from "@remit/mailbox-service";
+import { buildFilterConfig, type FilterConfigDeps } from "./filter-config.js";
+
+const anchorOnlyFilter = (destinationMailboxId: string): FilterItem =>
+	({
+		filterId: "flt-semantic",
+		accountConfigId: "cfg-1",
+		name: "receipts",
+		scope: "Standing",
+		state: FilterState.Active,
+		hasAnchor: true,
+		ruleChangedAt: 1,
+		matchOperator: FilterMatchOperator.Or,
+		literalClauses: [],
+		actionLabelId: NO_ACTION,
+		actionMailboxId: destinationMailboxId,
+		createdAt: 1,
+		updatedAt: 1,
+	}) as unknown as FilterItem;
+
+const repos = (filter: FilterItem): FilterConfigDeps => ({
+	filterService: {
+		listByAccountAndState: async () => [filter],
+		refreshExpiry: async (f: FilterItem) => f,
+	} as unknown as IFilterRepository,
+	filterAnchorService: {
+		get: async () => ({ anchorEmbedding: [1, 0, 0] }),
+	} as unknown as IFilterAnchorRepository,
+	messageLabelService: {} as unknown as IMessageLabelRepository,
+	placementMoveService: {} as unknown as PlacementMoveService,
+});
+
+const message: FilterMessage = {
+	from: "billing@stripe.com",
+	fromName: "Stripe",
+	subject: "Your receipt",
+	text: "Thanks for your payment",
+	listId: "",
+};
+
+describe("buildFilterConfig", () => {
+	it("returns undefined without a placement mover — no move path, filters stay off", () => {
+		const deps = repos(anchorOnlyFilter("mbx-archive"));
+		deps.placementMoveService = undefined;
+
+		assert.equal(buildFilterConfig(deps), undefined);
+	});
+
+	it("wires an embedder into the config", () => {
+		const config = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
+
+		assert.ok(config);
+		assert.ok(config.embedder);
+	});
+
+	it("lights up a semantic anchor-only filter on the body-sync pass", async () => {
+		let embedCalls = 0;
+		const embedder: MessageEmbedder = {
+			embed: async () => {
+				embedCalls += 1;
+				return [1, 0, 0];
+			},
+		};
+		const config = buildFilterConfig(
+			repos(anchorOnlyFilter("mbx-archive")),
+			embedder,
+		);
+		assert.ok(config);
+
+		const decision = await new FilterPipeline(config, {
+			info: () => {},
+		}).evaluate("cfg-1", "m-1", message);
+
+		assert.deepEqual(decision.move, {
+			destinationMailboxId: "mbx-archive",
+			filterId: "flt-semantic",
+		});
+		assert.equal(embedCalls, 1);
+	});
+
+	it("does not match when the message embedding diverges from the anchor", async () => {
+		const embedder: MessageEmbedder = {
+			embed: async () => [0, 1, 0],
+		};
+		const config = buildFilterConfig(
+			repos(anchorOnlyFilter("mbx-archive")),
+			embedder,
+		);
+		assert.ok(config);
+
+		const decision = await new FilterPipeline(config, {
+			info: () => {},
+		}).evaluate("cfg-1", "m-2", message);
+
+		assert.equal(decision.move, undefined);
+	});
+});

package/src/filter-config.ts

@@ -0,0 +1,43 @@
+import type {
+	IFilterAnchorRepository,
+	IFilterRepository,
+	IMessageLabelRepository,
+} from "@remit/data-ports";
+import type {
+	FilterConfig,
+	MessageEmbedder,
+	PlacementMoveService,
+} from "@remit/mailbox-service";
+import { getMessageEmbedder } from "./message-embedder.js";
+
+export interface FilterConfigDeps {
+	filterService: IFilterRepository;
+	filterAnchorService: IFilterAnchorRepository;
+	messageLabelService: IMessageLabelRepository;
+	placementMoveService?: PlacementMoveService;
+}
+
+/**
+ * Assemble the index-time filter config the body-sync pass runs (RFC 034). The
+ * embedder is provisioned from env exactly as the backend read-path and
+ * search-index worker provision theirs, so a semantic (anchor-only) filter is
+ * evaluated on incoming mail instead of silently skipped.
+ *
+ * Absent the placement mover there is no move path, so filters stay off — a
+ * matched filter's actions reuse the same enqueue plumbing the placement mover
+ * owns.
+ */
+export const buildFilterConfig = (
+	deps: FilterConfigDeps,
+	embedder?: MessageEmbedder,
+): FilterConfig | undefined => {
+	const { placementMoveService } = deps;
+	if (!placementMoveService) return undefined;
+	return {
+		filterService: deps.filterService,
+		filterAnchorService: deps.filterAnchorService,
+		messageLabelService: deps.messageLabelService,
+		placementMoveService,
+		embedder: embedder ?? getMessageEmbedder(),
+	};
+};

package/src/handlers/sync-message-body.ts

@@ -3,7 +3,6 @@ import type { Logger } from "@remit/logger-lambda";
 import { MetricUnit, metrics } from "@remit/logger-lambda";
 import {
 	BodySyncService,
-	type FilterConfig,
 	guardConnectionCursor,
 	isCursorRebuildNeeded,
 	MailboxCursorPausedError,
@@ -19,6 +18,7 @@ import {
 	createConnectionScopeWithCredentials,
 } from "../connection-scope.js";
 import type { SyncMessageBodyEvent } from "../events.js";
+import { buildFilterConfig } from "../filter-config.js";
 import { isNotFoundError } from "../is-not-found.js";
 import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
 import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
@@ -237,16 +237,14 @@ export const syncMessageBody = async (
 
 			// A matched filter's actions (label upsert, exclusive move) apply on
 			// the same body-sync pass, reusing the placement mover for the move
-			// (RFC 034 Decision 3.1). Absent the placement mover there is no move
-			// path, so filters stay off — the two share the same enqueue plumbing.
-			const filterConfig: FilterConfig | undefined = placementMoveService
-				? {
-						filterService,
-						filterAnchorService,
-						messageLabelService,
-						placementMoveService,
-					}
-				: undefined;
+			// (RFC 034 Decision 3.1). The env-provisioned embedder lets a semantic
+			// (anchor-only) filter match here instead of being silently skipped.
+			const filterConfig = buildFilterConfig({
+				filterService,
+				filterAnchorService,
+				messageLabelService,
+				placementMoveService,
+			});
 
 			const bodySyncService = new BodySyncService(
 				messageService,

package/src/message-embedder.test.ts

@@ -0,0 +1,39 @@
+import assert from "node:assert/strict";
+import { afterEach, describe, it } from "node:test";
+import {
+	_resetMessageEmbedderForTest,
+	getMessageEmbedder,
+} from "./message-embedder.js";
+
+describe("getMessageEmbedder", () => {
+	afterEach(() => {
+		_resetMessageEmbedderForTest();
+	});
+
+	it("adapts the batch embedding service into a single-text message embedder", async () => {
+		const vector = await getMessageEmbedder().embed("invoice from stripe");
+
+		assert.ok(Array.isArray(vector));
+		assert.ok(vector.length > 0);
+		assert.ok(vector.every((value) => typeof value === "number"));
+	});
+
+	it("embeds the same text deterministically", async () => {
+		const embedder = getMessageEmbedder();
+		const a = await embedder.embed("your monthly receipt");
+		const b = await embedder.embed("your monthly receipt");
+
+		assert.deepEqual(a, b);
+	});
+
+	it("memoizes the embedder across calls", () => {
+		assert.strictEqual(getMessageEmbedder(), getMessageEmbedder());
+	});
+
+	it("rebuilds after a reset", () => {
+		const first = getMessageEmbedder();
+		_resetMessageEmbedderForTest();
+
+		assert.notStrictEqual(getMessageEmbedder(), first);
+	});
+});

package/src/message-embedder.ts

@@ -0,0 +1,32 @@
+import type { MessageEmbedder } from "@remit/mailbox-service";
+import { buildEmbeddingServiceFromEnv } from "@remit/search-service/from-env";
+
+/**
+ * Adapt the batch {@link EmbeddingService} the rest of the stack composes from
+ * env into the single-text {@link MessageEmbedder} the filter pipeline needs: one
+ * incoming message becomes one message-level vector to compare against a filter's
+ * persisted anchor (RFC 034 Decision 2.1/2.3).
+ *
+ * The embedder is selected by the same `SEARCH_EMBEDDING_*` env the backend and
+ * search-index worker read, so the worker embeds under the identical model the
+ * anchors were built with. The instance is memoized across warm invocations so a
+ * local Transformers.js model loads once per container rather than per event.
+ */
+let cached: MessageEmbedder | undefined;
+
+export const getMessageEmbedder = (): MessageEmbedder => {
+	if (cached) return cached;
+	const service = buildEmbeddingServiceFromEnv();
+	cached = {
+		embed: async (text: string): Promise<number[]> => {
+			const [vector] = await service.embed([text]);
+			return vector;
+		},
+	};
+	return cached;
+};
+
+/** Reset the singleton — test use only. */
+export const _resetMessageEmbedderForTest = (): void => {
+	cached = undefined;
+};