@remit/imap-worker 0.0.71 → 0.0.72

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.71",
+  "version": "0.0.72",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -39,7 +39,6 @@
     "@remit/logger-lambda": "*",
     "@remit/mailbox-service": "*",
     "@remit/search-index-worker": "*",
-    "@remit/search-service": "*",
     "@remit/sqs-client": "*",
     "@remit/backend": "*",
     "@remit/domain-enums": "*",

package/src/handlers/sync-message-body.ts

@@ -11,6 +11,7 @@ import {
 	QuarantineService,
 	resolveExhaustedBodySyncFailures,
 } from "@remit/mailbox-service";
+import { buildFilterConfig } from "@remit/mailbox-service/filter-config";
 import { attemptBudget } from "@remit/sqs-client/attempt-budget";
 import { env } from "expect-env";
 import { isAccountDeleted } from "../account-check.js";
@@ -20,7 +21,6 @@ import {
 	createConnectionScopeWithCredentials,
 } from "../connection-scope.js";
 import type { SyncMessageBodyEvent } from "../events.js";
-import { buildFilterConfig } from "../filter-config.js";
 import { isNotFoundError } from "../is-not-found.js";
 import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
 import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";

package/src/filter-config.test.ts

@@ -1,171 +0,0 @@
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import type {
-	FilterItem,
-	IFilterAnchorRepository,
-	IFilterRepository,
-	IMessageLabelRepository,
-} from "@remit/data-ports";
-import { FilterMatchOperator, FilterState } from "@remit/domain-enums";
-import {
-	type FilterMessage,
-	FilterPipeline,
-	type MessageEmbedder,
-	NO_ACTION,
-	type PlacementMoveService,
-} from "@remit/mailbox-service";
-import { buildFilterConfig, type FilterConfigDeps } from "./filter-config.js";
-import { _resetMessageEmbedderForTest } from "./message-embedder.js";
-
-const anchorOnlyFilter = (destinationMailboxId: string): FilterItem =>
-	({
-		filterId: "flt-semantic",
-		accountConfigId: "cfg-1",
-		name: "receipts",
-		scope: "Standing",
-		state: FilterState.Active,
-		hasAnchor: true,
-		ruleChangedAt: 1,
-		matchOperator: FilterMatchOperator.Or,
-		literalClauses: [],
-		actionLabelId: NO_ACTION,
-		actionMailboxId: destinationMailboxId,
-		createdAt: 1,
-		updatedAt: 1,
-	}) as unknown as FilterItem;
-
-const CURRENT_MODEL = "test-model@3";
-
-const repos = (filter: FilterItem): FilterConfigDeps => ({
-	filterService: {
-		listByAccountAndState: async () => [filter],
-		refreshExpiry: async (f: FilterItem) => f,
-	} as unknown as IFilterRepository,
-	filterAnchorService: {
-		get: async () => ({
-			anchorEmbedding: [1, 0, 0],
-			anchorEmbeddingId: CURRENT_MODEL,
-		}),
-	} as unknown as IFilterAnchorRepository,
-	messageLabelService: {} as unknown as IMessageLabelRepository,
-	placementMoveService: {} as unknown as PlacementMoveService,
-});
-
-const message: FilterMessage = {
-	from: "billing@stripe.com",
-	fromName: "Stripe",
-	subject: "Your receipt",
-	text: "Thanks for your payment",
-	listId: "",
-};
-
-describe("buildFilterConfig", () => {
-	it("returns undefined without a placement mover — no move path, filters stay off", () => {
-		const deps = repos(anchorOnlyFilter("mbx-archive"));
-		deps.placementMoveService = undefined;
-
-		assert.equal(buildFilterConfig(deps), undefined);
-	});
-
-	it("wires an embedder into the config", () => {
-		const config = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
-
-		assert.ok(config);
-		assert.ok(config.embedder);
-	});
-
-	it("lights up a semantic anchor-only filter on the body-sync pass", async () => {
-		let embedCalls = 0;
-		const embedder: MessageEmbedder = {
-			embed: async () => {
-				embedCalls += 1;
-				return [1, 0, 0];
-			},
-			embeddingId: CURRENT_MODEL,
-		};
-		const config = buildFilterConfig(
-			repos(anchorOnlyFilter("mbx-archive")),
-			embedder,
-		);
-		assert.ok(config);
-
-		const decision = await new FilterPipeline(config, {
-			info: () => {},
-		}).evaluate("cfg-1", "m-1", message);
-
-		assert.deepEqual(decision.move, {
-			destinationMailboxId: "mbx-archive",
-			filterId: "flt-semantic",
-		});
-		assert.equal(embedCalls, 1);
-	});
-
-	it("does not match when the message embedding diverges from the anchor", async () => {
-		const embedder: MessageEmbedder = {
-			embed: async () => [0, 1, 0],
-			embeddingId: CURRENT_MODEL,
-		};
-		const config = buildFilterConfig(
-			repos(anchorOnlyFilter("mbx-archive")),
-			embedder,
-		);
-		assert.ok(config);
-
-		const decision = await new FilterPipeline(config, {
-			info: () => {},
-		}).evaluate("cfg-1", "m-2", message);
-
-		assert.equal(decision.move, undefined);
-	});
-});
-
-// An instance with semantic search off (issue #1068). `off` selects an embedder
-// that throws by design, and the throw lands in the pipeline's per-filter catch:
-// one error-level `filter_anchor_match_failed` for every semantic filter on
-// every message it syncs. The pipeline already has a designed skip for a
-// missing embedder, so the config passes none.
-describe("buildFilterConfig with semantic search off", () => {
-	const withProviderOff = async (body: () => Promise<void>): Promise<void> => {
-		const previous = process.env.SEARCH_EMBEDDING_PROVIDER;
-		process.env.SEARCH_EMBEDDING_PROVIDER = "off";
-		_resetMessageEmbedderForTest();
-		try {
-			await body();
-		} finally {
-			if (previous === undefined) {
-				delete process.env.SEARCH_EMBEDDING_PROVIDER;
-			} else {
-				process.env.SEARCH_EMBEDDING_PROVIDER = previous;
-			}
-			_resetMessageEmbedderForTest();
-		}
-	};
-
-	it("wires no embedder", async () => {
-		await withProviderOff(async () => {
-			const config = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
-
-			assert.ok(config);
-			assert.equal(config.embedder, undefined);
-		});
-	});
-
-	it("skips the semantic filter without logging an error", async () => {
-		await withProviderOff(async () => {
-			const config = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
-			assert.ok(config);
-
-			const errors: unknown[] = [];
-			const decision = await new FilterPipeline(config, {
-				info: () => {},
-				debug: () => {},
-				warn: () => {},
-				error: (fields) => errors.push(fields.alert),
-			}).evaluate("cfg-1", "m-off", message);
-
-			assert.deepEqual(decision.labels, []);
-			assert.equal(decision.move, undefined);
-			assert.deepEqual(errors, []);
-		});
-	});
-});

package/src/filter-config.ts

@@ -1,63 +0,0 @@
-import type {
-	IFilterAnchorRepository,
-	IFilterRepository,
-	IMessageLabelRepository,
-} from "@remit/data-ports";
-import type {
-	FilterConfig,
-	MessageEmbedder,
-	PlacementMoveService,
-} from "@remit/mailbox-service";
-import {
-	EMBEDDING_PROVIDER_OFF,
-	readEmbeddingProviderFromEnv,
-} from "@remit/search-service/from-env";
-import { getMessageEmbedder } from "./message-embedder.js";
-
-export interface FilterConfigDeps {
-	filterService: IFilterRepository;
-	filterAnchorService: IFilterAnchorRepository;
-	messageLabelService: IMessageLabelRepository;
-	placementMoveService?: PlacementMoveService;
-}
-
-/**
- * The env-selected embedder, or none on an instance with semantic search off.
- *
- * `off` embeds nothing by design, and its embedder says so by throwing
- * (EmbeddingDisabledError). The pipeline has a designed skip for a *missing*
- * embedder — semantic filters are passed over and logged at debug — but a
- * present embedder that throws lands in the per-filter catch instead, which is
- * an error-level `filter_anchor_match_failed` for every semantic filter on
- * every synced message. Same outcome, a log nobody can read.
- */
-const embedderFromEnv = (): MessageEmbedder | undefined =>
-	readEmbeddingProviderFromEnv() === EMBEDDING_PROVIDER_OFF
-		? undefined
-		: getMessageEmbedder();
-
-/**
- * Assemble the index-time filter config the body-sync pass runs (RFC 034). The
- * embedder is provisioned from env exactly as the backend read-path and
- * search-index worker provision theirs, so a semantic (anchor-only) filter is
- * evaluated on incoming mail instead of silently skipped.
- *
- * Absent the placement mover there is no move path, so filters stay off — a
- * matched filter's actions reuse the same enqueue plumbing the placement mover
- * owns.
- */
-export const buildFilterConfig = (
-	deps: FilterConfigDeps,
-	embedder?: MessageEmbedder,
-): FilterConfig | undefined => {
-	const { placementMoveService } = deps;
-	if (!placementMoveService) return undefined;
-	const resolved = embedder ?? embedderFromEnv();
-	return {
-		filterService: deps.filterService,
-		filterAnchorService: deps.filterAnchorService,
-		messageLabelService: deps.messageLabelService,
-		placementMoveService,
-		...(resolved ? { embedder: resolved } : {}),
-	};
-};

package/src/message-embedder.test.ts

@@ -1,39 +0,0 @@
-import assert from "node:assert/strict";
-import { afterEach, describe, it } from "node:test";
-import {
-	_resetMessageEmbedderForTest,
-	getMessageEmbedder,
-} from "./message-embedder.js";
-
-describe("getMessageEmbedder", () => {
-	afterEach(() => {
-		_resetMessageEmbedderForTest();
-	});
-
-	it("adapts the batch embedding service into a single-text message embedder", async () => {
-		const vector = await getMessageEmbedder().embed("invoice from stripe");
-
-		assert.ok(Array.isArray(vector));
-		assert.ok(vector.length > 0);
-		assert.ok(vector.every((value) => typeof value === "number"));
-	});
-
-	it("embeds the same text deterministically", async () => {
-		const embedder = getMessageEmbedder();
-		const a = await embedder.embed("your monthly receipt");
-		const b = await embedder.embed("your monthly receipt");
-
-		assert.deepEqual(a, b);
-	});
-
-	it("memoizes the embedder across calls", () => {
-		assert.strictEqual(getMessageEmbedder(), getMessageEmbedder());
-	});
-
-	it("rebuilds after a reset", () => {
-		const first = getMessageEmbedder();
-		_resetMessageEmbedderForTest();
-
-		assert.notStrictEqual(getMessageEmbedder(), first);
-	});
-});

package/src/message-embedder.ts

@@ -1,33 +0,0 @@
-import type { MessageEmbedder } from "@remit/mailbox-service";
-import { buildEmbeddingServiceFromEnv } from "@remit/search-service/from-env";
-
-/**
- * Adapt the batch {@link EmbeddingService} the rest of the stack composes from
- * env into the single-text {@link MessageEmbedder} the filter pipeline needs: one
- * incoming message becomes one message-level vector to compare against a filter's
- * persisted anchor (RFC 034 Decision 2.1/2.3).
- *
- * The embedder is selected by the same `SEARCH_EMBEDDING_*` env the backend and
- * search-index worker read, so the worker embeds under the identical model the
- * anchors were built with. The instance is memoized across warm invocations so a
- * local Transformers.js model loads once per container rather than per event.
- */
-let cached: MessageEmbedder | undefined;
-
-export const getMessageEmbedder = (): MessageEmbedder => {
-	if (cached) return cached;
-	const service = buildEmbeddingServiceFromEnv();
-	cached = {
-		embed: async (text: string): Promise<number[]> => {
-			const [vector] = await service.embed([text]);
-			return vector;
-		},
-		embeddingId: service.embeddingId,
-	};
-	return cached;
-};
-
-/** Reset the singleton — test use only. */
-export const _resetMessageEmbedderForTest = (): void => {
-	cached = undefined;
-};