@remit/mailbox-service 0.0.90 → 0.0.92

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.90",
+  "version": "0.0.92",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -9,6 +9,10 @@
       "types": "./src/index.ts",
       "default": "./src/index.ts"
     },
+    "./filter-config": {
+      "types": "./src/filters/filter-config.ts",
+      "default": "./src/filters/filter-config.ts"
+    },
     "./body-sync-queue": {
       "types": "./src/body-sync-queue.ts",
       "default": "./src/body-sync-queue.ts"
@@ -42,6 +46,7 @@
     "@remit/data-ports": "*",
     "@remit/domain-enums": "*",
     "@remit/mail-oauth-service": "*",
+    "@remit/search-service": "*",
     "@remit/secrets-service": "*",
     "@remit/sqs-client": "*",
     "@remit/storage-service": "*",

package/src/filters/filter-config.test.ts

@@ -0,0 +1,229 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	FilterItem,
+	IFilterAnchorRepository,
+	IFilterRepository,
+	IMessageLabelRepository,
+} from "@remit/data-ports";
+import { FilterMatchOperator, FilterState } from "@remit/domain-enums";
+import { _resetEmbeddingCapabilityForTest } from "@remit/search-service/capability";
+import type { PlacementMoveService } from "../placement-move.js";
+import { buildFilterConfig, type FilterConfigDeps } from "./filter-config.js";
+import { type FilterMessage, NO_ACTION } from "./match.js";
+import { _resetMessageEmbedderForTest } from "./message-embedder.js";
+import { FilterPipeline, type MessageEmbedder } from "./pipeline.js";
+
+const anchorOnlyFilter = (destinationMailboxId: string): FilterItem =>
+	({
+		filterId: "flt-semantic",
+		accountConfigId: "cfg-1",
+		name: "receipts",
+		scope: "Standing",
+		state: FilterState.Active,
+		hasAnchor: true,
+		ruleChangedAt: 1,
+		matchOperator: FilterMatchOperator.Or,
+		literalClauses: [],
+		actionLabelId: NO_ACTION,
+		actionMailboxId: destinationMailboxId,
+		createdAt: 1,
+		updatedAt: 1,
+	}) as unknown as FilterItem;
+
+const CURRENT_MODEL = "test-model@3";
+
+const repos = (filter: FilterItem): FilterConfigDeps => ({
+	filterService: {
+		listByAccountAndState: async () => [filter],
+		refreshExpiry: async (f: FilterItem) => f,
+	} as unknown as IFilterRepository,
+	filterAnchorService: {
+		get: async () => ({
+			anchorEmbedding: [1, 0, 0],
+			anchorEmbeddingId: CURRENT_MODEL,
+		}),
+	} as unknown as IFilterAnchorRepository,
+	messageLabelService: {} as unknown as IMessageLabelRepository,
+	placementMoveService: {} as unknown as PlacementMoveService,
+});
+
+const message: FilterMessage = {
+	from: "billing@stripe.com",
+	fromName: "Stripe",
+	subject: "Your receipt",
+	text: "Thanks for your payment",
+	listId: "",
+};
+
+describe("buildFilterConfig", () => {
+	it("returns undefined without a placement mover — no move path, filters stay off", () => {
+		const deps = repos(anchorOnlyFilter("mbx-archive"));
+		deps.placementMoveService = undefined;
+
+		assert.equal(buildFilterConfig(deps), undefined);
+	});
+
+	it("wires an embedder into the config", () => {
+		const config = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
+
+		assert.ok(config);
+		assert.ok(config.embedder);
+	});
+
+	it("lights up a semantic anchor-only filter on the body-sync pass", async () => {
+		let embedCalls = 0;
+		const embedder: MessageEmbedder = {
+			embed: async () => {
+				embedCalls += 1;
+				return [1, 0, 0];
+			},
+			embeddingId: CURRENT_MODEL,
+		};
+		const config = buildFilterConfig(
+			repos(anchorOnlyFilter("mbx-archive")),
+			embedder,
+		);
+		assert.ok(config);
+
+		const decision = await new FilterPipeline(config, {
+			info: () => {},
+		}).evaluate("cfg-1", "m-1", message);
+
+		assert.deepEqual(decision.move, {
+			destinationMailboxId: "mbx-archive",
+			filterId: "flt-semantic",
+		});
+		assert.equal(embedCalls, 1);
+	});
+
+	it("does not match when the message embedding diverges from the anchor", async () => {
+		const embedder: MessageEmbedder = {
+			embed: async () => [0, 1, 0],
+			embeddingId: CURRENT_MODEL,
+		};
+		const config = buildFilterConfig(
+			repos(anchorOnlyFilter("mbx-archive")),
+			embedder,
+		);
+		assert.ok(config);
+
+		const decision = await new FilterPipeline(config, {
+			info: () => {},
+		}).evaluate("cfg-1", "m-2", message);
+
+		assert.equal(decision.move, undefined);
+	});
+});
+
+// An instance with semantic search off (issue #1068). `off` selects an embedder
+// that throws by design, and the throw lands in the pipeline's per-filter catch:
+// one error-level `filter_anchor_match_failed` for every semantic filter on
+// every message it syncs. The pipeline already has a designed skip for a
+// missing embedder, so the config passes none.
+describe("buildFilterConfig with semantic search off", () => {
+	const withProviderOff = async (body: () => Promise<void>): Promise<void> => {
+		const previous = process.env.SEARCH_EMBEDDING_PROVIDER;
+		process.env.SEARCH_EMBEDDING_PROVIDER = "off";
+		_resetMessageEmbedderForTest();
+		try {
+			await body();
+		} finally {
+			if (previous === undefined) {
+				delete process.env.SEARCH_EMBEDDING_PROVIDER;
+			} else {
+				process.env.SEARCH_EMBEDDING_PROVIDER = previous;
+			}
+			_resetMessageEmbedderForTest();
+		}
+	};
+
+	it("wires no embedder", async () => {
+		await withProviderOff(async () => {
+			const config = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
+
+			assert.ok(config);
+			assert.equal(config.embedder, undefined);
+		});
+	});
+
+	it("skips the semantic filter without logging an error", async () => {
+		await withProviderOff(async () => {
+			const config = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
+			assert.ok(config);
+
+			const errors: unknown[] = [];
+			const decision = await new FilterPipeline(config, {
+				info: () => {},
+				debug: () => {},
+				warn: () => {},
+				error: (fields) => errors.push(fields.alert),
+			}).evaluate("cfg-1", "m-off", message);
+
+			assert.deepEqual(decision.labels, []);
+			assert.equal(decision.move, undefined);
+			assert.deepEqual(errors, []);
+		});
+	});
+});
+
+// The self-host `local` profile asks for the Transformers.js embedder, but the
+// backend and imap-worker images ship without `@huggingface/transformers`
+// (npm-scripts/docker-bundle.mjs). The first embed throws ERR_MODULE_NOT_FOUND,
+// which is the same capability absence `off` states and must take the same
+// designed skip — not an error-level `filter_anchor_match_failed` per semantic
+// filter per message on a user-facing GET.
+describe("buildFilterConfig when this deployment carries no embedder", () => {
+	const moduleNotFound = (): Error =>
+		Object.assign(
+			new Error("Cannot find package '@huggingface/transformers'"),
+			{ code: "ERR_MODULE_NOT_FOUND" },
+		);
+
+	it("skips the semantic filter, logs no error, and stops wiring an embedder", async () => {
+		const previous = process.env.SEARCH_EMBEDDING_PROVIDER;
+		process.env.SEARCH_EMBEDDING_PROVIDER = "local";
+		_resetMessageEmbedderForTest();
+		_resetEmbeddingCapabilityForTest();
+
+		try {
+			const unavailable: MessageEmbedder = {
+				embed: async () => {
+					throw moduleNotFound();
+				},
+				embeddingId: CURRENT_MODEL,
+			};
+			const config = buildFilterConfig(
+				repos(anchorOnlyFilter("mbx-archive")),
+				unavailable,
+			);
+			assert.ok(config);
+
+			const errors: unknown[] = [];
+			const decision = await new FilterPipeline(config, {
+				info: () => {},
+				debug: () => {},
+				warn: () => {},
+				error: (fields) => errors.push(fields.alert),
+			}).evaluate("cfg-1", "m-local", message);
+
+			assert.deepEqual(decision.labels, []);
+			assert.equal(decision.move, undefined);
+			assert.deepEqual(errors, []);
+
+			// Probed once, not per filter or per message: every later config on this
+			// process wires no embedder at all, so nothing else attempts the embed.
+			const next = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
+			assert.ok(next);
+			assert.equal(next.embedder, undefined);
+		} finally {
+			if (previous === undefined) {
+				delete process.env.SEARCH_EMBEDDING_PROVIDER;
+			} else {
+				process.env.SEARCH_EMBEDDING_PROVIDER = previous;
+			}
+			_resetMessageEmbedderForTest();
+			_resetEmbeddingCapabilityForTest();
+		}
+	});
+});

package/src/filters/filter-config.ts

@@ -0,0 +1,69 @@
+import type {
+	IFilterAnchorRepository,
+	IFilterRepository,
+	IMessageLabelRepository,
+} from "@remit/data-ports";
+import { isEmbeddingCapabilityUnavailable } from "@remit/search-service/capability";
+import {
+	EMBEDDING_PROVIDER_OFF,
+	readEmbeddingProviderFromEnv,
+} from "@remit/search-service/from-env";
+import type { PlacementMoveService } from "../placement-move.js";
+import { getMessageEmbedder } from "./message-embedder.js";
+import type { FilterConfig, MessageEmbedder } from "./pipeline.js";
+
+export interface FilterConfigDeps {
+	filterService: IFilterRepository;
+	filterAnchorService: IFilterAnchorRepository;
+	messageLabelService: IMessageLabelRepository;
+	placementMoveService?: PlacementMoveService;
+}
+
+/**
+ * The env-selected embedder, or none on an instance that cannot embed.
+ *
+ * `off` embeds nothing by design, and its embedder says so by throwing
+ * (EmbeddingDisabledError). The pipeline has a designed skip for a *missing*
+ * embedder — semantic filters are passed over and logged at debug — but a
+ * present embedder that throws lands in the per-filter catch instead, which is
+ * an error-level `filter_anchor_match_failed` for every semantic filter on
+ * every synced message. Same outcome, a log nobody can read.
+ *
+ * `local` on the self-host profiles reaches the same place by a different road:
+ * the backend and imap-worker images ship without `@huggingface/transformers`,
+ * so the first embed throws ERR_MODULE_NOT_FOUND. The pipeline classifies that
+ * one throw through `@remit/search-service/capability` and remembers it, and
+ * from then on the config here wires no embedder either — one probe per process,
+ * not one per filter per message.
+ */
+const embedderFromEnv = (): MessageEmbedder | undefined =>
+	readEmbeddingProviderFromEnv() === EMBEDDING_PROVIDER_OFF ||
+	isEmbeddingCapabilityUnavailable()
+		? undefined
+		: getMessageEmbedder();
+
+/**
+ * Assemble the filter config a body-materializing pass runs (RFC 034), shared by
+ * the imap-worker's sync path and the backend's read-path backfill so both embed
+ * under the same model that produced the anchors — a semantic (anchor-only)
+ * filter is evaluated on either path instead of silently skipped.
+ *
+ * Absent the placement mover there is no move path, so filters stay off — a
+ * matched filter's actions reuse the same enqueue plumbing the placement mover
+ * owns.
+ */
+export const buildFilterConfig = (
+	deps: FilterConfigDeps,
+	embedder?: MessageEmbedder,
+): FilterConfig | undefined => {
+	const { placementMoveService } = deps;
+	if (!placementMoveService) return undefined;
+	const resolved = embedder ?? embedderFromEnv();
+	return {
+		filterService: deps.filterService,
+		filterAnchorService: deps.filterAnchorService,
+		messageLabelService: deps.messageLabelService,
+		placementMoveService,
+		...(resolved ? { embedder: resolved } : {}),
+	};
+};

package/src/filters/message-embedder.test.ts

@@ -0,0 +1,39 @@
+import assert from "node:assert/strict";
+import { afterEach, describe, it } from "node:test";
+import {
+	_resetMessageEmbedderForTest,
+	getMessageEmbedder,
+} from "./message-embedder.js";
+
+describe("getMessageEmbedder", () => {
+	afterEach(() => {
+		_resetMessageEmbedderForTest();
+	});
+
+	it("adapts the batch embedding service into a single-text message embedder", async () => {
+		const vector = await getMessageEmbedder().embed("invoice from stripe");
+
+		assert.ok(Array.isArray(vector));
+		assert.ok(vector.length > 0);
+		assert.ok(vector.every((value) => typeof value === "number"));
+	});
+
+	it("embeds the same text deterministically", async () => {
+		const embedder = getMessageEmbedder();
+		const a = await embedder.embed("your monthly receipt");
+		const b = await embedder.embed("your monthly receipt");
+
+		assert.deepEqual(a, b);
+	});
+
+	it("memoizes the embedder across calls", () => {
+		assert.strictEqual(getMessageEmbedder(), getMessageEmbedder());
+	});
+
+	it("rebuilds after a reset", () => {
+		const first = getMessageEmbedder();
+		_resetMessageEmbedderForTest();
+
+		assert.notStrictEqual(getMessageEmbedder(), first);
+	});
+});

package/src/filters/message-embedder.ts

@@ -0,0 +1,33 @@
+import { buildEmbeddingServiceFromEnv } from "@remit/search-service/from-env";
+import type { MessageEmbedder } from "./pipeline.js";
+
+/**
+ * Adapt the batch {@link EmbeddingService} the rest of the stack composes from
+ * env into the single-text {@link MessageEmbedder} the filter pipeline needs: one
+ * incoming message becomes one message-level vector to compare against a filter's
+ * persisted anchor (RFC 034 Decision 2.1/2.3).
+ *
+ * The embedder is selected by the same `SEARCH_EMBEDDING_*` env the search-index
+ * worker reads, so every filter pass embeds under the identical model the anchors
+ * were built with. The instance is memoized across warm invocations so a local
+ * Transformers.js model loads once per container rather than per event.
+ */
+let cached: MessageEmbedder | undefined;
+
+export const getMessageEmbedder = (): MessageEmbedder => {
+	if (cached) return cached;
+	const service = buildEmbeddingServiceFromEnv();
+	cached = {
+		embed: async (text: string): Promise<number[]> => {
+			const [vector] = await service.embed([text]);
+			return vector;
+		},
+		embeddingId: service.embeddingId,
+	};
+	return cached;
+};
+
+/** Reset the singleton — test use only. */
+export const _resetMessageEmbedderForTest = (): void => {
+	cached = undefined;
+};

package/src/filters/pipeline.ts

@@ -6,6 +6,11 @@ import type {
 	IMessageLabelRepository,
 } from "@remit/data-ports";
 import { FilterState } from "@remit/domain-enums";
+import {
+	isEmbeddingCapabilityAbsence,
+	isEmbeddingCapabilityUnavailable,
+	recordEmbeddingCapabilityAbsence,
+} from "@remit/search-service/capability";
 import type { PlacementMoveService } from "../placement-move.js";
 import {
 	type AnchorEmbedder,
@@ -131,7 +136,29 @@ export class FilterPipeline {
 				messageEmbedding = null;
 				return null;
 			}
-			messageEmbedding = await embedder.embed(buildMatchText(msg));
+			messageEmbedding = await embedder
+				.embed(buildMatchText(msg))
+				.catch((error: unknown) => {
+					// This deployment carries no embedding pipeline at all — the
+					// container images ship without `@huggingface/transformers`. That is
+					// the same absence `SEARCH_EMBEDDING_PROVIDER=off` states, so it takes
+					// the same designed skip rather than an error-level
+					// `filter_anchor_match_failed` per semantic filter per message. The
+					// absence is remembered process-wide, so this is the only embed that
+					// pays for it: `buildFilterConfig` wires no embedder afterwards.
+					if (!isEmbeddingCapabilityAbsence(error)) throw error;
+					if (!isEmbeddingCapabilityUnavailable()) {
+						this.log.warn?.(
+							{
+								accountConfigId,
+								error: inspect(error),
+							},
+							"No embedding pipeline in this deployment; semantic filters are skipped (literal filters still run)",
+						);
+					}
+					recordEmbeddingCapabilityAbsence(error);
+					return null;
+				});
 			return messageEmbedding;
 		};
 

package/src/index.ts

@@ -193,6 +193,7 @@ export {
 	type SyncAccountInfo,
 } from "./mailbox-sync.js";
 export {
+	type DeleteMessagesOutcome,
 	type DeleteOptions,
 	type MessageMoveConfig,
 	type MessageMoveLogger,
@@ -267,6 +268,12 @@ export {
 	resolveMailboxRole,
 	shapeFromMessageData,
 } from "./quarantine.js";
+export {
+	type RestoreSourcePlacementDeps,
+	type RestoreSourcePlacementInput,
+	type RestoreSourcePlacementOutcome,
+	restoreSourcePlacement,
+} from "./restore-source-placement.js";
 export {
 	extractSnippetFromEmail,
 	generateSnippet,
@@ -286,6 +293,12 @@ export {
 	type StaleMessageReconcileDeps,
 	type StaleMessageReconcileResult,
 } from "./stale-message-reconcile.js";
+export {
+	buildThreadMessageMoveRevert,
+	buildThreadMessageTrashUpdate,
+	buildThreadMessageUndelete,
+	type ThreadMessageRowState,
+} from "./thread-message-rows.js";
 export type {
 	FlatMailboxInfo,
 	IImapConnection,

package/src/message-delete-junk-reconcile.test.ts

@@ -30,8 +30,11 @@ const buildWorld = (trashExists: boolean) => {
 		get: async () => [message],
 		update: async (_id: string, patch: Record<string, unknown>) =>
 			Object.assign(message, patch),
-		updateForMove: async (_id: string, patch: Record<string, unknown>) =>
-			Object.assign(message, patch),
+		transitionPlacement: async (
+			_id: string,
+			_expected: Record<string, unknown>,
+			patch: Record<string, unknown>,
+		) => Object.assign(message, patch),
 	} as unknown as IMessageRepository;
 
 	const threadMessageService = {

package/src/message-delete-no-trash.test.ts

@@ -52,7 +52,11 @@ const buildWorld = (
 			patches.push(patch);
 			return Object.assign(message, patch);
 		},
-		updateForMove: async (_id: string, patch: Record<string, unknown>) => {
+		transitionPlacement: async (
+			_id: string,
+			_expected: Record<string, unknown>,
+			patch: Record<string, unknown>,
+		) => {
 			patches.push(patch);
 			return Object.assign(message, patch);
 		},

package/src/message-delete-unsettled-placement.test.ts

@@ -68,11 +68,20 @@ const movingRow = () => ({
 	originalUid: INBOX_UID,
 });
 
-/** The same row after the trash-move handler gave up without confirming. */
+/**
+ * A row a delete gave up on. R3 defines the give-up as a hand-back: the row
+ * goes back on the placement the mail server confirmed, and only then carries
+ * `abandoned`. The pair is honest, so nothing downstream refuses it — the
+ * failure is a statement the reading pane makes, never a gate.
+ */
 const strandedRow = () => ({
-	...movingRow(),
 	messageId: STRANDED_ID,
-	syncStatus: "failed",
+	mailboxId: INBOX,
+	uid: INBOX_UID,
+	status: "active",
+	syncStatus: "abandoned",
+	abandonedMutation: "delete",
+	originalMailboxId: INBOX,
 });
 
 /** An ordinary settled row. */
@@ -127,7 +136,11 @@ const buildWorld = (seed: Array<Record<string, unknown>>) => {
 			patches.push({ messageId: id, patch });
 			return Object.assign(rows.get(id) ?? {}, patch);
 		},
-		updateForMove: async (id: string, patch: Record<string, unknown>) => {
+		transitionPlacement: async (
+			id: string,
+			_expected: Record<string, unknown>,
+			patch: Record<string, unknown>,
+		) => {
 			patches.push({ messageId: id, patch });
 			return Object.assign(rows.get(id) ?? {}, patch);
 		},
@@ -329,23 +342,18 @@ describe("the gate refuses only a uid that names somebody else (#845.3)", () =>
 		assert.ok(Date.now() - startedAt < 100, "and never entered the wait");
 	});
 
-	it("refuses a row stranded by a move that gave up, without spending the ceiling", async () => {
-		// `syncStatus: failed` with `status: moving` is the shape every handler's
-		// give-up path leaves behind, and only `updateUid` clears it. The pair is
-		// still a lie, so the delete is still refused — but under a reason whose
-		// remedy is a resync, and without a wait that could never succeed.
+	it("acts on a row a delete gave up on, rather than refusing it", async () => {
+		// A give-up hands the row back to the placement the server confirmed
+		// before it stops (R3), so the "Delete again" the reading pane offers is
+		// an ordinary delete. Refusing it — which is what a give-up left `moving`
+		// used to earn — made that button a dead end.
 		const { service, events } = buildWorld([strandedRow()]);
 
 		const startedAt = Date.now();
-		await assert.rejects(
-			() => service.deleteMessages(ACCOUNT_CONFIG, [STRANDED_ID], ACCOUNT),
-			(error: unknown) =>
-				error instanceof MessagePlacementUnsettledError &&
-				error.publicApiError?.details?.reason === "unverified",
-		);
+		await service.deleteMessages(ACCOUNT_CONFIG, [STRANDED_ID], ACCOUNT);
 
-		assert.deepEqual(events, []);
-		assert.ok(Date.now() - startedAt < 100, "refused without waiting");
+		assert.equal(events.length, 1);
+		assert.ok(Date.now() - startedAt < 100, "and never entered the wait");
 	});
 });
 

package/src/message-move-copy.test.ts

@@ -69,6 +69,11 @@ const buildWorld = () => {
 				category: "primary",
 				classificationState: "Examined",
 				hasListUnsubscribe: false,
+				// Settled, which is what a delete's predicate asks for: a row every
+				// repository read carries a status, and leaving it off made the
+				// fixture answer for a state no row is ever in.
+				status: "active",
+				syncStatus: "synced",
 			},
 		],
 	]);
@@ -130,6 +135,19 @@ const buildWorld = () => {
 			if (m) Object.assign(m, patch);
 			return m;
 		},
+		transitionPlacement: async (
+			id: string,
+			expected: Record<string, unknown>,
+			patch: Record<string, unknown>,
+		) => {
+			const m = messages.get(id) as Record<string, unknown> | undefined;
+			if (!m) return undefined;
+			for (const [field, want] of Object.entries(expected)) {
+				if (want !== undefined && m[field] !== want) return undefined;
+			}
+			Object.assign(m, patch);
+			return m;
+		},
 	} as unknown as IMessageRepository;
 
 	const threadMessageService = {
@@ -457,7 +475,7 @@ describe("MessageMoveService.copyMessage — deterministic per-folder identity (
 	});
 
 	it("copy then delete of the copy removes the copy row and leaves the original", async () => {
-		const { service, threadRows } = buildWorld();
+		const { service, threadRows, messages } = buildWorld();
 
 		await service.copyMessages(
 			ACCOUNT_CONFIG,
@@ -467,6 +485,15 @@ describe("MessageMoveService.copyMessage — deterministic per-folder identity (
 		);
 		const copyId = deriveCopyMessageId(SOURCE_ID, DEST_MAILBOX);
 
+		// A fresh copy row is `moving` until the server hands back its COPYUID, and
+		// a delete against it would name uid 0. Settle it the way `updateUid`
+		// does, which is also the only state in which a user can see it to delete.
+		Object.assign(messages.get(copyId) ?? {}, {
+			uid: 77,
+			status: "active",
+			syncStatus: "synced",
+		});
+
 		await service.deleteMessages(ACCOUNT_CONFIG, [copyId], ACCOUNT, {
 			permanent: true,
 		});