@remit/mailbox-service 0.0.33 → 0.0.34

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.33",
+  "version": "0.0.34",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/filters/pipeline.test.ts

@@ -61,6 +61,7 @@ const buildPipeline = (filter: FilterItem) => {
 				state.embedCalls += 1;
 				return [];
 			},
+			embeddingId: "test-model@0",
 		},
 	};
 	return { pipeline: new FilterPipeline(config, { info: () => {} }), state };
@@ -101,3 +102,193 @@ describe("FilterPipeline — anchorless From/Or filter at index time", () => {
 		assert.equal(state.embedCalls, 0);
 	});
 });
+
+describe("FilterPipeline — anchor lazy re-embed on model drift (reader #295)", () => {
+	const anchoredFilter = (): FilterItem =>
+		({
+			filterId: "flt-semantic",
+			accountConfigId: "cfg-1",
+			name: "receipts",
+			scope: "Standing",
+			state: FilterState.Active,
+			hasAnchor: true,
+			ruleChangedAt: 1,
+			matchOperator: FilterMatchOperator.And,
+			literalClauses: [],
+			actionLabelId: "lbl-1",
+			actionMailboxId: NO_ACTION,
+			createdAt: 1,
+			updatedAt: 1,
+		}) as unknown as FilterItem;
+
+	const message = {
+		from: "billing@stripe.com",
+		fromName: "Stripe",
+		subject: "Your receipt",
+		text: "Thanks for your payment",
+		listId: "",
+	};
+
+	it("re-embeds a stale anchor and matches on the very next evaluation, persisting the refreshed row", async () => {
+		const putCalls: Array<Record<string, unknown>> = [];
+		let embedCalls = 0;
+		const config: FilterConfig = {
+			filterService: {
+				listByAccountAndState: async () => [anchoredFilter()],
+				refreshExpiry: async (f: FilterItem) => f,
+			} as unknown as IFilterRepository,
+			filterAnchorService: {
+				get: async () => ({
+					accountConfigId: "cfg-1",
+					filterId: "flt-semantic",
+					// Stale-model space — orthogonal to what the current model
+					// produces, so a raw comparison against it would not match.
+					anchorEmbedding: [0, 1, 0],
+					anchorEmbeddingId: "old-model@3",
+					anchorSourceText: "your receipt is ready",
+					anchorMessageId: "msg-anchor",
+				}),
+				put: async (input: Record<string, unknown>) => {
+					putCalls.push(input);
+					return { ...input, createdAt: 1, updatedAt: 2 };
+				},
+			} as unknown as IFilterAnchorRepository,
+			messageLabelService: {} as unknown as IMessageLabelRepository,
+			placementMoveService: {} as unknown as PlacementMoveService,
+			embedder: {
+				embed: async () => {
+					embedCalls += 1;
+					return [1, 0, 0];
+				},
+				embeddingId: "new-model@3",
+			},
+		};
+
+		const decision = await new FilterPipeline(config, {
+			info: () => {},
+		}).evaluate("cfg-1", "m-1", message);
+
+		assert.deepEqual(
+			decision.labels,
+			[{ labelId: "lbl-1", filterId: "flt-semantic" }],
+			"the anchor matches once re-embedded into the current model's space",
+		);
+		assert.equal(
+			putCalls.length,
+			1,
+			"the refreshed anchor is written back exactly once",
+		);
+		assert.deepEqual(putCalls[0], {
+			accountConfigId: "cfg-1",
+			filterId: "flt-semantic",
+			anchorEmbedding: [1, 0, 0],
+			anchorEmbeddingId: "new-model@3",
+			anchorSourceText: "your receipt is ready",
+			anchorMessageId: "msg-anchor",
+		});
+		assert.equal(
+			embedCalls,
+			2,
+			"one embed re-embeds the anchor's source text, one embeds the candidate message",
+		);
+	});
+
+	it("never re-embeds when the anchor's embeddingId is already current", async () => {
+		let putCalls = 0;
+		let embedCalls = 0;
+		const config: FilterConfig = {
+			filterService: {
+				listByAccountAndState: async () => [anchoredFilter()],
+				refreshExpiry: async (f: FilterItem) => f,
+			} as unknown as IFilterRepository,
+			filterAnchorService: {
+				get: async () => ({
+					anchorEmbedding: [1, 0, 0],
+					anchorEmbeddingId: "current-model@3",
+					anchorSourceText: "your receipt is ready",
+					anchorMessageId: "msg-anchor",
+				}),
+				put: async () => {
+					putCalls += 1;
+					throw new Error("must not be called");
+				},
+			} as unknown as IFilterAnchorRepository,
+			messageLabelService: {} as unknown as IMessageLabelRepository,
+			placementMoveService: {} as unknown as PlacementMoveService,
+			embedder: {
+				embed: async () => {
+					embedCalls += 1;
+					return [1, 0, 0];
+				},
+				embeddingId: "current-model@3",
+			},
+		};
+
+		const decision = await new FilterPipeline(config, {
+			info: () => {},
+		}).evaluate("cfg-1", "m-1", message);
+
+		assert.deepEqual(decision.labels, [
+			{ labelId: "lbl-1", filterId: "flt-semantic" },
+		]);
+		assert.equal(putCalls, 0, "a current anchor is never rewritten");
+		assert.equal(
+			embedCalls,
+			1,
+			"only the candidate message is embedded — no wasted re-embed call",
+		);
+	});
+
+	it("degrades a re-embed failure to skipping just this filter, not the whole pass", async () => {
+		const literalFilter: FilterItem = {
+			filterId: "flt-literal",
+			accountConfigId: "cfg-1",
+			name: "stripe receipts",
+			scope: "Standing",
+			state: FilterState.Active,
+			hasAnchor: false,
+			ruleChangedAt: 1,
+			matchOperator: FilterMatchOperator.Or,
+			literalClauses: [{ field: FilterClauseField.From, value: "stripe.com" }],
+			actionLabelId: "lbl-literal",
+			actionMailboxId: NO_ACTION,
+			createdAt: 1,
+			updatedAt: 1,
+		} as unknown as FilterItem;
+
+		const config: FilterConfig = {
+			filterService: {
+				listByAccountAndState: async () => [anchoredFilter(), literalFilter],
+				refreshExpiry: async (f: FilterItem) => f,
+			} as unknown as IFilterRepository,
+			filterAnchorService: {
+				get: async () => ({
+					anchorEmbedding: [0, 1, 0],
+					anchorEmbeddingId: "old-model@3",
+					anchorSourceText: "your receipt is ready",
+					anchorMessageId: "msg-anchor",
+				}),
+				put: async () => {
+					throw new Error("SQLITE_BUSY");
+				},
+			} as unknown as IFilterAnchorRepository,
+			messageLabelService: {} as unknown as IMessageLabelRepository,
+			placementMoveService: {} as unknown as PlacementMoveService,
+			embedder: {
+				embed: async () => [1, 0, 0],
+				embeddingId: "new-model@3",
+			},
+		};
+
+		const decision = await new FilterPipeline(config, {
+			info: () => {},
+			error: () => {},
+		}).evaluate("cfg-1", "m-1", message);
+
+		assert.deepEqual(
+			decision.labels,
+			[{ labelId: "lbl-literal", filterId: "flt-literal" }],
+			"the anchored filter's re-embed failure is isolated — the literal filter still applies",
+		);
+	});
+});

package/src/filters/pipeline.ts

@@ -26,6 +26,15 @@ import {
  */
 export interface MessageEmbedder {
 	embed(text: string): Promise<number[]>;
+	/**
+	 * `<modelId>@<dimensions>` identifier of the model currently configured —
+	 * the same scheme `EmbeddingService.embeddingId`
+	 * (`packages/search-service/src/embeddings.ts`) derives. Compared against
+	 * `FilterAnchor.anchorEmbeddingId` to detect a model drift a same-dimension
+	 * swap would otherwise pass through silently (RFC 039 Decision 1a, reader
+	 * #295).
+	 */
+	readonly embeddingId: string;
 }
 
 export interface FilterLogger {
@@ -215,7 +224,7 @@ export class FilterPipeline {
 		// One deterministic point read per semantic candidate that survived the
 		// literal pre-filter — never a scan, never the anchor message itself
 		// (RFC 034 Decision 2.3).
-		const anchor = await this.config.filterAnchorService.get(
+		let anchor = await this.config.filterAnchorService.get(
 			accountConfigId,
 			filter.filterId,
 		);
@@ -227,6 +236,27 @@ export class FilterPipeline {
 			return false;
 		}
 
+		const embedder = this.config.embedder;
+		if (embedder && anchor.anchorEmbeddingId !== embedder.embeddingId) {
+			// The embedding model has drifted since this anchor was last written
+			// (RFC 039 Decision 1a) — re-embed the already-persisted
+			// `anchorSourceText` and write it back in place, lazily, on this read.
+			// No migration job walks these rows proactively; a re-embed failure
+			// here propagates to the per-filter catch in `match()`, which logs and
+			// skips this filter for this one evaluation exactly as an unrecoverable
+			// stale anchor does today — never a terminal state, so the next
+			// message that reaches this filter retries.
+			const anchorEmbedding = await embedder.embed(anchor.anchorSourceText);
+			anchor = await this.config.filterAnchorService.put({
+				accountConfigId,
+				filterId: filter.filterId,
+				anchorEmbedding,
+				anchorEmbeddingId: embedder.embeddingId,
+				anchorSourceText: anchor.anchorSourceText,
+				anchorMessageId: anchor.anchorMessageId,
+			});
+		}
+
 		const vector = await embed();
 		if (!vector) {
 			this.log.debug?.(