@remit/backend 0.0.107 → 0.0.108

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/backend",
-  "version": "0.0.107",
+  "version": "0.0.108",
   "description": "Remit Mail Inspector API backend",
   "license": "MIT",
   "author": "",

package/src/service/create-remit-client.ts

@@ -37,6 +37,7 @@ import {
 	BodySyncQueueService,
 	BodySyncService,
 	createConnection,
+	type FilterConfig,
 	FlagPushService,
 	FlagQueueService,
 	type IImapConnection,
@@ -47,6 +48,7 @@ import {
 	PlacementMoveService,
 	SpamReportService,
 } from "@remit/mailbox-service";
+import { buildFilterConfig } from "@remit/mailbox-service/filter-config";
 import { createSearchService, type SearchService } from "@remit/search-service";
 import {
 	buildEmbeddingServiceFromEnv,
@@ -339,6 +341,39 @@ export const buildSharedDeps = (): RemitClientSharedDeps => {
 	};
 };
 
+// The read-path body backfill (describeMessage / getRawMessage →
+// fetchAndGetBody) materializes a body that was never sync-synced, and must run
+// the account's standing filters the same way the imap-worker's sync path does.
+// Without this, a message a user opens before background body-sync is classified
+// but never filter-evaluated — and once its body is stored the filter-capable
+// sync path skips it, so an explicit standing filter silently never fires (issue
+// #223). Only the explicit user-rule half is wired here; heuristic classifier
+// placement stays on the sync path. It shares `buildFilterConfig` with
+// `sync-message-body.ts`, so both paths embed a candidate message under the same
+// env-selected model that produced the anchors and a semantic (anchor-only)
+// filter fires on either (issue #298). Over the message-management queue's
+// local-first PlacementMoveService; filters stay off when that queue is unset.
+export const buildReadPathFilterConfig = (
+	repositories: RemitClientRepositories,
+): FilterConfig | undefined => {
+	const placementMoveQueueUrl = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
+	return buildFilterConfig({
+		filterService: repositories.filter,
+		filterAnchorService: repositories.filterAnchor,
+		messageLabelService: repositories.messageLabel,
+		placementMoveService: placementMoveQueueUrl
+			? new PlacementMoveService({
+					messageService: repositories.message,
+					threadMessageService: repositories.threadMessage,
+					markerService: repositories.placementMove,
+					addressService: repositories.address,
+					mailboxSpecialUseService: repositories.mailboxSpecialUse,
+					sqsQueueUrl: placementMoveQueueUrl,
+				})
+			: undefined,
+	});
+};
+
 // Backend-neutral composition root: given repositories (from any data-ports
 // implementation) and the shared services, wire the domain and queue services
 // and assemble a RemitClient. Imports neither ElectroDB nor Drizzle — the
@@ -354,35 +389,7 @@ export const createRemitClient = (deps: RemitClientDeps): RemitClient => {
 		bodySyncQueue,
 	} = deps;
 
-	// The read-path body backfill (describeMessage / getRawMessage →
-	// fetchAndGetBody) materializes a body that was never sync-synced, and must
-	// run the account's standing filters the same way the imap-worker's sync path
-	// does. Without this, a message a user opens before background body-sync is
-	// classified but never filter-evaluated — and once its body is stored the
-	// filter-capable sync path skips it, so an explicit standing filter silently
-	// never fires (issue #223). Only the explicit user-rule half is wired here;
-	// heuristic classifier placement stays on the sync path. Wired like
-	// `sync-message-body.ts`, over the message-management queue's local-first
-	// PlacementMoveService; filters stay off when that queue is unset.
-	const placementMoveQueueUrl = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
-	const placementMoveService = placementMoveQueueUrl
-		? new PlacementMoveService({
-				messageService: repositories.message,
-				threadMessageService: repositories.threadMessage,
-				markerService: repositories.placementMove,
-				addressService: repositories.address,
-				mailboxSpecialUseService: repositories.mailboxSpecialUse,
-				sqsQueueUrl: placementMoveQueueUrl,
-			})
-		: undefined;
-	const filterConfig = placementMoveService
-		? {
-				filterService: repositories.filter,
-				filterAnchorService: repositories.filterAnchor,
-				messageLabelService: repositories.messageLabel,
-				placementMoveService,
-			}
-		: undefined;
+	const filterConfig = buildReadPathFilterConfig(repositories);
 
 	const flagPushService = new FlagPushService({
 		markerService: repositories.flagPush,

package/src/service/read-path-filter-config.test.ts

@@ -0,0 +1,243 @@
+/**
+ * The read-path body backfill must run a semantic (anchor-only) standing filter,
+ * which needs the env-selected embedder wired into the filter config
+ * `createRemitClient` hands its BodySyncService (issue #298).
+ *
+ * These drive the composed client — `createRemitClient(...).bodySync` — rather
+ * than `buildReadPathFilterConfig` alone, so replacing that call with an inline
+ * filter-config literal (which carries no embedder) fails here.
+ */
+
+import assert from "node:assert/strict";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import type {
+	CreateMessageLabelInput,
+	FilterAnchorItem,
+	FilterItem,
+	UpdateMessageInput,
+} from "@remit/data-ports";
+import { NotFoundError } from "@remit/data-ports/errors";
+import { FilterMatchOperator, FilterState } from "@remit/domain-enums";
+import {
+	buildMatchText,
+	type FilterMessage,
+	type IImapConnection,
+	NO_ACTION,
+} from "@remit/mailbox-service";
+import type { SearchService } from "@remit/search-service";
+import { buildEmbeddingServiceFromEnv } from "@remit/search-service/from-env";
+import type { SecretsService } from "@remit/secrets-service";
+import type { StorageService } from "@remit/storage-service";
+import {
+	buildReadPathFilterConfig,
+	createRemitClient,
+	type RemitClientRepositories,
+} from "./create-remit-client.js";
+
+const MESSAGE: FilterMessage = {
+	from: "billing@stripe.com",
+	fromName: "Stripe",
+	subject: "Your receipt",
+	text: "Thanks for your payment",
+	listId: "",
+};
+
+const eml = (message: FilterMessage): Buffer =>
+	Buffer.from(
+		[
+			`From: ${message.fromName} <${message.from}>`,
+			"To: me@example.com",
+			`Subject: ${message.subject}`,
+			"Content-Type: text/plain",
+			"",
+			message.text,
+		].join("\r\n"),
+	);
+
+/**
+ * Anchor-only: no literal clauses, so it matches on the embedding alone. Its
+ * action is a label, which the fake MessageLabel repository below records —
+ * a move would run the real PlacementMoveService against SQS.
+ */
+const anchorOnlyFilter: FilterItem = {
+	filterId: "flt-semantic",
+	accountConfigId: "cfg-1",
+	name: "receipts",
+	scope: "Standing",
+	state: FilterState.Active,
+	hasAnchor: true,
+	ruleChangedAt: 1,
+	matchOperator: FilterMatchOperator.Or,
+	literalClauses: [],
+	actionLabelId: "lbl-receipts",
+	actionMailboxId: NO_ACTION,
+	createdAt: 1,
+	updatedAt: 1,
+} as unknown as FilterItem;
+
+// The read path must embed a candidate message under the same model that wrote
+// the anchors, so an anchor written from this text matches it exactly.
+const anchorFor = async (message: FilterMessage): Promise<FilterAnchorItem> => {
+	const embedding = buildEmbeddingServiceFromEnv();
+	const [anchorEmbedding] = await embedding.embed([buildMatchText(message)]);
+	return {
+		accountConfigId: "cfg-1",
+		filterId: "flt-semantic",
+		anchorEmbedding,
+		anchorEmbeddingId: embedding.embeddingId,
+		anchorSourceText: buildMatchText(message),
+		anchorMessageId: "m-anchor",
+	} as unknown as FilterAnchorItem;
+};
+
+const repositoriesWithAnchor = (
+	anchor: FilterAnchorItem,
+	applied: CreateMessageLabelInput[] = [],
+): RemitClientRepositories =>
+	({
+		filter: {
+			listByAccountAndState: async () => [anchorOnlyFilter],
+			refreshExpiry: async (filter: FilterItem) => filter,
+		},
+		filterAnchor: { get: async () => anchor },
+		messageLabel: {
+			apply: async (input: CreateMessageLabelInput) => {
+				applied.push(input);
+			},
+		},
+		message: {
+			get: async (messageId: string) => ({
+				messageId,
+				mailboxId: "mb-inbox",
+				uid: 1,
+			}),
+			update: async (_messageId: string, _input: UpdateMessageInput) => {},
+		},
+		threadMessage: {
+			findAllByMessageId: async () => [
+				{
+					threadMessageId: "tm-1",
+					sentDate: 1,
+					mailboxId: "mb-inbox",
+					isRead: false,
+					isDeleted: false,
+					hasStars: false,
+					hasAttachment: false,
+				},
+			],
+			update: async () => {},
+		},
+		address: {
+			getAddress: async () => {
+				throw new NotFoundError("Address not found");
+			},
+			incrementInboundCount: async () => {},
+		},
+		envelope: { listBodyParts: async () => [] },
+		placementMove: {},
+		mailboxSpecialUse: {},
+	}) as unknown as RemitClientRepositories;
+
+const storage = {
+	storeMessageBody: async () => ({ uri: "s3://bodies/m-1" }),
+	storeParsedBody: async () => {},
+	listBodyParts: async () => [],
+} as unknown as StorageService;
+
+const readBodyThroughClient = async (
+	repositories: RemitClientRepositories,
+	message: FilterMessage,
+): Promise<void> => {
+	const client = createRemitClient({
+		repositories,
+		storage,
+		search: {} as unknown as SearchService,
+		secrets: {} as unknown as SecretsService,
+		sqsQueueUrl: "https://sqs.eu-west-1.amazonaws.com/0/test",
+		sqsSmtpQueueUrl: "https://sqs.eu-west-1.amazonaws.com/0/test-smtp",
+	});
+	const connection = {
+		openBox: async () => {},
+		fetchMessageBody: async () => eml(message),
+	} as unknown as IImapConnection;
+
+	await client.bodySync.fetchAndGetBody(
+		"m-1",
+		"acc-1",
+		"cfg-1",
+		"INBOX",
+		async () => connection,
+	);
+};
+
+describe("createRemitClient read-path filter config", () => {
+	let previousQueueUrl: string | undefined;
+
+	beforeEach(() => {
+		previousQueueUrl = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
+		process.env.SQS_QUEUE_URL_MESSAGE_MGMT =
+			"https://sqs.eu-west-1.amazonaws.com/0/test-message-mgmt";
+	});
+
+	afterEach(() => {
+		if (previousQueueUrl === undefined) {
+			delete process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
+			return;
+		}
+		process.env.SQS_QUEUE_URL_MESSAGE_MGMT = previousQueueUrl;
+	});
+
+	it("labels a message an anchor-only filter matches on a read-path backfill", async () => {
+		const applied: CreateMessageLabelInput[] = [];
+
+		await readBodyThroughClient(
+			repositoriesWithAnchor(await anchorFor(MESSAGE), applied),
+			MESSAGE,
+		);
+
+		assert.deepEqual(
+			applied.map(({ labelId, appliedByFilterId }) => ({
+				labelId,
+				appliedByFilterId,
+			})),
+			[{ labelId: "lbl-receipts", appliedByFilterId: "flt-semantic" }],
+		);
+	});
+
+	it("leaves a message the anchor does not match unlabelled", async () => {
+		const applied: CreateMessageLabelInput[] = [];
+
+		await readBodyThroughClient(
+			repositoriesWithAnchor(
+				await anchorFor({
+					...MESSAGE,
+					subject: "Scheduled maintenance window",
+					text: "Our service will be unavailable on Sunday",
+				}),
+				applied,
+			),
+			MESSAGE,
+		);
+
+		assert.deepEqual(applied, []);
+	});
+});
+
+describe("buildReadPathFilterConfig", () => {
+	it("keeps filters off when the message-management queue is unset", async () => {
+		const previous = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
+		delete process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
+		try {
+			assert.equal(
+				buildReadPathFilterConfig(
+					repositoriesWithAnchor(await anchorFor(MESSAGE)),
+				),
+				undefined,
+			);
+		} finally {
+			if (previous !== undefined) {
+				process.env.SQS_QUEUE_URL_MESSAGE_MGMT = previous;
+			}
+		}
+	});
+});

package/src/service/semantic-capability.ts

@@ -1,4 +1,10 @@
 import { logger } from "@remit/logger-lambda";
+import {
+	_resetEmbeddingCapabilityForTest,
+	isEmbeddingCapabilityAbsence,
+	isEmbeddingCapabilityUnavailable,
+	recordEmbeddingCapabilityAbsence,
+} from "@remit/search-service/capability";
 import { isSelfHostSqlBackend } from "../data-backend.js";
 
 /**
@@ -47,23 +53,20 @@ import { isSelfHostSqlBackend } from "../data-backend.js";
  * instead was considered and rejected — it would fabricate relevance scores and
  * matched-chunk labels for the "Related" UI, and cannot serve the cross-account
  * queries the daily brief issues.
+ *
+ * The error vocabulary and the memoized absence itself live in
+ * `@remit/search-service/capability`, shared with the filter pipeline's semantic
+ * skip: whichever path embeds first is the probe, and the other stops trying.
+ * This module adds the self-host scoping and the operator-facing log.
  */
 
-const CAPABILITY_ABSENCE_CODES = new Set([
-	"ERR_MODULE_NOT_FOUND",
-	"MODULE_NOT_FOUND",
-	"ERR_DLOPEN_FAILED",
-	"ERR_EMBEDDING_MODEL_UNAVAILABLE",
-]);
-
-let semanticUnavailable = false;
-
 /** Test-only reset for the memoized absence. */
 export const _resetSemanticCapabilityForTest = (): void => {
-	semanticUnavailable = false;
+	_resetEmbeddingCapabilityForTest();
 };
 
-export const isSemanticSearchUnavailable = (): boolean => semanticUnavailable;
+export const isSemanticSearchUnavailable = (): boolean =>
+	isEmbeddingCapabilityUnavailable();
 
 /**
  * Whether a failure is this deployment simply not carrying the semantic
@@ -71,11 +74,8 @@ export const isSemanticSearchUnavailable = (): boolean => semanticUnavailable;
  * Records nothing, for the caller that must degrade one operation without
  * disabling `/search/semantic` process-wide.
  */
-export const isSemanticCapabilityAbsence = (error: unknown): boolean => {
-	if (!isSelfHostSqlBackend()) return false;
-	const code = (error as { code?: unknown } | null)?.code;
-	return typeof code === "string" && CAPABILITY_ABSENCE_CODES.has(code);
-};
+export const isSemanticCapabilityAbsence = (error: unknown): boolean =>
+	isSelfHostSqlBackend() && isEmbeddingCapabilityAbsence(error);
 
 /**
  * Classify a semantic-search failure. Returns true — and remembers the
@@ -84,12 +84,12 @@ export const isSemanticCapabilityAbsence = (error: unknown): boolean => {
  */
 export const noteSemanticCapabilityAbsence = (error: unknown): boolean => {
 	if (!isSemanticCapabilityAbsence(error)) return false;
-	if (!semanticUnavailable) {
+	if (!isEmbeddingCapabilityUnavailable()) {
 		logger.warn(
 			{ error: error instanceof Error ? error.message : String(error) },
 			"Semantic search pipeline unavailable in this deployment — serving empty semantic results (see deploy/vps/README.md)",
 		);
 	}
-	semanticUnavailable = true;
+	recordEmbeddingCapabilityAbsence(error);
 	return true;
 };