@remit/backend 0.0.142 → 0.0.144

package/src/service/read-path-filter-config.test.ts

@@ -1,243 +0,0 @@
-/**
- * The read-path body backfill must run a semantic (anchor-only) standing filter,
- * which needs the env-selected embedder wired into the filter config
- * `createRemitClient` hands its BodySyncService (issue #298).
- *
- * These drive the composed client — `createRemitClient(...).bodySync` — rather
- * than `buildReadPathFilterConfig` alone, so replacing that call with an inline
- * filter-config literal (which carries no embedder) fails here.
- */
-
-import assert from "node:assert/strict";
-import { afterEach, beforeEach, describe, it } from "node:test";
-import type {
-	CreateMessageLabelInput,
-	FilterAnchorItem,
-	FilterItem,
-	UpdateMessageInput,
-} from "@remit/data-ports";
-import { NotFoundError } from "@remit/data-ports/errors";
-import { FilterMatchOperator, FilterState } from "@remit/domain-enums";
-import {
-	buildMatchText,
-	type FilterMessage,
-	type IImapConnection,
-	NO_ACTION,
-} from "@remit/mailbox-service";
-import type { SearchService } from "@remit/search-service";
-import { buildEmbeddingServiceFromEnv } from "@remit/search-service/from-env";
-import type { SecretsService } from "@remit/secrets-service";
-import type { StorageService } from "@remit/storage-service";
-import {
-	buildReadPathFilterConfig,
-	createRemitClient,
-	type RemitClientRepositories,
-} from "./create-remit-client.js";
-
-const MESSAGE: FilterMessage = {
-	from: "billing@stripe.com",
-	fromName: "Stripe",
-	subject: "Your receipt",
-	text: "Thanks for your payment",
-	listId: "",
-};
-
-const eml = (message: FilterMessage): Buffer =>
-	Buffer.from(
-		[
-			`From: ${message.fromName} <${message.from}>`,
-			"To: me@example.com",
-			`Subject: ${message.subject}`,
-			"Content-Type: text/plain",
-			"",
-			message.text,
-		].join("\r\n"),
-	);
-
-/**
- * Anchor-only: no literal clauses, so it matches on the embedding alone. Its
- * action is a label, which the fake MessageLabel repository below records —
- * a move would run the real PlacementMoveService against SQS.
- */
-const anchorOnlyFilter: FilterItem = {
-	filterId: "flt-semantic",
-	accountConfigId: "cfg-1",
-	name: "receipts",
-	scope: "Standing",
-	state: FilterState.Active,
-	hasAnchor: true,
-	ruleChangedAt: 1,
-	matchOperator: FilterMatchOperator.Or,
-	literalClauses: [],
-	actionLabelId: "lbl-receipts",
-	actionMailboxId: NO_ACTION,
-	createdAt: 1,
-	updatedAt: 1,
-} as unknown as FilterItem;
-
-// The read path must embed a candidate message under the same model that wrote
-// the anchors, so an anchor written from this text matches it exactly.
-const anchorFor = async (message: FilterMessage): Promise<FilterAnchorItem> => {
-	const embedding = buildEmbeddingServiceFromEnv();
-	const [anchorEmbedding] = await embedding.embed([buildMatchText(message)]);
-	return {
-		accountConfigId: "cfg-1",
-		filterId: "flt-semantic",
-		anchorEmbedding,
-		anchorEmbeddingId: embedding.embeddingId,
-		anchorSourceText: buildMatchText(message),
-		anchorMessageId: "m-anchor",
-	} as unknown as FilterAnchorItem;
-};
-
-const repositoriesWithAnchor = (
-	anchor: FilterAnchorItem,
-	applied: CreateMessageLabelInput[] = [],
-): RemitClientRepositories =>
-	({
-		filter: {
-			listByAccountAndState: async () => [anchorOnlyFilter],
-			refreshExpiry: async (filter: FilterItem) => filter,
-		},
-		filterAnchor: { get: async () => anchor },
-		messageLabel: {
-			apply: async (input: CreateMessageLabelInput) => {
-				applied.push(input);
-			},
-		},
-		message: {
-			get: async (messageId: string) => ({
-				messageId,
-				mailboxId: "mb-inbox",
-				uid: 1,
-			}),
-			update: async (_messageId: string, _input: UpdateMessageInput) => {},
-		},
-		threadMessage: {
-			findAllByMessageId: async () => [
-				{
-					threadMessageId: "tm-1",
-					sentDate: 1,
-					mailboxId: "mb-inbox",
-					isRead: false,
-					isDeleted: false,
-					hasStars: false,
-					hasAttachment: false,
-				},
-			],
-			update: async () => {},
-		},
-		address: {
-			getAddress: async () => {
-				throw new NotFoundError("Address not found");
-			},
-			incrementInboundCount: async () => {},
-		},
-		envelope: { listBodyParts: async () => [] },
-		placementMove: {},
-		mailboxSpecialUse: {},
-	}) as unknown as RemitClientRepositories;
-
-const storage = {
-	storeMessageBody: async () => ({ uri: "s3://bodies/m-1" }),
-	storeParsedBody: async () => {},
-	listBodyParts: async () => [],
-} as unknown as StorageService;
-
-const readBodyThroughClient = async (
-	repositories: RemitClientRepositories,
-	message: FilterMessage,
-): Promise<void> => {
-	const client = createRemitClient({
-		repositories,
-		storage,
-		search: {} as unknown as SearchService,
-		secrets: {} as unknown as SecretsService,
-		sqsQueueUrl: "https://sqs.eu-west-1.amazonaws.com/0/test",
-		sqsSmtpQueueUrl: "https://sqs.eu-west-1.amazonaws.com/0/test-smtp",
-	});
-	const connection = {
-		openBox: async () => {},
-		fetchMessageBody: async () => eml(message),
-	} as unknown as IImapConnection;
-
-	await client.bodySync.fetchAndGetBody(
-		"m-1",
-		"acc-1",
-		"cfg-1",
-		{ fullPath: "INBOX", placement: "inbox" },
-		async () => connection,
-	);
-};
-
-describe("createRemitClient read-path filter config", () => {
-	let previousQueueUrl: string | undefined;
-
-	beforeEach(() => {
-		previousQueueUrl = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
-		process.env.SQS_QUEUE_URL_MESSAGE_MGMT =
-			"https://sqs.eu-west-1.amazonaws.com/0/test-message-mgmt";
-	});
-
-	afterEach(() => {
-		if (previousQueueUrl === undefined) {
-			delete process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
-			return;
-		}
-		process.env.SQS_QUEUE_URL_MESSAGE_MGMT = previousQueueUrl;
-	});
-
-	it("labels a message an anchor-only filter matches on a read-path backfill", async () => {
-		const applied: CreateMessageLabelInput[] = [];
-
-		await readBodyThroughClient(
-			repositoriesWithAnchor(await anchorFor(MESSAGE), applied),
-			MESSAGE,
-		);
-
-		assert.deepEqual(
-			applied.map(({ labelId, appliedByFilterId }) => ({
-				labelId,
-				appliedByFilterId,
-			})),
-			[{ labelId: "lbl-receipts", appliedByFilterId: "flt-semantic" }],
-		);
-	});
-
-	it("leaves a message the anchor does not match unlabelled", async () => {
-		const applied: CreateMessageLabelInput[] = [];
-
-		await readBodyThroughClient(
-			repositoriesWithAnchor(
-				await anchorFor({
-					...MESSAGE,
-					subject: "Scheduled maintenance window",
-					text: "Our service will be unavailable on Sunday",
-				}),
-				applied,
-			),
-			MESSAGE,
-		);
-
-		assert.deepEqual(applied, []);
-	});
-});
-
-describe("buildReadPathFilterConfig", () => {
-	it("keeps filters off when the message-management queue is unset", async () => {
-		const previous = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
-		delete process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
-		try {
-			assert.equal(
-				buildReadPathFilterConfig(
-					repositoriesWithAnchor(await anchorFor(MESSAGE)),
-				),
-				undefined,
-			);
-		} finally {
-			if (previous !== undefined) {
-				process.env.SQS_QUEUE_URL_MESSAGE_MGMT = previous;
-			}
-		}
-	});
-});

package/src/service/semantic-capability.test.ts

@@ -1,78 +0,0 @@
-import assert from "node:assert/strict";
-import { afterEach, beforeEach, describe, it } from "node:test";
-import {
-	_resetSemanticCapabilityForTest,
-	isSemanticSearchUnavailable,
-	noteSemanticCapabilityAbsence,
-} from "./semantic-capability.js";
-
-const moduleNotFound = (): Error => {
-	const error = new Error(
-		"Cannot find package '@huggingface/transformers' imported from /app/server.mjs",
-	);
-	(error as Error & { code: string }).code = "ERR_MODULE_NOT_FOUND";
-	return error;
-};
-
-const modelUnavailable = (): Error => {
-	const error = new Error(
-		'Embedding model "Xenova/paraphrase-multilingual-MiniLM-L12-v2" could not be loaded',
-	);
-	(error as Error & { code: string }).code = "ERR_EMBEDDING_MODEL_UNAVAILABLE";
-	return error;
-};
-
-describe("noteSemanticCapabilityAbsence", () => {
-	const ORIGINAL = process.env.DATA_BACKEND;
-	beforeEach(() => {
-		_resetSemanticCapabilityForTest();
-	});
-	afterEach(() => {
-		if (ORIGINAL === undefined) delete process.env.DATA_BACKEND;
-		else process.env.DATA_BACKEND = ORIGINAL;
-		_resetSemanticCapabilityForTest();
-	});
-
-	it("absorbs a missing-module failure on the self-host SQL backend and remembers it", () => {
-		process.env.DATA_BACKEND = "sqlite";
-		assert.equal(isSemanticSearchUnavailable(), false);
-		assert.equal(noteSemanticCapabilityAbsence(moduleNotFound()), true);
-		assert.equal(isSemanticSearchUnavailable(), true);
-	});
-
-	it("absorbs a dlopen failure (musl loading a glibc extension)", () => {
-		process.env.DATA_BACKEND = "sqlite";
-		const error = new Error(
-			"Error loading shared library ld-linux-x86-64.so.2: No such file or directory",
-		);
-		(error as Error & { code: string }).code = "ERR_DLOPEN_FAILED";
-		assert.equal(noteSemanticCapabilityAbsence(error), true);
-	});
-
-	it("absorbs an embedding-model load failure (e2e-dev from source, HuggingFace fetch failed) and remembers it", () => {
-		process.env.DATA_BACKEND = "sqlite";
-		assert.equal(isSemanticSearchUnavailable(), false);
-		assert.equal(noteSemanticCapabilityAbsence(modelUnavailable()), true);
-		assert.equal(isSemanticSearchUnavailable(), true);
-	});
-
-	it("rethrows genuine query errors on the self-host SQL backend", () => {
-		process.env.DATA_BACKEND = "sqlite";
-		assert.equal(
-			noteSemanticCapabilityAbsence(new Error("SQLITE_BUSY")),
-			false,
-		);
-		assert.equal(isSemanticSearchUnavailable(), false);
-		assert.equal(noteSemanticCapabilityAbsence(undefined), false);
-	});
-
-	it("never engages on the AWS DynamoDB path — a missing module there is a broken deploy", () => {
-		delete process.env.DATA_BACKEND;
-		assert.equal(noteSemanticCapabilityAbsence(moduleNotFound()), false);
-		assert.equal(noteSemanticCapabilityAbsence(modelUnavailable()), false);
-		process.env.DATA_BACKEND = "dynamodb";
-		assert.equal(noteSemanticCapabilityAbsence(moduleNotFound()), false);
-		assert.equal(noteSemanticCapabilityAbsence(modelUnavailable()), false);
-		assert.equal(isSemanticSearchUnavailable(), false);
-	});
-});

package/src/service/semantic-capability.ts

@@ -1,95 +0,0 @@
-import { logger } from "@remit/logger-lambda";
-import {
-	_resetEmbeddingCapabilityForTest,
-	isEmbeddingCapabilityAbsence,
-	isEmbeddingCapabilityUnavailable,
-	recordEmbeddingCapabilityAbsence,
-} from "@remit/search-service/capability";
-import { isSelfHostSqlBackend } from "../data-backend.js";
-
-/**
- * Semantic-search capability gate for the self-host compose profiles.
- *
- * This gates the free-text `/search/semantic` path only. Answering that query
- * means embedding it in-process, and the backend container image ships without
- * `@huggingface/transformers` (the model plus its dependencies would roughly
- * quadruple the image — see npm-scripts/docker-bundle.mjs and
- * deploy/vps/README.md). The embedder is reached lazily through `runtimeImport`,
- * so the first semantic query — not startup — hits ERR_MODULE_NOT_FOUND. Without
- * this gate every search the web client issues fires a `/search/semantic`
- * request that 500s (the client fetches semantic hits alongside every literal
- * search), retried by the client on top.
- *
- * The vector STORE is a separate capability and is NOT gated here. The Organize
- * anchor widen (organize.ts matchSemantic) pools an anchor message's already
- * stored chunk vectors and runs a KNN read over the vector store — no embedding
- * model involved — so it needs only `sqlite-vec`, which the backend image now
- * carries as a musl build (Dockerfile sqlite-vec-musl stage,
- * SQLITE_VEC_EXTENSION_PATH). matchOrganize never consults the memoized flag
- * below; a missing embedder therefore never disables the widen. The one place
- * the widen would reach for an embedder is repairing a persisted FilterAnchor
- * whose model has drifted (organize.ts matchSemantic): with no embedder there
- * is no repair to be had, so it queries with the stored vector and classifies
- * the failure through {@link isSemanticCapabilityAbsence}, which deliberately
- * does not record the absence. Recording it there would take every later
- * free-text semantic query down on the strength of one filter's stale stamp.
- *
- * The e2e-dev lane runs this backend from source rather than the container, so
- * `@huggingface/transformers` IS present and the local embedder instead tries to
- * lazily download the model from HuggingFace; a transient `fetch failed` there
- * surfaces the same way. The search-service raises it as a typed
- * `EmbeddingModelUnavailableError` (code ERR_EMBEDDING_MODEL_UNAVAILABLE), which
- * this gate treats as the same capability absence — the model is not available,
- * empty semantic results are the truthful answer, and the lane never depends on
- * a live external download.
- *
- * When the pipeline is genuinely absent, empty hits are the truthful answer:
- * the FTS/literal engine is the primary search surface on these profiles and is
- * unaffected. The absence is remembered so subsequent requests short-circuit.
- *
- * Scoped to the self-host SQL backend: on AWS the pipeline (Bedrock +
- * S3 Vectors) is bundled, so a module-resolution failure there is a broken
- * deploy and must keep failing loud. Falling back to the FTS engine here
- * instead was considered and rejected — it would fabricate relevance scores and
- * matched-chunk labels for the "Related" UI, and cannot serve the cross-account
- * queries the daily brief issues.
- *
- * The error vocabulary and the memoized absence itself live in
- * `@remit/search-service/capability`, shared with the filter pipeline's semantic
- * skip: whichever path embeds first is the probe, and the other stops trying.
- * This module adds the self-host scoping and the operator-facing log.
- */
-
-/** Test-only reset for the memoized absence. */
-export const _resetSemanticCapabilityForTest = (): void => {
-	_resetEmbeddingCapabilityForTest();
-};
-
-export const isSemanticSearchUnavailable = (): boolean =>
-	isEmbeddingCapabilityUnavailable();
-
-/**
- * Whether a failure is this deployment simply not carrying the semantic
- * pipeline — the missing-module/extension shape on a self-host SQL backend.
- * Records nothing, for the caller that must degrade one operation without
- * disabling `/search/semantic` process-wide.
- */
-export const isSemanticCapabilityAbsence = (error: unknown): boolean =>
-	isSelfHostSqlBackend() && isEmbeddingCapabilityAbsence(error);
-
-/**
- * Classify a semantic-search failure. Returns true — and remembers the
- * absence — when it is the missing-module/extension shape on a self-host SQL
- * backend; any other error is the caller's to rethrow.
- */
-export const noteSemanticCapabilityAbsence = (error: unknown): boolean => {
-	if (!isSemanticCapabilityAbsence(error)) return false;
-	if (!isEmbeddingCapabilityUnavailable()) {
-		logger.warn(
-			{ error: error instanceof Error ? error.message : String(error) },
-			"Semantic search pipeline unavailable in this deployment — serving empty semantic results (see deploy/vps/README.md)",
-		);
-	}
-	recordEmbeddingCapabilityAbsence(error);
-	return true;
-};

package/src/service/sender-category.test.ts

@@ -1,235 +0,0 @@
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import type {
-	ThreadMessageFieldTerm,
-	ThreadMessageItem,
-	UpdateMessageInput,
-	UpdateThreadMessageInput,
-} from "@remit/data-ports";
-import {
-	backApplySenderCategory,
-	SENDER_CATEGORY_BACKAPPLY_LIMIT,
-	type SenderCategoryBackApplyDeps,
-} from "./sender-category.js";
-
-const ACCOUNT_CONFIG_ID = "cfg-415";
-const SENDER = "post@ns.nl";
-
-const row = (n: number, over: Partial<ThreadMessageItem> = {}) =>
-	({
-		threadMessageId: `tm-${n}`,
-		threadId: `th-${n}`,
-		messageId: `msg-${n}`,
-		accountConfigId: ACCOUNT_CONFIG_ID,
-		mailboxId: "mbx-inbox",
-		fromEmail: SENDER,
-		fromName: "NS Reizigers",
-		subject: `reisoverzicht ${n}`,
-		category: "personal",
-		sentDate: 2_000 - n,
-		isRead: false,
-		isDeleted: false,
-		hasStars: false,
-		hasAttachment: false,
-		...over,
-	}) as unknown as ThreadMessageItem;
-
-interface Writes {
-	messages: Array<{ messageId: string; input: UpdateMessageInput }>;
-	threadRows: Array<{
-		threadMessageId: string;
-		input: UpdateThreadMessageInput;
-	}>;
-	terms: ThreadMessageFieldTerm[][];
-	pageSizes: Array<number | undefined>;
-}
-
-/**
- * A store that pages its rows the way the real listing does — newest first,
- * `limit` rows per page, a continuation token while more remain — and that
- * answers `findAllByMessageId` from the same rows, so a re-label is provably
- * written to both halves of the pair.
- */
-const storeOf = (
-	rows: ThreadMessageItem[],
-	writes: Writes,
-): SenderCategoryBackApplyDeps => ({
-	client: {
-		message: {
-			update: async (messageId: string, input: UpdateMessageInput) => {
-				writes.messages.push({ messageId, input });
-				return {} as never;
-			},
-		},
-		threadMessage: {
-			listByFieldTerms: async (
-				_accountConfigId: string,
-				terms: readonly ThreadMessageFieldTerm[],
-				options?: { limit?: number; continuationToken?: string },
-			) => {
-				writes.terms.push([...terms]);
-				writes.pageSizes.push(options?.limit);
-				const from = Number(options?.continuationToken ?? 0);
-				const size = options?.limit ?? rows.length;
-				const page = rows.slice(from, from + size);
-				const next = from + page.length;
-				return {
-					items: page,
-					continuationToken: next < rows.length ? String(next) : undefined,
-				};
-			},
-			findAllByMessageId: async (_accountConfigId: string, messageId: string) =>
-				rows.filter((r) => r.messageId === messageId),
-			update: async (
-				_accountConfigId: string,
-				threadMessageId: string,
-				input: UpdateThreadMessageInput,
-			) => {
-				writes.threadRows.push({ threadMessageId, input });
-				return {} as never;
-			},
-		},
-	} as unknown as SenderCategoryBackApplyDeps["client"],
-});
-
-const emptyWrites = (): Writes => ({
-	messages: [],
-	threadRows: [],
-	terms: [],
-	pageSizes: [],
-});
-
-describe("backApplySenderCategory (#415)", () => {
-	it("re-labels mail the classifier already decided, on both halves of the pair", async () => {
-		const writes = emptyWrites();
-		const rows = [row(1), row(2), row(3)];
-
-		const result = await backApplySenderCategory(
-			storeOf(rows, writes),
-			ACCOUNT_CONFIG_ID,
-			SENDER,
-			"newsletter",
-		);
-
-		assert.deepEqual(result, { matched: 3, applied: 3, failed: 0 });
-		assert.deepEqual(
-			writes.messages.map((w) => w.messageId),
-			["msg-1", "msg-2", "msg-3"],
-		);
-		assert.deepEqual(
-			writes.messages.map((w) => w.input.category),
-			["newsletter", "newsletter", "newsletter"],
-		);
-		assert.deepEqual(
-			writes.threadRows.map((w) => w.threadMessageId),
-			["tm-1", "tm-2", "tm-3"],
-		);
-		assert.deepEqual(
-			writes.threadRows.map((w) => w.input.category),
-			["newsletter", "newsletter", "newsletter"],
-		);
-	});
-
-	it("stops at the bound, keeping the newest of the sender's mail", async () => {
-		const writes = emptyWrites();
-		const rows = Array.from(
-			{ length: SENDER_CATEGORY_BACKAPPLY_LIMIT + 25 },
-			(_, i) => row(i + 1),
-		);
-
-		const result = await backApplySenderCategory(
-			storeOf(rows, writes),
-			ACCOUNT_CONFIG_ID,
-			SENDER,
-			"newsletter",
-		);
-
-		assert.equal(result.matched, SENDER_CATEGORY_BACKAPPLY_LIMIT);
-		assert.equal(result.applied, SENDER_CATEGORY_BACKAPPLY_LIMIT);
-		assert.equal(writes.messages.length, SENDER_CATEGORY_BACKAPPLY_LIMIT);
-		assert.equal(writes.messages[0]?.messageId, "msg-1");
-		assert.equal(
-			writes.messages.at(-1)?.messageId,
-			`msg-${SENDER_CATEGORY_BACKAPPLY_LIMIT}`,
-		);
-	});
-
-	it("refines the store's substring narrowing to an exact sender match", async () => {
-		const writes = emptyWrites();
-		const rows = [
-			row(1),
-			row(2, { messageId: "msg-lookalike", fromEmail: "no-post@ns.nl" }),
-			row(3, { messageId: "msg-name-only", fromEmail: "spoof@example.com" }),
-		];
-
-		const result = await backApplySenderCategory(
-			storeOf(rows, writes),
-			ACCOUNT_CONFIG_ID,
-			SENDER,
-			"newsletter",
-		);
-
-		assert.equal(result.matched, 1);
-		assert.deepEqual(
-			writes.messages.map((w) => w.messageId),
-			["msg-1"],
-		);
-		assert.deepEqual(writes.terms[0], [{ field: "sender", contains: SENDER }]);
-	});
-
-	it("pages the narrowed listing until the bound fills", async () => {
-		const writes = emptyWrites();
-		const rows = Array.from({ length: 12 }, (_, i) =>
-			i % 2 === 0
-				? row(i + 1)
-				: row(i + 1, {
-						messageId: `msg-other-${i}`,
-						fromEmail: "someone@example.com",
-					}),
-		);
-
-		const result = await backApplySenderCategory(
-			storeOf(rows, writes),
-			ACCOUNT_CONFIG_ID,
-			SENDER,
-			"newsletter",
-			4,
-		);
-
-		assert.equal(result.matched, 4);
-		assert.ok(
-			writes.terms.length > 1,
-			"a page of narrowed rows that yields fewer matches than the bound must be followed by the next page",
-		);
-		assert.deepEqual(writes.pageSizes, Array(writes.terms.length).fill(4));
-	});
-
-	it("counts a poisoned message as failed and re-labels the rest", async () => {
-		const writes = emptyWrites();
-		const rows = [row(1), row(2), row(3)];
-		const store = storeOf(rows, writes);
-		const threadMessage = store.client.threadMessage as unknown as Record<
-			string,
-			unknown
-		>;
-		const findAll = threadMessage.findAllByMessageId as (
-			a: string,
-			b: string,
-		) => Promise<ThreadMessageItem[]>;
-		threadMessage.findAllByMessageId = async (a: string, b: string) =>
-			b === "msg-2" ? [] : findAll(a, b);
-
-		const result = await backApplySenderCategory(
-			store,
-			ACCOUNT_CONFIG_ID,
-			SENDER,
-			"newsletter",
-		);
-
-		assert.deepEqual(result, { matched: 3, applied: 2, failed: 1 });
-		assert.deepEqual(
-			writes.messages.map((w) => w.messageId),
-			["msg-1", "msg-3"],
-		);
-	});
-});