@remit/backend 0.0.28 → 0.0.30

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/backend",
-  "version": "0.0.28",
+  "version": "0.0.30",
   "description": "Remit Mail Inspector API backend",
   "license": "MIT",
   "author": "",

package/src/service/organize.test.ts

@@ -213,7 +213,14 @@ const candidate = (
 	over: Partial<OrganizeCandidate["message"]> = {},
 ): OrganizeCandidate => ({
 	messageId,
-	message: { from: "", fromName: "", subject: "", text: "", ...over },
+	message: {
+		from: "",
+		fromName: "",
+		subject: "",
+		text: "",
+		listId: "",
+		...over,
+	},
 });
 
 describe("matchOrganize", () => {
@@ -597,3 +604,115 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
 		);
 	});
 });
+
+describe("matchOrganize with ListId and FromDomain clauses", () => {
+	const senderChunk = (
+		messageId: string,
+		fromEmail: string,
+		over: Partial<ChunkMetadata> = {},
+	): VectorRecord => ({
+		chunkId: `${messageId}#sender`,
+		vector: ANCHOR_VECTOR,
+		metadata: metadata({
+			messageId,
+			chunkType: "sender",
+			textPreview: fromEmail,
+			...over,
+		}),
+	});
+
+	it("matches a ListId clause exactly off the vector-free projection", async () => {
+		const deps = vectorlessDeps([
+			candidate("msg-1", { listId: "weekly.news.example.com" }),
+			candidate("msg-2", { listId: "news.example.com" }),
+			candidate("msg-3", {}),
+		]);
+
+		const { messageIds } = await matchOrganize(deps, ACCOUNT_CONFIG_ID, {
+			...predicate(),
+			anchorMessageId: "None",
+			literalClauses: [{ field: "ListId", value: "weekly.news.example.com" }],
+		});
+
+		assert.deepEqual(messageIds, ["msg-1"]);
+	});
+
+	it("matches a FromDomain clause public-suffix aware off the vector-free projection", async () => {
+		const deps = vectorlessDeps([
+			candidate("msg-1", { from: "notifications@github.com" }),
+			candidate("msg-2", { from: "ci@sub.github.com" }),
+			candidate("msg-3", { from: "attacker@github.com.evil.example" }),
+		]);
+
+		const { messageIds } = await matchOrganize(deps, ACCOUNT_CONFIG_ID, {
+			...predicate(),
+			anchorMessageId: "None",
+			literalClauses: [{ field: "FromDomain", value: "github.com" }],
+		});
+
+		assert.deepEqual(messageIds, ["msg-1", "msg-2"]);
+	});
+
+	it("round-trips ListId and FromDomain through the semantic-arm chunk projection", async () => {
+		const store = createMemoryVectorStore();
+		await store.upsert([
+			senderChunk("msg-1", "ci@github.com", {
+				listId: "actions.github.com",
+			}),
+			bodyChunk("msg-1", ANCHOR_VECTOR, { listId: "actions.github.com" }),
+			senderChunk("msg-2", "hi@othersender.example", {
+				listId: "other.list.example",
+			}),
+			bodyChunk("msg-2", ANCHOR_VECTOR, { listId: "other.list.example" }),
+		]);
+
+		const byListId = await matchOrganize(matchDeps(store), ACCOUNT_CONFIG_ID, {
+			...predicate(),
+			literalClauses: [{ field: "ListId", value: "actions.github.com" }],
+		});
+		assert.deepEqual(byListId.messageIds, ["msg-1"]);
+
+		const byFromDomain = await matchOrganize(
+			matchDeps(store),
+			ACCOUNT_CONFIG_ID,
+			{
+				...predicate(),
+				literalClauses: [{ field: "FromDomain", value: "github.com" }],
+			},
+		);
+		assert.deepEqual(byFromDomain.messageIds, ["msg-1"]);
+	});
+
+	it("applies a ListId predicate to exactly the previewed set (preview == apply)", async () => {
+		const deps = vectorlessDeps([
+			candidate("msg-1", { listId: "weekly.news.example.com" }),
+			candidate("msg-2", { listId: "weekly.news.example.com" }),
+			candidate("msg-3", { listId: "other.example.com" }),
+		]);
+		const p = predicate({
+			anchorMessageId: "None",
+			actionLabelId: "lbl-list",
+			literalClauses: [{ field: "ListId", value: "weekly.news.example.com" }],
+		});
+
+		const previewed = await matchOrganize(deps, ACCOUNT_CONFIG_ID, p);
+		const applied = await matchOrganize(deps, ACCOUNT_CONFIG_ID, p);
+		assert.deepEqual(previewed, applied);
+		assert.deepEqual(previewed.messageIds, ["msg-1", "msg-2"]);
+
+		const tracked = trackingClient();
+		const result = await applyOrganize(
+			{ client: tracked.client },
+			ACCOUNT_CONFIG_ID,
+			applied.messageIds,
+			p,
+		);
+
+		assert.equal(result.applied, 2);
+		assert.equal(result.failed, 0);
+		assert.deepEqual(tracked.labeled.map((row) => row.messageId).sort(), [
+			"msg-1",
+			"msg-2",
+		]);
+	});
+});

package/src/service/organize.ts

@@ -131,11 +131,13 @@ const filterMessageFromChunks = (
 	let subject = "";
 	let fromName = "";
 	let from = "";
+	let listId = "";
 	const textParts: string[] = [];
 	for (const record of records) {
 		const meta = record.metadata;
 		if (meta.subject && !subject) subject = meta.subject;
 		if (meta.fromName && !fromName) fromName = meta.fromName;
+		if (meta.listId && !listId) listId = meta.listId;
 		if (meta.chunkType === "sender" && meta.textPreview && !from) {
 			from = meta.textPreview;
 		}
@@ -146,7 +148,7 @@ const filterMessageFromChunks = (
 			textParts.push(meta.textPreview);
 		}
 	}
-	return { from, fromName, subject, text: textParts.join("\n") };
+	return { from, fromName, subject, listId, text: textParts.join("\n") };
 };
 
 /**
@@ -355,8 +357,10 @@ const buildSemanticFromEnv = (): OrganizeSemanticDeps => {
  * deployment that ships no vector pipeline. Deduped by message id — the same
  * mail filed in two folders is one candidate.
  *
- * `From`/`Subject` come from the row verbatim (full fidelity). `text` (the body)
- * is left empty: the thread row carries only a truncated `snippet`, and matching
+ * `From`/`Subject`/`listId` come from the row verbatim (full fidelity), so a
+ * `From`, `Subject`, `FromDomain` or `ListId` clause matches here exactly as it
+ * does at index time. `text` (the body) is left empty: the thread row carries
+ * only a truncated `snippet`, and matching
  * a body-content clause against a preview would silently diverge from the live
  * index-time filter's full-body match. Body-content (`HasWords`) clauses are
  * rejected before this is read (see {@link assertNoBodyContentClause}), so the
@@ -384,6 +388,7 @@ const listAccountFilterMessagesFromClient =
 						fromName: row.fromName ?? "",
 						subject: row.subject ?? "",
 						text: "",
+						listId: row.listId ?? "",
 					},
 				});
 				if (candidates.length >= limit) return candidates;

package/src/service/semantic-capability.ts

@@ -4,14 +4,23 @@ import { isSelfHostSqlBackend } from "../data-backend.js";
 /**
  * Semantic-search capability gate for the self-host compose profiles.
  *
- * The backend container image ships neither `@huggingface/transformers` (query
- * embedding) nor `sqlite-vec` (the vector store's loadable extension, glibc-only
- * on top of that — see npm-scripts/docker-bundle.mjs and deploy/vps/README.md).
- * Both are reached lazily through `runtimeImport`, so the first semantic query —
- * not startup — hits ERR_MODULE_NOT_FOUND. Without this gate every search the
- * web client issues fires a `/search/semantic` request that 500s (the client
- * fetches semantic hits alongside every literal search), retried by the client
- * on top.
+ * This gates the free-text `/search/semantic` path only. Answering that query
+ * means embedding it in-process, and the backend container image ships without
+ * `@huggingface/transformers` (the model plus its dependencies would roughly
+ * quadruple the image — see npm-scripts/docker-bundle.mjs and
+ * deploy/vps/README.md). The embedder is reached lazily through `runtimeImport`,
+ * so the first semantic query — not startup — hits ERR_MODULE_NOT_FOUND. Without
+ * this gate every search the web client issues fires a `/search/semantic`
+ * request that 500s (the client fetches semantic hits alongside every literal
+ * search), retried by the client on top.
+ *
+ * The vector STORE is a separate capability and is NOT gated here. The Organize
+ * anchor widen (organize.ts matchSemantic) pools an anchor message's already
+ * stored chunk vectors and runs a KNN read over the vector store — no embedding
+ * model involved — so it needs only `sqlite-vec`, which the backend image now
+ * carries as a musl build (Dockerfile sqlite-vec-musl stage,
+ * SQLITE_VEC_EXTENSION_PATH). matchOrganize never consults the memoized flag
+ * below; a missing embedder therefore never disables the widen.
  *
  * The e2e-dev lane runs this backend from source rather than the container, so
  * `@huggingface/transformers` IS present and the local embedder instead tries to