@remit/backend 0.0.117 → 0.0.118

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/backend",
-  "version": "0.0.117",
+  "version": "0.0.118",
   "description": "Remit Mail Inspector API backend",
   "license": "MIT",
   "author": "",

package/src/handlers/organize.ts

@@ -177,6 +177,7 @@ export const OrganizeOperations: Record<
 			messageIds: result.messageIds,
 		};
 		if (result.semanticUnavailable) response.semanticUnavailable = true;
+		if (result.semanticIndexEmpty) response.semanticIndexEmpty = true;
 		return response;
 	},
 };

package/src/service/organize.test.ts

@@ -512,6 +512,105 @@ describe("matchOrganize honors the persisted FilterAnchor (reader #350)", () =>
 	});
 });
 
+/**
+ * A zero from the widen has two meanings and the preview could not tell them
+ * apart (issue #452): no mail resembles the anchor, or the index holds nothing
+ * to compare it against. Only the second is a reason to keep the rule.
+ */
+describe("matchOrganize reporting an empty semantic index", () => {
+	it("flags a widen answered by an index with no vectors at all", async () => {
+		const { messageIds, semanticIndexEmpty, semanticUnavailable } =
+			await matchAccepted(
+				matchDeps(createMemoryVectorStore()),
+				ACCOUNT_CONFIG_ID,
+				predicate(),
+			);
+
+		assert.deepEqual(messageIds, []);
+		assert.equal(semanticIndexEmpty, true);
+		assert.equal(
+			semanticUnavailable,
+			false,
+			"an empty index is not a missing vector pipeline",
+		);
+	});
+
+	it("flags a widen whose anchor message has no chunk vectors to pool", async () => {
+		const store = createMemoryVectorStore();
+		await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
+		const deps: OrganizeMatchDeps = {
+			...matchDeps(store),
+			semantic: () => ({
+				buildAnchor: async () => null,
+				vectorStore: store,
+				embed: async () => ANCHOR_VECTOR,
+				embeddingId: CURRENT_EMBEDDING_ID,
+			}),
+		};
+
+		const { messageIds, semanticIndexEmpty } = await matchAccepted(
+			deps,
+			ACCOUNT_CONFIG_ID,
+			predicate(),
+		);
+
+		assert.deepEqual(messageIds, []);
+		assert.equal(semanticIndexEmpty, true);
+	});
+
+	it("does not flag a populated index that simply holds nothing similar enough", async () => {
+		const store = createMemoryVectorStore();
+		await store.upsert([bodyChunk("msg-1", ORTHOGONAL_VECTOR)]);
+
+		const { messageIds, semanticIndexEmpty } = await matchAccepted(
+			matchDeps(store),
+			ACCOUNT_CONFIG_ID,
+			predicate(),
+		);
+
+		assert.deepEqual(
+			messageIds,
+			[],
+			"the one indexed message is below threshold",
+		);
+		assert.equal(
+			semanticIndexEmpty,
+			false,
+			"the index answered; the rule is what matched nothing",
+		);
+	});
+
+	it("does not flag a widen that matched", async () => {
+		const store = createMemoryVectorStore();
+		await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
+
+		const { messageIds, semanticIndexEmpty } = await matchAccepted(
+			matchDeps(store),
+			ACCOUNT_CONFIG_ID,
+			predicate(),
+		);
+
+		assert.deepEqual(messageIds, ["msg-1"]);
+		assert.equal(semanticIndexEmpty, false);
+	});
+
+	it("does not flag a literal-only predicate, which never consults the index", async () => {
+		const { semanticIndexEmpty } = await matchAccepted(
+			matchDeps(createMemoryVectorStore(), [
+				candidate("msg-1", { subject: "Weekly newsletter" }),
+			]),
+			ACCOUNT_CONFIG_ID,
+			{
+				...predicate(),
+				anchorMessageId: "None",
+				literalClauses: [{ field: "Subject", value: "reservation" }],
+			},
+		);
+
+		assert.equal(semanticIndexEmpty, false);
+	});
+});
+
 describe("matchOrganize on a deployment without the vector pipeline", () => {
 	const ORIGINAL = process.env.DATA_BACKEND;
 	beforeEach(() => {
@@ -610,14 +709,16 @@ describe("matchOrganize on a deployment without the vector pipeline", () => {
 	it("degrades an anchor-only widen to an empty flagged result instead of crashing", async () => {
 		const deps = vectorlessDeps([candidate("msg-1"), candidate("msg-2")]);
 
-		const { messageIds, semanticUnavailable } = await matchAccepted(
-			deps,
-			ACCOUNT_CONFIG_ID,
-			predicate(),
-		);
+		const { messageIds, semanticUnavailable, semanticIndexEmpty } =
+			await matchAccepted(deps, ACCOUNT_CONFIG_ID, predicate());
 
 		assert.deepEqual(messageIds, []);
 		assert.equal(semanticUnavailable, true);
+		assert.equal(
+			semanticIndexEmpty,
+			false,
+			"a deployment with no vector pipeline has no index to call empty",
+		);
 	});
 
 	it("keeps widening from a drifted persisted anchor when no embedding model is available", async () => {

package/src/service/organize.ts

@@ -183,11 +183,21 @@ export interface OrganizeRejection {
 	message: string;
 }
 
-/** The matched ids plus whether the semantic widen was skipped as unavailable. */
+/** The matched ids plus what the semantic side had to say about the zero it may have returned. */
 export interface OrganizeMatched {
 	rejected: null;
 	messageIds: string[];
 	semanticUnavailable: boolean;
+	/**
+	 * The widen ran and the index had nothing to answer with (issue #452): the
+	 * account holds no vectors at all, or none yet for the anchor message. It is
+	 * the difference between "this rule matches nothing" and "nothing is indexed
+	 * yet", which a bare zero collapses into one wrong sentence. Distinct from
+	 * {@link OrganizeMatched.semanticUnavailable}, which is the deployment
+	 * shipping no vector pipeline at all; an empty index is not an error, so it
+	 * cannot be inferred from a throw.
+	 */
+	semanticIndexEmpty: boolean;
 }
 
 /** The predicate was refused: nothing matched, and nothing may be applied. */
@@ -307,16 +317,21 @@ const anchorForWiden = async (
  * Fan out with a k-NN query gated on the cosine threshold, then refine by
  * literal clauses reconstructed from the same chunk vectors. Every read here
  * goes through the vector store; a deployment without the vector pipeline
- * fails on the first call, which {@link matchOrganize} catches. Returns null
- * when neither a persisted anchor nor the message's own chunk vectors exist
- * to pool.
+ * fails on the first call, which {@link matchOrganize} catches.
+ *
+ * `indexEmpty` separates the two ways this arm returns nothing (issue #452):
+ * neither a persisted anchor nor the anchor message's own chunk vectors exist
+ * to pool from, or the k-NN read came back with no rows at all for the account.
+ * Both mean the index has nothing to answer with — it is empty or still
+ * building — rather than that no mail resembles the anchor. A thresholded-away
+ * result set is a real answer and is not flagged.
  */
 const matchSemantic = async (
 	deps: OrganizeMatchDeps,
 	accountConfigId: string,
 	predicate: OrganizePredicate,
 	limit: number,
-): Promise<string[] | null> => {
+): Promise<{ messageIds: string[]; indexEmpty: boolean }> => {
 	const semantic = deps.semantic();
 	const persisted = await findPersistedAnchor(
 		deps.filterAnchors,
@@ -326,7 +341,7 @@ const matchSemantic = async (
 	const anchor: AnchorPayload | null = persisted
 		? await anchorForWiden(deps, semantic, persisted)
 		: await semantic.buildAnchor(accountConfigId, predicate.anchorMessageId);
-	if (!anchor) return null;
+	if (!anchor) return { messageIds: [], indexEmpty: true };
 	const threshold =
 		predicate.similarityThreshold ?? DEFAULT_SEMANTIC_MATCH_THRESHOLD;
 	const matches = await semantic.vectorStore.query({
@@ -334,6 +349,7 @@ const matchSemantic = async (
 		topK: limit * VECTOR_CHUNK_FACTOR,
 		filter: { accountConfigId },
 	});
+	if (matches.length === 0) return { messageIds: [], indexEmpty: true };
 	const bestScore = new Map<string, number>();
 	for (const match of matches) {
 		const messageId = match.metadata.messageId;
@@ -347,7 +363,8 @@ const matchSemantic = async (
 		.map(([messageId]) => messageId);
 
 	const clauses = predicate.literalClauses;
-	if (clauses.length === 0) return base.slice(0, limit);
+	if (clauses.length === 0)
+		return { messageIds: base.slice(0, limit), indexEmpty: false };
 
 	const matched: string[] = [];
 	for (const messageId of base) {
@@ -359,7 +376,7 @@ const matchSemantic = async (
 			matched.push(messageId);
 		}
 	}
-	return matched;
+	return { messageIds: matched, indexEmpty: false };
 };
 
 /**
@@ -475,7 +492,12 @@ export const matchOrganize = async (
 	const anchored = hasAnchor(predicate);
 	const clauses = predicate.literalClauses;
 	if (!anchored && clauses.length === 0) {
-		return { rejected: null, messageIds: [], semanticUnavailable: false };
+		return {
+			rejected: null,
+			messageIds: [],
+			semanticUnavailable: false,
+			semanticIndexEmpty: false,
+		};
 	}
 
 	if (!anchored) {
@@ -487,11 +509,16 @@ export const matchOrganize = async (
 			predicate,
 			limit,
 		);
-		return { rejected: null, messageIds, semanticUnavailable: false };
+		return {
+			rejected: null,
+			messageIds,
+			semanticUnavailable: false,
+			semanticIndexEmpty: false,
+		};
 	}
 
 	try {
-		const semanticIds = await matchSemantic(
+		const semantic = await matchSemantic(
 			deps,
 			accountConfigId,
 			predicate,
@@ -499,8 +526,9 @@ export const matchOrganize = async (
 		);
 		return {
 			rejected: null,
-			messageIds: semanticIds ?? [],
+			messageIds: semantic.messageIds,
 			semanticUnavailable: false,
+			semanticIndexEmpty: semantic.indexEmpty,
 		};
 	} catch (error) {
 		if (!noteSemanticCapabilityAbsence(error)) throw error;
@@ -508,7 +536,12 @@ export const matchOrganize = async (
 		// clauses over the corpus — nothing when the predicate is anchor-only —
 		// and flag the absence so the client can say so.
 		if (clauses.length === 0) {
-			return { rejected: null, messageIds: [], semanticUnavailable: true };
+			return {
+				rejected: null,
+				messageIds: [],
+				semanticUnavailable: true,
+				semanticIndexEmpty: false,
+			};
 		}
 		const rejection = bodyContentRejection(clauses);
 		if (rejection) return { rejected: rejection };
@@ -518,7 +551,12 @@ export const matchOrganize = async (
 			predicate,
 			limit,
 		);
-		return { rejected: null, messageIds, semanticUnavailable: true };
+		return {
+			rejected: null,
+			messageIds,
+			semanticUnavailable: true,
+			semanticIndexEmpty: false,
+		};
 	}
 };