@remit/backend 0.0.104 → 0.0.106

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/backend",
-  "version": "0.0.104",
+  "version": "0.0.106",
   "description": "Remit Mail Inspector API backend",
   "license": "MIT",
   "author": "",

package/src/handlers/message.test.ts

@@ -1,5 +1,6 @@
 import assert from "node:assert/strict";
 import { describe, it } from "node:test";
+import { MessagePlacementUnsettledError } from "@remit/data-ports/errors";
 import {
 	MoveNotSettledError,
 	NoJunkMailboxError,
@@ -41,6 +42,32 @@ describe("settleSpamReportBulk", () => {
 		assert.notEqual(reason, GENERIC_FAILURE_REASON);
 	});
 
+	// A refusal the move gate raises used to collapse to the generic retry copy,
+	// which is a dead end for the reason no retry clears (#665).
+	it("words a placement refusal from its reason, never from the server's sentence", async () => {
+		const outcome = await settleSpamReportBulk(
+			["msg-1", "msg-2"],
+			async (messageId) => {
+				throw new MessagePlacementUnsettledError(
+					`Message ${messageId} was not acted on: its folder and uid do not name the same message`,
+					"acc-1",
+					messageId,
+					messageId === "msg-1" ? "in_flight" : "unverified",
+				);
+			},
+		);
+
+		const [inFlight, unverified] = outcome.failures?.map((f) => f.reason) ?? [];
+		assert.match(inFlight ?? "", /try again in a moment/i);
+		assert.match(unverified ?? "", /sync the folder/i);
+		assert.doesNotMatch(unverified ?? "", /try again in a moment/i);
+		for (const reason of [inFlight, unverified]) {
+			assert.notEqual(reason, GENERIC_FAILURE_REASON);
+			assert.doesNotMatch(reason ?? "", /folder and uid/);
+			assert.doesNotMatch(reason ?? "", /acc-1/);
+		}
+	});
+
 	it("omits failures entirely when every message succeeds", async () => {
 		const outcome = await settleSpamReportBulk(
 			["msg-1", "msg-2"],

package/src/handlers/message.ts

@@ -7,6 +7,7 @@ import type { MailboxItem } from "@remit/data-ports";
 import {
 	BadRequestError,
 	ForbiddenError,
+	MessagePlacementUnsettledError,
 	NotFoundError,
 	UnrecoverableBodyError,
 } from "@remit/data-ports/errors";
@@ -356,13 +357,35 @@ export const GENERIC_FAILURE_REASON =
  * each names what happened and what the user can do about it, which the
  * generic text cannot.
  */
-const DESIGNED_FAILURES = [MoveNotSettledError, NoJunkMailboxError];
+const DESIGNED_FAILURES = [
+	MoveNotSettledError,
+	NoJunkMailboxError,
+	MessagePlacementUnsettledError,
+];
 
 const isDesignedFailure = (reason: unknown): reason is Error =>
 	DESIGNED_FAILURES.some((designed) => reason instanceof designed);
 
-const failureReason = (reason: unknown): string =>
-	isDesignedFailure(reason) ? reason.message : GENERIC_FAILURE_REASON;
+/**
+ * The placement refusal is the one designed outcome whose own message is not
+ * the text to ship: it names a uuid and no remedy, and the two reasons do not
+ * share a remedy — an in-flight move settles on its own, an abandoned one only
+ * clears once the folder is resynced. Worded here off `details.reason`, the way
+ * the delete path words the same refusal for the client.
+ */
+const placementRefusalReason = (
+	error: MessagePlacementUnsettledError,
+): string =>
+	error.publicApiError?.details?.reason === "unverified"
+		? "An earlier move of this message never finished, so where it sits is unknown. Sync the folder, then report it again."
+		: "This message is still being moved on the mail server. Try again in a moment.";
+
+const failureReason = (reason: unknown): string => {
+	if (!isDesignedFailure(reason)) return GENERIC_FAILURE_REASON;
+	if (reason instanceof MessagePlacementUnsettledError)
+		return placementRefusalReason(reason);
+	return reason.message;
+};
 
 /**
  * Drive one report-spam/not-spam operation per message, concurrently. Each

package/src/service/organize.test.ts

@@ -18,6 +18,7 @@ import {
 	applyOrganize,
 	BODY_CONTENT_REJECTION_MESSAGE,
 	matchOrganize,
+	ORGANIZE_MATCH_LIMIT,
 	type OrganizeCandidate,
 	type OrganizeMatchDeps,
 	type OrganizeMatched,
@@ -285,7 +286,7 @@ const matchDeps = (
 				embeddingId: CURRENT_EMBEDDING_ID,
 			};
 		},
-		listAccountFilterMessages: async () => corpus,
+		listAccountFilterMessages: async () => ({ items: corpus }),
 		filterAnchors: {
 			listByAccountConfig: async () => filterAnchorRows,
 			put: async () => {
@@ -326,7 +327,7 @@ const vectorlessDeps = (
 				embeddingId: CURRENT_EMBEDDING_ID,
 			};
 		},
-		listAccountFilterMessages: async () => corpus,
+		listAccountFilterMessages: async () => ({ items: corpus }),
 		filterAnchors: {
 			listByAccountConfig: async () => [],
 			put: async () => {
@@ -467,7 +468,7 @@ describe("matchOrganize honors the persisted FilterAnchor (reader #350)", () =>
 				embed: async () => ANCHOR_VECTOR,
 				embeddingId: CURRENT_EMBEDDING_ID,
 			}),
-			listAccountFilterMessages: async () => [],
+			listAccountFilterMessages: async () => ({ items: [] }),
 			filterAnchors: {
 				listByAccountConfig: async () => [persistedAnchor],
 				put: async () => {
@@ -652,7 +653,7 @@ describe("matchOrganize on a deployment without the vector pipeline", () => {
 				},
 				embeddingId: CURRENT_EMBEDDING_ID,
 			}),
-			listAccountFilterMessages: async () => [],
+			listAccountFilterMessages: async () => ({ items: [] }),
 			filterAnchors: {
 				listByAccountConfig: async () => [drifted],
 				put: async () => {
@@ -693,7 +694,7 @@ describe("matchOrganize on a deployment without the vector pipeline", () => {
 				embed: async () => [],
 				embeddingId: CURRENT_EMBEDDING_ID,
 			}),
-			listAccountFilterMessages: async () => [],
+			listAccountFilterMessages: async () => ({ items: [] }),
 			filterAnchors: {
 				listByAccountConfig: async () => [],
 				put: async () => {
@@ -1015,7 +1016,7 @@ describe("back-apply repairs a drifted anchor the way index-time matching does (
 				embed,
 				embeddingId: CURRENT_EMBEDDING_ID,
 			}),
-			listAccountFilterMessages: async () => [],
+			listAccountFilterMessages: async () => ({ items: [] }),
 			filterAnchors: {
 				listByAccountConfig: async () => anchors,
 				put: async (input: CreateFilterAnchorInput) => {
@@ -1167,3 +1168,175 @@ describe("matchOrganize with ListId and FromDomain clauses", () => {
 		]);
 	});
 });
+
+/**
+ * The corpus read a real deployment gets: the store evaluates the terms and
+ * the caller pages the MATCHES. Modelled here rather than stubbed with a fixed
+ * array so a matcher that reads only the newest page cannot pass.
+ */
+const storeBackedDeps = (
+	rows: readonly OrganizeCandidate[],
+): OrganizeMatchDeps & { pagesRead: () => number } => {
+	let pagesRead = 0;
+	const termMatches = (
+		term: { field: string; contains: string },
+		row: OrganizeCandidate,
+	): boolean => {
+		const needle = term.contains.toLowerCase();
+		if (term.field === "subject") {
+			return row.message.subject.toLowerCase().includes(needle);
+		}
+		if (term.field === "listId") {
+			return row.message.listId.toLowerCase().includes(needle);
+		}
+		return `${row.message.fromName} ${row.message.from}`
+			.toLowerCase()
+			.includes(needle);
+	};
+	return {
+		semantic: () => {
+			throw moduleNotFound();
+		},
+		listAccountFilterMessages: async (_accountConfigId, query) => {
+			pagesRead += 1;
+			const narrowed = rows.filter((row) =>
+				query.terms.length === 0
+					? true
+					: query.operator === "or"
+						? query.terms.some((term) => termMatches(term, row))
+						: query.terms.every((term) => termMatches(term, row)),
+			);
+			const offset = query.continuationToken
+				? Number(query.continuationToken)
+				: 0;
+			const items = narrowed.slice(offset, offset + query.limit);
+			const next = offset + items.length;
+			return {
+				items,
+				continuationToken: next < narrowed.length ? String(next) : undefined,
+			};
+		},
+		filterAnchors: {
+			listByAccountConfig: async () => [],
+			put: async () => {
+				throw new Error("unreachable");
+			},
+		},
+		pagesRead: () => pagesRead,
+	};
+};
+
+describe("literal matching bounds the result, not the corpus (reader #459)", () => {
+	const quietSender = candidate("quiet-sender", {
+		from: "noreply@bank.example",
+		fromName: "Statements",
+		subject: "Your March statement",
+	});
+	const newerNoise = Array.from({ length: ORGANIZE_MATCH_LIMIT + 1 }, (_, i) =>
+		candidate(`noise-${i}`, {
+			from: "digest@other.example",
+			fromName: "Digest",
+			subject: `Daily digest ${i}`,
+		}),
+	);
+
+	it("matches a domain rule against mail older than a whole page of newer misses", async () => {
+		const deps = storeBackedDeps([...newerNoise, quietSender]);
+
+		const { messageIds } = await matchAccepted(
+			deps,
+			ACCOUNT_CONFIG_ID,
+			predicate({
+				anchorMessageId: "None",
+				literalClauses: [{ field: "FromDomain", value: "bank.example" }],
+			}),
+		);
+
+		assert.deepEqual(messageIds, ["quiet-sender"]);
+	});
+
+	it("reads one narrowed page rather than paging the whole mailbox", async () => {
+		const deps = storeBackedDeps([...newerNoise, quietSender]);
+
+		await matchAccepted(
+			deps,
+			ACCOUNT_CONFIG_ID,
+			predicate({
+				anchorMessageId: "None",
+				literalClauses: [{ field: "From", value: "noreply@bank.example" }],
+			}),
+		);
+
+		assert.equal(deps.pagesRead(), 1);
+	});
+
+	it("still refines what the store returned: a term is a narrowing, not the verdict", async () => {
+		const deps = storeBackedDeps([
+			candidate("spoofed", { from: "billing@bank.example.evil.test" }),
+			quietSender,
+		]);
+
+		const { messageIds } = await matchAccepted(
+			deps,
+			ACCOUNT_CONFIG_ID,
+			predicate({
+				anchorMessageId: "None",
+				literalClauses: [{ field: "FromDomain", value: "bank.example" }],
+			}),
+		);
+
+		assert.deepEqual(messageIds, ["quiet-sender"]);
+	});
+
+	it("caps the matches at the limit when more of the corpus matches", async () => {
+		const matching = Array.from({ length: ORGANIZE_MATCH_LIMIT * 2 }, (_, i) =>
+			candidate(`hit-${i}`, { from: "noreply@bank.example" }),
+		);
+		const deps = storeBackedDeps(matching);
+
+		const { messageIds } = await matchAccepted(
+			deps,
+			ACCOUNT_CONFIG_ID,
+			predicate({
+				anchorMessageId: "None",
+				literalClauses: [{ field: "FromDomain", value: "bank.example" }],
+			}),
+		);
+
+		assert.equal(messageIds.length, ORGANIZE_MATCH_LIMIT);
+	});
+
+	it("never queries for clauses nothing can satisfy", async () => {
+		const deps = storeBackedDeps([quietSender]);
+
+		const { messageIds } = await matchAccepted(
+			deps,
+			ACCOUNT_CONFIG_ID,
+			predicate({
+				anchorMessageId: "None",
+				literalClauses: [{ field: "From", value: "   " }],
+			}),
+		);
+
+		assert.deepEqual(messageIds, []);
+		assert.equal(deps.pagesRead(), 0);
+	});
+
+	it("counts one match for mail filed in two folders", async () => {
+		const deps = storeBackedDeps([
+			candidate("filed-twice", { from: "noreply@bank.example" }),
+			candidate("filed-twice", { from: "noreply@bank.example" }),
+		]);
+
+		const { messageIds } = await matchAccepted(
+			deps,
+			ACCOUNT_CONFIG_ID,
+			predicate({
+				anchorMessageId: "None",
+				literalClauses: [{ field: "FromDomain", value: "bank.example" }],
+			}),
+		);
+
+		assert.deepEqual(messageIds, ["filed-twice"]);
+	});
+});

package/src/service/organize.ts

@@ -10,7 +10,9 @@ import { logger } from "@remit/logger-lambda";
 import {
 	DEFAULT_SEMANTIC_MATCH_THRESHOLD,
 	type FilterMessage,
+	type LiteralClauseNarrowing,
 	literalClausesMatch,
+	literalClauseTerms,
 	NO_ACTION,
 	PlacementMoveService,
 	refreshAnchorForEmbedder,
@@ -34,8 +36,10 @@ import {
 /**
  * Hard cap on both the previewed and the applied set. A back-apply is a
  * data-heavy corpus pass (remit-data-heavy: frugal): the semantic side is
- * bounded by the vector query's `topK`, the literal-only side by a paginated
- * message scan, and the final match set never exceeds this.
+ * bounded by the vector query's `topK`, the literal-only side by counting
+ * MATCHES as it pages a store-side narrowed listing, and the final match set
+ * never exceeds this. It bounds the answer, never the corpus the question is
+ * asked of (#459).
  */
 export const ORGANIZE_MATCH_LIMIT = 500;
 
@@ -117,6 +121,22 @@ export interface OrganizeCandidate {
 	message: FilterMessage;
 }
 
+/**
+ * One page of the narrowed corpus, cursored exactly as the underlying listing
+ * is: an absent `continuationToken` means the store has no more rows matching
+ * the terms, never that a read window ran out.
+ */
+export interface OrganizeCandidatePage {
+	items: OrganizeCandidate[];
+	continuationToken?: string;
+}
+
+/** What to ask the store for, and where to resume. */
+export interface OrganizeCandidateQuery extends LiteralClauseNarrowing {
+	limit: number;
+	continuationToken?: string;
+}
+
 export interface OrganizeMatchDeps {
 	/**
 	 * Build the semantic side on demand. Invoked only for an anchored predicate,
@@ -126,13 +146,16 @@ export interface OrganizeMatchDeps {
 	 */
 	semantic: () => OrganizeSemanticDeps;
 	/**
-	 * The account's messages as literal-match candidates, bounded and vector-free
-	 * — the corpus slice a purely-literal (or degraded) pass scans.
+	 * One page of the account's messages as literal-match candidates,
+	 * vector-free, with the clause terms evaluated by the store — the corpus a
+	 * purely-literal (or degraded) pass reads. Paged rather than capped, so the
+	 * caller's bound falls on matches instead of on how far back the read
+	 * reached (#459).
 	 */
 	listAccountFilterMessages: (
 		accountConfigId: string,
-		limit: number,
-	) => Promise<OrganizeCandidate[]>;
+		query: OrganizeCandidateQuery,
+	) => Promise<OrganizeCandidatePage>;
 	/**
 	 * Every persisted FilterAnchor for the account — read-only, always
 	 * available (never gated behind {@link semantic}, since it never touches
@@ -378,12 +401,18 @@ export const organizePredicateRejection = (
 	hasAnchor(predicate) ? null : bodyContentRejection(predicate.literalClauses);
 
 /**
- * The literal-only arm: scan a bounded, vector-free slice of the corpus and keep
- * the messages whose literal clauses match. Used both for a purely-literal
+ * The literal-only arm: the store selects on the clause terms, this refines
+ * what it returned and stops at `limit` MATCHES. Used both for a purely-literal
  * predicate and as the degraded fallback when a widen is requested on a
  * deployment without the vector pipeline. Serves `From`/`Subject` clauses at
  * full fidelity from the core thread rows; body-content (`HasWords`) clauses are
  * refused before this runs (see {@link bodyContentRejection}).
+ *
+ * The terms are a narrowing, not the verdict ({@link literalClauseTerms}), so
+ * every page is still run through {@link literalClausesMatch} — and paging
+ * continues until the cap fills or the store runs out, which is what makes the
+ * bound a bound on results rather than on the newest `limit` rows (#459).
+ * Deduped by message id: the same mail filed in two folders is one match.
  */
 const matchLiteral = async (
 	deps: OrganizeMatchDeps,
@@ -392,20 +421,29 @@ const matchLiteral = async (
 	limit: number,
 ): Promise<string[]> => {
 	const clauses = predicate.literalClauses;
-	const candidates = await deps.listAccountFilterMessages(
-		accountConfigId,
-		limit,
-	);
-	if (clauses.length === 0) {
-		return candidates.slice(0, limit).map((c) => c.messageId);
-	}
+	const narrowing = literalClauseTerms(clauses, predicate.matchOperator);
+	if (!narrowing) return [];
+
 	const matched: string[] = [];
-	for (const { messageId, message } of candidates) {
-		if (matched.length >= limit) break;
-		if (literalClausesMatch(clauses, predicate.matchOperator, message)) {
+	const seen = new Set<string>();
+	let continuationToken: string | undefined;
+	do {
+		const page = await deps.listAccountFilterMessages(accountConfigId, {
+			...narrowing,
+			limit,
+			continuationToken,
+		});
+		for (const { messageId, message } of page.items) {
+			if (seen.has(messageId)) continue;
+			seen.add(messageId);
+			if (!literalClausesMatch(clauses, predicate.matchOperator, message)) {
+				continue;
+			}
 			matched.push(messageId);
+			if (matched.length >= limit) return matched;
 		}
-	}
+		continuationToken = page.continuationToken;
+	} while (continuationToken);
 	return matched;
 };
 
@@ -508,11 +546,10 @@ const buildSemanticFromEnv = (): OrganizeSemanticDeps => {
 };
 
 /**
- * A bounded, vector-free corpus slice for a literal back-apply: the account's
- * messages projected onto the literal-match fields, gathered newest-first across
- * every mailbox and capped. Reads the core thread rows, so it runs on a
- * deployment that ships no vector pipeline. Deduped by message id — the same
- * mail filed in two folders is one candidate.
+ * The vector-free corpus read for a literal back-apply: one page of the
+ * account's messages, newest-first across every mailbox, with the clause terms
+ * evaluated in the query. Reads the core thread rows, so it runs on a
+ * deployment that ships no vector pipeline.
  *
  * `From`/`Subject`/`listId` come from the row verbatim (full fidelity), so a
  * `From`, `Subject`, `FromDomain` or `ListId` clause matches here exactly as it
@@ -525,34 +562,30 @@ const buildSemanticFromEnv = (): OrganizeSemanticDeps => {
  */
 const listAccountFilterMessagesFromClient =
 	(client: RemitClient): OrganizeMatchDeps["listAccountFilterMessages"] =>
-	async (accountConfigId, limit) => {
-		const candidates: OrganizeCandidate[] = [];
-		const seen = new Set<string>();
-		let continuationToken: string | undefined;
-		do {
-			const page = await client.threadMessage.listByDate(accountConfigId, {
-				limit,
-				continuationToken,
+	async (accountConfigId, query) => {
+		const page = await client.threadMessage.listByFieldTerms(
+			accountConfigId,
+			query.terms,
+			{
+				operator: query.operator,
+				limit: query.limit,
+				continuationToken: query.continuationToken,
 				excludeDeleted: true,
-			});
-			for (const row of page.items) {
-				if (seen.has(row.messageId)) continue;
-				seen.add(row.messageId);
-				candidates.push({
-					messageId: row.messageId,
-					message: {
-						from: row.fromEmail ?? "",
-						fromName: row.fromName ?? "",
-						subject: row.subject ?? "",
-						text: "",
-						listId: row.listId ?? "",
-					},
-				});
-				if (candidates.length >= limit) return candidates;
-			}
-			continuationToken = page.continuationToken;
-		} while (continuationToken);
-		return candidates;
+			},
+		);
+		return {
+			items: page.items.map((row) => ({
+				messageId: row.messageId,
+				message: {
+					from: row.fromEmail ?? "",
+					fromName: row.fromName ?? "",
+					subject: row.subject ?? "",
+					text: "",
+					listId: row.listId ?? "",
+				},
+			})),
+			continuationToken: page.continuationToken,
+		};
 	};
 
 /**