@remit/drizzle-service 0.0.78 → 0.0.79

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.78",
+  "version": "0.0.79",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/repos/thread-message-field-terms.sqlite.test.ts

@@ -0,0 +1,277 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import type { CreateThreadMessageInput } from "@remit/data-ports";
+import { threadMessageTable } from "../schema/thread-message.js";
+import { createSqliteTestDb } from "../test-db-sqlite.js";
+import {
+	DrizzleThreadMessageRepository,
+	THREAD_SEARCH_MAX_LIMIT,
+} from "./thread-message.js";
+
+// `listByFieldTerms` on sqlite (#459): the terms decide inside the query, so a
+// rule for a sender that has been quiet for months reaches its mail however
+// much newer mail sits above it. Filtering a date-ordered page instead answers
+// "matches among the newest N", which is the defect this pins.
+
+const ACCOUNT = "acct-terms";
+const OTHER_ACCOUNT = "acct-terms-other";
+const MAILBOX = "mbx-terms";
+const ARCHIVE = "mbx-terms-archive";
+
+// One more than the ceiling a back-apply reads with, so the single old match
+// sits strictly below any window the newest page could cover.
+const NEWER_NOISE = THREAD_SEARCH_MAX_LIMIT + 1;
+
+const OLD_DATE = 1_600_000_000_000;
+
+function makeInput(
+	overrides: Partial<CreateThreadMessageInput> = {},
+): CreateThreadMessageInput {
+	return {
+		accountConfigId: ACCOUNT,
+		threadId: `t-${Math.random().toString(36).slice(2)}`,
+		messageId: `m-${Math.random().toString(36).slice(2)}`,
+		mailboxId: MAILBOX,
+		uid: 1,
+		referenceOrder: 0,
+		internalDate: OLD_DATE,
+		sentDate: OLD_DATE,
+		isRead: false,
+		isDeleted: false,
+		hasAttachment: false,
+		hasStars: false,
+		...overrides,
+	};
+}
+
+describe("DrizzleThreadMessageRepository.listByFieldTerms (sqlite, #459)", () => {
+	let close: () => Promise<void>;
+	let repo: DrizzleThreadMessageRepository;
+
+	before(async () => {
+		const created = await createSqliteTestDb(
+			{ threadMessage: threadMessageTable },
+			{ searchIndex: true },
+		);
+		close = created.close;
+		repo = new DrizzleThreadMessageRepository(created.db);
+
+		await repo.create(
+			makeInput({
+				messageId: "quiet-sender",
+				subject: "Your March statement",
+				fromName: "Statements",
+				fromEmail: "noreply@bank.example",
+				listId: "statements.bank.example",
+			}),
+		);
+		await repo.create(
+			makeInput({
+				messageId: "accented",
+				subject: "CAFÉ closing early",
+				fromName: "Café",
+				fromEmail: "hello@paris.example",
+				sentDate: OLD_DATE + 9_000_000,
+				internalDate: OLD_DATE + 9_000_000,
+			}),
+		);
+		for (let index = 0; index < NEWER_NOISE; index++) {
+			await repo.create(
+				makeInput({
+					messageId: `noise-${index}`,
+					subject: `Daily digest ${index}`,
+					fromName: "Digest",
+					fromEmail: "digest@other.example",
+					sentDate: OLD_DATE + 1 + index,
+					internalDate: OLD_DATE + 1 + index,
+				}),
+			);
+		}
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	test("finds a match older than a whole page of newer non-matching mail", async () => {
+		const result = await repo.listByFieldTerms(
+			ACCOUNT,
+			[{ field: "sender", contains: "bank.example" }],
+			{ limit: THREAD_SEARCH_MAX_LIMIT },
+		);
+
+		assert.deepEqual(
+			result.items.map((item) => item.messageId),
+			["quiet-sender"],
+		);
+		assert.equal(result.continuationToken, undefined);
+	});
+
+	test("a page is a page of matches, not a page of rows", async () => {
+		const result = await repo.listByFieldTerms(
+			ACCOUNT,
+			[{ field: "sender", contains: "bank.example" }],
+			{ limit: 5 },
+		);
+
+		assert.equal(result.items.length, 1);
+	});
+
+	test("matches the subject and the List-Id columns", async () => {
+		const bySubject = await repo.listByFieldTerms(ACCOUNT, [
+			{ field: "subject", contains: "march statement" },
+		]);
+		const byListId = await repo.listByFieldTerms(ACCOUNT, [
+			{ field: "listId", contains: "statements.bank.example" },
+		]);
+
+		assert.deepEqual(
+			bySubject.items.map((item) => item.messageId),
+			["quiet-sender"],
+		);
+		assert.deepEqual(
+			byListId.items.map((item) => item.messageId),
+			["quiet-sender"],
+		);
+	});
+
+	test("`and` requires every term, `or` any of them", async () => {
+		const terms = [
+			{ field: "sender", contains: "bank.example" },
+			{ field: "subject", contains: "daily digest 0" },
+		] as const;
+
+		const conjunction = await repo.listByFieldTerms(ACCOUNT, terms, {
+			operator: "and",
+		});
+		const disjunction = await repo.listByFieldTerms(ACCOUNT, terms, {
+			operator: "or",
+		});
+
+		assert.deepEqual(conjunction.items, []);
+		assert.deepEqual(disjunction.items.map((item) => item.messageId).sort(), [
+			"noise-0",
+			"quiet-sender",
+		]);
+	});
+
+	// Below the trigram floor the predicate is the folded LIKE, and sqlite's
+	// lower() folds ASCII only — `é` never matches a stored `CAFÉ`. Applying it
+	// anyway would drop a row the caller's own matcher accepts, which is #459
+	// again for that clause shape, so a short non-ASCII term narrows nothing.
+	test("drops a short accented term rather than missing the row it should match", async () => {
+		const result = await repo.listByFieldTerms(
+			ACCOUNT,
+			[{ field: "subject", contains: "é" }],
+			{ limit: THREAD_SEARCH_MAX_LIMIT },
+		);
+
+		assert.ok(
+			result.items.some((item) => item.messageId === "accented"),
+			"the accented row survives",
+		);
+		assert.ok(result.items.length > 1, "the term narrowed nothing at all");
+	});
+
+	test("keeps narrowing on the other terms of an `and` around a dropped one", async () => {
+		const result = await repo.listByFieldTerms(
+			ACCOUNT,
+			[
+				{ field: "subject", contains: "é" },
+				{ field: "sender", contains: "bank.example" },
+			],
+			{ operator: "and" },
+		);
+
+		assert.deepEqual(
+			result.items.map((item) => item.messageId),
+			["quiet-sender"],
+		);
+	});
+
+	test("drops the whole narrowing when an `or` branch cannot be evaluated", async () => {
+		const result = await repo.listByFieldTerms(
+			ACCOUNT,
+			[
+				{ field: "subject", contains: "é" },
+				{ field: "sender", contains: "bank.example" },
+			],
+			{ operator: "or", limit: 5 },
+		);
+
+		const found = result.items.map((item) => item.messageId);
+		assert.ok(found.includes("accented"), "the dropped branch keeps its rows");
+		assert.ok(
+			found.some((messageId) => messageId.startsWith("noise-")),
+			"a narrowed `or` would have excluded these",
+		);
+	});
+
+	test("still narrows on a short ASCII term, which lower() folds correctly", async () => {
+		const result = await repo.listByFieldTerms(ACCOUNT, [
+			{ field: "sender", contains: "k." },
+		]);
+
+		assert.deepEqual(
+			result.items.map((item) => item.messageId),
+			["quiet-sender"],
+		);
+	});
+
+	test("no terms narrows nothing", async () => {
+		const result = await repo.listByFieldTerms(ACCOUNT, [], { limit: 3 });
+
+		assert.equal(result.items.length, 3);
+		assert.ok(result.continuationToken, "more rows remain");
+	});
+
+	test("stays inside the account and skips deleted rows on request", async () => {
+		await repo.create(
+			makeInput({
+				accountConfigId: OTHER_ACCOUNT,
+				messageId: "other-account",
+				fromEmail: "noreply@bank.example",
+			}),
+		);
+		await repo.create(
+			makeInput({
+				messageId: "deleted-match",
+				mailboxId: ARCHIVE,
+				fromEmail: "noreply@bank.example",
+				isDeleted: true,
+			}),
+		);
+
+		const result = await repo.listByFieldTerms(
+			ACCOUNT,
+			[{ field: "sender", contains: "bank.example" }],
+			{ excludeDeleted: true },
+		);
+
+		assert.deepEqual(
+			result.items.map((item) => item.messageId),
+			["quiet-sender"],
+		);
+	});
+
+	test("pages the matches with a keyset cursor", async () => {
+		const first = await repo.listByFieldTerms(
+			ACCOUNT,
+			[{ field: "sender", contains: "other.example" }],
+			{ limit: 2 },
+		);
+		assert.equal(first.items.length, 2);
+		assert.ok(first.continuationToken);
+
+		const second = await repo.listByFieldTerms(
+			ACCOUNT,
+			[{ field: "sender", contains: "other.example" }],
+			{ limit: 2, continuationToken: first.continuationToken },
+		);
+
+		const overlap = second.items.filter((item) =>
+			first.items.some((seen) => seen.messageId === item.messageId),
+		);
+		assert.deepEqual(overlap, [], "pages do not repeat a row");
+	});
+});

package/src/repos/thread-message.ts

@@ -3,6 +3,7 @@ import type {
 	IThreadMessageRepository,
 	ResultList,
 	SearchOptions,
+	ThreadMessageFieldTerm,
 	ThreadMessageItem,
 	UpdateThreadMessageInput,
 } from "@remit/data-ports";
@@ -24,7 +25,12 @@ import { NotFoundError } from "../error.js";
 import { deterministicBase36Id } from "../id.js";
 import { decodeToken } from "../pagination.js";
 import { threadMessageTable } from "../schema/thread-message.js";
-import { fromMatch, subjectMatch } from "./thread-search-predicates.js";
+import {
+	fromMatch,
+	isNarrowableTerm,
+	listIdMatch,
+	subjectMatch,
+} from "./thread-search-predicates.js";
 
 export const deriveThreadMessageId = (
 	threadId: string,
@@ -158,6 +164,30 @@ function buildSearchConditions(search: SearchOptions): SQL[] {
 	return conditions;
 }
 
+const fieldTermCondition = (term: ThreadMessageFieldTerm): SQL => {
+	if (term.field === "subject") return subjectMatch(term.contains);
+	if (term.field === "listId") return listIdMatch(term.contains);
+	return fromMatch(term.contains);
+};
+
+// Combine the caller's terms into one condition, or `undefined` when there are
+// none — an empty set narrows nothing, which is what both operators mean here.
+//
+// A term this engine cannot evaluate faithfully ({@link isNarrowableTerm}) is
+// left out rather than emitted: under `and` a missing conjunct only widens the
+// candidate set, which the caller refines anyway, but under `or` a missing
+// branch loses matches outright, so the whole narrowing goes.
+function buildFieldTermCondition(
+	terms: readonly ThreadMessageFieldTerm[],
+	operator: "and" | "or",
+): SQL | undefined {
+	const narrowable = terms.filter((term) => isNarrowableTerm(term.contains));
+	if (operator === "or" && narrowable.length !== terms.length) return undefined;
+	const conditions = narrowable.map(fieldTermCondition);
+	if (conditions.length === 0) return undefined;
+	return operator === "or" ? or(...conditions) : and(...conditions);
+}
+
 // Keyset cursor over (sent_date, thread_message_id). `desc` walks newest→oldest,
 // `asc` oldest→newest; the id tiebreak keeps paging stable across equal dates.
 function sentDateCursorCond(
@@ -530,6 +560,62 @@ export class DrizzleThreadMessageRepository
 		};
 	}
 
+	/**
+	 * Cross-mailbox narrowing for a rule back-apply. Same keyset cursor and
+	 * ordering as `searchByDate`, with the terms combined under the caller's
+	 * operator instead of the AND-only `SearchOptions` shape a search box needs.
+	 * The terms run in SQL over the whole config, so a page is a page of
+	 * narrowed rows and a rule for a sender that has been quiet for a month
+	 * reaches its mail (#459).
+	 */
+	async listByFieldTerms(
+		accountConfigId: string,
+		terms: readonly ThreadMessageFieldTerm[],
+		options?: {
+			operator?: "and" | "or";
+			order?: "asc" | "desc";
+			limit?: number;
+			continuationToken?: string;
+			excludeDeleted?: boolean;
+		},
+	): Promise<ResultList<ThreadMessageItem>> {
+		const order = options?.order ?? "desc";
+		const limit = clampThreadSearchLimit(options?.limit);
+		const cursor = options?.continuationToken
+			? decodeDateCursor(options.continuationToken)
+			: null;
+
+		const rows = await this.db
+			.select()
+			.from(threadMessageTable)
+			.where(
+				and(
+					eq(threadMessageTable.accountConfigId, accountConfigId),
+					options?.excludeDeleted
+						? eq(threadMessageTable.isDeleted, false)
+						: undefined,
+					buildFieldTermCondition(terms, options?.operator ?? "and"),
+					sentDateCursorCond(order, cursor),
+				),
+			)
+			.orderBy(
+				order === "desc"
+					? desc(threadMessageTable.sentDate)
+					: asc(threadMessageTable.sentDate),
+				asc(threadMessageTable.threadMessageId),
+			)
+			.limit(limit);
+
+		const lastRow = rows[rows.length - 1];
+		return {
+			items: rows.map(toItem),
+			continuationToken:
+				rows.length === limit && lastRow
+					? encodeDateCursor(lastRow.sentDate, lastRow.threadMessageId)
+					: undefined,
+		};
+	}
+
 	/**
 	 * COUNT of matching CONVERSATIONS over the SAME predicate as the
 	 * cross-account listings, across the caller's mailbox scope.

package/src/repos/thread-search-predicates.ts

@@ -31,11 +31,29 @@ const ftsPhrase = (term: string): string => `"${term.replace(/"/g, '""')}"`;
 // index floor.
 const isTrigramIndexable = (term: string): boolean => [...term].length >= 3;
 
+const isAscii = (term: string): boolean =>
+	[...term].every((char) => (char.codePointAt(0) ?? 0) < 128);
+
+/**
+ * Whether a term can be pushed into the query without dropping a row the
+ * caller's own matcher would accept.
+ *
+ * Below the trigram floor the predicate is the folded LIKE, and SQLite's
+ * `lower()` folds ASCII only: `é` never matches a stored `CAFÉ`. For a search
+ * box that is the accepted C10 difference between a short term and an indexed
+ * one. For a NARROWING it is not — a narrowing that misses is the #459 defect
+ * again, one clause shape at a time — so such a term is not narrowable at all
+ * and the caller widens instead.
+ */
+export const isNarrowableTerm = (term: string): boolean =>
+	isTrigramIndexable(term) || isAscii(term);
+
 const ftsRowidMatch = (matchExpr: string): SQL =>
 	sql`"thread_message"."rowid" in (select "rowid" from "thread_message_fts" where "thread_message_fts" match ${matchExpr})`;
 
 const SUBJECT_FOLDED = sql`lower(coalesce(subject, ''))`;
 const FROM_FOLDED = sql`lower(coalesce(from_name, '') || ' ' || coalesce(from_email, ''))`;
+const LIST_ID_FOLDED = sql`lower(coalesce(list_id, ''))`;
 
 const likePattern = (term: string): SQL =>
 	sql`'%' || lower(${escapeLike(term)}) || '%'`;
@@ -49,3 +67,9 @@ export const fromMatch = (term: string): SQL =>
 	isTrigramIndexable(term)
 		? ftsRowidMatch(`sender : ${ftsPhrase(term)}`)
 		: sql`${FROM_FOLDED} like ${likePattern(term)} escape '\\'`;
+
+// The FTS index carries subject and sender only, so a List-Id term is always
+// the folded LIKE scan. It is the narrowing half of a rule back-apply, where a
+// scan of one config's rows beats reading them all into the service (#459).
+export const listIdMatch = (term: string): SQL =>
+	sql`${LIST_ID_FOLDED} like ${likePattern(term)} escape '\\'`;