@remit/drizzle-service 0.0.86 → 0.0.87

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.86",
+  "version": "0.0.87",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/repos/message.sqlite.test.ts

@@ -1,6 +1,6 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
-import { eq } from "drizzle-orm";
+import { and, eq, isNull } from "drizzle-orm";
 import {
 	envelopeId as deriveEnvelopeId,
 	rootBodyPartId as deriveRootBodyPartId,
@@ -249,6 +249,63 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		});
 	});
 
+	test("requestReembed queues a forced re-index for each message that still exists", async () => {
+		const movedRows = () =>
+			db
+				.select()
+				.from(outboxTable)
+				.where(
+					and(
+						eq(outboxTable.messageId, MESSAGE_ID),
+						eq(outboxTable.event, "message.moved"),
+						isNull(outboxTable.processedAt),
+					),
+				);
+		const pendingBefore = (await movedRows()).length;
+
+		const queued = await repo.requestReembed([
+			MESSAGE_ID,
+			"00000000-0000-0000-2222-0000000000fe",
+		]);
+
+		assert.deepEqual(queued, { queued: 1, alreadyQueued: 0 });
+		assert.equal((await movedRows()).length, pendingBefore + 1);
+		const orphan = await db
+			.select()
+			.from(outboxTable)
+			.where(eq(outboxTable.messageId, "00000000-0000-0000-2222-0000000000fe"));
+		assert.equal(orphan.length, 0);
+	});
+
+	test("requestReembed skips a message whose forced re-index is still pending", async () => {
+		const pendingBefore = await db
+			.select()
+			.from(outboxTable)
+			.where(
+				and(
+					eq(outboxTable.messageId, MESSAGE_ID),
+					eq(outboxTable.event, "message.moved"),
+					isNull(outboxTable.processedAt),
+				),
+			);
+		assert.ok(pendingBefore.length > 0);
+
+		const again = await repo.requestReembed([MESSAGE_ID]);
+
+		assert.deepEqual(again, { queued: 0, alreadyQueued: 1 });
+		const pendingAfter = await db
+			.select()
+			.from(outboxTable)
+			.where(
+				and(
+					eq(outboxTable.messageId, MESSAGE_ID),
+					eq(outboxTable.event, "message.moved"),
+					isNull(outboxTable.processedAt),
+				),
+			);
+		assert.equal(pendingAfter.length, pendingBefore.length);
+	});
+
 	test("delete removes the message and appends a removal outbox row", async () => {
 		await repo.delete(MESSAGE_ID);
 		await assert.rejects(() => repo.get(MESSAGE_ID));

package/src/repos/message.ts

@@ -6,7 +6,7 @@ import type {
 	MessageItem,
 	PlacementPredicate,
 } from "@remit/data-ports";
-import { and, asc, eq, gt, inArray, or, type SQL } from "drizzle-orm";
+import { and, asc, eq, gt, inArray, isNull, or, type SQL } from "drizzle-orm";
 
 import type { Db } from "../db.js";
 import {
@@ -47,6 +47,13 @@ import {
 
 type DB = Db<MessageDataSchema>;
 
+const REEMBED_BATCH_SIZE = 500;
+
+export interface ReembedRequest {
+	queued: number;
+	alreadyQueued: number;
+}
+
 function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
 	return {
 		messageId: row.messageId,
@@ -632,4 +639,48 @@ export class DrizzleMessageRepository implements IMessageRepository {
 		});
 		return toMessageItem(rows[0]);
 	}
+
+	async requestReembed(messageIds: string[]): Promise<ReembedRequest> {
+		const request: ReembedRequest = { queued: 0, alreadyQueued: 0 };
+		for (
+			let start = 0;
+			start < messageIds.length;
+			start += REEMBED_BATCH_SIZE
+		) {
+			const batch = messageIds.slice(start, start + REEMBED_BATCH_SIZE);
+			await runInTransaction(this.db, async (tx) => {
+				const present = await tx
+					.select({ messageId: messageTable.messageId })
+					.from(messageTable)
+					.where(inArray(messageTable.messageId, batch));
+				const pending = await tx
+					.selectDistinct({ messageId: outboxTable.messageId })
+					.from(outboxTable)
+					.where(
+						and(
+							inArray(outboxTable.messageId, batch),
+							eq(outboxTable.event, "message.moved"),
+							isNull(outboxTable.processedAt),
+						),
+					);
+				const pendingIds = new Set(pending.map(({ messageId }) => messageId));
+				const toQueue = present.filter(
+					({ messageId }) => !pendingIds.has(messageId),
+				);
+				request.alreadyQueued += present.length - toQueue.length;
+				request.queued += toQueue.length;
+				if (toQueue.length === 0) return;
+				await tx.insert(outboxTable).values(
+					toQueue.map(({ messageId }) => ({
+						id: randomUUID(),
+						messageId,
+						event: "message.moved" as const,
+						payload: { messageId },
+						createdAt: new Date(),
+					})),
+				);
+			});
+		}
+		return request;
+	}
 }

package/src/repos/thread-message.test.ts

@@ -647,6 +647,258 @@ describe("DrizzleThreadMessageRepository.searchByDate", () => {
 			["epsilon mine"],
 		);
 	});
+	test("a sender match ranks above a newer mail that only mentions the term", async () => {
+		const acct = uuid();
+		const mbx = uuid();
+		const now = Date.now();
+		await seed(acct, mbx, [
+			{
+				subject: "lunch plans",
+				snippet: "I spoke to penny about it",
+				fromEmail: "bob@acme.test",
+				sentDate: now,
+				internalDate: now,
+			},
+			{
+				subject: "penny wise",
+				fromEmail: "carol@acme.test",
+				sentDate: now - 1,
+				internalDate: now - 1,
+			},
+			{
+				subject: "hello",
+				fromName: "Penny Lane",
+				fromEmail: "lane@acme.test",
+				sentDate: now - 2,
+				internalDate: now - 2,
+			},
+			{
+				subject: "older hello",
+				fromEmail: "penny@acme.test",
+				sentDate: now - 3,
+				internalDate: now - 3,
+			},
+		]);
+
+		const result = await repo.searchByDate(
+			acct,
+			{ query: "penny" },
+			{ excludeDeleted: true, mailboxIds: new Set([mbx]) },
+		);
+
+		assert.deepEqual(
+			result.items.map((r) => r.subject),
+			["hello", "older hello", "lunch plans", "penny wise"],
+			"sender matches first, newest first inside each tier",
+		);
+	});
+
+	test("one term hitting the sender lifts the mail into the sender tier", async () => {
+		const acct = uuid();
+		const mbx = uuid();
+		const now = Date.now();
+		await seed(acct, mbx, [
+			{
+				subject: "invoice for zeta",
+				fromEmail: "billing@acme.test",
+				sentDate: now,
+				internalDate: now,
+			},
+			{
+				subject: "your invoice",
+				fromEmail: "zeta@acme.test",
+				sentDate: now - 1,
+				internalDate: now - 1,
+			},
+		]);
+
+		const result = await repo.searchByDate(
+			acct,
+			{ query: "zeta invoice" },
+			{ excludeDeleted: true, mailboxIds: new Set([mbx]) },
+		);
+
+		assert.deepEqual(
+			result.items.map((r) => r.subject),
+			["your invoice", "invoice for zeta"],
+		);
+	});
+
+	test("paging walks the sender tier, then the mentions, without a gap or repeat", async () => {
+		const acct = uuid();
+		const mbx = uuid();
+		const now = Date.now();
+		await seed(acct, mbx, [
+			{ subject: "kappa news", sentDate: now, internalDate: now },
+			{
+				subject: "one",
+				fromEmail: "kappa@acme.test",
+				sentDate: now - 1,
+				internalDate: now - 1,
+			},
+			{ subject: "kappa digest", sentDate: now - 2, internalDate: now - 2 },
+			{
+				subject: "two",
+				fromEmail: "kappa@acme.test",
+				sentDate: now - 3,
+				internalDate: now - 3,
+			},
+			{ subject: "kappa recap", sentDate: now - 4, internalDate: now - 4 },
+		]);
+		const scope = { excludeDeleted: true, mailboxIds: new Set([mbx]) };
+
+		const walked: string[] = [];
+		let continuationToken: string | undefined;
+		do {
+			const page = await repo.searchByDate(
+				acct,
+				{ query: "kappa" },
+				{ ...scope, limit: 2, continuationToken },
+			);
+			walked.push(...page.items.map((r) => r.subject ?? ""));
+			continuationToken = page.continuationToken;
+		} while (continuationToken);
+
+		assert.deepEqual(walked, [
+			"one",
+			"two",
+			"kappa news",
+			"kappa digest",
+			"kappa recap",
+		]);
+	});
+
+	test("date ties at the tier boundary page without a gap or repeat, in either order", async () => {
+		const acct = uuid();
+		const mbx = uuid();
+		const tie = Date.now();
+		await seed(acct, mbx, [
+			{
+				subject: "sender a",
+				fromEmail: "lambda@acme.test",
+				sentDate: tie,
+				internalDate: tie,
+			},
+			{
+				subject: "sender b",
+				fromEmail: "lambda@acme.test",
+				sentDate: tie,
+				internalDate: tie,
+			},
+			{ subject: "lambda mention a", sentDate: tie, internalDate: tie },
+			{ subject: "lambda mention b", sentDate: tie, internalDate: tie },
+		]);
+		const scope = { excludeDeleted: true, mailboxIds: new Set([mbx]) };
+
+		for (const order of ["desc", "asc"] as const) {
+			const walked: string[] = [];
+			let continuationToken: string | undefined;
+			do {
+				const page = await repo.searchByDate(
+					acct,
+					{ query: "lambda" },
+					{ ...scope, order, limit: 1, continuationToken },
+				);
+				walked.push(...page.items.map((r) => r.subject ?? ""));
+				continuationToken = page.continuationToken;
+			} while (continuationToken);
+
+			assert.equal(walked.length, 4, `${order}: a row was skipped or repeated`);
+			assert.deepEqual(
+				walked.slice(0, 2).sort(),
+				["sender a", "sender b"],
+				`${order}: the sender tier comes first`,
+			);
+			assert.deepEqual(
+				walked.slice(2).sort(),
+				["lambda mention a", "lambda mention b"],
+				`${order}: the mentions follow`,
+			);
+		}
+	});
+
+	test("a ranked page hands back a versioned cursor", async () => {
+		const acct = uuid();
+		const mbx = uuid();
+		const now = Date.now();
+		await seed(acct, mbx, [
+			{ subject: "mu one", sentDate: now, internalDate: now },
+			{ subject: "mu two", sentDate: now - 1, internalDate: now - 1 },
+		]);
+
+		const page = await repo.searchByDate(
+			acct,
+			{ query: "mu one" },
+			{ excludeDeleted: true, mailboxIds: new Set([mbx]), limit: 1 },
+		);
+
+		assert.ok(page.continuationToken);
+		const decoded = JSON.parse(
+			Buffer.from(page.continuationToken, "base64").toString("utf8"),
+		);
+		assert.equal(decoded.v, 2);
+		assert.equal(typeof decoded.r, "number");
+	});
+
+	test("a cursor minted before the ranking finishes its session in date order", async () => {
+		const acct = uuid();
+		const mbx = uuid();
+		const now = Date.now();
+		const rows: Array<Partial<CreateThreadMessageInput>> = [
+			{ subject: "nu news", sentDate: now, internalDate: now },
+			{ subject: "nu digest", sentDate: now - 1, internalDate: now - 1 },
+			{
+				subject: "from nu",
+				fromEmail: "nu@acme.test",
+				sentDate: now - 2,
+				internalDate: now - 2,
+			},
+			{ subject: "nu recap", sentDate: now - 3, internalDate: now - 3 },
+			{
+				subject: "also from nu",
+				fromEmail: "nu@acme.test",
+				sentDate: now - 4,
+				internalDate: now - 4,
+			},
+		];
+		const ids: string[] = [];
+		for (const overrides of rows) {
+			const created = await repo.create(makeInput(acct, mbx, overrides));
+			ids.push(created.threadMessageId);
+			cleanup.push(() => repo.delete(acct, created.threadMessageId));
+		}
+
+		let continuationToken: string | undefined = Buffer.from(
+			JSON.stringify({ s: now - 1, id: ids[1] }),
+		).toString("base64");
+		const walked: string[] = [];
+		do {
+			const page = await repo.searchByDate(
+				acct,
+				{ query: "nu" },
+				{
+					excludeDeleted: true,
+					mailboxIds: new Set([mbx]),
+					limit: 2,
+					continuationToken,
+				},
+			);
+			walked.push(...page.items.map((r) => r.subject ?? ""));
+			continuationToken = page.continuationToken;
+			if (continuationToken) {
+				const decoded = JSON.parse(
+					Buffer.from(continuationToken, "base64").toString("utf8"),
+				);
+				assert.equal(
+					decoded.v,
+					undefined,
+					"the session keeps its cursor shape",
+				);
+			}
+		} while (continuationToken);
+
+		assert.deepEqual(walked, ["from nu", "nu recap", "also from nu"]);
+	});
 });
 
 // ─── Native text-search semantics ─────────────────────────────────────────────

package/src/repos/thread-message.ts

@@ -13,6 +13,7 @@ import {
 	asc,
 	desc,
 	eq,
+	getTableColumns,
 	gt,
 	inArray,
 	lt,
@@ -31,6 +32,7 @@ import {
 	fromMatch,
 	isNarrowableTerm,
 	listIdMatch,
+	senderMatchRank,
 	subjectMatch,
 } from "./thread-search-predicates.js";
 
@@ -57,6 +59,34 @@ export const clampThreadSearchLimit = (limit?: number): number => {
 
 type DateCursor = { s: number; id: string };
 type AccountCursor = { id: string };
+type RankedDateCursor = { v: 2; r: number; s: number; id: string };
+type SearchCursor = RankedDateCursor | (DateCursor & { v: 1 });
+
+function encodeRankedDateCursor(
+	rank: number,
+	sentDate: number,
+	threadMessageId: string,
+): string {
+	return Buffer.from(
+		JSON.stringify({ v: 2, r: rank, s: sentDate, id: threadMessageId }),
+	).toString("base64");
+}
+
+function decodeSearchCursor(token: string): SearchCursor {
+	const decoded = decodeToken(token, "base64");
+	if (decoded.v === undefined) {
+		return { v: 1, ...decodeDateCursor(token) };
+	}
+	if (
+		decoded.v !== 2 ||
+		typeof decoded.r !== "number" ||
+		typeof decoded.s !== "number" ||
+		typeof decoded.id !== "string"
+	) {
+		throw new BadRequestError("Invalid continuationToken");
+	}
+	return { v: 2, r: decoded.r, s: decoded.s, id: decoded.id };
+}
 
 function encodeDateCursor(sentDate: number, threadMessageId: string): string {
 	return Buffer.from(
@@ -144,6 +174,9 @@ function toItem(row: Row): ThreadMessageItem {
 const mutedSender = (): SQL =>
 	sql`exists (select 1 from ${addressTable} where ${addressTable.accountConfigId} = ${threadMessageTable.accountConfigId} and ${addressTable.normalizedEmail} = lower(coalesce(${threadMessageTable.fromEmail}, '')) and json_extract(coalesce(nullif(${addressTable.flags}, ''), '{}'), '$.muted.value') = 1)`;
 
+const queryTerms = (query: string | undefined): string[] =>
+	query ? query.split(/\s+/).filter(Boolean) : [];
+
 // Translate SearchOptions into SQL conditions: subject/from/query as indexed
 // text predicates, muted as a subquery over the sender's address, the rest as
 // plain column equalities. A multi-word `query`
@@ -157,8 +190,7 @@ function buildSearchConditions(search: SearchOptions): SQL[] {
 	if (search.from) conditions.push(fromMatch(search.from));
 
 	if (search.query) {
-		const tokens = search.query.split(/\s+/).filter(Boolean);
-		for (const token of tokens) {
+		for (const token of queryTerms(search.query)) {
 			conditions.push(
 				sql`(${subjectMatch(token)} or ${fromMatch(token)} or ${bodyMatch(token)})`,
 			);
@@ -233,6 +265,18 @@ function sentDateCursorCond(
 	);
 }
 
+function rankedDateCursorCond(
+	order: "asc" | "desc",
+	rank: SQL<number>,
+	cursor: RankedDateCursor | null,
+): SQL | undefined {
+	if (!cursor) return undefined;
+	return or(
+		lt(rank, cursor.r),
+		and(eq(rank, cursor.r), sentDateCursorCond(order, cursor)),
+	);
+}
+
 // ─── Repository ──────────────────────────────────────────────────────────────
 
 export class DrizzleThreadMessageRepository
@@ -528,8 +572,9 @@ export class DrizzleThreadMessageRepository
 
 	/**
 	 * Cross-mailbox search for the unified listing's search mode. Same predicate
-	 * builder and keyset cursor as `searchByMailboxWindow`, with the mailbox
-	 * equality swapped for the caller's scope set. Matching runs in SQL over the
+	 * builder as `searchByMailboxWindow`, with the mailbox equality swapped for
+	 * the caller's scope set. Sender matches rank first, then date order; a
+	 * cursor minted before the ranking finishes its session in date order. Matching runs in SQL over the
 	 * whole scope, so a short page means the matches are exhausted.
 	 */
 	async searchByDate(
@@ -546,15 +591,23 @@ export class DrizzleThreadMessageRepository
 		const order = options?.order ?? "desc";
 		const limit = clampThreadSearchLimit(options?.limit);
 		const cursor = options?.continuationToken
-			? decodeDateCursor(options.continuationToken)
+			? decodeSearchCursor(options.continuationToken)
 			: null;
+		const ranked = cursor?.v !== 1;
 
 		const mailboxCond = options?.mailboxIds?.size
 			? inArray(threadMessageTable.mailboxId, [...options.mailboxIds])
 			: undefined;
 
+		const senderRank = ranked
+			? senderMatchRank(queryTerms(search.query))
+			: sql<number>`0`;
+
 		const rows = await this.db
-			.select()
+			.select({
+				...getTableColumns(threadMessageTable),
+				senderRank,
+			})
 			.from(threadMessageTable)
 			.where(
 				and(
@@ -564,10 +617,13 @@ export class DrizzleThreadMessageRepository
 						? eq(threadMessageTable.isDeleted, false)
 						: undefined,
 					...buildSearchConditions(search),
-					sentDateCursorCond(order, cursor),
+					cursor?.v === 1
+						? sentDateCursorCond(order, cursor)
+						: rankedDateCursorCond(order, senderRank, cursor),
 				),
 			)
 			.orderBy(
+				...(ranked ? [desc(senderRank)] : []),
 				order === "desc"
 					? desc(threadMessageTable.sentDate)
 					: asc(threadMessageTable.sentDate),
@@ -580,14 +636,20 @@ export class DrizzleThreadMessageRepository
 			items: rows.map(toItem),
 			continuationToken:
 				rows.length === limit && lastRow
-					? encodeDateCursor(lastRow.sentDate, lastRow.threadMessageId)
+					? ranked
+						? encodeRankedDateCursor(
+								lastRow.senderRank,
+								lastRow.sentDate,
+								lastRow.threadMessageId,
+							)
+						: encodeDateCursor(lastRow.sentDate, lastRow.threadMessageId)
 					: undefined,
 		};
 	}
 
 	/**
-	 * Cross-mailbox narrowing for a rule back-apply. Same keyset cursor and
-	 * ordering as `searchByDate`, with the terms combined under the caller's
+	 * Cross-mailbox narrowing for a rule back-apply. Same date keyset cursor
+	 * and ordering as `listByDate`, with the terms combined under the caller's
 	 * operator instead of the AND-only `SearchOptions` shape a search box needs.
 	 * The terms run in SQL over the whole config, so a page is a page of
 	 * narrowed rows and a rule for a sender that has been quiet for a month

package/src/repos/thread-search-predicates.ts

@@ -91,3 +91,8 @@ export const bodyMatch = (term: string): SQL =>
 // scan of one config's rows beats reading them all into the service (#459).
 export const listIdMatch = (term: string): SQL =>
 	sql`${LIST_ID_FOLDED} like ${likePattern(term)} escape '\\'`;
+
+export const senderMatchRank = (terms: readonly string[]): SQL<number> =>
+	terms.length === 0
+		? sql<number>`0`
+		: sql<number>`case when ${sql.join(terms.map(fromMatch), sql` or `)} then 1 else 0 end`;