@remit/drizzle-service 0.0.103 → 0.0.105

package/src/repos/message.sqlite.test.ts

@@ -1,5 +1,6 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import {
 	envelopeId as deriveEnvelopeId,
 	rootBodyPartId as deriveRootBodyPartId,
@@ -43,7 +44,10 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 
 	before(async () => {
 		({ db, close } = await createSqliteTestDb(SCHEMA));
-		repo = createSqliteRepositories({ app: db, mail: db, search: db }).message;
+		repo = createSqliteRepositories(
+			{ app: db, mail: db, search: db },
+			BETTER_SQLITE3_LIMITS,
+		).message;
 	});
 
 	after(async () => {
@@ -279,4 +283,58 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		await repo.delete(MESSAGE_ID);
 		await assert.rejects(() => repo.get(MESSAGE_ID));
 	});
+
+	test("listMissingSearchText names stored bodies without search text, after the cursor", async () => {
+		const id = (n: number): string => `00000000-0000-0000-3333-00000000000${n}`;
+		for (const n of [1, 2, 3, 4]) {
+			await repo.create({ ...BASE_INPUT, messageId: id(n), uid: 100 + n });
+		}
+		for (const n of [1, 2, 4]) {
+			await repo.update(id(n), { bodyStorageKey: `file:///bodies/${n}` });
+		}
+		await repo.setSearchText(id(2), "already indexed");
+
+		const ours = (rows: Array<{ messageId: string }>): string[] =>
+			rows
+				.map((row) => row.messageId)
+				.filter((messageId) => messageId.includes("-3333-"));
+
+		assert.deepEqual(ours(await repo.listMissingSearchText("", 100)), [
+			id(1),
+			id(4),
+		]);
+		assert.deepEqual(ours(await repo.listMissingSearchText(id(1), 100)), [
+			id(4),
+		]);
+		assert.deepEqual(
+			await repo
+				.listMissingSearchText(id(1), 100)
+				.then((rows) => rows.find((row) => row.messageId === id(4))),
+			{
+				messageId: id(4),
+				bodyStorageKey: "file:///bodies/4",
+			},
+		);
+	});
+
+	test("copySearchText gives a copy the source's search text", async () => {
+		const source = "00000000-0000-0000-4444-000000000001";
+		const copy = "00000000-0000-0000-4444-000000000002";
+		for (const [messageId, uid] of [
+			[source, 201],
+			[copy, 202],
+		] as const) {
+			await repo.create({ ...BASE_INPUT, messageId, uid });
+			await repo.update(messageId, { bodyStorageKey: "file:///bodies/shared" });
+		}
+		await repo.setSearchText(source, "the shared body");
+
+		await repo.copySearchText(source, copy);
+
+		const missing = await repo.listMissingSearchText("", 100);
+		assert.equal(
+			missing.some((row) => row.messageId === copy),
+			false,
+		);
+	});
 });

package/src/repos/message.test.ts

@@ -1,5 +1,6 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import {
 	envelopeId as deriveEnvelopeId,
 	rootBodyPartId as deriveRootBodyPartId,
@@ -37,7 +38,10 @@ describe("DrizzleMessageRepository", () => {
 
 	before(async () => {
 		({ db, stop } = await createTestDb());
-		const repos = createSqliteRepositories({ app: db, mail: db, search: db });
+		const repos = createSqliteRepositories(
+			{ app: db, mail: db, search: db },
+			BETTER_SQLITE3_LIMITS,
+		);
 		messageRepo = repos.message;
 		envelopeRepo = repos.envelope;
 	});

package/src/repos/message.ts

@@ -39,6 +39,7 @@ import {
 	envelopeTable,
 	messageFlagTable,
 	messageReferenceTable,
+	messageSearchTextTable,
 	messageTable,
 	rawMessageStorageTable,
 } from "../schema/message-data.js";
@@ -153,6 +154,9 @@ export function deleteMessageSubtree(
 			.where(
 				and(inList(bodyStructureRefetchTable.messageId, messageIds), guard),
 			),
+		db
+			.delete(messageSearchTextTable)
+			.where(and(inList(messageSearchTextTable.messageId, messageIds), guard)),
 		db
 			.delete(envelopeTable)
 			.where(and(inList(envelopeTable.messageId, messageIds), guard)),
@@ -444,6 +448,68 @@ export class DrizzleMessageRepository implements IMessageRepository {
 		return this.get(messageId);
 	}
 
+	async setSearchText(messageId: string, text: string): Promise<void> {
+		const now = Date.now();
+		await this.db
+			.insert(messageSearchTextTable)
+			.values({ messageId, text, createdAt: now, updatedAt: now })
+			.onConflictDoUpdate({
+				target: messageSearchTextTable.messageId,
+				set: { text, updatedAt: now },
+			});
+	}
+
+	async copySearchText(
+		sourceMessageId: string,
+		targetMessageId: string,
+	): Promise<void> {
+		const [source] = await this.db
+			.select({ text: messageSearchTextTable.text })
+			.from(messageSearchTextTable)
+			.where(eq(messageSearchTextTable.messageId, sourceMessageId));
+		if (!source) return;
+		await this.setSearchText(targetMessageId, source.text);
+	}
+
+	async syncSearch(messageIds: string[]): Promise<void> {
+		if (messageIds.length === 0) return;
+		const listings = await this.db
+			.select({ threadMessageId: threadMessageTable.threadMessageId })
+			.from(threadMessageTable)
+			.where(inList(threadMessageTable.messageId, messageIds));
+		await this.search.sync(listings.map((row) => row.threadMessageId));
+	}
+
+	async listMissingSearchText(
+		after: string,
+		limit: number,
+	): ReturnType<IMessageRepository["listMissingSearchText"]> {
+		const rows = await this.db
+			.select({
+				messageId: messageTable.messageId,
+				bodyStorageKey: messageTable.bodyStorageKey,
+			})
+			.from(messageTable)
+			.leftJoin(
+				messageSearchTextTable,
+				eq(messageSearchTextTable.messageId, messageTable.messageId),
+			)
+			.where(
+				and(
+					gt(messageTable.messageId, after),
+					isNotNull(messageTable.bodyStorageKey),
+					isNull(messageSearchTextTable.messageId),
+				),
+			)
+			.orderBy(asc(messageTable.messageId))
+			.limit(limit);
+		return rows.flatMap((row) =>
+			row.bodyStorageKey
+				? [{ messageId: row.messageId, bodyStorageKey: row.bodyStorageKey }]
+				: [],
+		);
+	}
+
 	async delete(messageId: string): Promise<void> {
 		await this.deleteMany([messageId]);
 	}

package/src/repos/sync-warning.ts

@@ -6,14 +6,24 @@ import type {
 } from "@remit/data-ports";
 import { deriveSyncWarningId } from "@remit/data-ports/id";
 import { and, count, desc, eq, gte, sql } from "drizzle-orm";
-import type { RepositoryDb } from "../db.js";
+import type { WriteBatch } from "../batch.js";
+import { chunkRows } from "../bound-parameters.js";
+import type { RepositoryDb, SqliteStore } from "../db.js";
 import { syncWarningTable } from "../schema/sync-warning.js";
 import { inList } from "./in-list.js";
 
 type DB = RepositoryDb<Record<string, unknown>>;
 
 export class SyncWarningRepo implements ISyncWarningRepository {
-	constructor(private db: DB) {}
+	private readonly db: DB;
+	private readonly batch: WriteBatch;
+	private readonly maxBoundParameters: number;
+
+	constructor(store: SqliteStore<Record<string, unknown>>) {
+		this.db = store.db;
+		this.batch = store.batch;
+		this.maxBoundParameters = store.maxBoundParameters;
+	}
 
 	listByMailboxId = async (mailboxId: string): Promise<SyncWarningItem[]> =>
 		this.db
@@ -46,26 +56,29 @@ export class SyncWarningRepo implements ISyncWarningRepository {
 	recordFailures = async (keys: SyncWarningKey[]): Promise<void> => {
 		if (keys.length === 0) return;
 		const now = Date.now();
-		await this.db
-			.insert(syncWarningTable)
-			.values(
-				keys.map((key) => ({
-					syncWarningId: deriveSyncWarningId(
-						key.accountId,
-						key.mailboxId,
-						key.uidValidity,
-						key.uid,
-						key.stage,
-					),
-					...key,
-					attempts: 1,
-					createdAt: now,
-				})),
-			)
-			.onConflictDoUpdate({
-				target: syncWarningTable.syncWarningId,
-				set: { attempts: sql`${syncWarningTable.attempts} + 1` },
-			});
+		const rows = keys.map((key) => ({
+			syncWarningId: deriveSyncWarningId(
+				key.accountId,
+				key.mailboxId,
+				key.uidValidity,
+				key.uid,
+				key.stage,
+			),
+			...key,
+			attempts: 1,
+			createdAt: now,
+		}));
+		await this.batch(
+			chunkRows(syncWarningTable, rows, this.maxBoundParameters).map((slice) =>
+				this.db
+					.insert(syncWarningTable)
+					.values(slice)
+					.onConflictDoUpdate({
+						target: syncWarningTable.syncWarningId,
+						set: { attempts: sql`${syncWarningTable.attempts} + 1` },
+					}),
+			),
+		);
 	};
 
 	deleteMany = async (syncWarningIds: string[]): Promise<void> => {

package/src/repos/thread-message-body-muted.sqlite.test.ts

@@ -10,17 +10,14 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import type { CreateThreadMessageInput } from "@remit/data-ports";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { addressTable } from "../schema/i4-address.js";
+import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
-import { shippedIndexDdl } from "../test-shipped-sqlite-schema.js";
 import type { DrizzleThreadMessageRepository } from "./thread-message.js";
 
-// The index the mail baseline creates for this predicate, read from the one
-// committed source so the plan assertion below is about the real object.
-const mutedIndexDdl = shippedIndexDdl("address_by_normalized_email");
-
 const ACCOUNT = "acct-seam";
 const MAILBOX = "mbx-seam";
 const SCOPE = new Set([MAILBOX]);
@@ -57,15 +54,21 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
 
 	before(async () => {
 		({ db, sqlite, close } = await createSqliteTestDb(
-			{ threadMessage: threadMessageTable, address: addressTable },
+			{
+				threadMessage: threadMessageTable,
+				address: addressTable,
+				messageSearchText: messageSearchTextTable,
+			},
 			{ searchIndex: true },
 		));
-		sqlite.exec(mutedIndexDdl);
-		repo = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {
@@ -86,15 +89,20 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
 					}),
 				);
 			}
-			await repo.create(
-				makeInput({
-					subject: "unrelated note",
-					fromEmail: "noise@example.com",
-					fromName: "Noise",
-					snippet: "Your parcel was left with the concierge",
-					sentDate: BASE_DATE,
-				}),
-			);
+			const parcel = makeInput({
+				subject: "unrelated note",
+				fromEmail: "noise@example.com",
+				fromName: "Noise",
+				snippet: "Your parcel was left with the concierge",
+				sentDate: BASE_DATE,
+			});
+			await db.insert(messageSearchTextTable).values({
+				messageId: parcel.messageId,
+				text: "Your parcel was left with the concierge",
+				createdAt: 0,
+				updatedAt: 0,
+			});
+			await repo.create(parcel);
 		});
 
 		test("the term reaches the body text, not just subject and From", async () => {
@@ -123,16 +131,16 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
 			assert.equal(result.items.length, 1);
 		});
 
-		test("a term under the trigram floor reads the body too", async () => {
+		test("a short word prefix reads the body too", async () => {
 			const result = await repo.searchByDate(
 				ACCOUNT,
-				{ query: "ge" },
+				{ query: "co" },
 				{ mailboxIds: SCOPE, limit: 50 },
 			);
 
 			assert.ok(
 				result.items.some((item) => item.snippet?.includes("concierge")),
-				"the folded scan matched the body preview",
+				"the prefix matched the body preview",
 			);
 		});
 

package/src/repos/thread-message-field-terms.sqlite.test.ts

@@ -1,6 +1,8 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import type { CreateThreadMessageInput } from "@remit/data-ports";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
+import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
@@ -51,15 +53,21 @@ describe("DrizzleThreadMessageRepository.listByFieldTerms (sqlite, #459)", () =>
 
 	before(async () => {
 		const created = await createSqliteTestDb(
-			{ threadMessage: threadMessageTable },
+			{
+				threadMessage: threadMessageTable,
+				messageSearchText: messageSearchTextTable,
+			},
 			{ searchIndex: true },
 		);
 		close = created.close;
-		repo = createSqliteRepositories({
-			app: created.db,
-			mail: created.db,
-			search: created.db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: created.db,
+				mail: created.db,
+				search: created.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 
 		await repo.create(
 			makeInput({
@@ -160,10 +168,10 @@ describe("DrizzleThreadMessageRepository.listByFieldTerms (sqlite, #459)", () =>
 		]);
 	});
 
-	// Below the trigram floor the predicate is the folded LIKE, and sqlite's
-	// lower() folds ASCII only — `é` never matches a stored `CAFÉ`. Applying it
-	// anyway would drop a row the caller's own matcher accepts, which is #459
-	// again for that clause shape, so a short non-ASCII term narrows nothing.
+	// A field term is the folded LIKE, and sqlite's lower() folds ASCII only —
+	// `é` never matches a stored `CAFÉ`. Applying it anyway would drop a row
+	// the caller's own matcher accepts, which is #459 again for that clause
+	// shape, so a non-ASCII term narrows nothing.
 	test("drops a short accented term rather than missing the row it should match", async () => {
 		const result = await repo.listByFieldTerms(
 			ACCOUNT,

package/src/repos/thread-message.sqlite.test.ts

@@ -1,6 +1,8 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import type { CreateThreadMessageInput } from "@remit/data-ports";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
+import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
@@ -17,9 +19,9 @@ import {
 } from "./thread-message.js";
 
 // The thread-message repo on sqlite (RFC 036 D1): CRUD, keyset pagination, and
-// text search — the FTS5 trigram index for terms of three characters or more
-// (D4), the folded-LIKE scan for shorter terms. The search contract is accent-
-// and case-insensitive substring match over subject and sender.
+// text search — the FTS5 unicode61 index (D4), matching words by prefix, and
+// the folded-LIKE scan for terms with no word. The search contract is accent-
+// and case-insensitive word-prefix match over subject and sender.
 
 const ACCOUNT = "acct-thread";
 const MAILBOX = "mbx-thread";
@@ -55,14 +57,18 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
 		({ db, sqlite, close } = await createSqliteTestDb(
 			{
 				threadMessage: threadMessageTable,
+				messageSearchText: messageSearchTextTable,
 			},
 			{ searchIndex: true },
 		));
-		repo = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {
@@ -115,7 +121,7 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
 		assert.ok(byName.items.some((i) => i.fromName === "Alice Kramer"));
 	});
 
-	test("FTS trigram search folds diacritics both ways", async () => {
+	test("FTS search folds diacritics both ways", async () => {
 		await repo.create(makeInput({ subject: "Réunion budget" }));
 		const folded = await repo.searchByMailbox(
 			ACCOUNT,

package/src/repos/thread-message.test.ts

@@ -1,7 +1,12 @@
 import assert from "node:assert";
 import { after, before, describe, test } from "node:test";
-import type { CreateThreadMessageInput } from "@remit/data-ports";
+import type {
+	CreateThreadMessageInput,
+	IMessageRepository,
+} from "@remit/data-ports";
 import shortUuid from "short-uuid";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
+import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
@@ -40,11 +45,14 @@ function makeInput(
 	};
 }
 
-// A real better-sqlite3 database with the FTS5 trigram objects the migrator
+// A real better-sqlite3 database with the FTS5 objects the migrator
 // installs, so the search predicates run their shipped path.
 const setupDb = () =>
 	createSqliteTestDb(
-		{ threadMessage: threadMessageTable },
+		{
+			threadMessage: threadMessageTable,
+			messageSearchText: messageSearchTextTable,
+		},
 		{ searchIndex: true },
 	);
 
@@ -87,11 +95,14 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
 	before(async () => {
 		const harness = await setupDb();
 		close = harness.close;
-		repo = createSqliteRepositories({
-			app: harness.db,
-			mail: harness.db,
-			search: harness.db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: harness.db,
+				mail: harness.db,
+				search: harness.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {
@@ -194,7 +205,7 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
 	});
 
 	// ── Scenario 4 ────────────────────────────────────────────────────────────
-	// Matching runs over the WHOLE mailbox via the trigram index, so an old
+	// Matching runs over the WHOLE mailbox via the full-text index, so an old
 	// match well behind the recent rows is still found and paged (the DynamoDB
 	// recent-window bound does not apply here — the improvement over #443 on the
 	// DDB path).
@@ -386,17 +397,23 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
 
 describe("DrizzleThreadMessageRepository.searchByDate", () => {
 	let repo: DrizzleThreadMessageRepository;
+	let messages: IMessageRepository;
 	let close: () => Promise<void>;
 	const cleanup: Array<() => Promise<void>> = [];
 
 	before(async () => {
 		const harness = await setupDb();
 		close = harness.close;
-		repo = createSqliteRepositories({
-			app: harness.db,
-			mail: harness.db,
-			search: harness.db,
-		}).threadMessage;
+		const repositories = createSqliteRepositories(
+			{
+				app: harness.db,
+				mail: harness.db,
+				search: harness.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		);
+		repo = repositories.threadMessage;
+		messages = repositories.message;
 	});
 
 	after(async () => {
@@ -412,9 +429,11 @@ describe("DrizzleThreadMessageRepository.searchByDate", () => {
 		rows: Array<Partial<CreateThreadMessageInput>>,
 	): Promise<void> {
 		for (const overrides of rows) {
-			const created = await repo.create(
-				makeInput(accountConfigId, mailboxId, overrides),
-			);
+			const input = makeInput(accountConfigId, mailboxId, overrides);
+			if (overrides.snippet) {
+				await messages.setSearchText(input.messageId, overrides.snippet);
+			}
+			const created = await repo.create(input);
 			cleanup.push(() => repo.delete(accountConfigId, created.threadMessageId));
 		}
 	}
@@ -923,11 +942,14 @@ describe("DrizzleThreadMessageRepository — native text search", () => {
 	before(async () => {
 		const harness = await setupDb();
 		close = harness.close;
-		repo = createSqliteRepositories({
-			app: harness.db,
-			mail: harness.db,
-			search: harness.db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: harness.db,
+				mail: harness.db,
+				search: harness.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {
@@ -1053,7 +1075,7 @@ describe("DrizzleThreadMessageRepository — native text search", () => {
 		assert.deepEqual(subjectsOf(res), ["shared token here"]);
 	});
 
-	test("LIKE metacharacters in the query match literally", async () => {
+	test("LIKE metacharacters in a query with no word match literally", async () => {
 		const acct = uuid();
 		const mbx = uuid();
 		await seed(acct, mbx, [
@@ -1061,7 +1083,7 @@ describe("DrizzleThreadMessageRepository — native text search", () => {
 			{ subject: "50 percent off" },
 		]);
 
-		const res = await repo.searchByMailboxWindow(acct, mbx, { query: "50%" });
+		const res = await repo.searchByMailboxWindow(acct, mbx, { query: "%" });
 		assert.deepEqual(subjectsOf(res), ["50% off today"]);
 	});
 
@@ -1099,11 +1121,14 @@ describe("DrizzleThreadMessageRepository — core CRUD", () => {
 	before(async () => {
 		const harness = await setupDb();
 		close = harness.close;
-		repo = createSqliteRepositories({
-			app: harness.db,
-			mail: harness.db,
-			search: harness.db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: harness.db,
+				mail: harness.db,
+				search: harness.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {