@remit/drizzle-service 0.0.104 → 0.0.106

package/src/repos/thread-message-body-muted.sqlite.test.ts

@@ -10,6 +10,7 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import type { CreateThreadMessageInput } from "@remit/data-ports";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { addressTable } from "../schema/i4-address.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
@@ -60,11 +61,14 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
 			},
 			{ searchIndex: true },
 		));
-		repo = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {

package/src/repos/thread-message-field-terms.sqlite.test.ts

@@ -1,6 +1,7 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import type { CreateThreadMessageInput } from "@remit/data-ports";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
@@ -59,11 +60,14 @@ describe("DrizzleThreadMessageRepository.listByFieldTerms (sqlite, #459)", () =>
 			{ searchIndex: true },
 		);
 		close = created.close;
-		repo = createSqliteRepositories({
-			app: created.db,
-			mail: created.db,
-			search: created.db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: created.db,
+				mail: created.db,
+				search: created.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 
 		await repo.create(
 			makeInput({

package/src/repos/thread-message.sqlite.test.ts

@@ -1,6 +1,7 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import type { CreateThreadMessageInput } from "@remit/data-ports";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
@@ -60,11 +61,14 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
 			},
 			{ searchIndex: true },
 		));
-		repo = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {

package/src/repos/thread-message.test.ts

@@ -5,6 +5,7 @@ import type {
 	IMessageRepository,
 } from "@remit/data-ports";
 import shortUuid from "short-uuid";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
@@ -94,11 +95,14 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
 	before(async () => {
 		const harness = await setupDb();
 		close = harness.close;
-		repo = createSqliteRepositories({
-			app: harness.db,
-			mail: harness.db,
-			search: harness.db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: harness.db,
+				mail: harness.db,
+				search: harness.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {
@@ -400,11 +404,14 @@ describe("DrizzleThreadMessageRepository.searchByDate", () => {
 	before(async () => {
 		const harness = await setupDb();
 		close = harness.close;
-		const repositories = createSqliteRepositories({
-			app: harness.db,
-			mail: harness.db,
-			search: harness.db,
-		});
+		const repositories = createSqliteRepositories(
+			{
+				app: harness.db,
+				mail: harness.db,
+				search: harness.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		);
 		repo = repositories.threadMessage;
 		messages = repositories.message;
 	});
@@ -935,11 +942,14 @@ describe("DrizzleThreadMessageRepository — native text search", () => {
 	before(async () => {
 		const harness = await setupDb();
 		close = harness.close;
-		repo = createSqliteRepositories({
-			app: harness.db,
-			mail: harness.db,
-			search: harness.db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: harness.db,
+				mail: harness.db,
+				search: harness.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {
@@ -1111,11 +1121,14 @@ describe("DrizzleThreadMessageRepository — core CRUD", () => {
 	before(async () => {
 		const harness = await setupDb();
 		close = harness.close;
-		repo = createSqliteRepositories({
-			app: harness.db,
-			mail: harness.db,
-			search: harness.db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: harness.db,
+				mail: harness.db,
+				search: harness.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {

package/src/search/thread-message-search.sqlite.test.ts

@@ -3,6 +3,7 @@ import { after, before, describe, test } from "node:test";
 import type { CreateThreadMessageInput } from "@remit/data-ports";
 import Database from "better-sqlite3";
 import { drizzle } from "drizzle-orm/better-sqlite3";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { syncStore } from "../databases.js";
 import type { Db } from "../db.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
@@ -50,12 +51,18 @@ const open = (set: "mail" | "search") => {
 describe("the thread-message search index in its own database", () => {
 	const mail = open("mail");
 	const search = open("search");
-	const index = new SearchIndex(mail.db, syncStore(search.db));
-	const repo = createSqliteRepositories({
-		app: mail.db,
-		mail: mail.db,
-		search: search.db,
-	}).threadMessage;
+	const index = new SearchIndex(
+		mail.db,
+		syncStore(search.db, BETTER_SQLITE3_LIMITS),
+	);
+	const repo = createSqliteRepositories(
+		{
+			app: mail.db,
+			mail: mail.db,
+			search: search.db,
+		},
+		BETTER_SQLITE3_LIMITS,
+	).threadMessage;
 
 	const found = async (query: string): Promise<string[]> =>
 		(await repo.searchByMailboxWindow(ACCOUNT, MAILBOX, { query })).items.map(

package/src/search/thread-message-search.ts

@@ -13,6 +13,7 @@ import {
 } from "drizzle-orm";
 import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
 import type { SqliteBatchItem, WriteBatch } from "../batch.js";
+import { chunkRows } from "../bound-parameters.js";
 import type { RepositoryDb, SqliteStore } from "../db.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
@@ -37,10 +38,7 @@ type AnyDb = RepositoryDb<Record<string, unknown>>;
 
 type SearchDocument = typeof threadMessageSearchTable.$inferInsert;
 
-// D1 refuses a statement with more than 100 bound parameters. A document is
-// nine parameters, so ten documents per insert stay under it.
-const DOCUMENTS_PER_STATEMENT = 10;
-const DOCUMENTS_PER_REBUILD_PAGE = DOCUMENTS_PER_STATEMENT * 30;
+const DOCUMENTS_PER_REBUILD_PAGE = 300;
 
 // The document's rowid, derived from its thread message id so a write or a
 // removal finds the document without a lookup. 53 bits, the integers a number
@@ -96,14 +94,6 @@ const toDocument = (row: IndexedRow): SearchDocument => ({
 	body: row.searchText ?? "",
 });
 
-const chunk = <T>(items: readonly T[], size: number): T[][] => {
-	const chunks: T[][] = [];
-	for (let start = 0; start < items.length; start += size) {
-		chunks.push(items.slice(start, start + size));
-	}
-	return chunks;
-};
-
 /** Where a page of candidates resumes: the last one the previous page held. */
 export interface SearchPosition {
 	rank: number;
@@ -141,6 +131,7 @@ export interface CandidateQuery {
 export class SearchIndex {
 	private readonly search: AnyDb;
 	private readonly batch: WriteBatch;
+	private readonly maxBoundParameters: number;
 
 	constructor(
 		private readonly mail: AnyDb,
@@ -148,12 +139,17 @@ export class SearchIndex {
 	) {
 		this.search = store.db;
 		this.batch = store.batch;
+		this.maxBoundParameters = store.maxBoundParameters;
 	}
 
 	/** Make the index match the mail database for these thread messages. */
 	async sync(threadMessageIds: readonly string[]): Promise<void> {
 		const ids = [...new Set(threadMessageIds)];
-		for (const slice of chunk(ids, DOCUMENTS_PER_STATEMENT)) {
+		for (const slice of chunkRows(
+			threadMessageSearchTable,
+			ids,
+			this.maxBoundParameters,
+		)) {
 			const rows = await this.mail
 				.select(indexedColumns)
 				.from(threadMessageTable)
@@ -272,10 +268,11 @@ export class SearchIndex {
 			const last = rows.at(-1);
 			if (!last) return indexed;
 			await this.batch(
-				chunk(rows, DOCUMENTS_PER_STATEMENT).map((slice) =>
-					this.search
-						.insert(threadMessageSearchTable)
-						.values(slice.map(toDocument)),
+				chunkRows(threadMessageSearchTable, rows, this.maxBoundParameters).map(
+					(slice) =>
+						this.search
+							.insert(threadMessageSearchTable)
+							.values(slice.map(toDocument)),
 				),
 			);
 			indexed += rows.length;

package/src/sqlite-repositories.ts

@@ -1,3 +1,4 @@
+import type { SqliteLimits } from "./bound-parameters.js";
 import {
 	type SqliteDatabases,
 	type SqliteStores,
@@ -67,7 +68,7 @@ export const composeSqliteRepositories = ({
 		accountExportRequest: new AccountExportRequestRepo(app.db),
 		configImport: new ConfigImportRepo(app.db),
 		quarantine: new QuarantineRepo(mail.db),
-		syncWarning: new SyncWarningRepo(mail.db),
+		syncWarning: new SyncWarningRepo(mail),
 		organizeJobRequest: new OrganizeJobRequestRepo(mail.db),
 		placementMove: new MessagePlacementMoveRepo(mail.db),
 		flagPush: new MessageFlagPushRepo(mail.db),
@@ -77,7 +78,7 @@ export const composeSqliteRepositories = ({
 		senderSignerStanding: new SenderSignerStandingRepo(mail.db),
 		calendarCollection: new CalendarCollectionRepo(app.db),
 		calendarObject: new CalendarObjectRepo(app.db),
-		calendarEventIndex: new CalendarEventIndexRepo(app.db),
+		calendarEventIndex: new CalendarEventIndexRepo(app),
 		calendarFeedToken: new CalendarFeedTokenRepo(app.db),
 		calendarSuggestion,
 		calendarStore: new DrizzleCalendarStore(app, calendarSuggestion),
@@ -88,4 +89,6 @@ export type SqliteRepositories = ReturnType<typeof composeSqliteRepositories>;
 
 export const createSqliteRepositories = (
 	databases: SqliteDatabases,
-): SqliteRepositories => composeSqliteRepositories(syncStores(databases));
+	limits: SqliteLimits,
+): SqliteRepositories =>
+	composeSqliteRepositories(syncStores(databases, limits));

package/src/test-d1.ts

@@ -1,40 +1,57 @@
 import { drizzle } from "drizzle-orm/d1";
 import { Miniflare } from "miniflare";
 import { asyncBatch } from "./batch.js";
+import type { SqliteLimits } from "./bound-parameters.js";
 import { syncStore } from "./databases.js";
-import { migrationStatements } from "./test-shipped-sqlite-schema.js";
+import type { SqliteStore } from "./db.js";
+import {
+	type MigrationSet,
+	migrationStatements,
+} from "./test-shipped-sqlite-schema.js";
 import { createShippedSqliteDb } from "./test-sqlite.js";
 import type { OpenTestStores } from "./test-stores.js";
 
-/**
- * The stores a Cloudflare deployment runs the app repositories on: the app
- * database on an in-process D1, migrated with the committed app set, written
- * through `db.batch()`. The mail and search sets stay on a synchronous handle,
- * as they do in the tenant mail Durable Object.
- */
-export const openD1TestStores: OpenTestStores = async () => {
+const CLOUDFLARE_LIMITS: SqliteLimits = { maxBoundParameters: 100 };
+
+/** One in-process D1 database migrated with a committed set, written through `db.batch()`. */
+export const openD1TestStore = async (
+	set: MigrationSet,
+): Promise<{
+	store: SqliteStore<Record<string, unknown>>;
+	close: () => Promise<void>;
+}> => {
 	const miniflare = new Miniflare({
 		modules: true,
 		script: "export default {};",
-		d1Databases: { APP: "app" },
+		d1Databases: { DB: set },
 	});
-	const d1 = await miniflare.getD1Database("APP");
-	const statements = migrationStatements("app").filter(
+	const d1 = await miniflare.getD1Database("DB");
+	const statements = migrationStatements(set).filter(
 		(statement) => statement.trim() !== "",
 	);
 	await d1.batch(statements.map((statement) => d1.prepare(statement)));
-	const app = drizzle(d1);
+	const db = drizzle(d1);
+	return {
+		store: { db, batch: asyncBatch(db), ...CLOUDFLARE_LIMITS },
+		close: () => miniflare.dispose(),
+	};
+};
+
+/**
+ * The stores a Cloudflare deployment runs the app repositories on: the app
+ * database on an in-process D1, migrated with the committed app set, written
+ * through `db.batch()`. The mail and search sets stay on a synchronous handle,
+ * as they do in the tenant mail Durable Object.
+ */
+export const openD1TestStores: OpenTestStores = async () => {
+	const app = await openD1TestStore("app");
 	const shipped = createShippedSqliteDb();
-	const local = syncStore(shipped.db);
+	const local = syncStore(shipped.db, CLOUDFLARE_LIMITS);
 	return {
-		stores: {
-			app: { db: app, batch: asyncBatch(app) },
-			mail: local,
-			search: local,
-		},
+		stores: { app: app.store, mail: local, search: local },
 		close: async () => {
 			shipped.close();
-			await miniflare.dispose();
+			await app.close();
 		},
 	};
 };

package/src/test-stores.ts

@@ -1,3 +1,4 @@
+import { BETTER_SQLITE3_LIMITS } from "./bound-parameters.js";
 import { type SqliteStores, syncStores } from "./databases.js";
 import {
 	calendarEventIndexTable,
@@ -16,7 +17,13 @@ export type OpenTestStores = () => Promise<{
 /** The whole schema on one in-memory better-sqlite3 handle that plays all three databases. */
 export const openSqliteTestStores: OpenTestStores = async () => {
 	const { db, close } = await createTestDb();
-	return { stores: syncStores({ app: db, mail: db, search: db }), close };
+	return {
+		stores: syncStores(
+			{ app: db, mail: db, search: db },
+			BETTER_SQLITE3_LIMITS,
+		),
+		close,
+	};
 };
 
 /** The three calendar tables alone on one in-memory better-sqlite3 handle. */
@@ -26,5 +33,11 @@ export const openCalendarSqliteTestStores: OpenTestStores = async () => {
 		calendarObjects: calendarObjectTable,
 		calendarEventIndexes: calendarEventIndexTable,
 	});
-	return { stores: syncStores({ app: db, mail: db, search: db }), close };
+	return {
+		stores: syncStores(
+			{ app: db, mail: db, search: db },
+			BETTER_SQLITE3_LIMITS,
+		),
+		close,
+	};
 };