@remit/drizzle-service 0.0.104 → 0.0.105

package/src/repos/i4-account.test.ts

@@ -1,5 +1,6 @@
 import assert from "node:assert";
 import { after, before, describe, test } from "node:test";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createTestDb, randomId, type TestDb } from "../test-db.js";
 import type { AccountRepo } from "./i4-account.js";
@@ -26,11 +27,14 @@ describe("AccountRepo", () => {
 
 	before(async () => {
 		({ db, close } = await createTestDb());
-		repo = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).account;
+		repo = createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).account;
 	});
 
 	after(async () => {
@@ -164,11 +168,14 @@ describe("AccountRepo", () => {
 		const accountConfigId = randomId();
 		const account = await repo.create(makeAccountInput(accountConfigId));
 
-		const mailboxRepo = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).mailbox;
+		const mailboxRepo = createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).mailbox;
 		mailboxCreated(
 			await mailboxRepo.create({
 				accountId: account.accountId,

package/src/repos/i4-address-junk-move.sqlite.test.ts

@@ -1,6 +1,7 @@
 import assert from "node:assert/strict";
 import { after, before, beforeEach, describe, test } from "node:test";
 import type Database from "better-sqlite3";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createTestDb, type TestDb } from "../test-db.js";
 import { AddressRepo } from "./i4-address.js";
@@ -125,11 +126,14 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 	before(async () => {
 		({ db, sqlite, close } = await createTestDb());
 		repo = new AddressRepo(db as never);
-		specialUse = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).mailboxSpecialUse;
+		specialUse = createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).mailboxSpecialUse;
 	});
 
 	/**

package/src/repos/i4-mailbox-special-use.sqlite.test.ts

@@ -6,6 +6,7 @@ import {
 	meetsTrashAssurance,
 } from "@remit/data-ports/folder-role";
 import { CanonicalMailboxRole } from "@remit/domain-enums";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import {
 	accountSettingTable,
 	accountTable,
@@ -57,11 +58,14 @@ describe("MailboxSpecialUseRepo role lookups (sqlite)", () => {
 		});
 		close = testDb.close;
 		db = testDb.db;
-		const repos = createSqliteRepositories({
-			app: testDb.db,
-			mail: testDb.db,
-			search: testDb.db,
-		});
+		const repos = createSqliteRepositories(
+			{
+				app: testDb.db,
+				mail: testDb.db,
+				search: testDb.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		);
 		repo = repos.mailboxSpecialUse;
 		mailboxes = repos.mailbox;
 		accountSettings = repos.accountSetting;

package/src/repos/i4-mailbox.sqlite.test.ts

@@ -4,6 +4,7 @@ import { after, before, describe, test } from "node:test";
 import { MailboxSyncStatus } from "@remit/domain-enums";
 import Database from "better-sqlite3";
 import { drizzle } from "drizzle-orm/better-sqlite3";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { mailboxTable } from "../schema.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
@@ -41,11 +42,14 @@ describe("MailboxRepo (sqlite)", () => {
 	before(async () => {
 		const testDb = await createSqliteTestDb({ mailbox: mailboxTable });
 		close = testDb.close;
-		repo = createSqliteRepositories({
-			app: testDb.db,
-			mail: testDb.db,
-			search: testDb.db,
-		}).mailbox;
+		repo = createSqliteRepositories(
+			{
+				app: testDb.db,
+				mail: testDb.db,
+				search: testDb.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).mailbox;
 	});
 
 	after(async () => {
@@ -211,11 +215,14 @@ describe("MailboxRepo (sqlite, shipped migrations)", () => {
 		const sqlite = new Database(":memory:");
 		applyMigrationSet(sqlite, "mail");
 		const db = drizzle(sqlite, { schema: { mailbox: mailboxTable } });
-		repo = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).mailbox;
+		repo = createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).mailbox;
 		close = async () => {
 			sqlite.close();
 		};

package/src/repos/i4-mailbox.test.ts

@@ -3,6 +3,7 @@ import { after, before, describe, test } from "node:test";
 import { MailboxSyncStatus } from "@remit/domain-enums";
 import { eq } from "drizzle-orm";
 import { syncBatch } from "../batch.js";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import {
 	envelopeId as deriveEnvelopeId,
 	rootBodyPartId as deriveRootBodyPartId,
@@ -54,11 +55,14 @@ describe("MailboxRepo", () => {
 
 	before(async () => {
 		({ db, close } = await createTestDb());
-		repo = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).mailbox;
+		repo = createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).mailbox;
 	});
 
 	after(async () => {
@@ -543,11 +547,14 @@ describe("MailboxRepo", () => {
 	describe("deleteMailboxWithMail — the folder's mail goes with it (D8)", () => {
 		const seedMessage = async (mailboxId: string, uid: number) => {
 			const messageId = randomId();
-			await createSqliteRepositories({
-				app: db,
-				mail: db,
-				search: db,
-			}).message.create({
+			await createSqliteRepositories(
+				{
+					app: db,
+					mail: db,
+					search: db,
+				},
+				BETTER_SQLITE3_LIMITS,
+			).message.create({
 				messageId,
 				mailboxId,
 				uid,

package/src/repos/i4-outbox-attachment.test.ts

@@ -1,5 +1,6 @@
 import assert from "node:assert";
 import { after, before, describe, test } from "node:test";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createTestDb, randomId } from "../test-db.js";
 import type { OutboxAttachmentRepo } from "./i4-outbox-attachment.js";
@@ -28,11 +29,14 @@ const input = (
 
 before(async () => {
 	harness = await createTestDb();
-	repo = createSqliteRepositories({
-		app: harness.db,
-		mail: harness.db,
-		search: harness.db,
-	}).outboxAttachment;
+	repo = createSqliteRepositories(
+		{
+			app: harness.db,
+			mail: harness.db,
+			search: harness.db,
+		},
+		BETTER_SQLITE3_LIMITS,
+	).outboxAttachment;
 });
 
 after(async () => {

package/src/repos/message.sqlite.test.ts

@@ -1,5 +1,6 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import {
 	envelopeId as deriveEnvelopeId,
 	rootBodyPartId as deriveRootBodyPartId,
@@ -43,7 +44,10 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 
 	before(async () => {
 		({ db, close } = await createSqliteTestDb(SCHEMA));
-		repo = createSqliteRepositories({ app: db, mail: db, search: db }).message;
+		repo = createSqliteRepositories(
+			{ app: db, mail: db, search: db },
+			BETTER_SQLITE3_LIMITS,
+		).message;
 	});
 
 	after(async () => {

package/src/repos/message.test.ts

@@ -1,5 +1,6 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import {
 	envelopeId as deriveEnvelopeId,
 	rootBodyPartId as deriveRootBodyPartId,
@@ -37,7 +38,10 @@ describe("DrizzleMessageRepository", () => {
 
 	before(async () => {
 		({ db, stop } = await createTestDb());
-		const repos = createSqliteRepositories({ app: db, mail: db, search: db });
+		const repos = createSqliteRepositories(
+			{ app: db, mail: db, search: db },
+			BETTER_SQLITE3_LIMITS,
+		);
 		messageRepo = repos.message;
 		envelopeRepo = repos.envelope;
 	});

package/src/repos/sync-warning.ts

@@ -6,14 +6,24 @@ import type {
 } from "@remit/data-ports";
 import { deriveSyncWarningId } from "@remit/data-ports/id";
 import { and, count, desc, eq, gte, sql } from "drizzle-orm";
-import type { RepositoryDb } from "../db.js";
+import type { WriteBatch } from "../batch.js";
+import { chunkRows } from "../bound-parameters.js";
+import type { RepositoryDb, SqliteStore } from "../db.js";
 import { syncWarningTable } from "../schema/sync-warning.js";
 import { inList } from "./in-list.js";
 
 type DB = RepositoryDb<Record<string, unknown>>;
 
 export class SyncWarningRepo implements ISyncWarningRepository {
-	constructor(private db: DB) {}
+	private readonly db: DB;
+	private readonly batch: WriteBatch;
+	private readonly maxBoundParameters: number;
+
+	constructor(store: SqliteStore<Record<string, unknown>>) {
+		this.db = store.db;
+		this.batch = store.batch;
+		this.maxBoundParameters = store.maxBoundParameters;
+	}
 
 	listByMailboxId = async (mailboxId: string): Promise<SyncWarningItem[]> =>
 		this.db
@@ -46,26 +56,29 @@ export class SyncWarningRepo implements ISyncWarningRepository {
 	recordFailures = async (keys: SyncWarningKey[]): Promise<void> => {
 		if (keys.length === 0) return;
 		const now = Date.now();
-		await this.db
-			.insert(syncWarningTable)
-			.values(
-				keys.map((key) => ({
-					syncWarningId: deriveSyncWarningId(
-						key.accountId,
-						key.mailboxId,
-						key.uidValidity,
-						key.uid,
-						key.stage,
-					),
-					...key,
-					attempts: 1,
-					createdAt: now,
-				})),
-			)
-			.onConflictDoUpdate({
-				target: syncWarningTable.syncWarningId,
-				set: { attempts: sql`${syncWarningTable.attempts} + 1` },
-			});
+		const rows = keys.map((key) => ({
+			syncWarningId: deriveSyncWarningId(
+				key.accountId,
+				key.mailboxId,
+				key.uidValidity,
+				key.uid,
+				key.stage,
+			),
+			...key,
+			attempts: 1,
+			createdAt: now,
+		}));
+		await this.batch(
+			chunkRows(syncWarningTable, rows, this.maxBoundParameters).map((slice) =>
+				this.db
+					.insert(syncWarningTable)
+					.values(slice)
+					.onConflictDoUpdate({
+						target: syncWarningTable.syncWarningId,
+						set: { attempts: sql`${syncWarningTable.attempts} + 1` },
+					}),
+			),
+		);
 	};
 
 	deleteMany = async (syncWarningIds: string[]): Promise<void> => {

package/src/repos/thread-message-body-muted.sqlite.test.ts

@@ -10,6 +10,7 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import type { CreateThreadMessageInput } from "@remit/data-ports";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { addressTable } from "../schema/i4-address.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
@@ -60,11 +61,14 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
 			},
 			{ searchIndex: true },
 		));
-		repo = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {

package/src/repos/thread-message-field-terms.sqlite.test.ts

@@ -1,6 +1,7 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import type { CreateThreadMessageInput } from "@remit/data-ports";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
@@ -59,11 +60,14 @@ describe("DrizzleThreadMessageRepository.listByFieldTerms (sqlite, #459)", () =>
 			{ searchIndex: true },
 		);
 		close = created.close;
-		repo = createSqliteRepositories({
-			app: created.db,
-			mail: created.db,
-			search: created.db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: created.db,
+				mail: created.db,
+				search: created.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 
 		await repo.create(
 			makeInput({

package/src/repos/thread-message.sqlite.test.ts

@@ -1,6 +1,7 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import type { CreateThreadMessageInput } from "@remit/data-ports";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
@@ -60,11 +61,14 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
 			},
 			{ searchIndex: true },
 		));
-		repo = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {

package/src/repos/thread-message.test.ts

@@ -5,6 +5,7 @@ import type {
 	IMessageRepository,
 } from "@remit/data-ports";
 import shortUuid from "short-uuid";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
@@ -94,11 +95,14 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
 	before(async () => {
 		const harness = await setupDb();
 		close = harness.close;
-		repo = createSqliteRepositories({
-			app: harness.db,
-			mail: harness.db,
-			search: harness.db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: harness.db,
+				mail: harness.db,
+				search: harness.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {
@@ -400,11 +404,14 @@ describe("DrizzleThreadMessageRepository.searchByDate", () => {
 	before(async () => {
 		const harness = await setupDb();
 		close = harness.close;
-		const repositories = createSqliteRepositories({
-			app: harness.db,
-			mail: harness.db,
-			search: harness.db,
-		});
+		const repositories = createSqliteRepositories(
+			{
+				app: harness.db,
+				mail: harness.db,
+				search: harness.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		);
 		repo = repositories.threadMessage;
 		messages = repositories.message;
 	});
@@ -935,11 +942,14 @@ describe("DrizzleThreadMessageRepository — native text search", () => {
 	before(async () => {
 		const harness = await setupDb();
 		close = harness.close;
-		repo = createSqliteRepositories({
-			app: harness.db,
-			mail: harness.db,
-			search: harness.db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: harness.db,
+				mail: harness.db,
+				search: harness.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {
@@ -1111,11 +1121,14 @@ describe("DrizzleThreadMessageRepository — core CRUD", () => {
 	before(async () => {
 		const harness = await setupDb();
 		close = harness.close;
-		repo = createSqliteRepositories({
-			app: harness.db,
-			mail: harness.db,
-			search: harness.db,
-		}).threadMessage;
+		repo = createSqliteRepositories(
+			{
+				app: harness.db,
+				mail: harness.db,
+				search: harness.db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).threadMessage;
 	});
 
 	after(async () => {

package/src/search/thread-message-search.sqlite.test.ts

@@ -3,6 +3,7 @@ import { after, before, describe, test } from "node:test";
 import type { CreateThreadMessageInput } from "@remit/data-ports";
 import Database from "better-sqlite3";
 import { drizzle } from "drizzle-orm/better-sqlite3";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { syncStore } from "../databases.js";
 import type { Db } from "../db.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
@@ -50,12 +51,18 @@ const open = (set: "mail" | "search") => {
 describe("the thread-message search index in its own database", () => {
 	const mail = open("mail");
 	const search = open("search");
-	const index = new SearchIndex(mail.db, syncStore(search.db));
-	const repo = createSqliteRepositories({
-		app: mail.db,
-		mail: mail.db,
-		search: search.db,
-	}).threadMessage;
+	const index = new SearchIndex(
+		mail.db,
+		syncStore(search.db, BETTER_SQLITE3_LIMITS),
+	);
+	const repo = createSqliteRepositories(
+		{
+			app: mail.db,
+			mail: mail.db,
+			search: search.db,
+		},
+		BETTER_SQLITE3_LIMITS,
+	).threadMessage;
 
 	const found = async (query: string): Promise<string[]> =>
 		(await repo.searchByMailboxWindow(ACCOUNT, MAILBOX, { query })).items.map(

package/src/search/thread-message-search.ts

@@ -13,6 +13,7 @@ import {
 } from "drizzle-orm";
 import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
 import type { SqliteBatchItem, WriteBatch } from "../batch.js";
+import { chunkRows } from "../bound-parameters.js";
 import type { RepositoryDb, SqliteStore } from "../db.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
@@ -37,10 +38,7 @@ type AnyDb = RepositoryDb<Record<string, unknown>>;
 
 type SearchDocument = typeof threadMessageSearchTable.$inferInsert;
 
-// D1 refuses a statement with more than 100 bound parameters. A document is
-// nine parameters, so ten documents per insert stay under it.
-const DOCUMENTS_PER_STATEMENT = 10;
-const DOCUMENTS_PER_REBUILD_PAGE = DOCUMENTS_PER_STATEMENT * 30;
+const DOCUMENTS_PER_REBUILD_PAGE = 300;
 
 // The document's rowid, derived from its thread message id so a write or a
 // removal finds the document without a lookup. 53 bits, the integers a number
@@ -96,14 +94,6 @@ const toDocument = (row: IndexedRow): SearchDocument => ({
 	body: row.searchText ?? "",
 });
 
-const chunk = <T>(items: readonly T[], size: number): T[][] => {
-	const chunks: T[][] = [];
-	for (let start = 0; start < items.length; start += size) {
-		chunks.push(items.slice(start, start + size));
-	}
-	return chunks;
-};
-
 /** Where a page of candidates resumes: the last one the previous page held. */
 export interface SearchPosition {
 	rank: number;
@@ -141,6 +131,7 @@ export interface CandidateQuery {
 export class SearchIndex {
 	private readonly search: AnyDb;
 	private readonly batch: WriteBatch;
+	private readonly maxBoundParameters: number;
 
 	constructor(
 		private readonly mail: AnyDb,
@@ -148,12 +139,17 @@ export class SearchIndex {
 	) {
 		this.search = store.db;
 		this.batch = store.batch;
+		this.maxBoundParameters = store.maxBoundParameters;
 	}
 
 	/** Make the index match the mail database for these thread messages. */
 	async sync(threadMessageIds: readonly string[]): Promise<void> {
 		const ids = [...new Set(threadMessageIds)];
-		for (const slice of chunk(ids, DOCUMENTS_PER_STATEMENT)) {
+		for (const slice of chunkRows(
+			threadMessageSearchTable,
+			ids,
+			this.maxBoundParameters,
+		)) {
 			const rows = await this.mail
 				.select(indexedColumns)
 				.from(threadMessageTable)
@@ -272,10 +268,11 @@ export class SearchIndex {
 			const last = rows.at(-1);
 			if (!last) return indexed;
 			await this.batch(
-				chunk(rows, DOCUMENTS_PER_STATEMENT).map((slice) =>
-					this.search
-						.insert(threadMessageSearchTable)
-						.values(slice.map(toDocument)),
+				chunkRows(threadMessageSearchTable, rows, this.maxBoundParameters).map(
+					(slice) =>
+						this.search
+							.insert(threadMessageSearchTable)
+							.values(slice.map(toDocument)),
 				),
 			);
 			indexed += rows.length;

package/src/sqlite-repositories.ts

@@ -1,3 +1,4 @@
+import type { SqliteLimits } from "./bound-parameters.js";
 import {
 	type SqliteDatabases,
 	type SqliteStores,
@@ -67,7 +68,7 @@ export const composeSqliteRepositories = ({
 		accountExportRequest: new AccountExportRequestRepo(app.db),
 		configImport: new ConfigImportRepo(app.db),
 		quarantine: new QuarantineRepo(mail.db),
-		syncWarning: new SyncWarningRepo(mail.db),
+		syncWarning: new SyncWarningRepo(mail),
 		organizeJobRequest: new OrganizeJobRequestRepo(mail.db),
 		placementMove: new MessagePlacementMoveRepo(mail.db),
 		flagPush: new MessageFlagPushRepo(mail.db),
@@ -77,7 +78,7 @@ export const composeSqliteRepositories = ({
 		senderSignerStanding: new SenderSignerStandingRepo(mail.db),
 		calendarCollection: new CalendarCollectionRepo(app.db),
 		calendarObject: new CalendarObjectRepo(app.db),
-		calendarEventIndex: new CalendarEventIndexRepo(app.db),
+		calendarEventIndex: new CalendarEventIndexRepo(app),
 		calendarFeedToken: new CalendarFeedTokenRepo(app.db),
 		calendarSuggestion,
 		calendarStore: new DrizzleCalendarStore(app, calendarSuggestion),
@@ -88,4 +89,6 @@ export type SqliteRepositories = ReturnType<typeof composeSqliteRepositories>;
 
 export const createSqliteRepositories = (
 	databases: SqliteDatabases,
-): SqliteRepositories => composeSqliteRepositories(syncStores(databases));
+	limits: SqliteLimits,
+): SqliteRepositories =>
+	composeSqliteRepositories(syncStores(databases, limits));