@remit/drizzle-service 0.0.103 → 0.0.105

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.103",
+  "version": "0.0.105",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",
@@ -25,14 +25,15 @@
     "@remit/calendar-service": "*",
     "@remit/config-transfer": "*",
     "drizzle-kit": "^0.31.1",
+    "miniflare": "^4.20260730.0",
     "tsx": "*"
   },
   "dependencies": {
     "@remit/data-ports": "*",
     "@remit/domain-enums": "*",
     "drizzle-orm": "^0.45.2",
-    "short-uuid": "*",
-    "uuid": "*",
+    "short-uuid": "^6.0.3",
+    "uuid": "^13.0.2",
     "@remit/drizzle-sqlite-schema": "*",
     "better-sqlite3": "^12.11.1",
     "@types/better-sqlite3": "^7.6.13"

package/src/bound-parameters.test.ts

@@ -0,0 +1,57 @@
+import assert from "node:assert/strict";
+import { describe, test } from "node:test";
+import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
+import { chunkRows, rowsPerStatement } from "./bound-parameters.js";
+
+const tenColumns = sqliteTable("ten_columns", {
+	c0: text("c0"),
+	c1: text("c1"),
+	c2: text("c2"),
+	c3: text("c3"),
+	c4: text("c4"),
+	c5: text("c5"),
+	c6: text("c6"),
+	c7: text("c7"),
+	c8: text("c8"),
+	c9: integer("c9"),
+});
+
+const rows = (count: number): number[] =>
+	Array.from({ length: count }, (_, index) => index);
+
+describe("chunkRows", () => {
+	test("no rows make no statements", () => {
+		assert.deepEqual(chunkRows(tenColumns, [], 100), []);
+	});
+
+	test("rows that fill the limit exactly make one statement", () => {
+		const chunks = chunkRows(tenColumns, rows(10), 100);
+		assert.deepEqual(
+			chunks.map((slice) => slice.length),
+			[10],
+		);
+	});
+
+	test("one row over the limit makes a second statement", () => {
+		const chunks = chunkRows(tenColumns, rows(11), 100);
+		assert.deepEqual(
+			chunks.map((slice) => slice.length),
+			[10, 1],
+		);
+		assert.deepEqual(chunks.flat(), rows(11));
+	});
+
+	test("reserved parameters shrink every statement", () => {
+		const chunks = chunkRows(tenColumns, rows(10), 100, 1);
+		assert.deepEqual(
+			chunks.map((slice) => slice.length),
+			[9, 1],
+		);
+	});
+});
+
+describe("rowsPerStatement", () => {
+	test("a row wider than the limit is refused", () => {
+		assert.throws(() => rowsPerStatement(100, 101), RangeError);
+	});
+});

package/src/bound-parameters.ts

@@ -0,0 +1,49 @@
+import { getTableColumns, type Table } from "drizzle-orm";
+
+export interface SqliteLimits {
+	/** The most parameters the host's SQLite binds in one statement: 32766 on better-sqlite3, 100 on D1 and Durable Object storage. */
+	maxBoundParameters: number;
+}
+
+export const BETTER_SQLITE3_LIMITS: SqliteLimits = {
+	maxBoundParameters: 32766,
+};
+
+export const chunk = <T>(items: readonly T[], size: number): T[][] => {
+	const chunks: T[][] = [];
+	for (let start = 0; start < items.length; start += size) {
+		chunks.push(items.slice(start, start + size));
+	}
+	return chunks;
+};
+
+export const rowsPerStatement = (
+	maxBoundParameters: number,
+	parametersPerRow: number,
+	reservedParameters = 0,
+): number => {
+	const rows = Math.floor(
+		(maxBoundParameters - reservedParameters) / parametersPerRow,
+	);
+	if (rows < 1) {
+		throw new RangeError(
+			`a row of ${parametersPerRow} bound parameters does not fit a statement limit of ${maxBoundParameters}`,
+		);
+	}
+	return rows;
+};
+
+export const chunkRows = <T>(
+	table: Table,
+	rows: readonly T[],
+	maxBoundParameters: number,
+	reservedParameters = 0,
+): T[][] =>
+	chunk(
+		rows,
+		rowsPerStatement(
+			maxBoundParameters,
+			Object.keys(getTableColumns(table)).length,
+			reservedParameters,
+		),
+	);

package/src/databases.ts

@@ -1,4 +1,5 @@
 import { syncBatch } from "./batch.js";
+import type { SqliteLimits } from "./bound-parameters.js";
 import type { Db, SqliteStore } from "./db.js";
 import { messageDataSchema } from "./schema/message-data.js";
 import { createSqliteDatabase } from "./sqlite-client.js";
@@ -22,16 +23,16 @@ export interface SqliteStores {
 
 export const syncStore = <TSchema extends Record<string, unknown>>(
 	db: Db<TSchema>,
-): SqliteStore<TSchema> => ({ db, batch: syncBatch(db) });
+	{ maxBoundParameters }: SqliteLimits,
+): SqliteStore<TSchema> => ({ db, batch: syncBatch(db), maxBoundParameters });
 
-export const syncStores = ({
-	app,
-	mail,
-	search,
-}: SqliteDatabases): SqliteStores => ({
-	app: syncStore(app),
-	mail: syncStore(mail),
-	search: syncStore(search),
+export const syncStores = (
+	{ app, mail, search }: SqliteDatabases,
+	limits: SqliteLimits,
+): SqliteStores => ({
+	app: syncStore(app, limits),
+	mail: syncStore(mail, limits),
+	search: syncStore(search, limits),
 });
 
 export interface SqliteDatabasePaths {

package/src/db.ts

@@ -26,4 +26,5 @@ export type RepositoryDb<TSchema extends Record<string, unknown>> = Omit<
 export interface SqliteStore<TSchema extends Record<string, unknown>> {
 	db: RepositoryDb<TSchema>;
 	batch: WriteBatch;
+	maxBoundParameters: number;
 }

package/src/index.ts

@@ -5,6 +5,10 @@ export {
 	syncBatch,
 	type WriteBatch,
 } from "./batch.js";
+export {
+	BETTER_SQLITE3_LIMITS,
+	type SqliteLimits,
+} from "./bound-parameters.js";
 export {
 	hasSqliteDatabaseEnv,
 	openSqliteDatabases,

package/src/repair/junk-only-address.sqlite.test.ts

@@ -4,6 +4,7 @@ import { composeFolderRoleAppointmentName } from "@remit/data-ports/folder-role"
 import { CanonicalMailboxRole } from "@remit/domain-enums";
 import Database from "better-sqlite3";
 import { drizzle } from "drizzle-orm/better-sqlite3";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { AccountSettingRepo } from "../repos/i4-account-setting.js";
 import type { MailboxSpecialUseRepo } from "../repos/i4-mailbox-special-use.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
@@ -182,11 +183,14 @@ describe("addresses standing only on mail in Junk", () => {
 	before(() => {
 		sqlite = new Database(":memory:");
 		const db = drizzle(sqlite);
-		specialUse = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).mailboxSpecialUse;
+		specialUse = createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).mailboxSpecialUse;
 		accountSetting = new AccountSettingRepo(db as never);
 		applyMigrationSet(sqlite, "app");
 		applyMigrationSet(sqlite, "mail");

package/src/repos/account.conformance.d1.test.ts

@@ -0,0 +1,20 @@
+import { accountRepositoryConformance } from "@remit/data-ports/conformance";
+import { NotFoundError } from "../error.js";
+import { randomId } from "../id.js";
+import { composeSqliteRepositories } from "../sqlite-repositories.js";
+import { openD1TestStores } from "../test-d1.js";
+
+let close: (() => Promise<void>) | undefined;
+
+accountRepositoryConformance({
+	async createRepository() {
+		const opened = await openD1TestStores();
+		close = opened.close;
+		return composeSqliteRepositories(opened.stores).account;
+	},
+	teardown: async () => {
+		await close?.();
+	},
+	makeId: () => randomId(),
+	isNotFoundError: (error) => error instanceof NotFoundError,
+});

package/src/repos/account.conformance.sqlite.test.ts

@@ -1,4 +1,5 @@
 import { accountRepositoryConformance } from "@remit/data-ports/conformance";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { NotFoundError } from "../error.js";
 import { randomId } from "../id.js";
 import { accountTable } from "../schema.js";
@@ -13,11 +14,14 @@ accountRepositoryConformance({
 			accounts: accountTable,
 		});
 		close = closeDb;
-		return createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).account;
+		return createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).account;
 	},
 	teardown: async () => {
 		await close?.();

package/src/repos/calendar-collection.conformance.d1.test.ts

@@ -0,0 +1,20 @@
+import { calendarCollectionRepositoryConformance } from "@remit/data-ports/conformance";
+import { NotFoundError } from "../error.js";
+import { randomId } from "../id.js";
+import { openD1TestStores } from "../test-d1.js";
+import { CalendarCollectionRepo } from "./calendar-collection.js";
+
+let close: (() => Promise<void>) | undefined;
+
+calendarCollectionRepositoryConformance({
+	async createRepository() {
+		const opened = await openD1TestStores();
+		close = opened.close;
+		return new CalendarCollectionRepo(opened.stores.app.db);
+	},
+	teardown: async () => {
+		await close?.();
+	},
+	makeId: () => randomId(),
+	isNotFoundError: (error) => error instanceof NotFoundError,
+});

package/src/repos/calendar-default.d1.test.ts

@@ -0,0 +1,4 @@
+import { openD1TestStores } from "../test-d1.js";
+import { calendarDefaultSuite } from "./calendar-default.suite.js";
+
+calendarDefaultSuite(openD1TestStores);

package/src/repos/calendar-default.sqlite.test.ts

@@ -1,77 +1,4 @@
-import assert from "node:assert/strict";
-import { after, before, describe, test } from "node:test";
-import { provisionDefaultCalendar } from "@remit/calendar-service";
-import type { ICalendarStore } from "@remit/data-ports";
-import type { Db } from "../db.js";
-import {
-	calendarEventIndexTable,
-	calendarObjectTable,
-	calendarTable,
-} from "../schema.js";
-import { createSqliteRepositories } from "../sqlite-repositories.js";
-import { createSqliteTestDb } from "../test-db-sqlite.js";
-import { CalendarCollectionRepo } from "./calendar-collection.js";
+import { openCalendarSqliteTestStores } from "../test-stores.js";
+import { calendarDefaultSuite } from "./calendar-default.suite.js";
 
-const ACCOUNT_CONFIG_ID = "account-config-default";
-
-describe("provisioning the default calendar", () => {
-	let db: Db<Record<string, unknown>>;
-	let close: () => Promise<void>;
-	let store: ICalendarStore;
-
-	before(async () => {
-		const created = await createSqliteTestDb({
-			calendars: calendarTable,
-			calendarObjects: calendarObjectTable,
-			calendarEventIndexes: calendarEventIndexTable,
-		});
-		db = created.db as unknown as Db<Record<string, unknown>>;
-		close = created.close;
-		store = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).calendarStore;
-	});
-
-	after(() => close());
-
-	test("leaves one calendar behind when several first reads arrive together", async () => {
-		const provisioned = await Promise.all(
-			Array.from({ length: 8 }, () =>
-				provisionDefaultCalendar(store, ACCOUNT_CONFIG_ID),
-			),
-		);
-
-		const ids = new Set(provisioned.map((collection) => collection.calendarId));
-		assert.equal(ids.size, 1, "every caller was handed the same calendar");
-
-		const rows = await new CalendarCollectionRepo(db).listByAccountConfig(
-			ACCOUNT_CONFIG_ID,
-		);
-		assert.equal(rows.length, 1);
-		assert.equal(rows[0]?.urlSegment, "default");
-		assert.equal(
-			rows[0]?.syncSequence,
-			0,
-			"a losing provision did not reset a collection the winner may already have written to",
-		);
-	});
-
-	test("a later provision returns the stored collection rather than resetting it", async () => {
-		const repo = new CalendarCollectionRepo(db);
-		const [before] = await repo.listByAccountConfig(ACCOUNT_CONFIG_ID);
-		assert.ok(before);
-		await repo.bumpSyncSequence(ACCOUNT_CONFIG_ID, before.calendarId);
-
-		const again = await provisionDefaultCalendar(
-			store,
-			ACCOUNT_CONFIG_ID,
-			"A different name",
-		);
-
-		assert.equal(again.calendarId, before.calendarId);
-		assert.equal(again.displayName, before.displayName);
-		assert.equal(again.syncSequence, 1);
-	});
-});
+calendarDefaultSuite(openCalendarSqliteTestStores);

package/src/repos/calendar-default.suite.ts

@@ -0,0 +1,68 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import { provisionDefaultCalendar } from "@remit/calendar-service";
+import type { ICalendarStore } from "@remit/data-ports";
+import type { RepositoryDb } from "../db.js";
+import { composeSqliteRepositories } from "../sqlite-repositories.js";
+import type { OpenTestStores } from "../test-stores.js";
+import { CalendarCollectionRepo } from "./calendar-collection.js";
+
+const ACCOUNT_CONFIG_ID = "account-config-default";
+
+export const calendarDefaultSuite = (open: OpenTestStores): void => {
+	describe("provisioning the default calendar", () => {
+		let db: RepositoryDb<Record<string, unknown>>;
+		let close: () => Promise<void>;
+		let store: ICalendarStore;
+
+		before(async () => {
+			const opened = await open();
+			db = opened.stores.app.db;
+			close = opened.close;
+			store = composeSqliteRepositories(opened.stores).calendarStore;
+		});
+
+		after(() => close());
+
+		test("leaves one calendar behind when several first reads arrive together", async () => {
+			const provisioned = await Promise.all(
+				Array.from({ length: 8 }, () =>
+					provisionDefaultCalendar(store, ACCOUNT_CONFIG_ID),
+				),
+			);
+
+			const ids = new Set(
+				provisioned.map((collection) => collection.calendarId),
+			);
+			assert.equal(ids.size, 1, "every caller was handed the same calendar");
+
+			const rows = await new CalendarCollectionRepo(db).listByAccountConfig(
+				ACCOUNT_CONFIG_ID,
+			);
+			assert.equal(rows.length, 1);
+			assert.equal(rows[0]?.urlSegment, "default");
+			assert.equal(
+				rows[0]?.syncSequence,
+				0,
+				"a losing provision did not reset a collection the winner may already have written to",
+			);
+		});
+
+		test("a later provision returns the stored collection rather than resetting it", async () => {
+			const repo = new CalendarCollectionRepo(db);
+			const [before] = await repo.listByAccountConfig(ACCOUNT_CONFIG_ID);
+			assert.ok(before);
+			await repo.bumpSyncSequence(ACCOUNT_CONFIG_ID, before.calendarId);
+
+			const again = await provisionDefaultCalendar(
+				store,
+				ACCOUNT_CONFIG_ID,
+				"A different name",
+			);
+
+			assert.equal(again.calendarId, before.calendarId);
+			assert.equal(again.displayName, before.displayName);
+			assert.equal(again.syncSequence, 1);
+		});
+	});
+};

package/src/repos/calendar-event-index.conformance.d1.test.ts

@@ -0,0 +1,20 @@
+import { calendarEventIndexRepositoryConformance } from "@remit/data-ports/conformance";
+import { NotFoundError } from "../error.js";
+import { randomId } from "../id.js";
+import { openD1TestStores } from "../test-d1.js";
+import { CalendarEventIndexRepo } from "./calendar-event-index.js";
+
+let close: (() => Promise<void>) | undefined;
+
+calendarEventIndexRepositoryConformance({
+	async createRepository() {
+		const opened = await openD1TestStores();
+		close = opened.close;
+		return new CalendarEventIndexRepo(opened.stores.app);
+	},
+	teardown: async () => {
+		await close?.();
+	},
+	makeId: () => randomId(),
+	isNotFoundError: (error) => error instanceof NotFoundError,
+});

package/src/repos/calendar-event-index.conformance.sqlite.test.ts

@@ -1,4 +1,6 @@
 import { calendarEventIndexRepositoryConformance } from "@remit/data-ports/conformance";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
+import { syncStore } from "../databases.js";
 import { NotFoundError } from "../error.js";
 import { randomId } from "../id.js";
 import { calendarEventIndexTable } from "../schema.js";
@@ -13,7 +15,7 @@ calendarEventIndexRepositoryConformance({
 			calendarEventIndexes: calendarEventIndexTable,
 		});
 		close = closeDb;
-		return new CalendarEventIndexRepo(db as never);
+		return new CalendarEventIndexRepo(syncStore(db, BETTER_SQLITE3_LIMITS));
 	},
 	teardown: async () => {
 		await close?.();

package/src/repos/calendar-event-index.ts

@@ -4,7 +4,9 @@ import type {
 	ICalendarEventIndexRepository,
 } from "@remit/data-ports";
 import { and, asc, eq, gte, lt } from "drizzle-orm";
-import type { RepositoryDb } from "../db.js";
+import type { SqliteBatchItem, WriteBatch } from "../batch.js";
+import { chunkRows } from "../bound-parameters.js";
+import type { RepositoryDb, SqliteStore } from "../db.js";
 import { calendarEventIndexTable } from "../schema.js";
 
 type DB = RepositoryDb<Record<string, unknown>>;
@@ -28,39 +30,52 @@ function rowToOccurrence(
 }
 
 export class CalendarEventIndexRepo implements ICalendarEventIndexRepository {
-	constructor(private db: DB) {}
+	private readonly db: DB;
+	private readonly batch: WriteBatch;
+	private readonly maxBoundParameters: number;
+
+	constructor(store: SqliteStore<Record<string, unknown>>) {
+		this.db = store.db;
+		this.batch = store.batch;
+		this.maxBoundParameters = store.maxBoundParameters;
+	}
 
 	async replaceForObject(
 		calendarId: string,
 		calendarObjectId: string,
 		occurrences: CalendarOccurrenceInput[],
 	): Promise<void> {
-		await this.deleteForObject(calendarId, calendarObjectId);
-		if (occurrences.length === 0) return;
-
 		const now = Date.now();
-		await this.db.insert(calendarEventIndexTable).values(
-			occurrences.map((occurrence) => ({
-				calendarId,
-				calendarObjectId,
-				recurrenceId: occurrence.recurrenceId,
-				startAt: occurrence.startAt,
-				endAt: occurrence.endAt,
-				allDay: occurrence.allDay,
-				summary: occurrence.summary,
-				status: occurrence.status,
-				transparency: occurrence.transparency,
-				createdAt: now,
-				updatedAt: now,
-			})),
-		);
+		const rows = occurrences.map((occurrence) => ({
+			calendarId,
+			calendarObjectId,
+			recurrenceId: occurrence.recurrenceId,
+			startAt: occurrence.startAt,
+			endAt: occurrence.endAt,
+			allDay: occurrence.allDay,
+			summary: occurrence.summary,
+			status: occurrence.status,
+			transparency: occurrence.transparency,
+			createdAt: now,
+			updatedAt: now,
+		}));
+		const inserts: SqliteBatchItem[] = chunkRows(
+			calendarEventIndexTable,
+			rows,
+			this.maxBoundParameters,
+		).map((slice) => this.db.insert(calendarEventIndexTable).values(slice));
+		await this.batch([this.clear(calendarId, calendarObjectId), ...inserts]);
 	}
 
 	async deleteForObject(
 		calendarId: string,
 		calendarObjectId: string,
 	): Promise<void> {
-		await this.db
+		await this.clear(calendarId, calendarObjectId);
+	}
+
+	private clear(calendarId: string, calendarObjectId: string) {
+		return this.db
 			.delete(calendarEventIndexTable)
 			.where(
 				and(

package/src/repos/calendar-feed-token.conformance.d1.test.ts

@@ -0,0 +1,20 @@
+import { calendarFeedTokenRepositoryConformance } from "@remit/data-ports/conformance";
+import { NotFoundError } from "../error.js";
+import { randomId } from "../id.js";
+import { openD1TestStores } from "../test-d1.js";
+import { CalendarFeedTokenRepo } from "./calendar-feed-token.js";
+
+let close: (() => Promise<void>) | undefined;
+
+calendarFeedTokenRepositoryConformance({
+	async createRepository() {
+		const opened = await openD1TestStores();
+		close = opened.close;
+		return new CalendarFeedTokenRepo(opened.stores.app.db);
+	},
+	teardown: async () => {
+		await close?.();
+	},
+	makeId: () => randomId(),
+	isNotFoundError: (error) => error instanceof NotFoundError,
+});

package/src/repos/calendar-object.conformance.d1.test.ts

@@ -0,0 +1,20 @@
+import { calendarObjectRepositoryConformance } from "@remit/data-ports/conformance";
+import { NotFoundError } from "../error.js";
+import { randomId } from "../id.js";
+import { openD1TestStores } from "../test-d1.js";
+import { CalendarObjectRepo } from "./calendar-object.js";
+
+let close: (() => Promise<void>) | undefined;
+
+calendarObjectRepositoryConformance({
+	async createRepository() {
+		const opened = await openD1TestStores();
+		close = opened.close;
+		return new CalendarObjectRepo(opened.stores.app.db);
+	},
+	teardown: async () => {
+		await close?.();
+	},
+	makeId: () => randomId(),
+	isNotFoundError: (error) => error instanceof NotFoundError,
+});

package/src/repos/calendar-put.d1.test.ts

@@ -0,0 +1,4 @@
+import { openD1TestStores } from "../test-d1.js";
+import { calendarPutSuite } from "./calendar-put.suite.js";
+
+calendarPutSuite(openD1TestStores);