@remit/drizzle-service 0.0.104 → 0.0.106

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.104",
+  "version": "0.0.106",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/bound-parameters.test.ts

@@ -0,0 +1,57 @@
+import assert from "node:assert/strict";
+import { describe, test } from "node:test";
+import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
+import { chunkRows, rowsPerStatement } from "./bound-parameters.js";
+
+const tenColumns = sqliteTable("ten_columns", {
+	c0: text("c0"),
+	c1: text("c1"),
+	c2: text("c2"),
+	c3: text("c3"),
+	c4: text("c4"),
+	c5: text("c5"),
+	c6: text("c6"),
+	c7: text("c7"),
+	c8: text("c8"),
+	c9: integer("c9"),
+});
+
+const rows = (count: number): number[] =>
+	Array.from({ length: count }, (_, index) => index);
+
+describe("chunkRows", () => {
+	test("no rows make no statements", () => {
+		assert.deepEqual(chunkRows(tenColumns, [], 100), []);
+	});
+
+	test("rows that fill the limit exactly make one statement", () => {
+		const chunks = chunkRows(tenColumns, rows(10), 100);
+		assert.deepEqual(
+			chunks.map((slice) => slice.length),
+			[10],
+		);
+	});
+
+	test("one row over the limit makes a second statement", () => {
+		const chunks = chunkRows(tenColumns, rows(11), 100);
+		assert.deepEqual(
+			chunks.map((slice) => slice.length),
+			[10, 1],
+		);
+		assert.deepEqual(chunks.flat(), rows(11));
+	});
+
+	test("reserved parameters shrink every statement", () => {
+		const chunks = chunkRows(tenColumns, rows(10), 100, 1);
+		assert.deepEqual(
+			chunks.map((slice) => slice.length),
+			[9, 1],
+		);
+	});
+});
+
+describe("rowsPerStatement", () => {
+	test("a row wider than the limit is refused", () => {
+		assert.throws(() => rowsPerStatement(100, 101), RangeError);
+	});
+});

package/src/bound-parameters.ts

@@ -0,0 +1,49 @@
+import { getTableColumns, type Table } from "drizzle-orm";
+
+export interface SqliteLimits {
+	/** The most parameters the host's SQLite binds in one statement: 32766 on better-sqlite3, 100 on D1 and Durable Object storage. */
+	maxBoundParameters: number;
+}
+
+export const BETTER_SQLITE3_LIMITS: SqliteLimits = {
+	maxBoundParameters: 32766,
+};
+
+export const chunk = <T>(items: readonly T[], size: number): T[][] => {
+	const chunks: T[][] = [];
+	for (let start = 0; start < items.length; start += size) {
+		chunks.push(items.slice(start, start + size));
+	}
+	return chunks;
+};
+
+export const rowsPerStatement = (
+	maxBoundParameters: number,
+	parametersPerRow: number,
+	reservedParameters = 0,
+): number => {
+	const rows = Math.floor(
+		(maxBoundParameters - reservedParameters) / parametersPerRow,
+	);
+	if (rows < 1) {
+		throw new RangeError(
+			`a row of ${parametersPerRow} bound parameters does not fit a statement limit of ${maxBoundParameters}`,
+		);
+	}
+	return rows;
+};
+
+export const chunkRows = <T>(
+	table: Table,
+	rows: readonly T[],
+	maxBoundParameters: number,
+	reservedParameters = 0,
+): T[][] =>
+	chunk(
+		rows,
+		rowsPerStatement(
+			maxBoundParameters,
+			Object.keys(getTableColumns(table)).length,
+			reservedParameters,
+		),
+	);

package/src/databases.ts

@@ -1,4 +1,5 @@
 import { syncBatch } from "./batch.js";
+import type { SqliteLimits } from "./bound-parameters.js";
 import type { Db, SqliteStore } from "./db.js";
 import { messageDataSchema } from "./schema/message-data.js";
 import { createSqliteDatabase } from "./sqlite-client.js";
@@ -22,16 +23,16 @@ export interface SqliteStores {
 
 export const syncStore = <TSchema extends Record<string, unknown>>(
 	db: Db<TSchema>,
-): SqliteStore<TSchema> => ({ db, batch: syncBatch(db) });
+	{ maxBoundParameters }: SqliteLimits,
+): SqliteStore<TSchema> => ({ db, batch: syncBatch(db), maxBoundParameters });
 
-export const syncStores = ({
-	app,
-	mail,
-	search,
-}: SqliteDatabases): SqliteStores => ({
-	app: syncStore(app),
-	mail: syncStore(mail),
-	search: syncStore(search),
+export const syncStores = (
+	{ app, mail, search }: SqliteDatabases,
+	limits: SqliteLimits,
+): SqliteStores => ({
+	app: syncStore(app, limits),
+	mail: syncStore(mail, limits),
+	search: syncStore(search, limits),
 });
 
 export interface SqliteDatabasePaths {

package/src/db.ts

@@ -26,4 +26,5 @@ export type RepositoryDb<TSchema extends Record<string, unknown>> = Omit<
 export interface SqliteStore<TSchema extends Record<string, unknown>> {
 	db: RepositoryDb<TSchema>;
 	batch: WriteBatch;
+	maxBoundParameters: number;
 }

package/src/index.ts

@@ -5,6 +5,10 @@ export {
 	syncBatch,
 	type WriteBatch,
 } from "./batch.js";
+export {
+	BETTER_SQLITE3_LIMITS,
+	type SqliteLimits,
+} from "./bound-parameters.js";
 export {
 	hasSqliteDatabaseEnv,
 	openSqliteDatabases,
@@ -18,6 +22,11 @@ export {
 } from "./databases.js";
 export type { RepositoryDb, SqliteStore } from "./db.js";
 export { CreateFailedConflictError, NotFoundError } from "./error.js";
+export {
+	accountIdsOf,
+	eraseAccountConfigAppRows,
+	runDrizzleAccountConfigErase,
+} from "./repos/account-config-erase.js";
 export { CalendarCollectionRepo } from "./repos/calendar-collection.js";
 export { CalendarEventIndexRepo } from "./repos/calendar-event-index.js";
 export { CalendarFeedTokenRepo } from "./repos/calendar-feed-token.js";

package/src/repair/junk-only-address.sqlite.test.ts

@@ -4,6 +4,7 @@ import { composeFolderRoleAppointmentName } from "@remit/data-ports/folder-role"
 import { CanonicalMailboxRole } from "@remit/domain-enums";
 import Database from "better-sqlite3";
 import { drizzle } from "drizzle-orm/better-sqlite3";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { AccountSettingRepo } from "../repos/i4-account-setting.js";
 import type { MailboxSpecialUseRepo } from "../repos/i4-mailbox-special-use.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
@@ -182,11 +183,14 @@ describe("addresses standing only on mail in Junk", () => {
 	before(() => {
 		sqlite = new Database(":memory:");
 		const db = drizzle(sqlite);
-		specialUse = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).mailboxSpecialUse;
+		specialUse = createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).mailboxSpecialUse;
 		accountSetting = new AccountSettingRepo(db as never);
 		applyMigrationSet(sqlite, "app");
 		applyMigrationSet(sqlite, "mail");

package/src/repos/account-config-erase.test.ts

@@ -0,0 +1,127 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import { eq, getTableColumns } from "drizzle-orm";
+import {
+	getTableConfig,
+	type SQLiteColumn,
+	SQLiteTable,
+} from "drizzle-orm/sqlite-core";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
+import { syncStores } from "../databases.js";
+import * as appTables from "../schema-sets/app.js";
+import * as mailTables from "../schema-sets/mail.js";
+import { createTestDb, type TestDb } from "../test-db.js";
+import { runDrizzleAccountConfigErase } from "./account-config-erase.js";
+
+const log = { info: () => {} };
+
+const ERASED = "cfg-erased";
+const KEPT = "cfg-kept";
+
+const ownerIdsOf = (tenant: string): Record<string, string> => ({
+	account_config_id: tenant,
+	account_id: `${tenant}-account`,
+	mailbox_id: `${tenant}-mailbox`,
+	message_id: `${tenant}-message`,
+	calendar_id: `${tenant}-calendar`,
+});
+
+interface Owned {
+	readonly name: string;
+	readonly table: SQLiteTable;
+	readonly ownerColumn: SQLiteColumn;
+}
+
+const owned: Owned[] = Object.entries({ ...appTables, ...mailTables }).flatMap(
+	([name, table]) => {
+		if (!(table instanceof SQLiteTable)) return [];
+		const ownerKeys = Object.keys(ownerIdsOf(ERASED));
+		const ownerColumn = ownerKeys
+			.map((key) =>
+				getTableConfig(table).columns.find((column) => column.name === key),
+			)
+			.find((column) => column !== undefined);
+		return ownerColumn === undefined ? [] : [{ name, table, ownerColumn }];
+	},
+);
+
+const fillerOf = (column: SQLiteColumn, tenant: string): unknown => {
+	switch (column.dataType) {
+		case "number":
+			return 1;
+		case "boolean":
+			return false;
+		case "json":
+			return {};
+		case "buffer":
+			return Buffer.from(tenant);
+		default:
+			return `${tenant}-${column.name}`;
+	}
+};
+
+const rowOf = (table: SQLiteTable, tenant: string): Record<string, unknown> => {
+	const ownerIds = ownerIdsOf(tenant);
+	return Object.fromEntries(
+		Object.entries(getTableColumns(table)).flatMap(([key, column]) => {
+			const ownerId = ownerIds[column.name];
+			if (ownerId !== undefined) return [[key, ownerId]];
+			if (column.notNull && !column.hasDefault)
+				return [[key, fillerOf(column, tenant)]];
+			return [];
+		}),
+	);
+};
+
+const countOf = async (
+	db: TestDb,
+	{ table, ownerColumn }: Owned,
+	tenant: string,
+): Promise<number> => {
+	const ownerId = ownerIdsOf(tenant)[ownerColumn.name] ?? "";
+	return (await db.select().from(table).where(eq(ownerColumn, ownerId))).length;
+};
+
+describe("runDrizzleAccountConfigErase", () => {
+	let db: TestDb;
+	let close: () => Promise<void>;
+
+	before(async () => {
+		({ db, close } = await createTestDb());
+		for (const tenant of [ERASED, KEPT]) {
+			for (const { table } of owned) {
+				await db.insert(table).values(rowOf(table, tenant) as never);
+			}
+		}
+		const stores = syncStores(
+			{ app: db, mail: db, search: db } as never,
+			BETTER_SQLITE3_LIMITS,
+		);
+		await runDrizzleAccountConfigErase(stores, ERASED, log);
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	test("leaves no row of the erased account config in any table it owns", async () => {
+		for (const entry of owned) {
+			const expected = entry.table === appTables.accountConfigs ? 1 : 0;
+			assert.equal(
+				await countOf(db, entry, ERASED),
+				expected,
+				`${entry.name} keeps rows of the erased account config`,
+			);
+		}
+	});
+
+	test("keeps every row of another account config", async () => {
+		for (const entry of owned) {
+			assert.equal(
+				await countOf(db, entry, KEPT),
+				1,
+				`${entry.name} lost rows of another account config`,
+			);
+		}
+	});
+});

package/src/repos/account-config-erase.ts

@@ -0,0 +1,170 @@
+import { eq } from "drizzle-orm";
+import {
+	getTableConfig,
+	type SQLiteColumn,
+	SQLiteTable,
+} from "drizzle-orm/sqlite-core";
+import type { SqliteBatchItem } from "../batch.js";
+import { chunk } from "../bound-parameters.js";
+import type { SqliteStores } from "../databases.js";
+import type { SqliteStore } from "../db.js";
+import {
+	accountConfigTable,
+	accountTable,
+} from "../schema/i4-account-config.js";
+import { mailboxTable } from "../schema/i4-mailbox.js";
+import { calendarTable, messageTable } from "../schema.js";
+import * as appTables from "../schema-sets/app.js";
+import * as mailTables from "../schema-sets/mail.js";
+import { SearchIndex } from "../search/thread-message-search.js";
+import type { CascadeDeleteLogger } from "./cascade-delete.js";
+import { inList } from "./in-list.js";
+
+/**
+ * The column an account config owns a table's rows by: its own id where the
+ * table carries it, else the first id of the parent it hangs off.
+ */
+export const OWNER_KEYS = [
+	"account_config_id",
+	"account_id",
+	"mailbox_id",
+	"message_id",
+	"calendar_id",
+] as const;
+
+export type OwnerKey = (typeof OWNER_KEYS)[number];
+
+export interface OwnedTable {
+	readonly table: SQLiteTable;
+	readonly ownerKey: OwnerKey;
+	readonly ownerColumn: SQLiteColumn;
+}
+
+export const ownedTablesOf = (tables: Record<string, unknown>): OwnedTable[] =>
+	Object.values(tables).flatMap((table) => {
+		if (!(table instanceof SQLiteTable)) return [];
+		const { columns } = getTableConfig(table);
+		for (const ownerKey of OWNER_KEYS) {
+			const ownerColumn = columns.find((column) => column.name === ownerKey);
+			if (ownerColumn !== undefined) return [{ table, ownerKey, ownerColumn }];
+		}
+		return [];
+	});
+
+// Durable Object SQLite caps a bound value at 2 MB; a chunk of ids stays far
+// below it.
+const IDS_PER_STATEMENT = 5_000;
+
+type OwnerIds = Readonly<Record<OwnerKey, readonly string[]>>;
+
+const keyedDeletes = (
+	store: SqliteStore<Record<string, unknown>>,
+	tables: readonly OwnedTable[],
+	ids: OwnerIds,
+): SqliteBatchItem[] =>
+	tables
+		.filter(({ table }) => table !== accountConfigTable)
+		.flatMap(({ table, ownerKey, ownerColumn }) =>
+			ownerKey === "account_config_id"
+				? ids.account_config_id.map((id) =>
+						store.db.delete(table).where(eq(ownerColumn, id)),
+					)
+				: chunk(ids[ownerKey], IDS_PER_STATEMENT).map((slice) =>
+						store.db.delete(table).where(inList(ownerColumn, slice)),
+					),
+		);
+
+const APP_TABLES = ownedTablesOf(appTables);
+const MAIL_TABLES = ownedTablesOf(mailTables);
+
+const idsOf = async <Row>(
+	rows: PromiseLike<Row[]>,
+	pick: (row: Row) => string,
+): Promise<string[]> => (await rows).map(pick);
+
+/**
+ * Delete the app database rows an account config owns, except the
+ * AccountConfig row itself, which the caller deletes last.
+ */
+export const eraseAccountConfigAppRows = async (
+	app: SqliteStore<Record<string, unknown>>,
+	accountConfigId: string,
+	accountIds: readonly string[],
+): Promise<void> => {
+	const calendarIds = await idsOf(
+		app.db
+			.select({ calendarId: calendarTable.calendarId })
+			.from(calendarTable)
+			.where(eq(calendarTable.accountConfigId, accountConfigId)),
+		(row) => row.calendarId,
+	);
+	await app.batch(
+		keyedDeletes(app, APP_TABLES, {
+			account_config_id: [accountConfigId],
+			account_id: accountIds,
+			mailbox_id: [],
+			message_id: [],
+			calendar_id: calendarIds,
+		}),
+	);
+};
+
+export const accountIdsOf = (
+	app: SqliteStore<Record<string, unknown>>,
+	accountConfigId: string,
+): Promise<string[]> =>
+	idsOf(
+		app.db
+			.select({ accountId: accountTable.accountId })
+			.from(accountTable)
+			.where(eq(accountTable.accountConfigId, accountConfigId)),
+		(row) => row.accountId,
+	);
+
+/**
+ * Delete every row an account config owns across the app, mail and search
+ * databases, one keyed statement per table. A rerun deletes what is left.
+ */
+export const runDrizzleAccountConfigErase = async (
+	stores: SqliteStores,
+	accountConfigId: string,
+	log: CascadeDeleteLogger,
+): Promise<void> => {
+	const { app, mail, search } = stores;
+	const accountIds = await accountIdsOf(app, accountConfigId);
+	const mailboxIds = await idsOf(
+		mail.db
+			.select({ mailboxId: mailboxTable.mailboxId })
+			.from(mailboxTable)
+			.where(inList(mailboxTable.accountId, accountIds)),
+		(row) => row.mailboxId,
+	);
+	const messageIds = await idsOf(
+		mail.db
+			.select({ messageId: messageTable.messageId })
+			.from(messageTable)
+			.where(inList(messageTable.mailboxId, mailboxIds)),
+		(row) => row.messageId,
+	);
+
+	await new SearchIndex(mail.db, search).removeAccount(accountConfigId);
+	await mail.batch(
+		keyedDeletes(mail, MAIL_TABLES, {
+			account_config_id: [accountConfigId],
+			account_id: accountIds,
+			mailbox_id: mailboxIds,
+			message_id: messageIds,
+			calendar_id: [],
+		}),
+	);
+	await eraseAccountConfigAppRows(app, accountConfigId, accountIds);
+
+	log.info(
+		{
+			accountConfigId,
+			accountCount: accountIds.length,
+			messageCount: messageIds.length,
+		},
+		"Drizzle account config erase complete",
+	);
+};

package/src/repos/account.conformance.sqlite.test.ts

@@ -1,4 +1,5 @@
 import { accountRepositoryConformance } from "@remit/data-ports/conformance";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { NotFoundError } from "../error.js";
 import { randomId } from "../id.js";
 import { accountTable } from "../schema.js";
@@ -13,11 +14,14 @@ accountRepositoryConformance({
 			accounts: accountTable,
 		});
 		close = closeDb;
-		return createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).account;
+		return createSqliteRepositories(
+			{
+				app: db,
+				mail: db,
+				search: db,
+			},
+			BETTER_SQLITE3_LIMITS,
+		).account;
 	},
 	teardown: async () => {
 		await close?.();

package/src/repos/calendar-event-index.conformance.d1.test.ts

@@ -10,7 +10,7 @@ calendarEventIndexRepositoryConformance({
 	async createRepository() {
 		const opened = await openD1TestStores();
 		close = opened.close;
-		return new CalendarEventIndexRepo(opened.stores.app.db);
+		return new CalendarEventIndexRepo(opened.stores.app);
 	},
 	teardown: async () => {
 		await close?.();

package/src/repos/calendar-event-index.conformance.sqlite.test.ts

@@ -1,4 +1,6 @@
 import { calendarEventIndexRepositoryConformance } from "@remit/data-ports/conformance";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
+import { syncStore } from "../databases.js";
 import { NotFoundError } from "../error.js";
 import { randomId } from "../id.js";
 import { calendarEventIndexTable } from "../schema.js";
@@ -13,7 +15,7 @@ calendarEventIndexRepositoryConformance({
 			calendarEventIndexes: calendarEventIndexTable,
 		});
 		close = closeDb;
-		return new CalendarEventIndexRepo(db as never);
+		return new CalendarEventIndexRepo(syncStore(db, BETTER_SQLITE3_LIMITS));
 	},
 	teardown: async () => {
 		await close?.();

package/src/repos/calendar-event-index.ts

@@ -4,7 +4,9 @@ import type {
 	ICalendarEventIndexRepository,
 } from "@remit/data-ports";
 import { and, asc, eq, gte, lt } from "drizzle-orm";
-import type { RepositoryDb } from "../db.js";
+import type { SqliteBatchItem, WriteBatch } from "../batch.js";
+import { chunkRows } from "../bound-parameters.js";
+import type { RepositoryDb, SqliteStore } from "../db.js";
 import { calendarEventIndexTable } from "../schema.js";
 
 type DB = RepositoryDb<Record<string, unknown>>;
@@ -28,39 +30,52 @@ function rowToOccurrence(
 }
 
 export class CalendarEventIndexRepo implements ICalendarEventIndexRepository {
-	constructor(private db: DB) {}
+	private readonly db: DB;
+	private readonly batch: WriteBatch;
+	private readonly maxBoundParameters: number;
+
+	constructor(store: SqliteStore<Record<string, unknown>>) {
+		this.db = store.db;
+		this.batch = store.batch;
+		this.maxBoundParameters = store.maxBoundParameters;
+	}
 
 	async replaceForObject(
 		calendarId: string,
 		calendarObjectId: string,
 		occurrences: CalendarOccurrenceInput[],
 	): Promise<void> {
-		await this.deleteForObject(calendarId, calendarObjectId);
-		if (occurrences.length === 0) return;
-
 		const now = Date.now();
-		await this.db.insert(calendarEventIndexTable).values(
-			occurrences.map((occurrence) => ({
-				calendarId,
-				calendarObjectId,
-				recurrenceId: occurrence.recurrenceId,
-				startAt: occurrence.startAt,
-				endAt: occurrence.endAt,
-				allDay: occurrence.allDay,
-				summary: occurrence.summary,
-				status: occurrence.status,
-				transparency: occurrence.transparency,
-				createdAt: now,
-				updatedAt: now,
-			})),
-		);
+		const rows = occurrences.map((occurrence) => ({
+			calendarId,
+			calendarObjectId,
+			recurrenceId: occurrence.recurrenceId,
+			startAt: occurrence.startAt,
+			endAt: occurrence.endAt,
+			allDay: occurrence.allDay,
+			summary: occurrence.summary,
+			status: occurrence.status,
+			transparency: occurrence.transparency,
+			createdAt: now,
+			updatedAt: now,
+		}));
+		const inserts: SqliteBatchItem[] = chunkRows(
+			calendarEventIndexTable,
+			rows,
+			this.maxBoundParameters,
+		).map((slice) => this.db.insert(calendarEventIndexTable).values(slice));
+		await this.batch([this.clear(calendarId, calendarObjectId), ...inserts]);
 	}
 
 	async deleteForObject(
 		calendarId: string,
 		calendarObjectId: string,
 	): Promise<void> {
-		await this.db
+		await this.clear(calendarId, calendarObjectId);
+	}
+
+	private clear(calendarId: string, calendarObjectId: string) {
+		return this.db
 			.delete(calendarEventIndexTable)
 			.where(
 				and(