@remit/drizzle-service 0.0.99 → 0.0.101

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.99",
+  "version": "0.0.101",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/batch.sqlite.test.ts

@@ -0,0 +1,78 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import { eq } from "drizzle-orm";
+import { runBatch } from "./batch.js";
+import { calendarTable } from "./schema.js";
+import { createSqliteTestDb, type SqliteTestDb } from "./test-db-sqlite.js";
+
+const SCHEMA = { calendars: calendarTable };
+
+const collection = (calendarId: string): typeof calendarTable.$inferInsert => ({
+	calendarId,
+	accountConfigId: "account-config-batch",
+	urlSegment: calendarId,
+	displayName: calendarId,
+	color: "Cal1",
+	componentSet: "VeventOnly",
+	source: "UserCreated",
+	timezone: "",
+	subscriptionUrl: "",
+	subscriptionEnabled: false,
+	syncSequence: 0,
+	createdAt: 0,
+	updatedAt: 0,
+});
+
+describe("a batch on better-sqlite3", () => {
+	let db: SqliteTestDb<typeof SCHEMA>;
+	let close: () => Promise<void>;
+
+	before(async () => {
+		({ db, close } = await createSqliteTestDb(SCHEMA));
+	});
+
+	after(() => close());
+
+	const stored = () => db.select().from(calendarTable);
+
+	test("answers each statement in order, reads seeing the writes before them", async () => {
+		const [inserted, read] = await runBatch(db, [
+			db.insert(calendarTable).values(collection("first")).returning(),
+			db
+				.select()
+				.from(calendarTable)
+				.where(eq(calendarTable.calendarId, "first")),
+		] as const);
+
+		assert.deepEqual(
+			inserted.map((row) => row.calendarId),
+			["first"],
+		);
+		assert.deepEqual(
+			read.map((row) => row.calendarId),
+			["first"],
+		);
+	});
+
+	test("a statement that fails takes every earlier one with it", async () => {
+		const before = await stored();
+
+		await assert.rejects(
+			runBatch(db, [
+				db.insert(calendarTable).values(collection("second")),
+				db
+					.update(calendarTable)
+					.set({ syncSequence: 7 })
+					.where(eq(calendarTable.calendarId, "first")),
+				db.insert(calendarTable).values(collection("first")),
+			]),
+			/UNIQUE constraint failed/,
+		);
+
+		assert.deepEqual(await stored(), before);
+	});
+
+	test("an empty batch writes nothing and answers nothing", async () => {
+		assert.deepEqual(await runBatch(db, []), []);
+	});
+});

package/src/batch.ts

@@ -0,0 +1,46 @@
+import type { BatchItem, BatchResponse } from "drizzle-orm/batch";
+import type { Db } from "./db.js";
+
+export type SqliteBatchItem = BatchItem<"sqlite">;
+
+interface SyncPreparable {
+	_prepare(): { execute(): { sync(): unknown } };
+}
+
+const isSyncPreparable = (value: unknown): value is SyncPreparable =>
+	typeof value === "object" &&
+	value !== null &&
+	typeof Reflect.get(value, "_prepare") === "function";
+
+const runSync = (item: SqliteBatchItem): unknown => {
+	if (!isSyncPreparable(item)) {
+		throw new TypeError("batch item is not a drizzle sqlite query");
+	}
+	return item._prepare().execute().sync();
+};
+
+/**
+ * The shape of D1's `batch()`: every statement commits together or not at all,
+ * and none of them reads another's result mid-flight. A write that needs a
+ * value reads it before the batch and is written so a stale read loses safely
+ * (a conditional `WHERE`, an upsert, a deterministic id).
+ *
+ * On better-sqlite3 the batch is one synchronous IMMEDIATE transaction: it
+ * never yields the event loop, so no other write on the shared connection lands
+ * inside it, and IMMEDIATE takes the file's write lock before the first
+ * statement so a writer in another process waits out the busy timeout. On a
+ * Durable Object's `durable-sqlite` handle it is one `transactionSync`.
+ */
+export const runBatch = async <
+	TSchema extends Record<string, unknown>,
+	T extends readonly SqliteBatchItem[],
+>(
+	db: Db<TSchema>,
+	items: T,
+): Promise<BatchResponse<T>> => {
+	if (items.length === 0) return [] as unknown as BatchResponse<T>;
+	const results = db.transaction(() => items.map(runSync), {
+		behavior: "immediate",
+	});
+	return results as unknown as BatchResponse<T>;
+};

package/src/databases.ts

@@ -0,0 +1,70 @@
+import type { Db } from "./db.js";
+import {
+	type MessageDataSchema,
+	messageDataSchema,
+} from "./schema/message-data.js";
+import { createSqliteDatabase } from "./sqlite-client.js";
+
+// The three databases a self-host instance runs on (#1420). The app database
+// holds what lives in the client: accounts, config, rules, settings and auth.
+// The tenant mail database holds what lives on the IMAP server, plus the local
+// rows that refer to it. The tenant search database holds the full-text index,
+// rebuilt from the mail database, so it never commits together with it.
+export interface SqliteDatabases {
+	app: Db<Record<string, unknown>>;
+	mail: Db<MessageDataSchema>;
+	search: Db<Record<string, unknown>>;
+}
+
+export interface SqliteDatabasePaths {
+	app: string;
+	mail: string;
+	search: string;
+}
+
+export const SQLITE_DATABASE_ENV = {
+	app: "SQLITE_APP_DB_PATH",
+	mail: "SQLITE_MAIL_DB_PATH",
+	search: "SQLITE_SEARCH_DB_PATH",
+} as const satisfies Record<keyof SqliteDatabasePaths, string>;
+
+/**
+ * Whether this process is configured for the SQLite composition. The app path
+ * is the precondition every composition root checks before importing it.
+ */
+export const hasSqliteDatabaseEnv = (
+	env: NodeJS.ProcessEnv = process.env,
+): boolean => Boolean(env[SQLITE_DATABASE_ENV.app]);
+
+export const sqliteDatabasePathsFromEnv = (
+	env: NodeJS.ProcessEnv = process.env,
+): SqliteDatabasePaths => {
+	const read = (name: string): string => {
+		const value = env[name];
+		if (!value) throw new Error(`${name} is required`);
+		return value;
+	};
+	return {
+		app: read(SQLITE_DATABASE_ENV.app),
+		mail: read(SQLITE_DATABASE_ENV.mail),
+		search: read(SQLITE_DATABASE_ENV.search),
+	};
+};
+
+export const openSqliteDatabases = async (
+	paths: SqliteDatabasePaths,
+): Promise<SqliteDatabases & { close: () => Promise<void> }> => {
+	const app = await createSqliteDatabase({}, { filename: paths.app });
+	const mail = await createSqliteDatabase(messageDataSchema, {
+		filename: paths.mail,
+	});
+	const search = await createSqliteDatabase({}, { filename: paths.search });
+	return {
+		app: app.db,
+		mail: mail.db,
+		search: search.db,
+		close: async () => {
+			await Promise.all([app.close(), mail.close(), search.close()]);
+		},
+	};
+};

package/src/db.ts

@@ -1,13 +1,14 @@
-import type { BetterSQLite3Database } from "drizzle-orm/better-sqlite3";
+import type { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
 
 /**
- * A drizzle handle a repository can run on: the top-level database or a
- * transaction (or savepoint) bound to one. Both share the query-builder API, so
- * a repo constructed with either behaves the same — standalone, or enlisted in a
- * unit-of-work transaction.
+ * The drizzle handle a repository runs on: any synchronous SQLite driver.
  *
- * Internal to this adapter: it names better-sqlite3, so it is not exported from
- * the package index and never appears on a shared surface.
+ * better-sqlite3 is the one this package opens; a Cloudflare Durable Object's
+ * `durable-sqlite` handle satisfies it too, so its adapter constructs the same
+ * repositories over its own storage.
  */
-export type Db<TSchema extends Record<string, unknown>> =
-	BetterSQLite3Database<TSchema>;
+export type Db<TSchema extends Record<string, unknown>> = BaseSQLiteDatabase<
+	"sync",
+	unknown,
+	TSchema
+>;

package/src/generated-enum.test.ts

@@ -0,0 +1,8 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { MailboxAttribute, MessageSystemFlag } from "@remit/domain-enums";
+
+test("the generated enums keep the backslash IMAP puts on a system flag and a mailbox attribute", () => {
+	assert.equal(MessageSystemFlag.Seen, "\\Seen");
+	assert.equal(MailboxAttribute.NoSelect, "\\Noselect");
+});

package/src/generated-id.test.ts

@@ -0,0 +1,19 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { generateBase36Id } from "@remit/drizzle-sqlite-schema";
+import shortUuid from "short-uuid";
+
+const base36 = shortUuid.createTranslator(shortUuid.constants.uuid25Base36);
+
+test("the schema's id default writes the 25-char base36 v4 uuid that short-uuid reads and writes back unchanged", () => {
+	for (let i = 0; i < 5000; i += 1) {
+		const id = generateBase36Id();
+		assert.match(id, /^[0-9a-z]{25}$/);
+		const uuid = base36.toUUID(id);
+		assert.match(
+			uuid,
+			/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/,
+		);
+		assert.equal(base36.fromUUID(uuid), id);
+	}
+});

package/src/id.test.ts

@@ -1,7 +1,6 @@
 import assert from "node:assert/strict";
 import { describe, test } from "node:test";
 import {
-	bodyPartContentId,
 	bodyPartId,
 	bodyPartParameterId,
 	deterministicBase36Id,
@@ -59,11 +58,4 @@ describe("derived ids are stored primary keys: a changed value orphans every row
 			"5f4e930e-e418-51a2-9402-657822f8688b",
 		);
 	});
-
-	test("bodyPartContentId pins the body_part_content primary key", () => {
-		assert.equal(
-			bodyPartContentId(MESSAGE, "bodypart-golden"),
-			"c0ee5f7d-1894-5957-9f65-d9aafdf306ab",
-		);
-	});
 });

package/src/id.ts

@@ -42,9 +42,3 @@ export const envelopeAddressId = (
 	order: number,
 ): string =>
 	generateDeterministicId(`envelopeaddress:${messageId}:${role}:${order}`);
-
-export const bodyPartContentId = (
-	messageId: string,
-	bodyPartId: string,
-): string =>
-	generateDeterministicId(`bodypartcontent:${messageId}:${bodyPartId}`);

package/src/index.ts

@@ -1,10 +1,18 @@
+export {
+	hasSqliteDatabaseEnv,
+	openSqliteDatabases,
+	SQLITE_DATABASE_ENV,
+	type SqliteDatabasePaths,
+	type SqliteDatabases,
+	sqliteDatabasePathsFromEnv,
+} from "./databases.js";
 export { CreateFailedConflictError, NotFoundError } from "./error.js";
 export { CalendarCollectionRepo } from "./repos/calendar-collection.js";
 export { CalendarEventIndexRepo } from "./repos/calendar-event-index.js";
 export { CalendarFeedTokenRepo } from "./repos/calendar-feed-token.js";
 export { CalendarObjectRepo } from "./repos/calendar-object.js";
+export { DrizzleCalendarStore } from "./repos/calendar-store.js";
 export { CalendarSuggestionRepo } from "./repos/calendar-suggestion.js";
-export { DrizzleCalendarUnitOfWork } from "./repos/calendar-unit-of-work.js";
 export {
 	type CascadeDeleteLogger,
 	type CascadeDeleter,
@@ -14,8 +22,6 @@ export {
 } from "./repos/cascade-delete.js";
 export { DrizzleEnvelopeRepository } from "./repos/envelope.js";
 export { FilterRepo } from "./repos/filter.js";
-export { FilterAnchorRepo } from "./repos/filter-anchor.js";
-export { DrizzleFilterAnchorTransaction } from "./repos/filter-anchor-transaction.js";
 export * from "./repos/i4-account.js";
 export * from "./repos/i4-account-config.js";
 export * from "./repos/i4-account-export-request.js";
@@ -45,13 +51,12 @@ export { LabelRepo } from "./repos/label.js";
 export {
 	DrizzleMessageRepository,
 	deleteMessageSubtree,
-	MESSAGE_REMOVED_EVENT,
 } from "./repos/message.js";
 export { DrizzleMessageFlagRepository } from "./repos/message-flag.js";
 export { MessageLabelRepo } from "./repos/message-label.js";
 export { QuarantineRepo } from "./repos/quarantine.js";
+export { SyncWarningRepo } from "./repos/sync-warning.js";
 export { DrizzleThreadMessageRepository } from "./repos/thread-message.js";
-export { DrizzleUnitOfWork } from "./repos/unit-of-work.js";
 export * from "./schema/i4-account-config.js";
 export * from "./schema/i4-account-export-request.js";
 export * from "./schema/i4-account-setting.js";
@@ -66,9 +71,14 @@ export * from "./schema/i4-outbox-message.js";
 export * from "./schema/message-data.js";
 export { messageDataSchema } from "./schema/message-data.js";
 export * from "./schema/quarantine.js";
+export * from "./schema/sync-warning.js";
+export { SearchIndex } from "./search/thread-message-search.js";
 export {
 	createSqliteDatabase,
 	type SqliteClient,
 	type SqliteClientOptions,
 } from "./sqlite-client.js";
-export { runInTransaction, serializeSqliteWrites } from "./tx.js";
+export {
+	createSqliteRepositories,
+	type SqliteRepositories,
+} from "./sqlite-repositories.js";

package/src/repair/address-display-name.sqlite.test.ts

@@ -17,11 +17,9 @@ const clientOver = (sqlite: Database.Database): DisplayNameRepairClient => ({
 });
 
 const seed = (sqlite: Database.Database): void => {
-	sqlite.exec(shippedTableDdl("0000_happy_roland_deschain", "address"));
-	sqlite.exec(
-		shippedTableDdl("0000_happy_roland_deschain", "envelope_address"),
-	);
-	sqlite.exec(shippedTableDdl("0000_happy_roland_deschain", "thread_message"));
+	sqlite.exec(shippedTableDdl("address"));
+	sqlite.exec(shippedTableDdl("envelope_address"));
+	sqlite.exec(shippedTableDdl("thread_message"));
 
 	const address = sqlite.prepare(
 		`INSERT INTO address (

package/src/repair/address-display-name.ts

@@ -48,6 +48,8 @@ export interface SiteResult {
 	readonly scanned: number;
 	readonly claiming: number;
 	readonly rewritten: number;
+	/** The keys of the rows rewritten, so a caller can follow them elsewhere. */
+	readonly rewrittenIds: readonly string[];
 }
 
 export interface DisplayNameReport {
@@ -112,7 +114,7 @@ export const sweepDisplayNames = async (
 	for (const site of SITES) {
 		const rows = await candidates(client, site);
 		let claiming = 0;
-		let rewritten = 0;
+		const rewrittenIds: string[] = [];
 
 		for (const row of rows) {
 			const name = row.name ?? "";
@@ -120,7 +122,9 @@ export const sweepDisplayNames = async (
 			if (stored === name) continue;
 			claiming += 1;
 			if (mode === "repair") {
-				rewritten += await rewrite(client, site, row, stored);
+				if ((await rewrite(client, site, row, stored)) > 0) {
+					rewrittenIds.push(row.id);
+				}
 			}
 		}
 
@@ -128,7 +132,8 @@ export const sweepDisplayNames = async (
 			table: site.table,
 			scanned: rows.length,
 			claiming,
-			rewritten,
+			rewritten: rewrittenIds.length,
+			rewrittenIds,
 		});
 	}
 

package/src/repair/junk-only-address.sqlite.test.ts

@@ -1,5 +1,4 @@
 import assert from "node:assert/strict";
-import { readFileSync } from "node:fs";
 import { after, before, beforeEach, describe, test } from "node:test";
 import { composeFolderRoleAppointmentName } from "@remit/data-ports/folder-role";
 import { CanonicalMailboxRole } from "@remit/domain-enums";
@@ -7,10 +6,7 @@ import Database from "better-sqlite3";
 import { drizzle } from "drizzle-orm/better-sqlite3";
 import { AccountSettingRepo } from "../repos/i4-account-setting.js";
 import { MailboxSpecialUseRepo } from "../repos/i4-mailbox-special-use.js";
-import {
-	applyMigration,
-	shippedTableDdl,
-} from "../test-shipped-sqlite-schema.js";
+import { applyMigrationSet } from "../test-shipped-sqlite-schema.js";
 import {
 	type JunkOnlyRepairClient,
 	type JunkOnlyRepairMode,
@@ -18,8 +14,6 @@ import {
 	sweepJunkOnlyAddresses,
 } from "./junk-only-address.js";
 
-const DDL_TAG = "0000_happy_roland_deschain";
-
 const clientOver = (sqlite: Database.Database): JunkOnlyRepairClient => ({
 	all: async (sql, params) => sqlite.prepare(sql).all(...params),
 	run: async (sql, params) => sqlite.prepare(sql).run(...params).changes,
@@ -53,11 +47,11 @@ describe("addresses standing only on mail in Junk", () => {
 		sqlite
 			.prepare(
 				`INSERT OR IGNORE INTO account (
-					account_id, account_config_id, username, email, imap_host,
-					imap_port, imap_tls, imap_start_tls, smtp_port, is_active,
+					account_id, account_config_id, username, email, synced_services,
+					imap_host, imap_port, imap_tls, imap_start_tls, smtp_port, is_active,
 					connection_state, created_at, updated_at
-				) VALUES (?, 'cfg-1', 'user', 'user@example.com', 'imap.example.com',
-					993, 1, 0, 465, 1, 'disconnected', 0, 0)`,
+				) VALUES (?, 'cfg-1', 'user', 'user@example.com', '["Mail"]',
+					'imap.example.com', 993, 1, 0, 465, 1, 'disconnected', 0, 0)`,
 			)
 			.run(accountId);
 	};
@@ -187,32 +181,10 @@ describe("addresses standing only on mail in Junk", () => {
 	before(() => {
 		sqlite = new Database(":memory:");
 		const db = drizzle(sqlite);
-		specialUse = new MailboxSpecialUseRepo(db as never);
+		specialUse = new MailboxSpecialUseRepo(db as never, db as never);
 		accountSetting = new AccountSettingRepo(db as never);
-		for (const table of [
-			"address",
-			"envelope_address",
-			"message",
-			"mailbox",
-			"mailbox_special_use_entry",
-			"account_setting",
-			"account",
-		]) {
-			sqlite.exec(shippedTableDdl(DDL_TAG, table));
-		}
-		// `mailbox` is read through the repo, which selects every column the
-		// entity declares — including the rename target added after this DDL.
-		applyMigration(sqlite, "0028_mailbox_pending_path");
-		applyMigration(sqlite, "0033_mailbox_sync_failure_reason");
-		sqlite.exec(
-			readFileSync(
-				new URL(
-					"../../../../npm-scripts/sqlite-address-sightings-index.sql",
-					import.meta.url,
-				),
-				"utf8",
-			),
-		);
+		applyMigrationSet(sqlite, "app");
+		applyMigrationSet(sqlite, "mail");
 	});
 
 	after(() => {

package/src/repair/stranded-sent-outbox.sqlite.test.ts

@@ -10,10 +10,7 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import Database from "better-sqlite3";
-import {
-	applyMigration,
-	shippedTableDdl,
-} from "../test-shipped-sqlite-schema.js";
+import { applyMigrationSet } from "../test-shipped-sqlite-schema.js";
 import {
 	STRANDED_AFTER_MILLIS,
 	type StrandedSentRepairClient,
@@ -82,11 +79,7 @@ describe("outbox rows stranded at sent", () => {
 
 	before(() => {
 		sqlite = new Database(":memory:");
-		sqlite.exec(
-			shippedTableDdl("0000_happy_roland_deschain", "outbox_message"),
-		);
-		sqlite.exec(shippedTableDdl("0013_low_harpoon", "outbox_attachment"));
-		applyMigration(sqlite, "0015_true_meggan");
+		applyMigrationSet(sqlite, "mail");
 	});
 
 	after(() => {

package/src/repos/account.conformance.sqlite.test.ts

@@ -13,7 +13,7 @@ accountRepositoryConformance({
 			accounts: accountTable,
 		});
 		close = closeDb;
-		return new AccountRepo(db as never);
+		return new AccountRepo(db as never, db as never);
 	},
 	teardown: async () => {
 		await close?.();

package/src/repos/calendar-default.sqlite.test.ts

@@ -1,7 +1,7 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import { provisionDefaultCalendar } from "@remit/calendar-service";
-import type { ICalendarUnitOfWork } from "@remit/data-ports";
+import type { ICalendarStore } from "@remit/data-ports";
 import type { Db } from "../db.js";
 import {
 	calendarEventIndexTable,
@@ -10,14 +10,14 @@ import {
 } from "../schema.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
 import { CalendarCollectionRepo } from "./calendar-collection.js";
-import { DrizzleCalendarUnitOfWork } from "./calendar-unit-of-work.js";
+import { DrizzleCalendarStore } from "./calendar-store.js";
 
 const ACCOUNT_CONFIG_ID = "account-config-default";
 
 describe("provisioning the default calendar", () => {
 	let db: Db<Record<string, unknown>>;
 	let close: () => Promise<void>;
-	let unitOfWork: ICalendarUnitOfWork;
+	let store: ICalendarStore;
 
 	before(async () => {
 		const created = await createSqliteTestDb({
@@ -27,7 +27,7 @@ describe("provisioning the default calendar", () => {
 		});
 		db = created.db as unknown as Db<Record<string, unknown>>;
 		close = created.close;
-		unitOfWork = new DrizzleCalendarUnitOfWork(db);
+		store = new DrizzleCalendarStore(db, db);
 	});
 
 	after(() => close());
@@ -35,7 +35,7 @@ describe("provisioning the default calendar", () => {
 	test("leaves one calendar behind when several first reads arrive together", async () => {
 		const provisioned = await Promise.all(
 			Array.from({ length: 8 }, () =>
-				provisionDefaultCalendar(unitOfWork, ACCOUNT_CONFIG_ID),
+				provisionDefaultCalendar(store, ACCOUNT_CONFIG_ID),
 			),
 		);
 
@@ -61,7 +61,7 @@ describe("provisioning the default calendar", () => {
 		await repo.bumpSyncSequence(ACCOUNT_CONFIG_ID, before.calendarId);
 
 		const again = await provisionDefaultCalendar(
-			unitOfWork,
+			store,
 			ACCOUNT_CONFIG_ID,
 			"A different name",
 		);