@remit/drizzle-service 0.0.69 → 0.0.71

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.69",
+  "version": "0.0.71",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/index.ts

@@ -1,6 +1,7 @@
 export { CreateFailedConflictError, NotFoundError } from "./error.js";
 export { CalendarCollectionRepo } from "./repos/calendar-collection.js";
 export { CalendarEventIndexRepo } from "./repos/calendar-event-index.js";
+export { CalendarFeedTokenRepo } from "./repos/calendar-feed-token.js";
 export { CalendarObjectRepo } from "./repos/calendar-object.js";
 export { CalendarSuggestionRepo } from "./repos/calendar-suggestion.js";
 export { DrizzleCalendarUnitOfWork } from "./repos/calendar-unit-of-work.js";

package/src/repos/calendar-feed-token.conformance.sqlite.test.ts

@@ -0,0 +1,23 @@
+import { calendarFeedTokenRepositoryConformance } from "@remit/data-ports/conformance";
+import { NotFoundError } from "../error.js";
+import { randomId } from "../id.js";
+import { calendarFeedTokenTable } from "../schema.js";
+import { createSqliteTestDb } from "../test-db-sqlite.js";
+import { CalendarFeedTokenRepo } from "./calendar-feed-token.js";
+
+let close: (() => Promise<void>) | undefined;
+
+calendarFeedTokenRepositoryConformance({
+	async createRepository() {
+		const { db, close: closeDb } = await createSqliteTestDb({
+			calendarFeedTokens: calendarFeedTokenTable,
+		});
+		close = closeDb;
+		return new CalendarFeedTokenRepo(db as never);
+	},
+	teardown: async () => {
+		await close?.();
+	},
+	makeId: () => randomId(),
+	isNotFoundError: (error) => error instanceof NotFoundError,
+});

package/src/repos/calendar-feed-token.ts

@@ -0,0 +1,97 @@
+import type {
+	CalendarFeedTokenItem,
+	ICalendarFeedTokenRepository,
+	PutCalendarFeedTokenInput,
+} from "@remit/data-ports";
+import { deriveCalendarFeedTokenId } from "@remit/data-ports/id";
+import { and, eq } from "drizzle-orm";
+import type { Db } from "../db.js";
+import { calendarFeedTokenTable } from "../schema.js";
+
+type DB = Db<Record<string, unknown>>;
+
+function rowToFeedToken(
+	row: typeof calendarFeedTokenTable.$inferSelect,
+): CalendarFeedTokenItem {
+	return {
+		feedTokenId: row.feedTokenId,
+		accountConfigId: row.accountConfigId,
+		calendarId: row.calendarId,
+		tokenHash: row.tokenHash,
+		createdAt: row.createdAt,
+		rotatedAt: row.rotatedAt,
+		updatedAt: row.updatedAt,
+	};
+}
+
+export class CalendarFeedTokenRepo implements ICalendarFeedTokenRepository {
+	constructor(private db: DB) {}
+
+	/**
+	 * One statement, so a rotation can never be observed as two live addresses.
+	 * The conflict target is the derived primary key, which is what makes "one
+	 * active token per calendar" a property of the schema rather than of this
+	 * method: `createdAt` is left where it was and `rotatedAt` takes the clock,
+	 * so the row itself says whether this address is the calendar's first.
+	 */
+	async put(input: PutCalendarFeedTokenInput): Promise<CalendarFeedTokenItem> {
+		const now = Date.now();
+		const [row] = await this.db
+			.insert(calendarFeedTokenTable)
+			.values({
+				feedTokenId: deriveCalendarFeedTokenId(input.calendarId),
+				accountConfigId: input.accountConfigId,
+				calendarId: input.calendarId,
+				tokenHash: input.tokenHash,
+				createdAt: now,
+				rotatedAt: 0,
+				updatedAt: now,
+			})
+			.onConflictDoUpdate({
+				target: [
+					calendarFeedTokenTable.accountConfigId,
+					calendarFeedTokenTable.feedTokenId,
+				],
+				set: { tokenHash: input.tokenHash, rotatedAt: now, updatedAt: now },
+			})
+			.returning();
+		return rowToFeedToken(row);
+	}
+
+	async findByCalendar(
+		accountConfigId: string,
+		calendarId: string,
+	): Promise<CalendarFeedTokenItem | null> {
+		const [row] = await this.db
+			.select()
+			.from(calendarFeedTokenTable)
+			.where(
+				and(
+					eq(calendarFeedTokenTable.accountConfigId, accountConfigId),
+					eq(calendarFeedTokenTable.calendarId, calendarId),
+				),
+			);
+		return row ? rowToFeedToken(row) : null;
+	}
+
+	async findByTokenHash(
+		tokenHash: string,
+	): Promise<CalendarFeedTokenItem | null> {
+		const [row] = await this.db
+			.select()
+			.from(calendarFeedTokenTable)
+			.where(eq(calendarFeedTokenTable.tokenHash, tokenHash));
+		return row ? rowToFeedToken(row) : null;
+	}
+
+	async delete(accountConfigId: string, calendarId: string): Promise<void> {
+		await this.db
+			.delete(calendarFeedTokenTable)
+			.where(
+				and(
+					eq(calendarFeedTokenTable.accountConfigId, accountConfigId),
+					eq(calendarFeedTokenTable.calendarId, calendarId),
+				),
+			);
+	}
+}

package/src/repos/calendar-migrations.sqlite.test.ts

@@ -0,0 +1,258 @@
+/**
+ * The calendar tables against the shape a deployment actually runs: every
+ * committed entity migration applied in journal order to an empty database,
+ * rather than a schema pushed from the drizzle table objects.
+ *
+ * Nothing else ran these files. Every calendar SQLite test builds its database
+ * with `createSqliteTestDb`, which regenerates the DDL from the table objects on
+ * each run, and the drift guard only diffs the latest snapshot — so a migration
+ * that fails to apply, or applies to a different shape than its snapshot claims,
+ * passes the whole suite while a self-host upgrade breaks. That is reader#73 one
+ * layer up: there the shipped DDL was wrong, here the shipped DDL was never
+ * executed at all.
+ *
+ * Everything below walks the journal, so a migration added tomorrow is covered
+ * the moment it is committed.
+ */
+
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import {
+	CalendarEventStatus,
+	CalendarInviteMethod,
+	CalendarSuggestionSource,
+	CalendarSuggestionState,
+	CalendarTransparency,
+	ZoneCertainty,
+} from "@remit/domain-enums";
+import Database from "better-sqlite3";
+import { drizzle } from "drizzle-orm/better-sqlite3";
+import type { Db } from "../db.js";
+import {
+	calendarEventIndexTable,
+	calendarObjectTable,
+	calendarSuggestionTable,
+	calendarTable,
+} from "../schema.js";
+import {
+	applyMigration,
+	migrationJournal,
+	migrationSnapshot,
+	migrationTagsOnDisk,
+} from "../test-shipped-sqlite-schema.js";
+import { CalendarCollectionRepo } from "./calendar-collection.js";
+import { CalendarEventIndexRepo } from "./calendar-event-index.js";
+import { CalendarObjectRepo } from "./calendar-object.js";
+import { CalendarSuggestionRepo } from "./calendar-suggestion.js";
+
+const ACCOUNT_CONFIG_ID = "account-config-shipped";
+const ROOT_SNAPSHOT_PREV_ID = "00000000-0000-0000-0000-000000000000";
+
+const ICAL_DATA = [
+	"BEGIN:VCALENDAR",
+	"VERSION:2.0",
+	"BEGIN:VEVENT",
+	"UID:shipped@example.test",
+	"DTSTART:20260901T080000Z",
+	"DTEND:20260901T090000Z",
+	"SUMMARY:Shipped-migrations round trip",
+	"END:VEVENT",
+	"END:VCALENDAR",
+	"",
+].join("\r\n");
+
+describe("the committed sqlite migration journal", () => {
+	const entries = migrationJournal();
+
+	test("names each migration once, in a strictly rising order", () => {
+		// Not "consecutive": drizzle-kit skipped 0009, and a withdrawn number is
+		// harmless. A repeated or reordered one is not — two branches generating
+		// against the same predecessor is the collision this pins.
+		for (const [position, entry] of entries.entries()) {
+			const previous = entries[position - 1];
+			assert.ok(
+				previous === undefined || entry.idx > previous.idx,
+				`journal entry ${entry.tag} does not follow ${previous?.tag}`,
+			);
+			assert.equal(
+				entry.tag.slice(0, 4),
+				String(entry.idx).padStart(4, "0"),
+				`journal entry ${entry.tag} is filed under idx ${entry.idx}`,
+			);
+		}
+	});
+
+	test("accounts for every migration file on disk", () => {
+		// An orphan .sql is a migration a deployment never runs; a journal entry
+		// with no file is one that crashes the migrator on the next upgrade.
+		assert.deepEqual(
+			migrationTagsOnDisk(),
+			entries.map((entry) => entry.tag).sort(),
+		);
+	});
+
+	test("chains every snapshot to its predecessor", () => {
+		for (const [position, entry] of entries.entries()) {
+			const snapshot = migrationSnapshot(entry.idx);
+			const previous = entries[position - 1];
+			assert.equal(
+				snapshot.prevId,
+				previous === undefined
+					? ROOT_SNAPSHOT_PREV_ID
+					: migrationSnapshot(previous.idx).id,
+				`snapshot ${entry.tag} does not follow ${previous?.tag ?? "the root"}`,
+			);
+		}
+	});
+});
+
+describe("the calendar tables under the shipped migrations", () => {
+	let sqlite: Database.Database;
+	let db: Db<Record<string, unknown>>;
+
+	before(() => {
+		sqlite = new Database(":memory:");
+		for (const entry of migrationJournal()) {
+			applyMigration(sqlite, entry.tag);
+		}
+		sqlite.pragma("foreign_keys = ON");
+		db = drizzle(sqlite, {
+			schema: {
+				calendars: calendarTable,
+				calendarObjects: calendarObjectTable,
+				calendarEventIndexes: calendarEventIndexTable,
+				calendarSuggestions: calendarSuggestionTable,
+			},
+		}) as unknown as Db<Record<string, unknown>>;
+	});
+
+	after(() => {
+		sqlite.close();
+	});
+
+	test("leaves all four calendar tables behind", () => {
+		const present = new Set(
+			sqlite
+				.prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
+				.all()
+				.map((row) => (row as { name: string }).name),
+		);
+
+		for (const table of [
+			"calendar",
+			"calendar_object",
+			"calendar_event_index",
+			"calendar_suggestion",
+		]) {
+			assert.ok(present.has(table), `${table} was never created`);
+		}
+	});
+
+	test("round-trips a collection", async () => {
+		const repo = new CalendarCollectionRepo(db);
+
+		const created = await repo.create({
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			urlSegment: "shipped",
+			displayName: "Shipped",
+		});
+
+		const fetched = await repo.get(ACCOUNT_CONFIG_ID, created.calendarId);
+		assert.equal(fetched.displayName, "Shipped");
+		assert.equal(fetched.urlSegment, "shipped");
+		assert.equal(fetched.syncSequence, 0);
+	});
+
+	test("round-trips an object and its occurrence", async () => {
+		const collection = await new CalendarCollectionRepo(db).create({
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			urlSegment: "objects",
+			displayName: "Objects",
+		});
+		const objects = new CalendarObjectRepo(db);
+
+		const written = await objects.put({
+			calendarId: collection.calendarId,
+			resourceName: "shipped.ics",
+			icalUid: "shipped@example.test",
+			icalData: ICAL_DATA,
+			etag: "b".repeat(64),
+			sequence: 0,
+			syncSequence: 1,
+			summary: "Shipped-migrations round trip",
+			dtStart: "2026-09-01T08:00:00+00:00",
+			dtEnd: "2026-09-01T09:00:00+00:00",
+			allDay: false,
+			zoneCertainty: ZoneCertainty.Explicit,
+			status: CalendarEventStatus.Confirmed,
+			transparency: CalendarTransparency.Opaque,
+			hasRecurrence: false,
+			expandedThrough: "",
+		});
+
+		const fetched = await objects.get(
+			collection.calendarId,
+			written.calendarObjectId,
+		);
+		assert.equal(fetched.icalData, ICAL_DATA);
+		assert.equal(fetched.summary, "Shipped-migrations round trip");
+		assert.equal(fetched.allDay, false);
+
+		const index = new CalendarEventIndexRepo(db);
+		await index.replaceForObject(
+			collection.calendarId,
+			written.calendarObjectId,
+			[
+				{
+					recurrenceId: "",
+					startAt: "2026-09-01T08:00:00Z",
+					endAt: "2026-09-01T09:00:00Z",
+					allDay: false,
+					// The three columns 0018 added. A migration that never ran leaves
+					// the repo selecting columns the table does not have.
+					summary: "Shipped-migrations round trip",
+					status: CalendarEventStatus.Confirmed,
+					transparency: CalendarTransparency.Opaque,
+				},
+			],
+		);
+
+		const occurrences = await index.listForObject(
+			collection.calendarId,
+			written.calendarObjectId,
+		);
+		assert.equal(occurrences.length, 1);
+		assert.equal(occurrences[0]?.startAt, "2026-09-01T08:00:00Z");
+		assert.equal(occurrences[0]?.summary, "Shipped-migrations round trip");
+		assert.equal(occurrences[0]?.status, CalendarEventStatus.Confirmed);
+		assert.equal(occurrences[0]?.transparency, CalendarTransparency.Opaque);
+	});
+
+	test("round-trips a suggestion", async () => {
+		const repo = new CalendarSuggestionRepo(db);
+
+		const written = await repo.put({
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			messageId: "message-1",
+			bodyPartId: "body-part-1",
+			icalUid: "invite@example.test",
+			sequence: 0,
+			method: CalendarInviteMethod.Request,
+			source: CalendarSuggestionSource.IcalendarPart,
+			summary: "Design review",
+			dtStart: "2026-09-01T10:00:00+02:00",
+			dtEnd: "2026-09-01T11:00:00+02:00",
+			allDay: false,
+			location: "",
+			organizer: "organizer@example.test",
+			zoneCertainty: ZoneCertainty.Explicit,
+			icalData: "BEGIN:VCALENDAR\r\nEND:VCALENDAR\r\n",
+		});
+
+		const fetched = await repo.get(ACCOUNT_CONFIG_ID, written.suggestionId);
+		assert.equal(fetched.summary, "Design review");
+		assert.equal(fetched.state, CalendarSuggestionState.Pending);
+		assert.equal(fetched.acceptedCalendarObjectId, "");
+		assert.equal(fetched.organizer, "organizer@example.test");
+	});
+});

package/src/repos/calendar-unit-of-work.ts

@@ -6,6 +6,7 @@ import type { Db } from "../db.js";
 import { runInTransaction } from "../tx.js";
 import { CalendarCollectionRepo } from "./calendar-collection.js";
 import { CalendarEventIndexRepo } from "./calendar-event-index.js";
+import { CalendarFeedTokenRepo } from "./calendar-feed-token.js";
 import { CalendarObjectRepo } from "./calendar-object.js";
 import { CalendarSuggestionRepo } from "./calendar-suggestion.js";
 
@@ -23,6 +24,7 @@ export class DrizzleCalendarUnitOfWork implements ICalendarUnitOfWork {
 				calendarObject: new CalendarObjectRepo(tx as never),
 				calendarEventIndex: new CalendarEventIndexRepo(tx as never),
 				calendarSuggestion: new CalendarSuggestionRepo(tx as never),
+				calendarFeedToken: new CalendarFeedTokenRepo(tx as never),
 			}),
 		);
 	}

package/src/schema.ts

@@ -11,6 +11,7 @@ import * as entities from "@remit/drizzle-sqlite-schema";
 export const calendarTable = entities.calendarCollections;
 export const calendarObjectTable = entities.calendarObjects;
 export const calendarEventIndexTable = entities.calendarEventIndexes;
+export const calendarFeedTokenTable = entities.calendarFeedTokens;
 export const calendarSuggestionTable = entities.calendarSuggestions;
 export const filterAnchorTable = entities.filterAnchors;
 export const filterTable = entities.filters;

package/src/test-shipped-sqlite-schema.ts

@@ -1,4 +1,4 @@
-import { readFileSync } from "node:fs";
+import { readdirSync, readFileSync } from "node:fs";
 import type Database from "better-sqlite3";
 
 // Read the committed SQLite entity migrations — the DDL a self-host deployment
@@ -17,6 +17,42 @@ const MIGRATIONS_DIR = new URL(
 export const migrationSql = (tag: string): string =>
 	readFileSync(new URL(`${tag}.sql`, MIGRATIONS_DIR), "utf8");
 
+export interface MigrationJournalEntry {
+	idx: number;
+	tag: string;
+}
+
+/** Every committed entity migration, in the order the migrator runs them. */
+export const migrationJournal = (): MigrationJournalEntry[] =>
+	(
+		JSON.parse(
+			readFileSync(new URL("meta/_journal.json", MIGRATIONS_DIR), "utf8"),
+		) as { entries: MigrationJournalEntry[] }
+	).entries;
+
+/** The `.sql` files present on disk, whatever the journal says about them. */
+export const migrationTagsOnDisk = (): string[] =>
+	readdirSync(MIGRATIONS_DIR)
+		.filter((name) => name.endsWith(".sql"))
+		.map((name) => name.slice(0, -".sql".length))
+		.sort();
+
+export interface MigrationSnapshot {
+	id: string;
+	prevId: string;
+}
+
+export const migrationSnapshot = (idx: number): MigrationSnapshot =>
+	JSON.parse(
+		readFileSync(
+			new URL(
+				`meta/${String(idx).padStart(4, "0")}_snapshot.json`,
+				MIGRATIONS_DIR,
+			),
+			"utf8",
+		),
+	) as MigrationSnapshot;
+
 /** The `CREATE TABLE` block for one table, as that migration declares it. */
 export const shippedTableDdl = (tag: string, table: string): string => {
 	const match = migrationSql(tag).match(

package/src/tx.sqlite.test.ts

@@ -0,0 +1,83 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { after, before, describe, test } from "node:test";
+import Database from "better-sqlite3";
+import { sql } from "drizzle-orm";
+import { drizzle } from "drizzle-orm/better-sqlite3";
+import type { Db } from "./db.js";
+import { runInTransaction } from "./tx.js";
+
+/**
+ * A write set that reads before it writes, against the file four processes
+ * share (RFC 036 D3).
+ *
+ * SQLite gives a deferred transaction its write lock at its first write, and by
+ * then a unit that read first is holding a snapshot. Another process committing
+ * in that gap makes the upgrade impossible, and SQLite refuses it at once —
+ * `busy_timeout` waits for a lock, not for a snapshot that is already stale. The
+ * unit then dies with "database is locked" however long that timeout is, which
+ * is what answered a calendar write 500 while the imap-worker was syncing.
+ */
+
+const PRAGMAS = [
+	"journal_mode = WAL",
+	"busy_timeout = 5000",
+	"synchronous = NORMAL",
+	"foreign_keys = ON",
+];
+
+let directory: string;
+let sqlite: Database.Database;
+let other: Database.Database;
+let db: Db<Record<string, never>>;
+
+before(() => {
+	directory = mkdtempSync(join(tmpdir(), "remit-tx-"));
+	const path = join(directory, "remit.db");
+
+	sqlite = new Database(path);
+	for (const pragma of PRAGMAS) sqlite.pragma(pragma);
+	sqlite.exec("CREATE TABLE row (id INTEGER PRIMARY KEY, v TEXT NOT NULL)");
+	sqlite.exec("INSERT INTO row (id, v) VALUES (1, 'read'), (2, 'other')");
+
+	// The other writer, as a second connection. It refuses instead of waiting,
+	// so the run finishes in its own time whichever of the two holds the lock.
+	other = new Database(path);
+	other.pragma("busy_timeout = 0");
+
+	db = drizzle(sqlite) as unknown as Db<Record<string, never>>;
+});
+
+after(() => {
+	other.close();
+	sqlite.close();
+	rmSync(directory, { recursive: true, force: true });
+});
+
+describe("a top-level sqlite unit that reads before it writes", () => {
+	test("commits through another connection's write landing between the two", async () => {
+		const outcome = await runInTransaction(db, async (tx) => {
+			await tx.get(sql`SELECT v FROM row WHERE id = 1`);
+
+			// The other process, arriving in the gap an async write set leaves
+			// open. Whether its own write lands is not the claim here — only that
+			// this unit can still finish the one it came to make.
+			try {
+				other.prepare("UPDATE row SET v = 'moved' WHERE id = 2").run();
+			} catch {
+				// Refused because this unit holds the write lock, which is the
+				// outcome that keeps the unit below alive.
+			}
+
+			await tx.run(sql`UPDATE row SET v = 'written' WHERE id = 1`);
+			return "committed";
+		});
+
+		assert.equal(outcome, "committed");
+		assert.deepEqual(sqlite.prepare("SELECT v FROM row WHERE id = 1").get(), {
+			v: "written",
+		});
+	});
+});

package/src/tx.ts

@@ -7,9 +7,19 @@ import type { Db } from "./db.js";
 //
 // better-sqlite3's native transaction runner rejects a callback that returns a
 // promise, and the repos' write sets are async, so drizzle's own
-// `db.transaction()` cannot be used. The callback is bracketed with a SAVEPOINT
-// instead — a savepoint opens a transaction when none is active and commits
-// when the outermost one is released.
+// `db.transaction()` cannot be used. A top-level unit is bracketed with
+// `BEGIN IMMEDIATE` / `COMMIT` instead, and a nested one with a SAVEPOINT.
+//
+// IMMEDIATE, not the deferred transaction a bare `BEGIN` or `SAVEPOINT` opens.
+// A deferred transaction takes its write lock at its first write, and a unit
+// that reads before it writes (every unit that checks a row before updating it)
+// has by then taken a read snapshot. If another process commits in between,
+// SQLite refuses the upgrade with SQLITE_BUSY_SNAPSHOT — and that refusal is
+// immediate, because `busy_timeout` cannot wait out a snapshot that is already
+// stale. Four processes write this file (RFC 036 D3), so the gap is real: it
+// answered a calendar write 500 while the imap-worker was syncing. Taking the
+// write lock at BEGIN closes it — an IMMEDIATE that meets another writer waits
+// out the `busy_timeout` and then proceeds, which is what that timeout is for.
 //
 // All writers on this backend share one better-sqlite3 connection (RFC 036 D3),
 // and every query runs synchronously, but an async callback still yields the
@@ -61,6 +71,27 @@ async function runSqliteSavepoint<TSchema extends Record<string, unknown>, T>(
 	}
 }
 
+async function runSqliteImmediate<TSchema extends Record<string, unknown>, T>(
+	db: Db<TSchema>,
+	fn: (tx: Db<TSchema>) => Promise<T>,
+): Promise<T> {
+	const runner = db as unknown as { run: (query: SQL) => unknown };
+	runner.run(sql.raw("BEGIN IMMEDIATE"));
+	try {
+		const result = await fn(db);
+		runner.run(sql.raw("COMMIT"));
+		return result;
+	} catch (error) {
+		try {
+			runner.run(sql.raw("ROLLBACK"));
+		} catch {
+			// A failed rollback must not mask the error that caused it; surface
+			// the original below.
+		}
+		throw error;
+	}
+}
+
 export async function runInTransaction<
 	TSchema extends Record<string, unknown>,
 	T,
@@ -72,7 +103,7 @@ export async function runInTransaction<
 	}
 
 	return serializeSqlite(() =>
-		inSqliteTx.run(true, () => runSqliteSavepoint(db, fn)),
+		inSqliteTx.run(true, () => runSqliteImmediate(db, fn)),
 	);
 }