@remit/drizzle-service 0.0.12 → 0.0.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.12",
+  "version": "0.0.13",
   "description": "Drizzle ORM service parameterized by dialect (Postgres / SQLite)",
   "type": "module",
   "main": "src/index.ts",

package/src/repos/i4-mailbox.sqlite.test.ts

@@ -1,38 +1,16 @@
 import assert from "node:assert/strict";
 import { randomUUID } from "node:crypto";
-import { readFileSync } from "node:fs";
 import { after, before, describe, test } from "node:test";
 import Database from "better-sqlite3";
 import { drizzle } from "drizzle-orm/better-sqlite3";
 import { mailboxTable } from "../schema.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
+import {
+	applyMigration,
+	shippedTableDdl,
+} from "../test-shipped-sqlite-schema.js";
 import { MailboxRepo } from "./i4-mailbox.js";
 
-/**
- * The `mailbox` DDL as it actually ships, read from the committed migration
- * rather than pushed from the drizzle table objects.
- *
- * The two disagree: the table object declares `highest_modseq` as text, the
- * shipped migration still declares it `integer` (reader#73). Every other
- * SQLite test in this package runs against the pushed shape, so none of them
- * has ever exercised the one deployments run on — and SQLite hands a column
- * with numeric affinity back as a number regardless of what the schema says.
- * Reading the committed file keeps this test honest as the migration changes.
- */
-const shippedMailboxDdl = (): string => {
-	const sql = readFileSync(
-		new URL(
-			"../../../../deploy/vps/migrations-sqlite/entities/0000_happy_roland_deschain.sql",
-			import.meta.url,
-		),
-		"utf8",
-	);
-	const match = sql.match(/CREATE TABLE `mailbox` \([\s\S]*?\n\);/);
-	if (!match)
-		throw new Error("mailbox DDL not found in the committed migration");
-	return match[0];
-};
-
 function makeMailboxInput(accountId: string, fullPath = "INBOX") {
 	return {
 		accountId,
@@ -90,13 +68,25 @@ describe("MailboxRepo (sqlite)", () => {
 	});
 });
 
-describe("MailboxRepo (sqlite, shipped column shape)", () => {
+/**
+ * The same repository against the shape a deployment actually runs: the
+ * committed migrations applied in order, rather than the schema pushed from the
+ * drizzle table objects.
+ *
+ * The two used to disagree — `highest_modseq` shipped as `integer` while the
+ * table object said `text` (reader#73) — and SQLite hands a column with numeric
+ * affinity back as a number whatever the declared type, so the repo returned a
+ * number where its own type said string. Reading the committed files keeps this
+ * honest as the migration set changes.
+ */
+describe("MailboxRepo (sqlite, shipped migrations)", () => {
 	let close: () => Promise<void>;
 	let repo: MailboxRepo;
 
-	before(async () => {
+	before(() => {
 		const sqlite = new Database(":memory:");
-		sqlite.exec(shippedMailboxDdl());
+		sqlite.exec(shippedTableDdl("0000_happy_roland_deschain", "mailbox"));
+		applyMigration(sqlite, "0002_highest_modseq_text");
 		const db = drizzle(sqlite, { schema: { mailbox: mailboxTable } });
 		repo = new MailboxRepo(db as never);
 		close = async () => {
@@ -108,11 +98,14 @@ describe("MailboxRepo (sqlite, shipped column shape)", () => {
 		await close();
 	});
 
-	test("reads the sync cursor back as a string, whatever the column stores", async () => {
-		// A plain-digit cursor lands in a column with numeric affinity and comes
-		// back a number. Callers compare the value they wrote against the value
-		// they read — `"900" === 900` is false — so an unnormalised read makes a
-		// stalled cursor undetectable.
+	test("declares highest_modseq as text", () => {
+		assert.match(
+			shippedTableDdl("0002_highest_modseq_text", "__new_mailbox"),
+			/`highest_modseq` text NOT NULL/,
+		);
+	});
+
+	test("reads a plain-digit cursor back as a string", async () => {
 		const accountId = randomUUID();
 		const created = await repo.create({
 			...makeMailboxInput(accountId),
@@ -123,10 +116,9 @@ describe("MailboxRepo (sqlite, shipped column shape)", () => {
 
 		const fetched = await repo.get(accountId, created.mailboxId);
 		assert.strictEqual(fetched.highestModseq, "900");
-		assert.strictEqual(fetched.highestModseq === "900", true);
 	});
 
-	test("keeps a resumable cursor intact through the same column", async () => {
+	test("keeps a resumable cursor intact", async () => {
 		const accountId = randomUUID();
 		const created = await repo.create({
 			...makeMailboxInput(accountId, "Archive"),
@@ -136,4 +128,17 @@ describe("MailboxRepo (sqlite, shipped column shape)", () => {
 		const fetched = await repo.get(accountId, created.mailboxId);
 		assert.strictEqual(fetched.highestModseq, "900:149");
 	});
+
+	test("round-trips a cursor above 2^53 with its exact digits", async () => {
+		const accountId = randomUUID();
+		const modseq = "18446744073709551615";
+		const created = await repo.create({
+			...makeMailboxInput(accountId, "Sent"),
+			highestModseq: modseq,
+		});
+
+		assert.strictEqual(created.highestModseq, modseq);
+		const fetched = await repo.get(accountId, created.mailboxId);
+		assert.strictEqual(fetched.highestModseq, modseq);
+	});
 });

package/src/repos/i4-mailbox.ts

@@ -33,15 +33,7 @@ export function rowToMailbox(
 		fullPath: row.fullPath,
 		uidValidity: row.uidValidity,
 		uidNext: row.uidNext,
-		// SQLite returns a column with numeric affinity as a number whatever the
-		// schema declares, and the shipped self-host migration still declares
-		// this one `integer` (reader#73). A cursor read back as a number is not
-		// merely awkward to parse: `"900" === 900` is false, so code comparing
-		// the value it just wrote against the value it read would conclude
-		// nothing had changed — which is how a stalled cursor goes unreported.
-		// Normalising here means every consumer sees the declared type instead
-		// of each one guarding separately.
-		highestModseq: String(row.highestModseq),
+		highestModseq: row.highestModseq,
 		messageCount: row.messageCount,
 		unseenCount: row.unseenCount,
 		deletedCount: row.deletedCount,

package/src/test-shipped-sqlite-schema.ts

@@ -0,0 +1,39 @@
+import { readFileSync } from "node:fs";
+import type Database from "better-sqlite3";
+
+// Read the committed SQLite entity migrations — the DDL a self-host deployment
+// actually runs — so a test can exercise the shipped shape instead of the one
+// `pushSQLiteSchema` derives from the drizzle table objects. The two are
+// generated from the same entities but only the pushed one is regenerated on
+// every run, so drift between them is invisible to any test that pushes
+// (reader#73). Reading the files means a test fails when they drift, and
+// tracks them when they change.
+
+const MIGRATIONS_DIR = new URL(
+	"../../../deploy/vps/migrations-sqlite/entities/",
+	import.meta.url,
+);
+
+export const migrationSql = (tag: string): string =>
+	readFileSync(new URL(`${tag}.sql`, MIGRATIONS_DIR), "utf8");
+
+/** The `CREATE TABLE` block for one table, as that migration declares it. */
+export const shippedTableDdl = (tag: string, table: string): string => {
+	const match = migrationSql(tag).match(
+		new RegExp(`CREATE TABLE \`${table}\` \\([\\s\\S]*?\\n\\);`),
+	);
+	if (!match) {
+		throw new Error(`${table} DDL not found in migration ${tag}`);
+	}
+	return match[0];
+};
+
+/** Run every statement of a committed migration against an open database. */
+export const applyMigration = (
+	sqlite: Database.Database,
+	tag: string,
+): void => {
+	for (const statement of migrationSql(tag).split("--> statement-breakpoint")) {
+		sqlite.exec(statement);
+	}
+};

package/src/vps-migrations-drift.sqlite.test.ts

@@ -0,0 +1,101 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { describe, test } from "node:test";
+import {
+	generateSQLiteDrizzleJson,
+	generateSQLiteMigration,
+} from "drizzle-kit/api";
+
+/**
+ * Every committed SQLite migration set must describe the schema its drizzle
+ * source declares.
+ *
+ * Nothing else checked this. The previous guard shelled out to
+ * `npm-scripts/check-vps-migrations.mjs`, which is stripped from this tree, so
+ * it skipped on every run. Every other SQLite test pushes its schema from the
+ * drizzle table objects, so a migration set that has fallen behind those
+ * objects still passes the whole suite while deployments run the stale shape —
+ * which is how `mailbox.highest_modseq` shipped as `integer` for as long as it
+ * did (reader#73). SQLite column types are affinity rather than constraint, so
+ * a wrong declaration corrupts values instead of rejecting them.
+ *
+ * This is the same diff `drizzle-kit generate` takes, run in-process against
+ * each set's latest committed snapshot. A non-empty result means someone
+ * changed a schema without regenerating:
+ *
+ *   npx drizzle-kit generate --config <the config named below>
+ *
+ * The schema and output paths come from the configs themselves, so a set stays
+ * covered when either moves.
+ */
+
+const REPO_ROOT = new URL("../../../", import.meta.url);
+
+const CONFIGS = [
+	"deploy/vps/migrate/drizzle.entities.sqlite.config.ts",
+	"deploy/vps/migrate/drizzle.auth.sqlite.config.ts",
+];
+
+type DrizzleConfig = { schema: string; out: string };
+
+const loadConfig = async (path: string): Promise<DrizzleConfig> =>
+	(
+		(await import(new URL(path, REPO_ROOT).href)) as {
+			default: DrizzleConfig;
+		}
+	).default;
+
+const latestSnapshot = (out: string): Record<string, unknown> => {
+	const dir = new URL(`${out}/`, REPO_ROOT);
+	const journal = JSON.parse(
+		readFileSync(new URL("meta/_journal.json", dir), "utf8"),
+	) as { entries: Array<{ idx: number }> };
+	const idx = Math.max(...journal.entries.map((entry) => entry.idx));
+	return JSON.parse(
+		readFileSync(
+			new URL(`meta/${String(idx).padStart(4, "0")}_snapshot.json`, dir),
+			"utf8",
+		),
+	) as Record<string, unknown>;
+};
+
+describe("committed sqlite migrations", () => {
+	for (const configPath of CONFIGS) {
+		test(`${configPath} — the set matches its schema`, async () => {
+			const config = await loadConfig(configPath);
+			const schema = (await import(
+				new URL(config.schema, REPO_ROOT).href
+			)) as Record<string, unknown>;
+
+			const drift = await generateSQLiteMigration(
+				latestSnapshot(config.out) as unknown as Parameters<
+					typeof generateSQLiteMigration
+				>[0],
+				await generateSQLiteDrizzleJson(schema),
+			);
+
+			assert.deepEqual(
+				drift,
+				[],
+				`the committed migrations in ${config.out} no longer match ${config.schema} — regenerate them with drizzle-kit generate`,
+			);
+		});
+	}
+
+	test("declare mailbox.highest_modseq as text", async () => {
+		const { out } = await loadConfig(CONFIGS[0]);
+		const snapshot = latestSnapshot(out) as {
+			tables: Record<
+				string,
+				{ columns: Record<string, { type: string; notNull: boolean }> }
+			>;
+		};
+		const column = snapshot.tables.mailbox.columns.highest_modseq;
+
+		// A mod-sequence is an unsigned 63-bit value carried as decimal digits and
+		// parsed to BigInt, and the stored cursor also takes a `<group>:<uid>`
+		// form. Numeric affinity would hand both back as numbers.
+		assert.equal(column.type, "text");
+		assert.equal(column.notNull, true);
+	});
+});

package/src/repos/message-flag-wire-format-migration.sqlite.test.ts

@@ -1,186 +0,0 @@
-import assert from "node:assert/strict";
-import { readFileSync } from "node:fs";
-import { afterEach, beforeEach, describe, test } from "node:test";
-import { MessageSystemFlag } from "@remit/domain-enums";
-import type Database from "better-sqlite3";
-import {
-	type MessageDataSchema,
-	messageDataSchema,
-} from "../schema/message-data.js";
-import { createSqliteTestDb, type SqliteTestDb } from "../test-db-sqlite.js";
-import { DrizzleMessageFlagRepository } from "./message-flag.js";
-
-/**
- * The one-time `message_flag.flag_name` rename shipped for issue #64, run
- * against a real database rather than a hand-copied twin: the SQL is read
- * from the committed migration so this test fails if that file drifts.
- *
- * Before the generated-enum fix, `MessageSystemFlag.Seen` held `Seen` and
- * every row landed under the unprefixed spelling. `hasFlag` is an exact
- * string match, so once the corrected code queries `\Seen` those rows go
- * invisible — which is a re-star on unstar, and a silent no-op on
- * mark-as-unread.
- */
-const MIGRATION_SQL = readFileSync(
-	new URL(
-		"../../../../deploy/vps/migrations-sqlite/entities/0002_system_flag_wire_format.sql",
-		import.meta.url,
-	),
-	"utf8",
-);
-
-const applyMigration = (sqlite: Database.Database): void => {
-	for (const statement of MIGRATION_SQL.split("--> statement-breakpoint")) {
-		sqlite.exec(statement);
-	}
-};
-
-const MESSAGE_ID = "00000000-0000-0000-6464-000000000001";
-const OTHER_MESSAGE_ID = "00000000-0000-0000-6464-000000000002";
-
-describe("message_flag wire-format migration (issue #64, sqlite)", () => {
-	let db: SqliteTestDb<MessageDataSchema>;
-	let sqlite: Database.Database;
-	let close: () => Promise<void>;
-	let repo: DrizzleMessageFlagRepository;
-
-	const insertLegacyRow = (messageId: string, flagName: string): void => {
-		sqlite
-			.prepare(
-				`INSERT INTO message_flag
-				   (message_flag_id, message_id, flag_name, set_at, created_at, updated_at)
-				 VALUES (?, ?, ?, 1000, 1000, 1000)`,
-			)
-			.run(`${messageId}:${flagName}`, messageId, flagName);
-	};
-
-	const flagNames = (messageId: string): string[] =>
-		(
-			sqlite
-				.prepare(
-					"SELECT flag_name FROM message_flag WHERE message_id = ? ORDER BY flag_name",
-				)
-				.all(messageId) as Array<{ flag_name: string }>
-		).map((r) => r.flag_name);
-
-	beforeEach(async () => {
-		({ db, sqlite, close } = await createSqliteTestDb(messageDataSchema));
-		repo = new DrizzleMessageFlagRepository(
-			db as unknown as ConstructorParameters<
-				typeof DrizzleMessageFlagRepository
-			>[0],
-		);
-	});
-
-	afterEach(async () => {
-		await close();
-	});
-
-	test("a row written under the old spelling is found by the corrected enum", async () => {
-		insertLegacyRow(MESSAGE_ID, "Flagged");
-		assert.equal(
-			await repo.hasFlag(MESSAGE_ID, MessageSystemFlag.Flagged),
-			false,
-		);
-
-		applyMigration(sqlite);
-
-		assert.equal(
-			await repo.hasFlag(MESSAGE_ID, MessageSystemFlag.Flagged),
-			true,
-		);
-		assert.deepEqual(flagNames(MESSAGE_ID), ["\\Flagged"]);
-	});
-
-	test("unstarring a migrated message removes the star instead of re-adding it", async () => {
-		insertLegacyRow(MESSAGE_ID, "Flagged");
-		applyMigration(sqlite);
-
-		// The toggleFlagged decision: hasFlag true => operation "remove".
-		const hadFlag = await repo.hasFlag(MESSAGE_ID, MessageSystemFlag.Flagged);
-		assert.equal(hadFlag, true, "pre-migration row must read as starred");
-
-		await repo.removeFlag(MESSAGE_ID, MessageSystemFlag.Flagged);
-		assert.equal(
-			await repo.hasFlag(MESSAGE_ID, MessageSystemFlag.Flagged),
-			false,
-		);
-		assert.deepEqual(flagNames(MESSAGE_ID), []);
-	});
-
-	test("mark-as-unread on a migrated message clears the read state", async () => {
-		insertLegacyRow(MESSAGE_ID, "Seen");
-		applyMigration(sqlite);
-
-		assert.equal(await repo.hasFlag(MESSAGE_ID, MessageSystemFlag.Seen), true);
-		await repo.removeFlag(MESSAGE_ID, MessageSystemFlag.Seen);
-		assert.equal(await repo.hasFlag(MESSAGE_ID, MessageSystemFlag.Seen), false);
-	});
-
-	test("renames every RFC 9051 system flag", () => {
-		for (const name of ["Seen", "Answered", "Flagged", "Deleted", "Draft"]) {
-			insertLegacyRow(MESSAGE_ID, name);
-		}
-
-		applyMigration(sqlite);
-
-		assert.deepEqual(
-			flagNames(MESSAGE_ID).sort(),
-			Object.values(MessageSystemFlag).slice().sort(),
-		);
-	});
-
-	test("leaves keyword and custom flags untouched", () => {
-		insertLegacyRow(MESSAGE_ID, "$Forwarded");
-		insertLegacyRow(MESSAGE_ID, "$Junk");
-		insertLegacyRow(MESSAGE_ID, "project-invoices");
-
-		applyMigration(sqlite);
-
-		assert.deepEqual(flagNames(MESSAGE_ID), [
-			"$Forwarded",
-			"$Junk",
-			"project-invoices",
-		]);
-	});
-
-	test("is idempotent — a second and third run change nothing", () => {
-		insertLegacyRow(MESSAGE_ID, "Seen");
-		insertLegacyRow(OTHER_MESSAGE_ID, "Flagged");
-		insertLegacyRow(OTHER_MESSAGE_ID, "$Forwarded");
-
-		applyMigration(sqlite);
-		const afterFirst = [
-			...flagNames(MESSAGE_ID),
-			...flagNames(OTHER_MESSAGE_ID),
-		];
-
-		applyMigration(sqlite);
-		applyMigration(sqlite);
-
-		assert.deepEqual(
-			[...flagNames(MESSAGE_ID), ...flagNames(OTHER_MESSAGE_ID)],
-			afterFirst,
-		);
-		assert.deepEqual(afterFirst, ["\\Seen", "$Forwarded", "\\Flagged"]);
-	});
-
-	test("collapses a message already carrying both spellings to one row", () => {
-		insertLegacyRow(MESSAGE_ID, "Seen");
-		insertLegacyRow(MESSAGE_ID, "\\Seen");
-
-		applyMigration(sqlite);
-
-		assert.deepEqual(flagNames(MESSAGE_ID), ["\\Seen"]);
-	});
-
-	test("does not touch other messages' rows", () => {
-		insertLegacyRow(MESSAGE_ID, "Seen");
-		insertLegacyRow(OTHER_MESSAGE_ID, "Seen");
-
-		applyMigration(sqlite);
-
-		assert.deepEqual(flagNames(MESSAGE_ID), ["\\Seen"]);
-		assert.deepEqual(flagNames(OTHER_MESSAGE_ID), ["\\Seen"]);
-	});
-});

package/src/vps-migrations-drift.test.ts

@@ -1,28 +0,0 @@
-import assert from "node:assert/strict";
-import { execFileSync } from "node:child_process";
-import { existsSync } from "node:fs";
-import { dirname, resolve } from "node:path";
-import { test } from "node:test";
-import { fileURLToPath } from "node:url";
-
-const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
-
-// The migration-check script is stripped from the open-core tree; skip there and
-// run where it ships.
-const hasCheckScript = existsSync(
-	resolve(repoRoot, "npm-scripts/check-vps-migrations.mjs"),
-);
-
-// Fails when the committed VPS migrations (deploy/vps/migrations/*) no longer
-// produce the schema drizzle would generate from the entity + auth schemas.
-// See npm-scripts/check-vps-migrations.mjs for the mechanism.
-test("committed VPS migrations match the drizzle schema", {
-	skip: !hasCheckScript,
-}, () => {
-	assert.doesNotThrow(() => {
-		execFileSync("node", ["npm-scripts/check-vps-migrations.mjs", "--check"], {
-			cwd: repoRoot,
-			stdio: "inherit",
-		});
-	}, "committed VPS migrations are stale — run `npm run migrations:generate`");
-});