@remit/drizzle-service 0.0.99 → 0.0.101

package/src/repos/search-index-shape.test.ts

@@ -1,82 +0,0 @@
-/**
- * The guard that keeps an instance installed by an earlier build from running a
- * predicate its index cannot answer.
- *
- * `CREATE VIRTUAL TABLE IF NOT EXISTS` is a no-op against an existing table, so
- * adding an indexed column changes nothing on a database that already has the
- * old one — and a MATCH naming that column raises rather than missing. The
- * migrator compares the shipped shape with the installed one and rebuilds.
- */
-import assert from "node:assert/strict";
-import { readFileSync } from "node:fs";
-import { describe, test } from "node:test";
-import {
-	searchIndexColumns,
-	searchIndexShapeIsCurrent,
-} from "./search-index-shape.js";
-
-const shipped = readFileSync(
-	new URL("../../../../npm-scripts/sqlite-search-index.sql", import.meta.url),
-	"utf8",
-);
-
-const installed = (columns: string[]): string =>
-	`CREATE VIRTUAL TABLE thread_message_fts USING fts5(${columns.join(", ")}, content='thread_message', content_rowid='rowid', tokenize='trigram remove_diacritics 1')`;
-
-describe("searchIndexColumns", () => {
-	test("reads the indexed columns and leaves the options out", () => {
-		assert.deepEqual(searchIndexColumns(installed(["subject", "sender"])), [
-			"subject",
-			"sender",
-		]);
-	});
-
-	test("reads the shipped index, triggers and all", () => {
-		assert.deepEqual(searchIndexColumns(shipped), [
-			"subject",
-			"sender",
-			"body",
-		]);
-	});
-
-	test("a table that is not an fts5 index has no columns", () => {
-		assert.deepEqual(searchIndexColumns("CREATE TABLE t (a, b)"), []);
-	});
-});
-
-describe("searchIndexShapeIsCurrent", () => {
-	test("the shipped index matches itself", () => {
-		assert.equal(searchIndexShapeIsCurrent(shipped, shipped), true);
-	});
-
-	// The upgrade this exists for: an instance carrying the two-column index the
-	// previous build installed must be rebuilt, or every search raises.
-	test("an index missing an indexed column is stale", () => {
-		assert.equal(
-			searchIndexShapeIsCurrent(installed(["subject", "sender"]), shipped),
-			false,
-		);
-	});
-
-	test("an index carrying every shipped column is current", () => {
-		assert.equal(
-			searchIndexShapeIsCurrent(
-				installed(["subject", "sender", "body"]),
-				shipped,
-			),
-			true,
-		);
-	});
-
-	// An extra column indexes text nothing reads. It costs space and matches
-	// nothing wrong, so it is not a reason to drop and re-tokenize the table.
-	test("an index carrying more than the shipped columns is current", () => {
-		assert.equal(
-			searchIndexShapeIsCurrent(
-				installed(["subject", "sender", "body", "labels"]),
-				shipped,
-			),
-			true,
-		);
-	});
-});

package/src/repos/search-index-shape.ts

@@ -1,56 +0,0 @@
-/**
- * Whether an installed FTS5 search index carries the columns the shipped one
- * declares.
- *
- * `CREATE VIRTUAL TABLE IF NOT EXISTS` is a no-op against a table that already
- * exists, so a database installed by an earlier build keeps that build's
- * columns for good. A predicate naming a column that build never indexed does
- * not degrade — SQLite raises, and every search on that instance fails — so the
- * migrator compares the two shapes and rebuilds where they differ.
- *
- * Both sides are read from DDL rather than from a hand-kept list: the shipped
- * side is the committed `.sql`, the installed side is what `sqlite_master`
- * holds, and neither can drift from what is actually there.
- */
-
-/** The text between the parentheses of a `USING fts5(...)` clause. */
-const fts5Arguments = (ddl: string): string => {
-	const clause = ddl.search(/using\s+fts5\s*\(/i);
-	if (clause === -1) return "";
-	const open = ddl.indexOf("(", clause);
-	let depth = 0;
-	for (let index = open; index < ddl.length; index++) {
-		const char = ddl[index];
-		if (char === "(") depth++;
-		if (char !== ")") continue;
-		depth--;
-		if (depth === 0) return ddl.slice(open + 1, index);
-	}
-	return "";
-};
-
-/**
- * The indexed column names an FTS5 table declares. An argument carrying an `=`
- * is an option (`content=`, `tokenize=`), not a column, and anything quoted or
- * otherwise unusual is left out rather than guessed at — a name this cannot
- * read is a name the comparison must not claim is missing.
- */
-export const searchIndexColumns = (ddl: string): string[] =>
-	fts5Arguments(ddl)
-		.split(",")
-		.map((argument) => argument.trim())
-		.filter((argument) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(argument));
-
-/**
- * Whether `installed` indexes every column `shipped` does. A superset passes:
- * an extra column indexes text no predicate reads, which costs space and
- * matches nothing wrong.
- */
-export const searchIndexShapeIsCurrent = (
-	installed: string,
-	shipped: string,
-): boolean => {
-	const columns = new Set(searchIndexColumns(installed));
-	const expected = searchIndexColumns(shipped);
-	return expected.length > 0 && expected.every((column) => columns.has(column));
-};

package/src/repos/serialized-writes.sqlite.test.ts

@@ -1,199 +0,0 @@
-import assert from "node:assert/strict";
-import { after, before, describe, test } from "node:test";
-import { eq } from "drizzle-orm";
-import type { Db } from "../db.js";
-import {
-	type MessageDataSchema,
-	messageDataSchema,
-	outboxTable,
-} from "../schema/message-data.js";
-import { createSqliteTestDb, type SqliteTestDb } from "../test-db-sqlite.js";
-import { runInTransaction, serializeSqliteWrites } from "../tx.js";
-
-// The RFC 036 D3 hard requirement: every write on the shared better-sqlite3
-// connection routes through the wave-1 serialization, so a plain single
-// statement can never land inside another unit's open SAVEPOINT (the
-// uncommitted-read / rollback hazard named on PR #1310). The db handle repos
-// hold is `serializeSqliteWrites(db)` — unserialized access is impossible by
-// construction. The `outbox` table stands in for any write target here (few
-// required columns, already dialect-cast for the repos).
-
-const row = (id: string) => ({
-	id,
-	messageId: `msg-${id}`,
-	event: "message.body_synced" as const,
-	payload: { messageId: `msg-${id}` },
-	createdAt: new Date(),
-});
-
-const nextTick = (): Promise<void> =>
-	new Promise((resolve) => setImmediate(resolve));
-
-describe("serialized SQLite writes (RFC 036 D3)", () => {
-	let raw: SqliteTestDb<MessageDataSchema>;
-	let db: Db<MessageDataSchema>;
-	let close: () => Promise<void>;
-
-	before(async () => {
-		({ db: raw, close } =
-			await createSqliteTestDb<MessageDataSchema>(messageDataSchema));
-		db = serializeSqliteWrites(raw);
-	});
-
-	after(async () => {
-		await close();
-	});
-
-	const byId = (id: string) =>
-		raw.select().from(outboxTable).where(eq(outboxTable.id, id));
-
-	test("a single-statement write does not interleave into an open transaction", async () => {
-		const order: string[] = [];
-		let release: () => void = () => {};
-		const barrier = new Promise<void>((resolve) => {
-			release = resolve;
-		});
-
-		// A top-level transaction: insert TX_ROW, park on a barrier (yielding the
-		// event loop with the SAVEPOINT open), then throw so the whole unit rolls
-		// back.
-		const txDone = runInTransaction(db, async (tx) => {
-			await tx.insert(outboxTable).values(row("tx-row"));
-			order.push("tx-insert");
-			await barrier;
-			order.push("tx-throw");
-			throw new Error("rollback");
-		}).catch(() => {
-			order.push("tx-rolledback");
-		});
-
-		// Let the transaction reach the barrier before the single write is issued.
-		await nextTick();
-		await nextTick();
-
-		const singleWrite = db
-			.insert(outboxTable)
-			.values(row("single-row"))
-			.then(() => {
-				order.push("single-committed");
-			});
-
-		// While the transaction holds the write queue, the single write must not
-		// have executed — it is queued behind the open unit, not interleaved into
-		// its savepoint.
-		await nextTick();
-		await nextTick();
-		assert.equal(
-			order.includes("single-committed"),
-			false,
-			"single write ran while a top-level transaction was open",
-		);
-
-		release();
-		await txDone;
-		await singleWrite;
-
-		assert.deepEqual(order, [
-			"tx-insert",
-			"tx-throw",
-			"tx-rolledback",
-			"single-committed",
-		]);
-
-		// The transaction rolled back; the single write committed independently —
-		// it was never part of the rolled-back savepoint.
-		assert.equal(
-			(await byId("tx-row")).length,
-			0,
-			"transaction must roll back",
-		);
-		assert.equal(
-			(await byId("single-row")).length,
-			1,
-			"single write must commit, not be swallowed by the rolled-back transaction",
-		);
-	});
-
-	test("concurrent single-statement writes all commit", async () => {
-		const ids = Array.from({ length: 20 }, (_, i) => `w-${i}`);
-		await Promise.all(ids.map((id) => db.insert(outboxTable).values(row(id))));
-		for (const id of ids) {
-			assert.equal((await byId(id)).length, 1, `${id} must be committed`);
-		}
-	});
-
-	test("the explicit .run() executor is also serialized, not just await", async () => {
-		// A write executed via `.run()` (not `await`) must still go through the
-		// queue — the hardened wrapper intercepts every terminal executor.
-		const builder = db
-			.insert(outboxTable)
-			.values(row("run-row")) as unknown as {
-			run: () => Promise<unknown>;
-		};
-		await builder.run();
-		assert.equal((await byId("run-row")).length, 1, "run() committed the row");
-	});
-
-	test("the .values() executor is serialized, not run inline", async () => {
-		// `values` doubles as the insert chain method and (on a runnable with
-		// `.returning()`) a terminal executor. The executor form must queue like
-		// every other terminal — issued while a transaction is parked, it must
-		// wait for the unit to finish instead of interleaving into its savepoint.
-		const order: string[] = [];
-		let release: () => void = () => {};
-		const barrier = new Promise<void>((resolve) => {
-			release = resolve;
-		});
-
-		const txDone = runInTransaction(db, async (tx) => {
-			await tx.insert(outboxTable).values(row("values-tx-row"));
-			order.push("tx-insert");
-			await barrier;
-		});
-
-		await nextTick();
-		await nextTick();
-
-		const runnable = db
-			.insert(outboxTable)
-			.values(row("values-row"))
-			.returning() as unknown as { values: () => Promise<unknown[]> };
-		const executorWrite = runnable.values().then(() => {
-			order.push("values-committed");
-		});
-
-		await nextTick();
-		await nextTick();
-		assert.equal(
-			order.includes("values-committed"),
-			false,
-			".values() executor ran while a top-level transaction was open",
-		);
-
-		release();
-		await txDone;
-		await executorWrite;
-		assert.deepEqual(order, ["tx-insert", "values-committed"]);
-		assert.equal((await byId("values-row")).length, 1);
-	});
-
-	test("writes inside a transaction still commit atomically", async () => {
-		await runInTransaction(db, async (tx) => {
-			await tx.insert(outboxTable).values(row("a"));
-			await tx
-				.update(outboxTable)
-				.set({ event: "message.moved" })
-				.where(eq(outboxTable.id, "a"));
-		});
-		const [committed] = await byId("a");
-		assert.equal(committed?.event, "message.moved");
-
-		await assert.rejects(() =>
-			runInTransaction(db, async (tx) => {
-				await tx.insert(outboxTable).values(row("b"));
-				throw new Error("boom");
-			}),
-		);
-		assert.equal((await byId("b")).length, 0, "transaction rollback drops b");
-	});
-});

package/src/repos/unit-of-work.sqlite.test.ts

@@ -1,138 +0,0 @@
-import assert from "node:assert/strict";
-import { after, before, describe, test } from "node:test";
-import { eq } from "drizzle-orm";
-import {
-	envelopeId as deriveEnvelopeId,
-	rootBodyPartId as deriveRootBodyPartId,
-} from "../id.js";
-import { addressTable } from "../schema/i4-address.js";
-import {
-	type MessageDataSchema,
-	messageDataSchema,
-	messageTable,
-} from "../schema/message-data.js";
-import { threadMessageTable } from "../schema/thread-message.js";
-import { createSqliteTestDb, type SqliteTestDb } from "../test-db-sqlite.js";
-import { DrizzleUnitOfWork } from "./unit-of-work.js";
-
-// The unit-of-work runs its write set inside one SAVEPOINT on sqlite (RFC 036
-// D1) — and each repo's own `create` opens a nested SAVEPOINT, so this also
-// covers savepoint nesting. Commit persists the whole set; a throw rolls all of
-// it back, message and thread-message together.
-
-const SCHEMA = {
-	...messageDataSchema,
-	threadMessage: threadMessageTable,
-	address: addressTable,
-};
-
-describe("DrizzleUnitOfWork (sqlite)", () => {
-	let db: SqliteTestDb<MessageDataSchema>;
-	let close: () => Promise<void>;
-	let uow: DrizzleUnitOfWork;
-
-	before(async () => {
-		({ db, close } = await createSqliteTestDb<MessageDataSchema>(SCHEMA));
-		uow = new DrizzleUnitOfWork(db);
-	});
-
-	after(async () => {
-		await close();
-	});
-
-	const messageInput = (id: string) => ({
-		messageId: id,
-		mailboxId: "00000000-0000-0000-3333-000000000002",
-		uid: 7,
-		sequenceNumber: 1,
-		rfc822Size: 512,
-		internalDate: 1700000000000,
-		envelopeId: deriveEnvelopeId(id),
-		rootBodyPartId: deriveRootBodyPartId(id),
-		status: "active" as const,
-		syncStatus: "synced" as const,
-	});
-
-	const threadInput = (id: string) => ({
-		accountConfigId: "acct-1",
-		threadId: `thread-${id}`,
-		messageId: id,
-		mailboxId: "mbx-1",
-		uid: 7,
-		referenceOrder: 0,
-		internalDate: 1700000000000,
-		sentDate: 1700000000000,
-		isRead: false,
-		isDeleted: false,
-		hasAttachment: false,
-		hasStars: false,
-	});
-
-	test("commits message + thread-message together", async () => {
-		const id = "00000000-0000-0000-3333-000000000010";
-		await uow.transaction(async (repos) => {
-			await repos.message.create(messageInput(id));
-			await repos.threadMessage.create(threadInput(id));
-		});
-
-		const msg = await db
-			.select()
-			.from(messageTable)
-			.where(eq(messageTable.messageId, id));
-		const thread = await db
-			.select()
-			.from(threadMessageTable)
-			.where(eq(threadMessageTable.messageId, id));
-		assert.equal(msg.length, 1);
-		assert.equal(thread.length, 1);
-	});
-
-	test("a throw rolls the whole set back", async () => {
-		const id = "00000000-0000-0000-3333-000000000011";
-		await assert.rejects(() =>
-			uow.transaction(async (repos) => {
-				await repos.message.create(messageInput(id));
-				await repos.threadMessage.create(threadInput(id));
-				throw new Error("boom");
-			}),
-		);
-
-		const msg = await db
-			.select()
-			.from(messageTable)
-			.where(eq(messageTable.messageId, id));
-		const thread = await db
-			.select()
-			.from(threadMessageTable)
-			.where(eq(threadMessageTable.messageId, id));
-		assert.equal(msg.length, 0, "message insert must roll back");
-		assert.equal(thread.length, 0, "thread-message insert must roll back");
-	});
-
-	test("concurrent transactions serialize without corrupting savepoints", async () => {
-		const ids = Array.from(
-			{ length: 12 },
-			(_, i) =>
-				`00000000-0000-0000-3333-0000000001${i.toString().padStart(2, "0")}`,
-		);
-
-		// The failure this guards: shared-connection savepoint interleaving when
-		// callers run under concurrency (message-sync's pMap). All must commit.
-		await Promise.all(
-			ids.map((id) =>
-				uow.transaction(async (repos) => {
-					await repos.message.create(messageInput(id));
-					await repos.threadMessage.create(threadInput(id));
-				}),
-			),
-		);
-
-		for (const id of ids) {
-			const msg = await db
-				.select()
-				.from(messageTable)
-				.where(eq(messageTable.messageId, id));
-			assert.equal(msg.length, 1, `message ${id} must be committed`);
-		}
-	});
-});

package/src/repos/unit-of-work.test.ts

@@ -1,109 +0,0 @@
-import assert from "node:assert/strict";
-import { after, before, describe, test } from "node:test";
-import { eq } from "drizzle-orm";
-import {
-	envelopeId as deriveEnvelopeId,
-	rootBodyPartId as deriveRootBodyPartId,
-} from "../id.js";
-import {
-	envelopeTable,
-	messageTable,
-	outboxTable,
-} from "../schema/message-data.js";
-import { createTestDb, type TestDb } from "./test-helpers.js";
-import { DrizzleUnitOfWork } from "./unit-of-work.js";
-
-describe("DrizzleUnitOfWork", () => {
-	let db: TestDb;
-	let stop: () => Promise<void>;
-	let unitOfWork: DrizzleUnitOfWork;
-
-	const MESSAGE_ID = "00000000-0000-0000-2222-000000000001";
-	const MAILBOX_ID = "00000000-0000-0000-2222-000000000002";
-	const NOW = 1700000000000;
-
-	const writeEnvelopeMessageAndBody = async (
-		repos: Parameters<Parameters<DrizzleUnitOfWork["transaction"]>[0]>[0],
-	) => {
-		await repos.envelope.upsertEnvelope({
-			envelopeId: deriveEnvelopeId(MESSAGE_ID),
-			messageId: MESSAGE_ID,
-			dateValue: NOW,
-			dateRaw: "Tue, 14 Nov 2023 22:13:20 +0000",
-			subject: "hello",
-			messageIdValue: "<hello@example.com>",
-		});
-		await repos.message.upsertWithStatus({
-			messageId: MESSAGE_ID,
-			mailboxId: MAILBOX_ID,
-			uid: 7,
-			sequenceNumber: 1,
-			rfc822Size: 1024,
-			internalDate: NOW,
-			envelopeId: deriveEnvelopeId(MESSAGE_ID),
-			rootBodyPartId: deriveRootBodyPartId(MESSAGE_ID),
-		});
-		// Body-sync appends the transactional-outbox row; the create emits none.
-		await repos.message.update(MESSAGE_ID, {
-			bodyStorageKey: "body/hello.json",
-		});
-	};
-
-	const rows = async () => {
-		const envelopes = await db
-			.select()
-			.from(envelopeTable)
-			.where(eq(envelopeTable.messageId, MESSAGE_ID));
-		const messages = await db
-			.select()
-			.from(messageTable)
-			.where(eq(messageTable.messageId, MESSAGE_ID));
-		const outbox = await db
-			.select()
-			.from(outboxTable)
-			.where(eq(outboxTable.messageId, MESSAGE_ID));
-		return { envelopes, messages, outbox };
-	};
-
-	before(async () => {
-		({ db, stop } = await createTestDb());
-		unitOfWork = new DrizzleUnitOfWork(db);
-	});
-
-	after(async () => {
-		await stop();
-	});
-
-	test("a mid-save throw rolls back the whole write set, outbox included", async () => {
-		await assert.rejects(
-			() =>
-				unitOfWork.transaction(async (repos) => {
-					await writeEnvelopeMessageAndBody(repos);
-					// A later write in the set fails after the envelope, message and its
-					// transactional-outbox row have already been written.
-					throw new Error("thread write failed");
-				}),
-			/thread write failed/,
-		);
-
-		const { envelopes, messages, outbox } = await rows();
-		assert.equal(envelopes.length, 0, "envelope must be rolled back");
-		assert.equal(messages.length, 0, "message must be rolled back");
-		assert.equal(
-			outbox.length,
-			0,
-			"outbox row must be rolled back with the message",
-		);
-	});
-
-	test("a successful transaction commits the data rows and the outbox row", async () => {
-		await unitOfWork.transaction(writeEnvelopeMessageAndBody);
-
-		const { envelopes, messages, outbox } = await rows();
-		assert.equal(envelopes.length, 1);
-		assert.equal(messages.length, 1);
-		assert.equal(outbox.length, 1);
-		assert.equal(outbox[0].event, "message.body_synced");
-		assert.deepStrictEqual(outbox[0].payload, { messageId: MESSAGE_ID });
-	});
-});

package/src/repos/unit-of-work.ts

@@ -1,31 +0,0 @@
-import type { IUnitOfWork, UnitOfWorkRepositories } from "@remit/data-ports";
-import type { Db } from "../db.js";
-import type { MessageDataSchema } from "../schema/message-data.js";
-import { runInTransaction } from "../tx.js";
-import { DrizzleEnvelopeRepository } from "./envelope.js";
-import { AddressRepo } from "./i4-address.js";
-import { DrizzleMessageRepository } from "./message.js";
-import { DrizzleThreadMessageRepository } from "./thread-message.js";
-
-/**
- * Runs a write set inside a single transaction. The repositories handed
- * to the callback are bound to that transaction, so the data rows and the
- * transactional-outbox rows the message write appends commit atomically — a
- * throw anywhere rolls the whole set back, outbox included.
- */
-export class DrizzleUnitOfWork implements IUnitOfWork {
-	constructor(private db: Db<MessageDataSchema>) {}
-
-	transaction<T>(
-		fn: (repos: UnitOfWorkRepositories) => Promise<T>,
-	): Promise<T> {
-		return runInTransaction(this.db, (tx) =>
-			fn({
-				message: new DrizzleMessageRepository(tx),
-				envelope: new DrizzleEnvelopeRepository(tx),
-				address: new AddressRepo(tx),
-				threadMessage: new DrizzleThreadMessageRepository(tx),
-			}),
-		);
-	}
-}

package/src/schema/outbox.ts

@@ -1,46 +0,0 @@
-import { sql } from "drizzle-orm";
-import { index, integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
-
-/**
- * Transactional outbox. It has no TypeSpec entity — it is infrastructure for
- * the search-index worker (append a row per body change / move, drain by id,
- * mark `processed_at`). The partial index selects unprocessed rows for the
- * short-cadence poll (RFC 036 D2).
- *
- * Exported raw for committed-migration generation (schema-full-sqlite) as well
- * as for the repos and the drain logic.
- */
-/**
- * The event vocabulary a producer may append. Every kind here is drained by the
- * search-index relay (`DRAIN_EVENTS`); a kind nothing drains accumulates
- * undrained rows forever, which is what `message.created` did (reader#1063).
- * The column carries the union so a new producer cannot invent a kind without
- * adding it here, where the drain-coverage test picks it up.
- */
-export const OUTBOX_EVENTS = [
-	"message.body_synced",
-	"message.moved",
-	"message.removed",
-] as const;
-
-export type OutboxEvent = (typeof OUTBOX_EVENTS)[number];
-
-export const outboxTable = sqliteTable(
-	"outbox",
-	{
-		id: text("id").primaryKey(),
-		messageId: text("message_id").notNull(),
-		event: text("event", { enum: OUTBOX_EVENTS }).notNull(),
-		payload: text("payload", { mode: "json" }).notNull(),
-		createdAt: integer("created_at", { mode: "timestamp_ms" })
-			.$defaultFn(() => new Date())
-			.notNull(),
-		processedAt: integer("processed_at", { mode: "number" }),
-	},
-	(t) => [
-		index("outbox_message_id_idx").on(t.messageId),
-		index("outbox_unprocessed_idx")
-			.on(t.createdAt)
-			.where(sql`${t.processedAt} IS NULL`),
-	],
-);

package/src/schema-full-sqlite.ts

@@ -1,13 +0,0 @@
-// Complete SQLite drizzle schema for committed-migration GENERATION only
-// (RFC 036 D5) — consumed by packages/migrate/drizzle.entities.sqlite.config.ts
-// and the drift guard (npm-scripts/check-vps-migrations.mjs), never at runtime.
-//
-// It pulls the generated entity package in wholesale, so a new TypeSpec entity
-// flows into the committed migration with nothing to hand-maintain. The only
-// addition is the `outbox` infra table, which has no entity.
-//
-// schema.ts stays the app/dev surface (single `*Table` alias per table, what
-// the repos and `pushSchema` need); this file exposes canonical names, so it
-// must not be fed to `pushSchema` alongside schema.ts (duplicate index names).
-export * from "@remit/drizzle-sqlite-schema";
-export { outboxTable } from "./schema/outbox.js";