@remit/drizzle-service 0.0.1

package/src/repos/serialized-writes.sqlite.test.ts

@@ -0,0 +1,199 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import { eq } from "drizzle-orm";
+import type { Db } from "../db.js";
+import {
+	type MessageDataSchema,
+	messageDataSchema,
+	outboxTable,
+} from "../schema/message-data.js";
+import { createSqliteTestDb, type SqliteTestDb } from "../test-db-sqlite.js";
+import { runInTransaction, serializeSqliteWrites } from "../tx.js";
+
+// The RFC 036 D3 hard requirement: every write on the shared better-sqlite3
+// connection routes through the wave-1 serialization, so a plain single
+// statement can never land inside another unit's open SAVEPOINT (the
+// uncommitted-read / rollback hazard named on PR #1310). The db handle repos
+// hold is `serializeSqliteWrites(db)` — unserialized access is impossible by
+// construction. The `outbox` table stands in for any write target here (few
+// required columns, already dialect-cast for the repos).
+
+const row = (id: string) => ({
+	id,
+	messageId: `msg-${id}`,
+	event: "message.body_synced",
+	payload: { messageId: `msg-${id}` },
+	createdAt: new Date(),
+});
+
+const nextTick = (): Promise<void> =>
+	new Promise((resolve) => setImmediate(resolve));
+
+describe("serialized SQLite writes (RFC 036 D3)", () => {
+	let raw: SqliteTestDb<MessageDataSchema>;
+	let db: Db<MessageDataSchema>;
+	let close: () => Promise<void>;
+
+	before(async () => {
+		({ db: raw, close } =
+			await createSqliteTestDb<MessageDataSchema>(messageDataSchema));
+		db = serializeSqliteWrites(raw);
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	const byId = (id: string) =>
+		raw.select().from(outboxTable).where(eq(outboxTable.id, id));
+
+	test("a single-statement write does not interleave into an open transaction", async () => {
+		const order: string[] = [];
+		let release: () => void = () => {};
+		const barrier = new Promise<void>((resolve) => {
+			release = resolve;
+		});
+
+		// A top-level transaction: insert TX_ROW, park on a barrier (yielding the
+		// event loop with the SAVEPOINT open), then throw so the whole unit rolls
+		// back.
+		const txDone = runInTransaction(db, async (tx) => {
+			await tx.insert(outboxTable).values(row("tx-row"));
+			order.push("tx-insert");
+			await barrier;
+			order.push("tx-throw");
+			throw new Error("rollback");
+		}).catch(() => {
+			order.push("tx-rolledback");
+		});
+
+		// Let the transaction reach the barrier before the single write is issued.
+		await nextTick();
+		await nextTick();
+
+		const singleWrite = db
+			.insert(outboxTable)
+			.values(row("single-row"))
+			.then(() => {
+				order.push("single-committed");
+			});
+
+		// While the transaction holds the write queue, the single write must not
+		// have executed — it is queued behind the open unit, not interleaved into
+		// its savepoint.
+		await nextTick();
+		await nextTick();
+		assert.equal(
+			order.includes("single-committed"),
+			false,
+			"single write ran while a top-level transaction was open",
+		);
+
+		release();
+		await txDone;
+		await singleWrite;
+
+		assert.deepEqual(order, [
+			"tx-insert",
+			"tx-throw",
+			"tx-rolledback",
+			"single-committed",
+		]);
+
+		// The transaction rolled back; the single write committed independently —
+		// it was never part of the rolled-back savepoint.
+		assert.equal(
+			(await byId("tx-row")).length,
+			0,
+			"transaction must roll back",
+		);
+		assert.equal(
+			(await byId("single-row")).length,
+			1,
+			"single write must commit, not be swallowed by the rolled-back transaction",
+		);
+	});
+
+	test("concurrent single-statement writes all commit", async () => {
+		const ids = Array.from({ length: 20 }, (_, i) => `w-${i}`);
+		await Promise.all(ids.map((id) => db.insert(outboxTable).values(row(id))));
+		for (const id of ids) {
+			assert.equal((await byId(id)).length, 1, `${id} must be committed`);
+		}
+	});
+
+	test("the explicit .run() executor is also serialized, not just await", async () => {
+		// A write executed via `.run()` (not `await`) must still go through the
+		// queue — the hardened wrapper intercepts every terminal executor.
+		const builder = db
+			.insert(outboxTable)
+			.values(row("run-row")) as unknown as {
+			run: () => Promise<unknown>;
+		};
+		await builder.run();
+		assert.equal((await byId("run-row")).length, 1, "run() committed the row");
+	});
+
+	test("the .values() executor is serialized, not run inline", async () => {
+		// `values` doubles as the insert chain method and (on a runnable with
+		// `.returning()`) a terminal executor. The executor form must queue like
+		// every other terminal — issued while a transaction is parked, it must
+		// wait for the unit to finish instead of interleaving into its savepoint.
+		const order: string[] = [];
+		let release: () => void = () => {};
+		const barrier = new Promise<void>((resolve) => {
+			release = resolve;
+		});
+
+		const txDone = runInTransaction(db, async (tx) => {
+			await tx.insert(outboxTable).values(row("values-tx-row"));
+			order.push("tx-insert");
+			await barrier;
+		});
+
+		await nextTick();
+		await nextTick();
+
+		const runnable = db
+			.insert(outboxTable)
+			.values(row("values-row"))
+			.returning() as unknown as { values: () => Promise<unknown[]> };
+		const executorWrite = runnable.values().then(() => {
+			order.push("values-committed");
+		});
+
+		await nextTick();
+		await nextTick();
+		assert.equal(
+			order.includes("values-committed"),
+			false,
+			".values() executor ran while a top-level transaction was open",
+		);
+
+		release();
+		await txDone;
+		await executorWrite;
+		assert.deepEqual(order, ["tx-insert", "values-committed"]);
+		assert.equal((await byId("values-row")).length, 1);
+	});
+
+	test("writes inside a transaction still commit atomically", async () => {
+		await runInTransaction(db, async (tx) => {
+			await tx.insert(outboxTable).values(row("a"));
+			await tx
+				.update(outboxTable)
+				.set({ event: "message.moved" })
+				.where(eq(outboxTable.id, "a"));
+		});
+		const [committed] = await byId("a");
+		assert.equal(committed?.event, "message.moved");
+
+		await assert.rejects(() =>
+			runInTransaction(db, async (tx) => {
+				await tx.insert(outboxTable).values(row("b"));
+				throw new Error("boom");
+			}),
+		);
+		assert.equal((await byId("b")).length, 0, "transaction rollback drops b");
+	});
+});

package/src/repos/test-helpers.ts

@@ -0,0 +1,222 @@
+import { rmSync } from "node:fs";
+import { sql } from "drizzle-orm";
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+import { drizzle } from "drizzle-orm/node-postgres";
+import EmbeddedPostgres from "embedded-postgres";
+import {
+	type MessageDataSchema,
+	messageDataSchema,
+} from "../schema/message-data.js";
+
+export type TestDb = NodePgDatabase<MessageDataSchema>;
+
+function randomPort(): number {
+	return 54500 + Math.floor(Math.random() * 400);
+}
+
+export async function createTestDb(): Promise<{
+	db: TestDb;
+	stop: () => Promise<void>;
+}> {
+	const port = randomPort();
+	const databaseDir = `/tmp/drizzle-test-pg-${port}-${Date.now()}`;
+
+	const pg = new EmbeddedPostgres({
+		databaseDir,
+		user: "test",
+		password: "test",
+		port,
+		persistent: false,
+	});
+
+	await pg.initialise();
+	await pg.start();
+
+	const connectionString = `postgresql://test:test@localhost:${port}/postgres`;
+	const db = drizzle(connectionString, { schema: messageDataSchema }) as TestDb;
+
+	await db.execute(sql.raw(DDL));
+
+	return {
+		db,
+		stop: async () => {
+			const client = (
+				db as unknown as { $client: { end: () => Promise<void> } }
+			).$client;
+			await client.end();
+			await pg.stop();
+			try {
+				rmSync(databaseDir, { recursive: true, force: true });
+			} catch {
+				// best-effort cleanup
+			}
+		},
+	};
+}
+
+const DDL = `
+CREATE TABLE IF NOT EXISTS envelope (
+  envelope_id       UUID PRIMARY KEY,
+  message_id        UUID NOT NULL,
+  date_value        BIGINT NOT NULL,
+  date_raw          TEXT NOT NULL,
+  subject           TEXT,
+  message_id_value  TEXT,
+  created_at        BIGINT NOT NULL,
+  updated_at        BIGINT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS envelope_message_id_idx ON envelope (message_id);
+
+CREATE TABLE IF NOT EXISTS message_reference (
+  message_reference_id  UUID PRIMARY KEY,
+  message_id            UUID NOT NULL,
+  envelope_id           UUID NOT NULL,
+  message_id_value      TEXT NOT NULL,
+  reference_type        TEXT NOT NULL,
+  reference_order       INTEGER NOT NULL,
+  created_at            BIGINT NOT NULL,
+  updated_at            BIGINT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS message_reference_message_id_idx ON message_reference (message_id);
+
+CREATE TABLE IF NOT EXISTS envelope_address (
+  envelope_address_id UUID PRIMARY KEY,
+  message_id          UUID NOT NULL,
+  address_id          UUID NOT NULL,
+  display_name        TEXT,
+  normalized_email    TEXT NOT NULL,
+  address_role        TEXT NOT NULL,
+  address_order       INTEGER NOT NULL,
+  created_at          BIGINT NOT NULL,
+  updated_at          BIGINT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS envelope_address_message_id_idx ON envelope_address (message_id);
+
+CREATE TABLE IF NOT EXISTS body_part (
+  body_part_id          UUID PRIMARY KEY,
+  message_id            UUID NOT NULL,
+  parent_body_part_id   UUID,
+  part_path             TEXT NOT NULL,
+  media_type            TEXT NOT NULL,
+  media_subtype         TEXT NOT NULL,
+  content_id            TEXT,
+  content_description   TEXT,
+  transfer_encoding     TEXT NOT NULL,
+  size_octets           INTEGER NOT NULL,
+  line_count            INTEGER,
+  md5_hash              TEXT,
+  disposition           TEXT,
+  disposition_filename  TEXT,
+  language              TEXT,
+  location              TEXT,
+  is_multipart          BOOLEAN NOT NULL,
+  multipart_subtype     TEXT,
+  created_at            BIGINT NOT NULL,
+  updated_at            BIGINT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS body_part_message_id_idx ON body_part (message_id);
+
+CREATE TABLE IF NOT EXISTS body_part_parameter (
+  body_part_parameter_id  UUID PRIMARY KEY,
+  message_id              UUID NOT NULL,
+  body_part_id            UUID NOT NULL,
+  parameter_name          TEXT NOT NULL,
+  parameter_value         TEXT NOT NULL,
+  created_at              BIGINT NOT NULL,
+  updated_at              BIGINT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS body_part_parameter_message_id_idx ON body_part_parameter (message_id);
+
+CREATE TABLE IF NOT EXISTS raw_message_storage (
+  raw_storage_id    UUID PRIMARY KEY,
+  message_id        UUID NOT NULL,
+  storage_type      TEXT NOT NULL,
+  storage_location  TEXT NOT NULL,
+  storage_key       TEXT NOT NULL,
+  size_bytes        INTEGER NOT NULL,
+  checksum_sha256   TEXT NOT NULL,
+  content_encoding  TEXT NOT NULL,
+  stored_at         BIGINT NOT NULL,
+  expires_at        BIGINT,
+  created_at        BIGINT NOT NULL,
+  updated_at        BIGINT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS raw_message_storage_message_id_idx ON raw_message_storage (message_id);
+
+CREATE TABLE IF NOT EXISTS body_part_storage (
+  body_part_storage_id  UUID PRIMARY KEY,
+  message_id            UUID NOT NULL,
+  body_part_id          UUID NOT NULL,
+  storage_type          TEXT NOT NULL,
+  storage_location      TEXT NOT NULL,
+  storage_key           TEXT NOT NULL,
+  decoded_size_bytes    INTEGER NOT NULL,
+  checksum_sha256       TEXT NOT NULL,
+  content_encoding      TEXT NOT NULL,
+  is_deduped            BOOLEAN NOT NULL,
+  dedup_hash            TEXT,
+  stored_at             BIGINT NOT NULL,
+  created_at            BIGINT NOT NULL,
+  updated_at            BIGINT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS body_part_storage_message_id_idx ON body_part_storage (message_id);
+
+CREATE TABLE IF NOT EXISTS body_part_content (
+  body_part_content_id  UUID PRIMARY KEY,
+  message_id            UUID NOT NULL,
+  body_part_id          UUID NOT NULL,
+  content               TEXT NOT NULL,
+  content_length        INTEGER NOT NULL,
+  created_at            BIGINT NOT NULL,
+  updated_at            BIGINT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS body_part_content_message_id_idx ON body_part_content (message_id);
+
+CREATE TABLE IF NOT EXISTS message (
+  message_id              UUID PRIMARY KEY,
+  mailbox_id              UUID NOT NULL,
+  uid                     INTEGER NOT NULL,
+  sequence_number         INTEGER NOT NULL,
+  rfc822_size             INTEGER NOT NULL,
+  internal_date           BIGINT NOT NULL,
+  message_id_header       TEXT,
+  envelope_id             UUID NOT NULL,
+  root_body_part_id       UUID NOT NULL,
+  body_storage_key        TEXT,
+  status                  TEXT NOT NULL DEFAULT 'active',
+  sync_status             TEXT NOT NULL DEFAULT 'pending',
+  original_mailbox_id     UUID,
+  original_uid            INTEGER,
+  category                TEXT NOT NULL DEFAULT 'uncategorized',
+  authenticity            JSONB,
+  auth_result             JSONB,
+  provider_spam           JSONB,
+  has_list_unsubscribe    BOOLEAN NOT NULL DEFAULT false,
+  moved_by_remit          BOOLEAN NOT NULL DEFAULT false,
+  placement_verdict       JSONB,
+  created_at              BIGINT NOT NULL,
+  updated_at              BIGINT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS message_mailbox_id_idx ON message (mailbox_id);
+
+CREATE TABLE IF NOT EXISTS message_flag (
+  message_flag_id   UUID PRIMARY KEY,
+  message_id        UUID NOT NULL,
+  flag_name         TEXT NOT NULL,
+  set_at            BIGINT NOT NULL,
+  created_at        BIGINT NOT NULL,
+  updated_at        BIGINT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS message_flag_message_id_idx ON message_flag (message_id);
+
+CREATE TABLE IF NOT EXISTS outbox (
+  id           UUID PRIMARY KEY,
+  message_id   UUID NOT NULL,
+  event        TEXT NOT NULL,
+  payload      JSONB NOT NULL,
+  created_at   TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  processed_at BIGINT
+);
+CREATE INDEX IF NOT EXISTS outbox_message_id_idx ON outbox (message_id);
+CREATE INDEX IF NOT EXISTS outbox_unprocessed_idx ON outbox (created_at) WHERE processed_at IS NULL;
+`;

package/src/repos/thread-message.sqlite.test.ts

@@ -0,0 +1,198 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import type { CreateThreadMessageInput } from "@remit/data-ports";
+import { threadMessageTable } from "../schema/thread-message.js";
+import { createSqliteTestDb } from "../test-db-sqlite.js";
+import { DrizzleThreadMessageRepository } from "./thread-message.js";
+
+// The thread-message repo on sqlite (RFC 036 D1): CRUD, keyset pagination, and
+// text search — the FTS5 trigram index for terms of three characters or more
+// (D4), the folded-LIKE scan for shorter terms. The search contract is accent-
+// and case-insensitive substring match over subject and sender.
+
+const ACCOUNT = "acct-thread";
+const MAILBOX = "mbx-thread";
+
+function makeInput(
+	overrides: Partial<CreateThreadMessageInput> = {},
+): CreateThreadMessageInput {
+	const now = Date.now();
+	return {
+		accountConfigId: ACCOUNT,
+		threadId: `t-${Math.random().toString(36).slice(2)}`,
+		messageId: `m-${Math.random().toString(36).slice(2)}`,
+		mailboxId: MAILBOX,
+		uid: 1,
+		referenceOrder: 0,
+		internalDate: now,
+		sentDate: now,
+		isRead: false,
+		isDeleted: false,
+		hasAttachment: false,
+		hasStars: false,
+		...overrides,
+	};
+}
+
+describe("DrizzleThreadMessageRepository (sqlite)", () => {
+	let db: Awaited<ReturnType<typeof createSqliteTestDb>>["db"];
+	let close: () => Promise<void>;
+	let repo: DrizzleThreadMessageRepository;
+
+	before(async () => {
+		({ db, close } = await createSqliteTestDb(
+			{
+				threadMessage: threadMessageTable,
+			},
+			{ searchIndex: true },
+		));
+		repo = new DrizzleThreadMessageRepository(db);
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	test("create then get round-trips", async () => {
+		const created = await repo.create(
+			makeInput({ subject: "Hello world", fromEmail: "a@example.com" }),
+		);
+		const got = await repo.get(ACCOUNT, created.threadMessageId);
+		assert.equal(got.subject, "Hello world");
+		assert.equal(got.isRead, false);
+	});
+
+	test("search matches a case-insensitive substring of the subject", async () => {
+		await repo.create(makeInput({ subject: "Invoice for March" }));
+		await repo.create(makeInput({ subject: "unrelated note" }));
+
+		const result = await repo.searchByMailbox(
+			ACCOUNT,
+			MAILBOX,
+			{ subject: "invoice" },
+			{ count: 50 },
+		);
+		assert.ok(
+			result.items.every((i) => /invoice/i.test(i.subject ?? "")),
+			"every match contains the needle",
+		);
+		assert.ok(
+			result.items.some((i) => i.subject === "Invoice for March"),
+			"the matching subject is returned",
+		);
+	});
+
+	test("search matches the sender fields", async () => {
+		await repo.create(
+			makeInput({
+				subject: "x",
+				fromName: "Alice Kramer",
+				fromEmail: "alice@corp.test",
+			}),
+		);
+		const byName = await repo.searchByMailbox(
+			ACCOUNT,
+			MAILBOX,
+			{ from: "kramer" },
+			{ count: 50 },
+		);
+		assert.ok(byName.items.some((i) => i.fromName === "Alice Kramer"));
+	});
+
+	test("FTS trigram search folds diacritics both ways", async () => {
+		await repo.create(makeInput({ subject: "Réunion budget" }));
+		const folded = await repo.searchByMailbox(
+			ACCOUNT,
+			MAILBOX,
+			{ subject: "reunion" },
+			{ count: 50 },
+		);
+		assert.ok(
+			folded.items.some((i) => i.subject === "Réunion budget"),
+			"an unaccented needle matches an accented subject",
+		);
+	});
+
+	test("a sub-3-character term falls back to the folded LIKE scan", async () => {
+		await repo.create(makeInput({ subject: "Q2 results" }));
+		const short = await repo.searchByMailbox(
+			ACCOUNT,
+			MAILBOX,
+			{ subject: "q2" },
+			{ count: 50 },
+		);
+		assert.ok(
+			short.items.some((i) => i.subject === "Q2 results"),
+			"a two-character term still matches via LIKE",
+		);
+	});
+
+	test("an updated subject is re-indexed by the FTS triggers", async () => {
+		const created = await repo.create(makeInput({ subject: "draft proposal" }));
+		await repo.update(ACCOUNT, created.threadMessageId, {
+			subject: "final proposal",
+		});
+		const stale = await repo.searchByMailbox(
+			ACCOUNT,
+			MAILBOX,
+			{ subject: "draft" },
+			{ count: 50 },
+		);
+		assert.ok(
+			!stale.items.some((i) => i.threadMessageId === created.threadMessageId),
+			"the old subject no longer matches after re-index",
+		);
+		const fresh = await repo.searchByMailbox(
+			ACCOUNT,
+			MAILBOX,
+			{ subject: "final" },
+			{ count: 50 },
+		);
+		assert.ok(
+			fresh.items.some((i) => i.threadMessageId === created.threadMessageId),
+			"the new subject matches after re-index",
+		);
+	});
+
+	test("countByMailbox counts matches under the same predicate", async () => {
+		const n = await repo.countByMailbox(
+			ACCOUNT,
+			MAILBOX,
+			{ subject: "invoice" },
+			{ limit: 100 },
+		);
+		assert.ok(n >= 1);
+	});
+
+	test("listByDate paginates with a stable keyset cursor", async () => {
+		const acct = "acct-page";
+		const base = Date.now();
+		for (let i = 0; i < 5; i++) {
+			await repo.create(
+				makeInput({
+					accountConfigId: acct,
+					mailboxId: "mbx-page",
+					subject: `page ${i}`,
+					sentDate: base - i,
+					internalDate: base - i,
+				}),
+			);
+		}
+
+		const first = await repo.listByDate(acct, { limit: 2, order: "desc" });
+		assert.equal(first.items.length, 2);
+		assert.ok(first.continuationToken);
+
+		const second = await repo.listByDate(acct, {
+			limit: 2,
+			order: "desc",
+			continuationToken: first.continuationToken,
+		});
+		assert.equal(second.items.length, 2);
+		const firstIds = new Set(first.items.map((i) => i.threadMessageId));
+		assert.ok(
+			second.items.every((i) => !firstIds.has(i.threadMessageId)),
+			"pages do not overlap",
+		);
+	});
+});