@remit/drizzle-service 0.0.1

package/src/repos/cascade-delete.test.ts

@@ -0,0 +1,423 @@
+import assert from "node:assert";
+import { after, before, describe, test } from "node:test";
+import { MailboxCursorState } from "@remit/domain-enums";
+import { eq } from "drizzle-orm";
+import {
+	accountSettingTable,
+	accountTable,
+	addressTable,
+	bodyPartTable,
+	envelopeTable,
+	mailboxLockTable,
+	mailboxTable,
+	messageFlagPushTable,
+	messageFlagTable,
+	messagePlacementMoveTable,
+	messageTable,
+	outboxMessageTable,
+	outboxTable,
+	threadMessageTable,
+} from "../schema.js";
+import { createTestDb, type TestDb } from "../test-db.js";
+import {
+	type CascadeEntity,
+	runDrizzleCascadeDelete,
+} from "./cascade-delete.js";
+
+const log = { info: () => {} };
+
+const NOW = 1700000000000;
+const ACCOUNT_CONFIG_ID = "cfg-cascade-1";
+const ACCOUNT_ID = "acc-cascade-1";
+const MAILBOX_ID = "mbx-cascade-1";
+const MESSAGE_ID = "msg-cascade-1";
+const THREAD_MESSAGE_ID = "tm-cascade-1";
+const OUTBOX_MESSAGE_ID = "outbox-cascade-1";
+const ACCOUNT_SETTING_ID = "setting-cascade-1";
+const ADDRESS_ID = "address-cascade-1";
+const ENVELOPE_ID = "00000000-0000-4000-8000-000000000001";
+const ROOT_BODY_PART_ID = "00000000-0000-4000-8000-000000000002";
+const MESSAGE_FLAG_ID = "00000000-0000-4000-8000-000000000003";
+const BODY_PART_ID = "00000000-0000-4000-8000-000000000004";
+const OTHER_MAILBOX_ID = "mbx-cascade-2";
+
+describe("runDrizzleCascadeDelete", () => {
+	let db: TestDb;
+	let close: () => Promise<void>;
+
+	before(async () => {
+		({ db, close } = await createTestDb());
+
+		await db.insert(accountTable).values({
+			accountId: ACCOUNT_ID,
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			username: "u",
+			email: "u@example.com",
+			authType: "password",
+			imapHost: "imap.example.com",
+			imapPort: 993,
+			imapTls: true,
+			imapStartTls: false,
+			smtpPort: 587,
+			isActive: true,
+			connectionState: "authenticated",
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
+		await db.insert(mailboxTable).values({
+			mailboxId: MAILBOX_ID,
+			accountId: ACCOUNT_ID,
+			namespaceType: "personal",
+			namespacePrefix: "",
+			hierarchyDelimiter: "/",
+			fullPath: "INBOX",
+			uidValidity: 1,
+			uidNext: 1,
+			highestModseq: 0,
+			messageCount: 0,
+			unseenCount: 0,
+			deletedCount: 0,
+			totalSize: 0,
+			lastSyncUid: 0,
+			highWaterMarkUid: 0,
+			lastMessageSyncAt: NOW,
+			cursorState: MailboxCursorState.normal,
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
+		await db.insert(messageTable).values({
+			messageId: MESSAGE_ID,
+			mailboxId: MAILBOX_ID,
+			uid: 1,
+			sequenceNumber: 1,
+			rfc822Size: 10,
+			internalDate: NOW,
+			envelopeId: ENVELOPE_ID,
+			rootBodyPartId: ROOT_BODY_PART_ID,
+			status: "active",
+			syncStatus: "synced",
+			category: "uncategorized",
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
+		await db.insert(envelopeTable).values({
+			envelopeId: ENVELOPE_ID,
+			messageId: MESSAGE_ID,
+			dateValue: NOW,
+			dateRaw: "Mon, 14 Nov 2023 08:00:00 +0000",
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
+		await db.insert(messageFlagTable).values({
+			messageFlagId: MESSAGE_FLAG_ID,
+			messageId: MESSAGE_ID,
+			flagName: "\\Seen",
+			setAt: NOW,
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
+		await db.insert(bodyPartTable).values({
+			bodyPartId: BODY_PART_ID,
+			messageId: MESSAGE_ID,
+			partPath: "1",
+			mediaType: "TEXT",
+			mediaSubtype: "plain",
+			transferEncoding: "7BIT",
+			sizeOctets: 4,
+			isMultipart: false,
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
+		await db.insert(threadMessageTable).values({
+			threadMessageId: THREAD_MESSAGE_ID,
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			threadId: "thread-1",
+			messageId: MESSAGE_ID,
+			mailboxId: MAILBOX_ID,
+			uid: 1,
+			referenceOrder: 0,
+			internalDate: NOW,
+			sentDate: NOW,
+			isRead: false,
+			hasAttachment: false,
+			star: "none",
+			hasStars: false,
+			isDeleted: false,
+			category: "uncategorized",
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
+		await db.insert(outboxMessageTable).values({
+			outboxMessageId: OUTBOX_MESSAGE_ID,
+			accountId: ACCOUNT_ID,
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			fromAddress: "u@example.com",
+			toAddresses: ["b@example.com"],
+			ccAddresses: [],
+			bccAddresses: [],
+			messageIdValue: "<x@example.com>",
+			references: [],
+			status: "queued",
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
+		await db.insert(mailboxLockTable).values({
+			mailboxId: MAILBOX_ID,
+			eventName: "SYNC_MESSAGES",
+			accountId: ACCOUNT_ID,
+			lockId: "lock-1",
+			acquiredAt: NOW,
+			lockedBy: "worker-1",
+			ttl: NOW + 60000,
+		});
+		await db.insert(accountSettingTable).values({
+			accountSettingId: ACCOUNT_SETTING_ID,
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			name: "Theme",
+			value: { mode: "dark" },
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
+		await db.insert(addressTable).values({
+			addressId: ADDRESS_ID,
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			localPart: "u",
+			domain: "example.com",
+			normalizedEmail: "u@example.com",
+			normalizedCompound: "u u@example.com",
+			flags: {},
+			inboundCount: 0,
+			outboundCount: 0,
+			replyCount: 0,
+			lastInboundAt: NOW,
+			lastReplyAt: NOW,
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
+		await db.insert(messagePlacementMoveTable).values({
+			messageId: MESSAGE_ID,
+			accountId: ACCOUNT_ID,
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			sourceMailboxId: MAILBOX_ID,
+			destinationMailboxId: OTHER_MAILBOX_ID,
+			state: "pending",
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
+		await db.insert(messageFlagPushTable).values([
+			{
+				messageId: MESSAGE_ID,
+				flagName: "\\Seen",
+				accountId: ACCOUNT_ID,
+				accountConfigId: ACCOUNT_CONFIG_ID,
+				mailboxId: MAILBOX_ID,
+				operation: "add",
+				state: "pending",
+				createdAt: NOW,
+				updatedAt: NOW,
+			},
+			{
+				messageId: MESSAGE_ID,
+				flagName: "\\Flagged",
+				accountId: ACCOUNT_ID,
+				accountConfigId: ACCOUNT_CONFIG_ID,
+				mailboxId: MAILBOX_ID,
+				operation: "remove",
+				state: "pending",
+				createdAt: NOW,
+				updatedAt: NOW,
+			},
+		]);
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	const entities: CascadeEntity[] = [
+		// Message + its enumerated children — the children are removed by message id.
+		{ entityType: "Message", key: { messageId: MESSAGE_ID } },
+		{ entityType: "MessageFlag", key: { messageFlagId: MESSAGE_FLAG_ID } },
+		{ entityType: "Envelope", key: { envelopeId: ENVELOPE_ID } },
+		{ entityType: "BodyPart", key: { bodyPartId: BODY_PART_ID } },
+		{
+			entityType: "ThreadMessage",
+			key: {
+				accountConfigId: ACCOUNT_CONFIG_ID,
+				threadMessageId: THREAD_MESSAGE_ID,
+			},
+		},
+		{ entityType: "Mailbox", key: { mailboxId: MAILBOX_ID } },
+		{
+			entityType: "OutboxMessage",
+			key: { outboxMessageId: OUTBOX_MESSAGE_ID },
+		},
+		{
+			entityType: "MailboxLock",
+			key: { mailboxId: MAILBOX_ID, eventName: "SYNC_MESSAGES" },
+		},
+		{
+			entityType: "AccountSetting",
+			key: { accountSettingId: ACCOUNT_SETTING_ID },
+		},
+		{ entityType: "Address", key: { addressId: ADDRESS_ID } },
+		{ entityType: "Account", key: { accountId: ACCOUNT_ID } },
+		{
+			entityType: "MessagePlacementMove",
+			key: { messageId: MESSAGE_ID },
+		},
+		{
+			entityType: "MessageFlagPush",
+			key: { messageId: MESSAGE_ID, flagName: "\\Seen" },
+		},
+		{
+			entityType: "MessageFlagPush",
+			key: { messageId: MESSAGE_ID, flagName: "\\Flagged" },
+		},
+	];
+
+	test("deletes every enumerated row across all tables", async () => {
+		await runDrizzleCascadeDelete(db as never, entities, log);
+
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(messageTable)
+					.where(eq(messageTable.messageId, MESSAGE_ID))
+			).length,
+			0,
+		);
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(messageFlagTable)
+					.where(eq(messageFlagTable.messageId, MESSAGE_ID))
+			).length,
+			0,
+		);
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(bodyPartTable)
+					.where(eq(bodyPartTable.messageId, MESSAGE_ID))
+			).length,
+			0,
+		);
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(envelopeTable)
+					.where(eq(envelopeTable.messageId, MESSAGE_ID))
+			).length,
+			0,
+		);
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(threadMessageTable)
+					.where(eq(threadMessageTable.threadMessageId, THREAD_MESSAGE_ID))
+			).length,
+			0,
+		);
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(mailboxTable)
+					.where(eq(mailboxTable.mailboxId, MAILBOX_ID))
+			).length,
+			0,
+		);
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(outboxMessageTable)
+					.where(eq(outboxMessageTable.outboxMessageId, OUTBOX_MESSAGE_ID))
+			).length,
+			0,
+		);
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(mailboxLockTable)
+					.where(eq(mailboxLockTable.mailboxId, MAILBOX_ID))
+			).length,
+			0,
+		);
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(accountSettingTable)
+					.where(eq(accountSettingTable.accountSettingId, ACCOUNT_SETTING_ID))
+			).length,
+			0,
+		);
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(addressTable)
+					.where(eq(addressTable.addressId, ADDRESS_ID))
+			).length,
+			0,
+		);
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(accountTable)
+					.where(eq(accountTable.accountId, ACCOUNT_ID))
+			).length,
+			0,
+		);
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(messagePlacementMoveTable)
+					.where(eq(messagePlacementMoveTable.messageId, MESSAGE_ID))
+			).length,
+			0,
+		);
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(messageFlagPushTable)
+					.where(eq(messageFlagPushTable.messageId, MESSAGE_ID))
+			).length,
+			0,
+			"both the Seen and Flagged markers (composite key) must be deleted",
+		);
+	});
+
+	test("emits a message.removed outbox row for search-index cleanup", async () => {
+		const rows = await db
+			.select()
+			.from(outboxTable)
+			.where(eq(outboxTable.messageId, MESSAGE_ID));
+		assert.equal(rows.length, 1);
+		assert.equal(rows[0].event, "message.removed");
+		assert.equal(rows[0].processedAt, null);
+	});
+
+	test("rejects an unknown entity type without deleting", async () => {
+		await assert.rejects(
+			() =>
+				runDrizzleCascadeDelete(
+					db as never,
+					[{ entityType: "Nonsense", key: { id: "x" } }],
+					log,
+				),
+			/Unknown entity type in cascade: Nonsense/,
+		);
+	});
+});

package/src/repos/cascade-delete.ts

@@ -0,0 +1,219 @@
+import { and, eq, inArray, or, type SQL } from "drizzle-orm";
+import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres";
+import { accountTable } from "../schema/i4-account-config.js";
+import { accountSettingTable } from "../schema/i4-account-setting.js";
+import { addressTable } from "../schema/i4-address.js";
+import { mailboxTable } from "../schema/i4-mailbox.js";
+import { mailboxLockTable } from "../schema/i4-mailbox-lock.js";
+import { messageFlagPushTable } from "../schema/i4-message-flag-push.js";
+import { messagePlacementMoveTable } from "../schema/i4-message-placement-move.js";
+import { outboxMessageTable } from "../schema/i4-outbox-message.js";
+import { messageDataSchema } from "../schema/message-data.js";
+import { threadMessageTable } from "../schema/thread-message.js";
+import { createSqliteDatabase } from "../sqlite-client.js";
+import { runInTransaction } from "../tx.js";
+import { deleteMessageSubtree, type SubtreeDb } from "./message.js";
+
+export interface CascadeEntity {
+	entityType: string;
+	key: Record<string, string>;
+}
+
+export interface CascadeDeleteLogger {
+	info(obj: Record<string, unknown>, msg: string): void;
+}
+
+type CascadeDb = NodePgDatabase<Record<string, unknown>>;
+
+/**
+ * Per-message entity types. Their rows are removed with the message subtree by
+ * message id, so their individually-enumerated keys are not deleted one by one —
+ * the DynamoDB cascade enumerates them only because BatchWriteItem needs each
+ * full key; Postgres deletes them set-wise by message id.
+ */
+const MESSAGE_CHILD_TYPES = new Set([
+	"MessageFlag",
+	"MessageReference",
+	"EnvelopeAddress",
+	"Envelope",
+	"BodyPart",
+	"BodyPartParameter",
+	"BodyPartStorage",
+	"BodyPartContent",
+	"RawMessageStorage",
+]);
+
+const KEYED_TYPES = new Set([
+	"Message",
+	"ThreadMessage",
+	"Mailbox",
+	"OutboxMessage",
+	"MailboxLock",
+	"AccountSetting",
+	"Address",
+	"Account",
+	"MessagePlacementMove",
+	"MessageFlagPush",
+]);
+
+const groupByType = (
+	entities: CascadeEntity[],
+): Map<string, Record<string, string>[]> => {
+	const grouped = new Map<string, Record<string, string>[]>();
+	for (const entity of entities) {
+		const list = grouped.get(entity.entityType) ?? [];
+		list.push(entity.key);
+		grouped.set(entity.entityType, list);
+	}
+	return grouped;
+};
+
+/**
+ * Postgres equivalent of the DynamoDB BatchWriteItem cascade: delete every
+ * enumerated row for an account/tenant in one transaction. Message subtrees
+ * (message + nine child tables) are deleted set-wise by message id and emit a
+ * `message.removed` outbox row each for search-index cleanup; the remaining
+ * entity types are deleted by their enumerated primary keys. AccountConfig is
+ * never deleted here — the caller removes it last through its repository, the
+ * cascade-in-progress marker.
+ */
+export const runDrizzleCascadeDelete = async (
+	db: CascadeDb,
+	entities: CascadeEntity[],
+	log: CascadeDeleteLogger,
+): Promise<void> => {
+	const grouped = groupByType(entities);
+
+	for (const entityType of grouped.keys()) {
+		if (MESSAGE_CHILD_TYPES.has(entityType)) continue;
+		if (!KEYED_TYPES.has(entityType)) {
+			throw new Error(`Unknown entity type in cascade: ${entityType}`);
+		}
+	}
+
+	await runInTransaction(db, async (tx) => {
+		const messageIds = (grouped.get("Message") ?? []).map((k) => k.messageId);
+		await deleteMessageSubtree(tx as unknown as SubtreeDb, messageIds);
+
+		const threadMessageIds = (grouped.get("ThreadMessage") ?? []).map(
+			(k) => k.threadMessageId,
+		);
+		if (threadMessageIds.length > 0) {
+			await tx
+				.delete(threadMessageTable)
+				.where(inArray(threadMessageTable.threadMessageId, threadMessageIds));
+		}
+
+		const mailboxIds = (grouped.get("Mailbox") ?? []).map((k) => k.mailboxId);
+		if (mailboxIds.length > 0) {
+			await tx
+				.delete(mailboxTable)
+				.where(inArray(mailboxTable.mailboxId, mailboxIds));
+		}
+
+		const outboxMessageIds = (grouped.get("OutboxMessage") ?? []).map(
+			(k) => k.outboxMessageId,
+		);
+		if (outboxMessageIds.length > 0) {
+			await tx
+				.delete(outboxMessageTable)
+				.where(inArray(outboxMessageTable.outboxMessageId, outboxMessageIds));
+		}
+
+		const lockKeys = grouped.get("MailboxLock") ?? [];
+		if (lockKeys.length > 0) {
+			const conditions: SQL[] = lockKeys.map((k) =>
+				and(
+					eq(mailboxLockTable.mailboxId, k.mailboxId),
+					eq(mailboxLockTable.eventName, k.eventName),
+				),
+			) as SQL[];
+			await tx.delete(mailboxLockTable).where(or(...conditions));
+		}
+
+		const accountSettingIds = (grouped.get("AccountSetting") ?? []).map(
+			(k) => k.accountSettingId,
+		);
+		if (accountSettingIds.length > 0) {
+			await tx
+				.delete(accountSettingTable)
+				.where(
+					inArray(accountSettingTable.accountSettingId, accountSettingIds),
+				);
+		}
+
+		const addressIds = (grouped.get("Address") ?? []).map((k) => k.addressId);
+		if (addressIds.length > 0) {
+			await tx
+				.delete(addressTable)
+				.where(inArray(addressTable.addressId, addressIds));
+		}
+
+		const accountIds = (grouped.get("Account") ?? []).map((k) => k.accountId);
+		if (accountIds.length > 0) {
+			await tx
+				.delete(accountTable)
+				.where(inArray(accountTable.accountId, accountIds));
+		}
+
+		const placementMoveMessageIds = (
+			grouped.get("MessagePlacementMove") ?? []
+		).map((k) => k.messageId);
+		if (placementMoveMessageIds.length > 0) {
+			await tx
+				.delete(messagePlacementMoveTable)
+				.where(
+					inArray(messagePlacementMoveTable.messageId, placementMoveMessageIds),
+				);
+		}
+
+		// Composite key (messageId, flagName) — a message can carry an
+		// independent pending read-state marker and pending star marker at once
+		// (issue #1273), so a single-column inArray cannot target one row.
+		const flagPushKeys = grouped.get("MessageFlagPush") ?? [];
+		if (flagPushKeys.length > 0) {
+			const conditions: SQL[] = flagPushKeys.map((k) =>
+				and(
+					eq(messageFlagPushTable.messageId, k.messageId),
+					eq(messageFlagPushTable.flagName, k.flagName),
+				),
+			) as SQL[];
+			await tx.delete(messageFlagPushTable).where(or(...conditions));
+		}
+	});
+
+	log.info({ entityCount: entities.length }, "Drizzle cascade delete complete");
+};
+
+export type CascadeDeleter = (
+	entities: CascadeEntity[],
+	log: CascadeDeleteLogger,
+) => Promise<void>;
+
+/**
+ * Bind a cascade deleter to one Postgres connection. The account worker builds
+ * this once at module load on the `DATA_BACKEND=postgres` path and reuses it
+ * across invocations; the underlying pool is shared with no per-purge setup.
+ */
+export const createCascadeDeleter = (connectionUrl: string): CascadeDeleter => {
+	const db = drizzle(connectionUrl) as CascadeDb;
+	return (entities, log) => runDrizzleCascadeDelete(db, entities, log);
+};
+
+/**
+ * SQLite twin of {@link createCascadeDeleter} (RFC 036). Opens the shared SQLite
+ * file through the same serialized connection every writer uses and binds a
+ * cascade deleter to it. `runDrizzleCascadeDelete` deletes through the
+ * dialect-selected entity tables (`active-entities`, SQLite on this backend), so
+ * the same enumerated `CascadeEntity[]` drives an identical cascade over SQLite.
+ * Async because opening the SQLite handle loads the native binding lazily.
+ */
+export const createSqliteCascadeDeleter = async (
+	dbPath: string,
+): Promise<CascadeDeleter> => {
+	const { db } = await createSqliteDatabase(messageDataSchema, {
+		filename: dbPath,
+	});
+	const cascadeDb = db as unknown as CascadeDb;
+	return (entities, log) => runDrizzleCascadeDelete(cascadeDb, entities, log);
+};