@remit/drizzle-service 0.0.99 → 0.0.101

package/src/repos/cascade-delete.ts

@@ -1,5 +1,10 @@
 import { and, eq, inArray, or, type SQL } from "drizzle-orm";
-import type { Db } from "../db.js";
+import { runBatch, type SqliteBatchItem } from "../batch.js";
+import {
+	openSqliteDatabases,
+	type SqliteDatabasePaths,
+	type SqliteDatabases,
+} from "../databases.js";
 import { accountTable } from "../schema/i4-account-config.js";
 import { accountSettingTable } from "../schema/i4-account-setting.js";
 import { addressTable } from "../schema/i4-address.js";
@@ -8,12 +13,11 @@ import { mailboxLockTable } from "../schema/i4-mailbox-lock.js";
 import { messageFlagPushTable } from "../schema/i4-message-flag-push.js";
 import { messagePlacementMoveTable } from "../schema/i4-message-placement-move.js";
 import { outboxMessageTable } from "../schema/i4-outbox-message.js";
-import { messageDataSchema } from "../schema/message-data.js";
 import { quarantineTable } from "../schema/quarantine.js";
+import { syncWarningTable } from "../schema/sync-warning.js";
 import { threadMessageTable } from "../schema/thread-message.js";
-import { createSqliteDatabase } from "../sqlite-client.js";
-import { runInTransaction } from "../tx.js";
-import { deleteMessageSubtree, type SubtreeDb } from "./message.js";
+import { SearchIndex } from "../search/thread-message-search.js";
+import { deleteMessageSubtree } from "./message.js";
 
 export interface CascadeEntity {
 	entityType: string;
@@ -24,8 +28,6 @@ export interface CascadeDeleteLogger {
 	info(obj: Record<string, unknown>, msg: string): void;
 }
 
-type CascadeDb = Db<Record<string, unknown>>;
-
 /**
  * Per-message entity types. Their rows are removed with the message subtree by
  * message id, so their individually-enumerated keys are not deleted one by one —
@@ -40,7 +42,6 @@ const MESSAGE_CHILD_TYPES = new Set([
 	"BodyPart",
 	"BodyPartParameter",
 	"BodyPartStorage",
-	"BodyPartContent",
 	"RawMessageStorage",
 ]);
 
@@ -71,15 +72,17 @@ const groupByType = (
 
 /**
  * Relational equivalent of the DynamoDB BatchWriteItem cascade: delete every
- * enumerated row for an account/tenant in one transaction. Message subtrees
- * (message + nine child tables) are deleted set-wise by message id and emit a
- * `message.removed` outbox row each for search-index cleanup; the remaining
+ * enumerated row for an account/tenant, one batch per database — the mail rows
+ * first, then the app rows, then the search documents of the deleted thread
+ * messages. Every statement is keyed, so a rerun after a failure between two
+ * batches deletes what is left. Message subtrees
+ * (message + nine child tables) are deleted set-wise by message id; the remaining
  * entity types are deleted by their enumerated primary keys. AccountConfig is
  * never deleted here — the caller removes it last through its repository, the
  * cascade-in-progress marker.
  */
 export const runDrizzleCascadeDelete = async (
-	db: CascadeDb,
+	{ app, mail, search }: SqliteDatabases,
 	entities: CascadeEntity[],
 	log: CascadeDeleteLogger,
 ): Promise<void> => {
@@ -92,103 +95,132 @@ export const runDrizzleCascadeDelete = async (
 		}
 	}
 
-	await runInTransaction(db, async (tx) => {
-		const messageIds = (grouped.get("Message") ?? []).map((k) => k.messageId);
-		await deleteMessageSubtree(tx as unknown as SubtreeDb, messageIds);
-
-		const threadMessageIds = (grouped.get("ThreadMessage") ?? []).map(
-			(k) => k.threadMessageId,
-		);
-		if (threadMessageIds.length > 0) {
-			await tx
+	const items: SqliteBatchItem[] = [];
+	const appItems: SqliteBatchItem[] = [];
+	const messageIds = (grouped.get("Message") ?? []).map((k) => k.messageId);
+	items.push(...deleteMessageSubtree(mail, messageIds));
+
+	const threadMessageIds = (grouped.get("ThreadMessage") ?? []).map(
+		(k) => k.threadMessageId,
+	);
+	if (threadMessageIds.length > 0) {
+		items.push(
+			mail
 				.delete(threadMessageTable)
-				.where(inArray(threadMessageTable.threadMessageId, threadMessageIds));
-		}
+				.where(inArray(threadMessageTable.threadMessageId, threadMessageIds)),
+		);
+	}
 
-		const mailboxIds = (grouped.get("Mailbox") ?? []).map((k) => k.mailboxId);
-		if (mailboxIds.length > 0) {
-			await tx
+	const mailboxIds = (grouped.get("Mailbox") ?? []).map((k) => k.mailboxId);
+	if (mailboxIds.length > 0) {
+		items.push(
+			mail
+				.delete(syncWarningTable)
+				.where(inArray(syncWarningTable.mailboxId, mailboxIds)),
+		);
+		items.push(
+			mail
 				.delete(mailboxTable)
-				.where(inArray(mailboxTable.mailboxId, mailboxIds));
-		}
-
-		const outboxMessageIds = (grouped.get("OutboxMessage") ?? []).map(
-			(k) => k.outboxMessageId,
+				.where(inArray(mailboxTable.mailboxId, mailboxIds)),
 		);
-		if (outboxMessageIds.length > 0) {
-			await tx
+	}
+
+	const outboxMessageIds = (grouped.get("OutboxMessage") ?? []).map(
+		(k) => k.outboxMessageId,
+	);
+	if (outboxMessageIds.length > 0) {
+		items.push(
+			mail
 				.delete(outboxMessageTable)
-				.where(inArray(outboxMessageTable.outboxMessageId, outboxMessageIds));
-		}
+				.where(inArray(outboxMessageTable.outboxMessageId, outboxMessageIds)),
+		);
+	}
 
-		const lockKeys = grouped.get("MailboxLock") ?? [];
-		if (lockKeys.length > 0) {
-			const conditions: SQL[] = lockKeys.map((k) =>
-				and(
-					eq(mailboxLockTable.mailboxId, k.mailboxId),
-					eq(mailboxLockTable.eventName, k.eventName),
-				),
-			) as SQL[];
-			await tx.delete(mailboxLockTable).where(or(...conditions));
-		}
+	const lockKeys = grouped.get("MailboxLock") ?? [];
+	if (lockKeys.length > 0) {
+		const conditions: SQL[] = lockKeys.map((k) =>
+			and(
+				eq(mailboxLockTable.mailboxId, k.mailboxId),
+				eq(mailboxLockTable.eventName, k.eventName),
+			),
+		) as SQL[];
+		items.push(mail.delete(mailboxLockTable).where(or(...conditions)));
+	}
 
-		const accountSettingIds = (grouped.get("AccountSetting") ?? []).map(
-			(k) => k.accountSettingId,
-		);
-		if (accountSettingIds.length > 0) {
-			await tx
+	const accountSettingIds = (grouped.get("AccountSetting") ?? []).map(
+		(k) => k.accountSettingId,
+	);
+	if (accountSettingIds.length > 0) {
+		appItems.push(
+			app
 				.delete(accountSettingTable)
 				.where(
 					inArray(accountSettingTable.accountSettingId, accountSettingIds),
-				);
-		}
+				),
+		);
+	}
 
-		const addressIds = (grouped.get("Address") ?? []).map((k) => k.addressId);
-		if (addressIds.length > 0) {
-			await tx
+	const addressIds = (grouped.get("Address") ?? []).map((k) => k.addressId);
+	if (addressIds.length > 0) {
+		items.push(
+			mail
 				.delete(addressTable)
-				.where(inArray(addressTable.addressId, addressIds));
-		}
+				.where(inArray(addressTable.addressId, addressIds)),
+		);
+	}
 
-		const accountIds = (grouped.get("Account") ?? []).map((k) => k.accountId);
-		if (accountIds.length > 0) {
-			// Quarantine rows are deleted set-wise by account rather than from an
-			// enumerated key, the way message subtrees are. They carry the user's
-			// own folder names and parser output quoting the message, so account
-			// deletion has to take them even though nothing enumerates them.
-			await tx
+	const accountIds = (grouped.get("Account") ?? []).map((k) => k.accountId);
+	if (accountIds.length > 0) {
+		// Quarantine rows are deleted set-wise by account rather than from an
+		// enumerated key, the way message subtrees are. They carry the user's
+		// own folder names and parser output quoting the message, so account
+		// deletion has to take them even though nothing enumerates them.
+		items.push(
+			mail
 				.delete(quarantineTable)
-				.where(inArray(quarantineTable.accountId, accountIds));
-			await tx
+				.where(inArray(quarantineTable.accountId, accountIds)),
+		);
+		items.push(
+			mail
+				.delete(syncWarningTable)
+				.where(inArray(syncWarningTable.accountId, accountIds)),
+		);
+		appItems.push(
+			app
 				.delete(accountTable)
-				.where(inArray(accountTable.accountId, accountIds));
-		}
+				.where(inArray(accountTable.accountId, accountIds)),
+		);
+	}
 
-		const placementMoveMessageIds = (
-			grouped.get("MessagePlacementMove") ?? []
-		).map((k) => k.messageId);
-		if (placementMoveMessageIds.length > 0) {
-			await tx
+	const placementMoveMessageIds = (
+		grouped.get("MessagePlacementMove") ?? []
+	).map((k) => k.messageId);
+	if (placementMoveMessageIds.length > 0) {
+		items.push(
+			mail
 				.delete(messagePlacementMoveTable)
 				.where(
 					inArray(messagePlacementMoveTable.messageId, placementMoveMessageIds),
-				);
-		}
-
-		// Composite key (messageId, flagName) — a message can carry an
-		// independent pending read-state marker and pending star marker at once
-		// (issue #1273), so a single-column inArray cannot target one row.
-		const flagPushKeys = grouped.get("MessageFlagPush") ?? [];
-		if (flagPushKeys.length > 0) {
-			const conditions: SQL[] = flagPushKeys.map((k) =>
-				and(
-					eq(messageFlagPushTable.messageId, k.messageId),
-					eq(messageFlagPushTable.flagName, k.flagName),
 				),
-			) as SQL[];
-			await tx.delete(messageFlagPushTable).where(or(...conditions));
-		}
-	});
+		);
+	}
+
+	// Composite key (messageId, flagName) — a message can carry an
+	// independent pending read-state marker and pending star marker at once
+	// (issue #1273), so a single-column inArray cannot target one row.
+	const flagPushKeys = grouped.get("MessageFlagPush") ?? [];
+	if (flagPushKeys.length > 0) {
+		const conditions: SQL[] = flagPushKeys.map((k) =>
+			and(
+				eq(messageFlagPushTable.messageId, k.messageId),
+				eq(messageFlagPushTable.flagName, k.flagName),
+			),
+		) as SQL[];
+		items.push(mail.delete(messageFlagPushTable).where(or(...conditions)));
+	}
+	await runBatch(mail, items);
+	await runBatch(app, appItems);
+	await new SearchIndex(mail, search).sync(threadMessageIds);
 
 	log.info({ entityCount: entities.length }, "Drizzle cascade delete complete");
 };
@@ -199,15 +231,13 @@ export type CascadeDeleter = (
 ) => Promise<void>;
 
 /**
- * Open the shared SQLite file through the same serialized connection every
- * writer uses and bind a cascade deleter to it. Async because opening the
- * SQLite handle loads the native binding lazily.
+ * Open the SQLite databases through the same connection settings every writer
+ * uses and bind a cascade deleter to them. Async because opening a SQLite handle
+ * loads the native binding lazily.
  */
 export const createSqliteCascadeDeleter = async (
-	dbPath: string,
+	paths: SqliteDatabasePaths,
 ): Promise<CascadeDeleter> => {
-	const { db } = await createSqliteDatabase(messageDataSchema, {
-		filename: dbPath,
-	});
-	return (entities, log) => runDrizzleCascadeDelete(db, entities, log);
+	const databases = await openSqliteDatabases(paths);
+	return (entities, log) => runDrizzleCascadeDelete(databases, entities, log);
 };

package/src/repos/envelope.test.ts

@@ -34,22 +34,7 @@ describe("DrizzleEnvelopeRepository", () => {
 		await stop();
 	});
 
-	describe("getMessageData — MessageData shape (8 members)", () => {
-		test("returns all 8 collection members", async () => {
-			const data = await repo.getMessageData(MESSAGE_ID);
-			const keys = Object.keys(data).sort();
-			assert.deepStrictEqual(keys, [
-				"bodyPart",
-				"bodyPartContent",
-				"bodyPartParameter",
-				"bodyPartStorage",
-				"envelope",
-				"envelopeAddress",
-				"messageReference",
-				"rawMessageStorage",
-			]);
-		});
-
+	describe("getMessageData — MessageData shape", () => {
 		test("envelope has 1 row with correct fields", async () => {
 			const data = await repo.getMessageData(MESSAGE_ID);
 			assert.equal(data.envelope.length, 1);
@@ -70,7 +55,6 @@ describe("DrizzleEnvelopeRepository", () => {
 			assert.equal(data.bodyPartParameter.length, 0);
 			assert.equal(data.rawMessageStorage.length, 0);
 			assert.equal(data.bodyPartStorage.length, 0);
-			assert.equal(data.bodyPartContent.length, 0);
 		});
 
 		test("throws NotFoundError for unknown messageId", async () => {

package/src/repos/envelope.ts

@@ -1,14 +1,15 @@
 import type {
-	BodyPartContentUpsertInput,
 	BodyPartItem,
 	BodyPartUpsertInput,
 	CreateEnvelopeInput,
 	EnvelopeItem,
 	IEnvelopeRepository,
 	MessageData,
+	MessageReferenceInput,
 	UpdateEnvelopeInput,
 } from "@remit/data-ports";
 import { eq, inArray } from "drizzle-orm";
+import { runBatch } from "../batch.js";
 import type { Db } from "../db.js";
 import {
 	CreateFailedConflictError,
@@ -16,25 +17,24 @@ import {
 	NotFoundError,
 } from "../error.js";
 import {
-	bodyPartContentId as deriveBodyPartContentId,
 	bodyPartId as deriveBodyPartId,
 	bodyPartParameterId as deriveBodyPartParameterId,
 	envelopeId as deriveEnvelopeId,
+	deterministicBase36Id,
 } from "../id.js";
 import {
-	bodyPartContentTable,
 	bodyPartParameterTable,
 	bodyPartStorageTable,
 	bodyPartTable,
+	bodyStructureRefetchTable,
 	envelopeAddressTable,
 	envelopeTable,
 	type MessageDataSchema,
 	messageReferenceTable,
+	messageTable,
 	rawMessageStorageTable,
 } from "../schema/message-data.js";
-import { runInTransaction } from "../tx.js";
 import {
-	toBodyPartContentItem,
 	toBodyPartItem,
 	toBodyPartParameterItem,
 	toBodyPartStorageItem,
@@ -66,7 +66,6 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
 			bodyPartParameters,
 			rawMessageStorages,
 			bodyPartStorages,
-			bodyPartContents,
 		] = await Promise.all([
 			this.db
 				.select()
@@ -96,10 +95,6 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
 				.select()
 				.from(bodyPartStorageTable)
 				.where(eq(bodyPartStorageTable.messageId, messageId)),
-			this.db
-				.select()
-				.from(bodyPartContentTable)
-				.where(eq(bodyPartContentTable.messageId, messageId)),
 		]);
 
 		if (envelopes.length === 0) {
@@ -114,7 +109,6 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
 			bodyPartParameter: bodyPartParameters.map(toBodyPartParameterItem),
 			rawMessageStorage: rawMessageStorages.map(toRawMessageStorageItem),
 			bodyPartStorage: bodyPartStorages.map(toBodyPartStorageItem),
-			bodyPartContent: bodyPartContents.map(toBodyPartContentItem),
 		};
 	}
 
@@ -180,38 +174,68 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
 			(r) => r.bodyPartParameterId,
 		);
 
-		await runInTransaction(this.db, async (tx) => {
-			await tx
-				.insert(bodyPartTable)
-				.values(dedupedBodyPartRows)
+		const insertParts = this.db
+			.insert(bodyPartTable)
+			.values(dedupedBodyPartRows)
+			.onConflictDoUpdate({
+				target: bodyPartTable.bodyPartId,
+				set: { updatedAt: now },
+			});
+		if (dedupedParameterRows.length === 0) {
+			await runBatch(this.db, [insertParts]);
+			return;
+		}
+		await runBatch(this.db, [
+			insertParts,
+			this.db
+				.insert(bodyPartParameterTable)
+				.values(dedupedParameterRows)
 				.onConflictDoUpdate({
-					target: bodyPartTable.bodyPartId,
-					set: { updatedAt: now },
-				});
+					target: bodyPartParameterTable.bodyPartParameterId,
+					set: {
+						parameterValue: bodyPartParameterTable.parameterValue,
+						updatedAt: now,
+					},
+				}),
+		]);
+	}
 
-			if (dedupedParameterRows.length > 0) {
-				await tx
-					.insert(bodyPartParameterTable)
-					.values(dedupedParameterRows)
-					.onConflictDoUpdate({
-						target: bodyPartParameterTable.bodyPartParameterId,
-						set: {
-							parameterValue: bodyPartParameterTable.parameterValue,
-							updatedAt: now,
-						},
-					});
-			}
-		});
+	async replaceMessageReferences(
+		messageId: string,
+		references: MessageReferenceInput[],
+	): Promise<void> {
+		const now = Date.now();
+		const envelopeId = deriveEnvelopeId(messageId);
+		const clear = this.db
+			.delete(messageReferenceTable)
+			.where(eq(messageReferenceTable.messageId, messageId));
+		if (references.length === 0) {
+			await runBatch(this.db, [clear]);
+			return;
+		}
+		await runBatch(this.db, [
+			clear,
+			this.db.insert(messageReferenceTable).values(
+				references.map((reference) => ({
+					...reference,
+					messageReferenceId: deterministicBase36Id(
+						`msgref:${messageId}:${reference.referenceType}:${reference.referenceOrder}`,
+					),
+					messageId,
+					envelopeId,
+					createdAt: now,
+					updatedAt: now,
+				})),
+			),
+		]);
 	}
 
 	async deleteManyEnvelopes(envelopeIds: string[]): Promise<void> {
 		if (envelopeIds.length === 0) return;
 
-		await runInTransaction(this.db, async (tx) => {
-			await tx
-				.delete(envelopeTable)
-				.where(inArray(envelopeTable.envelopeId, envelopeIds));
-		});
+		await this.db
+			.delete(envelopeTable)
+			.where(inArray(envelopeTable.envelopeId, envelopeIds));
 	}
 
 	async createEnvelope(input: CreateEnvelopeInput): Promise<EnvelopeItem> {
@@ -301,37 +325,6 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
 			.where(eq(envelopeTable.envelopeId, envelopeId));
 	}
 
-	async upsertBodyPartContents(
-		messageId: string,
-		contents: BodyPartContentUpsertInput[],
-	): Promise<void> {
-		if (contents.length === 0) return;
-
-		const now = Date.now();
-
-		const rows = contents.map((c) => ({
-			bodyPartContentId: deriveBodyPartContentId(messageId, c.bodyPartId),
-			messageId,
-			bodyPartId: c.bodyPartId,
-			content: c.content,
-			contentLength: c.content.length,
-			createdAt: now,
-			updatedAt: now,
-		}));
-
-		await this.db
-			.insert(bodyPartContentTable)
-			.values(rows)
-			.onConflictDoUpdate({
-				target: bodyPartContentTable.bodyPartContentId,
-				set: {
-					content: bodyPartContentTable.content,
-					contentLength: bodyPartContentTable.contentLength,
-					updatedAt: now,
-				},
-			});
-	}
-
 	async listBodyParts(messageId: string): Promise<BodyPartItem[]> {
 		const rows = await this.db
 			.select()
@@ -339,4 +332,27 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
 			.where(eq(bodyPartTable.messageId, messageId));
 		return rows.map(toBodyPartItem);
 	}
+
+	async listQueuedStructureRefetches(
+		mailboxId: string,
+		limit: number,
+	): Promise<Array<{ messageId: string; uid: number }>> {
+		return this.db
+			.select({ messageId: messageTable.messageId, uid: messageTable.uid })
+			.from(bodyStructureRefetchTable)
+			.innerJoin(
+				messageTable,
+				eq(messageTable.messageId, bodyStructureRefetchTable.messageId),
+			)
+			.where(eq(messageTable.mailboxId, mailboxId))
+			.orderBy(messageTable.uid)
+			.limit(limit);
+	}
+
+	async clearStructureRefetches(messageIds: string[]): Promise<void> {
+		if (messageIds.length === 0) return;
+		await this.db
+			.delete(bodyStructureRefetchTable)
+			.where(inArray(bodyStructureRefetchTable.messageId, messageIds));
+	}
 }