@remit/drizzle-service 0.0.107 → 0.0.109

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.107",
+  "version": "0.0.109",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/id.ts

@@ -42,3 +42,12 @@ export const envelopeAddressId = (
 	order: number,
 ): string =>
 	generateDeterministicId(`envelopeaddress:${messageId}:${role}:${order}`);
+
+export const messageReferenceId = (
+	messageId: string,
+	referenceType: string,
+	referenceOrder: number,
+): string =>
+	deterministicBase36Id(
+		`msgref:${messageId}:${referenceType}:${referenceOrder}`,
+	);

package/src/index.ts

@@ -75,7 +75,6 @@ export {
 export { DrizzleMessageFlagRepository } from "./repos/message-flag.js";
 export { MessageLabelRepo } from "./repos/message-label.js";
 export {
-	DEFAULT_MESSAGE_OBJECT_DELETE_CONCURRENCY,
 	type MessageObjectStore,
 	messageObjectsInMemory,
 } from "./repos/message-objects.js";

package/src/repos/cascade-delete.ts

@@ -43,7 +43,6 @@ const MESSAGE_CHILD_TYPES = new Set([
 	"Envelope",
 	"BodyPart",
 	"BodyPartParameter",
-	"BodyPartStorage",
 	"RawMessageStorage",
 ]);
 

package/src/repos/envelope.test.ts

@@ -61,7 +61,6 @@ describe("DrizzleEnvelopeRepository", () => {
 			assert.equal(data.bodyPart.length, 0);
 			assert.equal(data.bodyPartParameter.length, 0);
 			assert.equal(data.rawMessageStorage.length, 0);
-			assert.equal(data.bodyPartStorage.length, 0);
 		});
 
 		test("throws NotFoundError for unknown messageId", async () => {

package/src/repos/envelope.ts

@@ -21,11 +21,10 @@ import {
 	bodyPartId as deriveBodyPartId,
 	bodyPartParameterId as deriveBodyPartParameterId,
 	envelopeId as deriveEnvelopeId,
-	deterministicBase36Id,
+	messageReferenceId as deriveMessageReferenceId,
 } from "../id.js";
 import {
 	bodyPartParameterTable,
-	bodyPartStorageTable,
 	bodyPartTable,
 	bodyStructureRefetchTable,
 	envelopeAddressTable,
@@ -38,7 +37,6 @@ import { inList } from "./in-list.js";
 import {
 	toBodyPartItem,
 	toBodyPartParameterItem,
-	toBodyPartStorageItem,
 	toEnvelopeAddressItem,
 	toEnvelopeItem,
 	toMessageReferenceItem,
@@ -74,7 +72,6 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
 			bodyParts,
 			bodyPartParameters,
 			rawMessageStorages,
-			bodyPartStorages,
 		] = await Promise.all([
 			this.db
 				.select()
@@ -100,10 +97,6 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
 				.select()
 				.from(rawMessageStorageTable)
 				.where(eq(rawMessageStorageTable.messageId, messageId)),
-			this.db
-				.select()
-				.from(bodyPartStorageTable)
-				.where(eq(bodyPartStorageTable.messageId, messageId)),
 		]);
 
 		if (envelopes.length === 0) {
@@ -117,7 +110,6 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
 			bodyPart: bodyParts.map(toBodyPartItem),
 			bodyPartParameter: bodyPartParameters.map(toBodyPartParameterItem),
 			rawMessageStorage: rawMessageStorages.map(toRawMessageStorageItem),
-			bodyPartStorage: bodyPartStorages.map(toBodyPartStorageItem),
 		};
 	}
 
@@ -229,8 +221,10 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
 			.where(eq(messageReferenceTable.messageId, messageId));
 		const rows = references.map((reference) => ({
 			...reference,
-			messageReferenceId: deterministicBase36Id(
-				`msgref:${messageId}:${reference.referenceType}:${reference.referenceOrder}`,
+			messageReferenceId: deriveMessageReferenceId(
+				messageId,
+				reference.referenceType,
+				reference.referenceOrder,
 			),
 			messageId,
 			envelopeId,

package/src/repos/i4-mailbox.test.ts

@@ -564,8 +564,6 @@ describe("MailboxRepo", () => {
 				sequenceNumber: uid,
 				rfc822Size: 10,
 				internalDate: 1700000000000,
-				envelopeId: deriveEnvelopeId(messageId),
-				rootBodyPartId: deriveRootBodyPartId(messageId),
 			});
 			await db.insert(threadMessageTable).values({
 				threadMessageId: randomId(),

package/src/repos/i4-mailbox.ts

@@ -42,11 +42,8 @@ import { messageFlagPushTable } from "../schema/i4-message-flag-push.js";
 import { messagePlacementMoveTable } from "../schema/i4-message-placement-move.js";
 import { messageTable } from "../schema/message-data.js";
 import { syncWarningTable } from "../schema/sync-warning.js";
-import { threadMessageTable } from "../schema/thread-message.js";
-import type { SearchIndex } from "../search/thread-message-search.js";
 import { inList } from "./in-list.js";
-import { deleteMessageSubtree } from "./message.js";
-import type { MessageObjects } from "./message-objects.js";
+import type { DrizzleMessageRepository } from "./message.js";
 
 const base36Translator = shortUuid.createTranslator(
 	shortUuid.constants.uuid25Base36,
@@ -61,7 +58,6 @@ type DB = RepositoryDb<Record<string, unknown>>;
  * the file's write lock, so the bound is what keeps a large folder's delete from
  * parking every other writer behind it (D8).
  */
-const MAIL_DELETE_BATCH_SIZE = 100;
 
 /**
  * The folders under `path`. A flat namespace nests nothing: with no delimiter
@@ -155,8 +151,7 @@ export class MailboxRepo implements IMailboxRepository {
 
 	constructor(
 		store: SqliteStore<Record<string, unknown>>,
-		private search: SearchIndex,
-		private messageObjects: MessageObjects,
+		private messages: Pick<DrizzleMessageRepository, "deleteMany">,
 	) {
 		this.db = store.db;
 		this.batch = store.batch;
@@ -549,32 +544,11 @@ export class MailboxRepo implements IMailboxRepository {
 		// Ordered, batched and resumable rather than one transaction (D8). The
 		// caller keeps the row `deleting` until the last commit, so an interrupted
 		// run re-enters here and continues against whatever is left.
-		for (;;) {
-			const rows = await this.db
-				.select({ messageId: messageTable.messageId })
-				.from(messageTable)
-				.where(eq(messageTable.mailboxId, mailboxId))
-				.limit(MAIL_DELETE_BATCH_SIZE);
-			if (rows.length === 0) break;
-			const messageIds = rows.map((row) => row.messageId);
-			const listings = await this.db
-				.select({ threadMessageId: threadMessageTable.threadMessageId })
-				.from(threadMessageTable)
-				.where(inList(threadMessageTable.messageId, messageIds));
-
-			// The primitive the rest of the codebase deletes mail with: nine
-			// per-message child tables. A bespoke table list would orphan them.
-			const subtree = deleteMessageSubtree(this.db, messageIds);
-			const [removed] = await this.batch([
-				subtree.removed,
-				...subtree.statements,
-				this.db
-					.delete(threadMessageTable)
-					.where(inList(threadMessageTable.messageId, messageIds)),
-			]);
-			await this.search.sync(listings.map((row) => row.threadMessageId));
-			await this.messageObjects.reclaim(removed);
-		}
+		const rows = await this.db
+			.select({ messageId: messageTable.messageId })
+			.from(messageTable)
+			.where(eq(messageTable.mailboxId, mailboxId));
+		await this.messages.deleteMany(rows.map((row) => row.messageId));
 
 		await this.db
 			.delete(mailboxSpecialUseTable)

package/src/repos/mappers.ts

@@ -1,7 +1,6 @@
 import type {
 	BodyPartItem,
 	BodyPartParameterItem,
-	BodyPartStorageItem,
 	EnvelopeAddressItem,
 	EnvelopeItem,
 	MessageReferenceItem,
@@ -9,7 +8,6 @@ import type {
 } from "@remit/data-ports";
 import type {
 	bodyPartParameterTable,
-	bodyPartStorageTable,
 	bodyPartTable,
 	envelopeAddressTable,
 	envelopeTable,
@@ -132,24 +130,3 @@ export function toRawMessageStorageItem(
 		...(row.expiresAt !== null ? { expiresAt: row.expiresAt } : {}),
 	};
 }
-
-export function toBodyPartStorageItem(
-	row: typeof bodyPartStorageTable.$inferSelect,
-): BodyPartStorageItem {
-	return {
-		bodyPartStorageId: row.bodyPartStorageId,
-		messageId: row.messageId,
-		bodyPartId: row.bodyPartId,
-		storageType: row.storageType,
-		storageLocation: row.storageLocation,
-		storageKey: row.storageKey,
-		decodedSizeBytes: row.decodedSizeBytes,
-		checksumSha256: row.checksumSha256,
-		contentEncoding: row.contentEncoding,
-		isDeduped: row.isDeduped,
-		storedAt: row.storedAt,
-		createdAt: row.createdAt,
-		updatedAt: row.updatedAt,
-		...(row.dedupHash !== null ? { dedupHash: row.dedupHash } : {}),
-	};
-}

package/src/repos/message-objects.sqlite.test.ts

@@ -12,10 +12,6 @@ import { after, before, describe, test } from "node:test";
 import type { StorageService } from "@remit/storage-service";
 import { createFilesystemStorageService } from "@remit/storage-service/filesystem";
 import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
-import {
-	envelopeId as deriveEnvelopeId,
-	rootBodyPartId as deriveRootBodyPartId,
-} from "../id.js";
 import {
 	createSqliteRepositories,
 	type SqliteRepositories,
@@ -25,6 +21,17 @@ import { mailboxCreated } from "./test-helpers.js";
 
 const ACCOUNT_CONFIG_ID = "cfg-objects";
 
+const bodyPart = (partPath: string) => ({
+	partPath,
+	parentPartPath: null,
+	mediaType: "TEXT" as const,
+	mediaSubtype: "plain",
+	transferEncoding: "7BIT" as const,
+	sizeOctets: 4,
+	isMultipart: false,
+	parameters: [],
+});
+
 describe("message objects follow the last row that references them", () => {
 	let db: TestDb;
 	let close: () => Promise<void>;
@@ -32,26 +39,26 @@ describe("message objects follow the last row that references them", () => {
 	let storage: StorageService;
 	let failDeletesAfter = Number.POSITIVE_INFINITY;
 	let repos: SqliteRepositories;
+	let deleteCalls: number[] = [];
 
 	before(async () => {
 		({ db, close } = await createTestDb());
 		directory = mkdtempSync(join(tmpdir(), "message-objects-"));
 		storage = createFilesystemStorageService(directory);
+		const counted: Pick<StorageService, "deleteKeys"> = {
+			deleteKeys: (keys) => {
+				deleteCalls.push(keys.length);
+				if (failDeletesAfter <= 0) {
+					return Promise.reject(new Error("storage went away"));
+				}
+				failDeletesAfter -= 1;
+				return storage.deleteKeys(keys);
+			},
+		};
 		repos = createSqliteRepositories(
 			{ app: db, mail: db, search: db },
 			BETTER_SQLITE3_LIMITS,
-			{
-				storage: {
-					deletePrefix: (prefix) => {
-						if (failDeletesAfter <= 0) {
-							return Promise.reject(new Error("storage went away"));
-						}
-						failDeletesAfter -= 1;
-						return storage.deletePrefix(prefix);
-					},
-				},
-				messageObjectDeleteConcurrency: 1,
-			},
+			{ storage: counted },
 		);
 	});
 
@@ -109,10 +116,9 @@ describe("message objects follow the last row that references them", () => {
 			sequenceNumber: uid,
 			rfc822Size: 10,
 			internalDate: 1700000000000,
-			envelopeId: deriveEnvelopeId(messageId),
-			rootBodyPartId: deriveRootBodyPartId(messageId),
 			bodyStorageKey: stored.uri,
 		});
+		await repos.envelope.upsertBodyParts(messageId, [bodyPart("1")]);
 		return { messageId, accountId, bodyStorageKey: stored.uri };
 	};
 
@@ -184,10 +190,9 @@ describe("message objects follow the last row that references them", () => {
 			sequenceNumber: 1,
 			rfc822Size: 10,
 			internalDate: 1700000000000,
-			envelopeId: deriveEnvelopeId(messageId),
-			rootBodyPartId: deriveRootBodyPartId(messageId),
 			bodyStorageKey: `file://${join(legacy, "body.eml")}`,
 		});
+		await repos.envelope.upsertBodyParts(messageId, [bodyPart("1")]);
 
 		await repos.message.delete(messageId);
 
@@ -201,7 +206,7 @@ describe("message objects follow the last row that references them", () => {
 		for (let uid = 1; uid <= 101; uid += 1) {
 			rows.push(await message(accountId, doomed.mailboxId, uid));
 		}
-		failDeletesAfter = 100;
+		failDeletesAfter = 1;
 
 		await assert.rejects(
 			repos.mailbox.deleteMailboxWithMail(accountId, doomed.mailboxId),
@@ -214,4 +219,53 @@ describe("message objects follow the last row that references them", () => {
 		);
 		assert.equal(gone.filter((exists) => !exists).length, 100);
 	});
+
+	const gone = async (
+		rows: readonly {
+			accountId: string;
+			messageId: string;
+			bodyStorageKey: string;
+		}[],
+	) => {
+		for (const row of rows) {
+			assert.equal(await storage.exists(row.bodyStorageKey), false);
+			assert.equal(await part(row), null);
+		}
+	};
+
+	test("deleting many messages makes one storage call per batch and keeps a body a copy shares", async () => {
+		const accountId = randomId();
+		const inbox = await folder(accountId, "INBOX");
+		const rows = [];
+		for (let uid = 1; uid <= 300; uid += 1) {
+			rows.push(await message(accountId, inbox.mailboxId, uid));
+		}
+		const [shared, ...unshared] = rows;
+		const copy = await message(accountId, inbox.mailboxId, 301, {
+			from: shared.messageId,
+		});
+		deleteCalls = [];
+
+		await repos.message.deleteMany(rows.map((row) => row.messageId));
+
+		assert.equal(deleteCalls.length, 3);
+		assert.ok(await storage.retrieve(copy.bodyStorageKey));
+		assert.ok(await part(copy));
+		await gone(unshared);
+	});
+
+	test("a folder delete makes one storage call per batch and removes every object", async () => {
+		const accountId = randomId();
+		const doomed = await folder(accountId, "Archive");
+		const rows = [];
+		for (let uid = 1; uid <= 250; uid += 1) {
+			rows.push(await message(accountId, doomed.mailboxId, uid));
+		}
+		deleteCalls = [];
+
+		await repos.mailbox.deleteMailboxWithMail(accountId, doomed.mailboxId);
+
+		assert.equal(deleteCalls.length, 3);
+		await gone(rows);
+	});
 });

package/src/repos/message-objects.ts

@@ -1,23 +1,18 @@
 import {
 	createMockStorageService,
-	messageObjectPrefixOf,
+	messageObjectKeysOf,
 	type StorageService,
 } from "@remit/storage-service";
-import pMap from "p-map";
 import type { RepositoryDb } from "../db.js";
 import { messageTable } from "../schema/message-data.js";
 import { inList } from "./in-list.js";
 
-export const DEFAULT_MESSAGE_OBJECT_DELETE_CONCURRENCY = 8;
-
 export interface MessageObjectStore {
-	storage: Pick<StorageService, "deletePrefix">;
-	messageObjectDeleteConcurrency: number;
+	storage: Pick<StorageService, "deleteKeys">;
 }
 
 export const messageObjectsInMemory = (): MessageObjectStore => ({
 	storage: createMockStorageService(),
-	messageObjectDeleteConcurrency: 1,
 });
 
 export interface RemovedMessageRow {
@@ -25,52 +20,51 @@ export interface RemovedMessageRow {
 	bodyStorageKey: string | null;
 }
 
-interface Candidate {
-	prefix: string;
-	bodyStorageKey: string;
+export interface RemovedBodyPartRow {
+	messageId: string;
+	partPath: string;
 }
 
-const candidatesOf = (row: RemovedMessageRow): Candidate[] => {
-	const { messageId, bodyStorageKey } = row;
-	if (bodyStorageKey === null) return [];
-	const ownerPrefix = messageObjectPrefixOf(bodyStorageKey);
-	const ownPrefix = ownerPrefix.replace(/[^/]+\/$/, `${messageId}/`);
-	const owner = { prefix: ownerPrefix, bodyStorageKey };
-	if (ownPrefix === ownerPrefix) return [owner];
-	const at = bodyStorageKey.lastIndexOf(ownerPrefix);
-	const ownBodyStorageKey = `${bodyStorageKey.slice(0, at)}${ownPrefix}${bodyStorageKey.slice(at + ownerPrefix.length)}`;
-	return [owner, { prefix: ownPrefix, bodyStorageKey: ownBodyStorageKey }];
-};
-
 export class MessageObjects {
 	constructor(
 		private readonly db: Pick<RepositoryDb<Record<string, unknown>>, "select">,
 		private readonly store: MessageObjectStore,
 	) {}
 
-	async reclaim(removed: readonly RemovedMessageRow[]): Promise<void> {
-		const candidates = new Map(
-			removed.flatMap(candidatesOf).map((c) => [c.prefix, c]),
-		);
-		if (candidates.size === 0) return;
-		const keys = [
-			...new Set([...candidates.values()].map((c) => c.bodyStorageKey)),
-		];
+	async reclaim(
+		removed: readonly RemovedMessageRow[],
+		removedParts: readonly RemovedBodyPartRow[],
+	): Promise<void> {
+		const partPathsOf = new Map<string, string[]>();
+		for (const { messageId, partPath } of removedParts) {
+			partPathsOf.set(messageId, [
+				...(partPathsOf.get(messageId) ?? []),
+				partPath,
+			]);
+		}
+		const own: string[] = [];
+		const sharedBy = new Map<string, string[]>();
+		for (const { messageId, bodyStorageKey } of removed) {
+			if (bodyStorageKey === null) continue;
+			const keys = messageObjectKeysOf(
+				bodyStorageKey,
+				messageId,
+				partPathsOf.get(messageId) ?? [],
+			);
+			own.push(...keys.own);
+			sharedBy.set(bodyStorageKey, keys.shared);
+		}
+		if (sharedBy.size === 0) return;
 		const referenced = await this.db
 			.select({ bodyStorageKey: messageTable.bodyStorageKey })
 			.from(messageTable)
-			.where(inList(messageTable.bodyStorageKey, keys));
-		const stillReferenced = new Set(
-			referenced.flatMap((row) =>
-				row.bodyStorageKey === null ? [] : [row.bodyStorageKey],
-			),
-		);
-		await pMap(
-			[...candidates.values()].filter(
-				(c) => !stillReferenced.has(c.bodyStorageKey),
-			),
-			(c) => this.store.storage.deletePrefix(c.prefix),
-			{ concurrency: this.store.messageObjectDeleteConcurrency },
-		);
+			.where(inList(messageTable.bodyStorageKey, [...sharedBy.keys()]));
+		for (const { bodyStorageKey } of referenced) {
+			if (bodyStorageKey !== null) sharedBy.delete(bodyStorageKey);
+		}
+		await this.store.storage.deleteKeys([
+			...own,
+			...[...sharedBy.values()].flat(),
+		]);
 	}
 }

package/src/repos/message.sqlite.test.ts

@@ -85,8 +85,6 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		const created = await repo.create({
 			...BASE_INPUT,
 			messageId: AUTO_MOVED_ID,
-			envelopeId: deriveEnvelopeId(AUTO_MOVED_ID),
-			rootBodyPartId: deriveRootBodyPartId(AUTO_MOVED_ID),
 			movedByRemit: true,
 			filterMove: filterMove as unknown as never,
 		});
@@ -160,8 +158,6 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 			sequenceNumber: 2,
 			rfc822Size: 1024,
 			internalDate: NOW,
-			envelopeId: deriveEnvelopeId(inboundId),
-			rootBodyPartId: deriveRootBodyPartId(inboundId),
 		});
 
 		assert.equal(created, true);
@@ -182,8 +178,6 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 				...BASE_INPUT,
 				messageId: LOCK_ID,
 				uid: 7,
-				envelopeId: deriveEnvelopeId(LOCK_ID),
-				rootBodyPartId: deriveRootBodyPartId(LOCK_ID),
 				status: "moving" as const,
 				syncStatus: "pending" as const,
 			});
@@ -255,8 +249,6 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 				...BASE_INPUT,
 				messageId: GONE_ID,
 				uid: 9,
-				envelopeId: deriveEnvelopeId(GONE_ID),
-				rootBodyPartId: deriveRootBodyPartId(GONE_ID),
 			});
 		});
 
@@ -319,19 +311,19 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		);
 	});
 
-	test("copySearchText gives a copy the source's search text", async () => {
+	test("copyMessage gives a copy the source's search text", async () => {
 		const source = "00000000-0000-0000-4444-000000000001";
 		const copy = "00000000-0000-0000-4444-000000000002";
-		for (const [messageId, uid] of [
-			[source, 201],
-			[copy, 202],
-		] as const) {
-			await repo.create({ ...BASE_INPUT, messageId, uid });
-			await repo.update(messageId, { bodyStorageKey: "file:///bodies/shared" });
-		}
+		await repo.create({ ...BASE_INPUT, messageId: source, uid: 201 });
+		await repo.update(source, { bodyStorageKey: "file:///bodies/shared" });
 		await repo.setSearchText(source, "the shared body");
 
-		await repo.copySearchText(source, copy);
+		await repo.copyMessage(source, {
+			...BASE_INPUT,
+			messageId: copy,
+			uid: 202,
+			bodyStorageKey: "file:///bodies/shared",
+		});
 
 		const missing = await repo.listMissingSearchText("", 100);
 		assert.equal(

package/src/repos/message.test.ts

@@ -6,10 +6,10 @@ import {
 	rootBodyPartId as deriveRootBodyPartId,
 } from "../id.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
+import { createTestDb, type TestDb } from "../test-db.js";
 import type { DrizzleEnvelopeRepository } from "./envelope.js";
 import type { DrizzleMessageRepository } from "./message.js";
 import { messageObjectsInMemory } from "./message-objects.js";
-import { createTestDb, type TestDb } from "./test-helpers.js";
 
 describe("DrizzleMessageRepository", () => {
 	let db: TestDb;
@@ -38,7 +38,7 @@ describe("DrizzleMessageRepository", () => {
 	};
 
 	before(async () => {
-		({ db, stop } = await createTestDb());
+		({ db, close: stop } = await createTestDb());
 		const repos = createSqliteRepositories(
 			{ app: db, mail: db, search: db },
 			BETTER_SQLITE3_LIMITS,
@@ -122,7 +122,6 @@ describe("DrizzleMessageRepository", () => {
 			assert.equal(desc.bodyPart.length, 0);
 			assert.equal(desc.bodyPartParameter.length, 0);
 			assert.equal(desc.rawMessageStorage.length, 0);
-			assert.equal(desc.bodyPartStorage.length, 0);
 		});
 
 		test("throws NotFoundError for unknown messageId", async () => {
@@ -149,8 +148,6 @@ describe("DrizzleMessageRepository", () => {
 				sequenceNumber: 1,
 				rfc822Size: 512,
 				internalDate: NOW,
-				envelopeId: deriveEnvelopeId(MOVE_MESSAGE_ID),
-				rootBodyPartId: deriveRootBodyPartId(MOVE_MESSAGE_ID),
 				status: "active" as const,
 				syncStatus: "synced" as const,
 			});
@@ -264,8 +261,6 @@ describe("DrizzleMessageRepository", () => {
 				sequenceNumber: 1,
 				rfc822Size: 10,
 				internalDate: NOW,
-				envelopeId: DEL_ENVELOPE_ID,
-				rootBodyPartId: DEL_ROOT_BODY_PART_ID,
 				status: "active",
 				syncStatus: "synced",
 			});

package/src/repos/message.ts

@@ -8,6 +8,7 @@ import type {
 import {
 	and,
 	asc,
+	type Column,
 	eq,
 	exists,
 	gt,
@@ -19,6 +20,7 @@ import {
 	type SQL,
 } from "drizzle-orm";
 import type { SqliteBatchItem, WriteBatch } from "../batch.js";
+import { chunk, chunkRows } from "../bound-parameters.js";
 import type { RepositoryDb, SqliteStore } from "../db.js";
 import {
 	CreateFailedConflictError,
@@ -26,13 +28,16 @@ import {
 	NotFoundError,
 } from "../error.js";
 import {
+	bodyPartId as deriveBodyPartId,
+	bodyPartParameterId as deriveBodyPartParameterId,
+	envelopeAddressId as deriveEnvelopeAddressId,
 	envelopeId as deriveEnvelopeId,
+	messageReferenceId as deriveMessageReferenceId,
 	rootBodyPartId as deriveRootBodyPartId,
 } from "../id.js";
 import { decodeToken, resultList } from "../pagination.js";
 import {
 	bodyPartParameterTable,
-	bodyPartStorageTable,
 	bodyPartTable,
 	bodyStructureRefetchTable,
 	envelopeAddressTable,
@@ -45,11 +50,10 @@ import {
 } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import type { SearchIndex } from "../search/thread-message-search.js";
-import { inList } from "./in-list.js";
+import { inList, pairInList } from "./in-list.js";
 import {
 	toBodyPartItem,
 	toBodyPartParameterItem,
-	toBodyPartStorageItem,
 	toEnvelopeAddressItem,
 	toEnvelopeItem,
 	toMessageReferenceItem,
@@ -118,18 +122,23 @@ export type SubtreeDb = Pick<
 	"delete" | "select"
 >;
 
+export type SubtreeGuard = (messageId: Column) => SQL;
+
+export const MAIL_DELETE_BATCH_SIZE = 100;
+
 /**
  * The statements that delete a message and its whole per-message subtree by
  * message id: the child tables, then the message row. The caller runs them as
- * one batch so the subtree goes atomically. `removed` reads the rows the batch
- * deletes; it goes first, so it sees them under the same guard. `guard` rides
- * every statement, so a guard that reads the message row holds until the last
- * statement removes it.
+ * one batch so the subtree goes atomically. `removed` and `removedParts` read
+ * the message and body-part rows the batch deletes; they go first, so they see
+ * them under the same guard. `guard` rides every statement on that
+ * statement's own `messageId` column, so a guard that reads the message row
+ * holds until the last statement removes it.
  */
 export function deleteMessageSubtree(
 	db: SubtreeDb,
-	messageIds: string[],
-	guard?: SQL,
+	messageIds: readonly string[],
+	guard?: SubtreeGuard,
 ) {
 	const removed = db
 		.select({
@@ -137,46 +146,110 @@ export function deleteMessageSubtree(
 			bodyStorageKey: messageTable.bodyStorageKey,
 		})
 		.from(messageTable)
-		.where(and(inList(messageTable.messageId, messageIds), guard));
-	if (messageIds.length === 0) return { removed, statements: [] };
+		.where(
+			and(
+				inList(messageTable.messageId, messageIds),
+				guard?.(messageTable.messageId),
+			),
+		);
+	const removedParts = db
+		.select({
+			messageId: bodyPartTable.messageId,
+			partPath: bodyPartTable.partPath,
+		})
+		.from(bodyPartTable)
+		.where(
+			and(
+				inList(bodyPartTable.messageId, messageIds),
+				guard?.(bodyPartTable.messageId),
+			),
+		);
+	if (messageIds.length === 0) {
+		return { removed, removedParts, statements: [] };
+	}
 	const statements: SqliteBatchItem[] = [
 		db
 			.delete(messageFlagTable)
-			.where(and(inList(messageFlagTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(messageFlagTable.messageId, messageIds),
+					guard?.(messageFlagTable.messageId),
+				),
+			),
 		db
 			.delete(messageReferenceTable)
-			.where(and(inList(messageReferenceTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(messageReferenceTable.messageId, messageIds),
+					guard?.(messageReferenceTable.messageId),
+				),
+			),
 		db
 			.delete(envelopeAddressTable)
-			.where(and(inList(envelopeAddressTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(envelopeAddressTable.messageId, messageIds),
+					guard?.(envelopeAddressTable.messageId),
+				),
+			),
 		db
 			.delete(bodyPartParameterTable)
-			.where(and(inList(bodyPartParameterTable.messageId, messageIds), guard)),
-		db
-			.delete(bodyPartStorageTable)
-			.where(and(inList(bodyPartStorageTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(bodyPartParameterTable.messageId, messageIds),
+					guard?.(bodyPartParameterTable.messageId),
+				),
+			),
 		db
 			.delete(rawMessageStorageTable)
-			.where(and(inList(rawMessageStorageTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(rawMessageStorageTable.messageId, messageIds),
+					guard?.(rawMessageStorageTable.messageId),
+				),
+			),
 		db
 			.delete(bodyPartTable)
-			.where(and(inList(bodyPartTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(bodyPartTable.messageId, messageIds),
+					guard?.(bodyPartTable.messageId),
+				),
+			),
 		db
 			.delete(bodyStructureRefetchTable)
 			.where(
-				and(inList(bodyStructureRefetchTable.messageId, messageIds), guard),
+				and(
+					inList(bodyStructureRefetchTable.messageId, messageIds),
+					guard?.(bodyStructureRefetchTable.messageId),
+				),
 			),
 		db
 			.delete(messageSearchTextTable)
-			.where(and(inList(messageSearchTextTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(messageSearchTextTable.messageId, messageIds),
+					guard?.(messageSearchTextTable.messageId),
+				),
+			),
 		db
 			.delete(envelopeTable)
-			.where(and(inList(envelopeTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(envelopeTable.messageId, messageIds),
+					guard?.(envelopeTable.messageId),
+				),
+			),
 		db
 			.delete(messageTable)
-			.where(and(inList(messageTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(messageTable.messageId, messageIds),
+					guard?.(messageTable.messageId),
+				),
+			),
 	];
-	return { removed, statements };
+	return { removed, removedParts, statements };
 }
 
 /**
@@ -210,6 +283,7 @@ const placementTerms = (expected: PlacementPredicate): SQL[] => {
 export class DrizzleMessageRepository implements IMessageRepository {
 	private readonly db: DB;
 	private readonly batch: WriteBatch;
+	private readonly maxBoundParameters: number;
 
 	constructor(
 		store: SqliteStore<Record<string, unknown>>,
@@ -218,6 +292,7 @@ export class DrizzleMessageRepository implements IMessageRepository {
 	) {
 		this.db = store.db;
 		this.batch = store.batch;
+		this.maxBoundParameters = store.maxBoundParameters;
 	}
 
 	private rowOf(input: CreateMessageInput) {
@@ -322,7 +397,6 @@ export class DrizzleMessageRepository implements IMessageRepository {
 			bodyParts,
 			bodyPartParameters,
 			rawMessageStorages,
-			bodyPartStorages,
 		] = await Promise.all([
 			this.db
 				.select()
@@ -356,10 +430,6 @@ export class DrizzleMessageRepository implements IMessageRepository {
 				.select()
 				.from(rawMessageStorageTable)
 				.where(eq(rawMessageStorageTable.messageId, messageId)),
-			this.db
-				.select()
-				.from(bodyPartStorageTable)
-				.where(eq(bodyPartStorageTable.messageId, messageId)),
 		]);
 
 		if (messages.length === 0) {
@@ -382,7 +452,6 @@ export class DrizzleMessageRepository implements IMessageRepository {
 			bodyPart: bodyParts.map(toBodyPartItem),
 			bodyPartParameter: bodyPartParameters.map(toBodyPartParameterItem),
 			rawMessageStorage: rawMessageStorages.map(toRawMessageStorageItem),
-			bodyPartStorage: bodyPartStorages.map(toBodyPartStorageItem),
 		};
 	}
 
@@ -472,16 +541,172 @@ export class DrizzleMessageRepository implements IMessageRepository {
 			});
 	}
 
-	async copySearchText(
+	async copyMessage(
 		sourceMessageId: string,
-		targetMessageId: string,
-	): Promise<void> {
-		const [source] = await this.db
-			.select({ text: messageSearchTextTable.text })
-			.from(messageSearchTextTable)
-			.where(eq(messageSearchTextTable.messageId, sourceMessageId));
-		if (!source) return;
-		await this.setSearchText(targetMessageId, source.text);
+		input: CreateMessageInput,
+	): Promise<{ item: MessageItem; created: boolean }> {
+		const targetMessageId = input.messageId;
+		const [searchTexts, envelopes, references, addresses, parts, parameters] =
+			await Promise.all([
+				this.db
+					.select({ text: messageSearchTextTable.text })
+					.from(messageSearchTextTable)
+					.where(eq(messageSearchTextTable.messageId, sourceMessageId)),
+				this.db
+					.select()
+					.from(envelopeTable)
+					.where(eq(envelopeTable.messageId, sourceMessageId)),
+				this.db
+					.select()
+					.from(messageReferenceTable)
+					.where(eq(messageReferenceTable.messageId, sourceMessageId)),
+				this.db
+					.select()
+					.from(envelopeAddressTable)
+					.where(eq(envelopeAddressTable.messageId, sourceMessageId)),
+				this.db
+					.select()
+					.from(bodyPartTable)
+					.where(eq(bodyPartTable.messageId, sourceMessageId)),
+				this.db
+					.select()
+					.from(bodyPartParameterTable)
+					.where(eq(bodyPartParameterTable.messageId, sourceMessageId)),
+			]);
+
+		const now = Date.now();
+		const envelopeId = deriveEnvelopeId(targetMessageId);
+		const copiedPartIds = new Map(
+			parts.map((part) => [
+				part.bodyPartId,
+				deriveBodyPartId(targetMessageId, part.partPath),
+			]),
+		);
+		const partPaths = new Map(
+			parts.map((part) => [part.bodyPartId, part.partPath]),
+		);
+
+		const searchTextRows = searchTexts.map((row) => ({
+			messageId: targetMessageId,
+			text: row.text,
+			createdAt: now,
+			updatedAt: now,
+		}));
+		const envelopeRows = envelopes.map((row) => ({
+			...row,
+			envelopeId,
+			messageId: targetMessageId,
+			createdAt: now,
+			updatedAt: now,
+		}));
+		const referenceRows = references.map((row) => ({
+			...row,
+			messageReferenceId: deriveMessageReferenceId(
+				targetMessageId,
+				row.referenceType,
+				row.referenceOrder,
+			),
+			messageId: targetMessageId,
+			envelopeId,
+			createdAt: now,
+			updatedAt: now,
+		}));
+		const addressRows = addresses.map((row) => ({
+			...row,
+			envelopeAddressId: deriveEnvelopeAddressId(
+				targetMessageId,
+				row.addressRole,
+				row.addressOrder,
+			),
+			messageId: targetMessageId,
+			createdAt: now,
+			updatedAt: now,
+		}));
+		const partRows = parts.map((row) => ({
+			...row,
+			bodyPartId: deriveBodyPartId(targetMessageId, row.partPath),
+			messageId: targetMessageId,
+			parentBodyPartId:
+				row.parentBodyPartId === null
+					? null
+					: (copiedPartIds.get(row.parentBodyPartId) ?? null),
+			createdAt: now,
+			updatedAt: now,
+		}));
+		const parameterRows = parameters.flatMap((row) => {
+			const partPath = partPaths.get(row.bodyPartId);
+			if (partPath === undefined) return [];
+			return [
+				{
+					...row,
+					bodyPartParameterId: deriveBodyPartParameterId(
+						targetMessageId,
+						partPath,
+						row.parameterName,
+					),
+					messageId: targetMessageId,
+					bodyPartId: deriveBodyPartId(targetMessageId, partPath),
+					createdAt: now,
+					updatedAt: now,
+				},
+			];
+		});
+
+		const insertCopy = this.db
+			.insert(messageTable)
+			.values(this.rowOf(input))
+			.onConflictDoNothing({ target: messageTable.messageId })
+			.returning();
+		const clones: SqliteBatchItem[] = [
+			...searchTextRows.map((row) =>
+				this.db
+					.insert(messageSearchTextTable)
+					.values(row)
+					.onConflictDoNothing({ target: messageSearchTextTable.messageId }),
+			),
+			...chunkRows(envelopeTable, envelopeRows, this.maxBoundParameters).map(
+				(slice) =>
+					this.db.insert(envelopeTable).values(slice).onConflictDoNothing(),
+			),
+			...chunkRows(
+				messageReferenceTable,
+				referenceRows,
+				this.maxBoundParameters,
+			).map((slice) =>
+				this.db
+					.insert(messageReferenceTable)
+					.values(slice)
+					.onConflictDoNothing(),
+			),
+			...chunkRows(
+				envelopeAddressTable,
+				addressRows,
+				this.maxBoundParameters,
+			).map((slice) =>
+				this.db
+					.insert(envelopeAddressTable)
+					.values(slice)
+					.onConflictDoNothing(),
+			),
+			...chunkRows(bodyPartTable, partRows, this.maxBoundParameters).map(
+				(slice) =>
+					this.db.insert(bodyPartTable).values(slice).onConflictDoNothing(),
+			),
+			...chunkRows(
+				bodyPartParameterTable,
+				parameterRows,
+				this.maxBoundParameters,
+			).map((slice) =>
+				this.db
+					.insert(bodyPartParameterTable)
+					.values(slice)
+					.onConflictDoNothing(),
+			),
+		];
+		const [inserted] = await this.batch([insertCopy, ...clones]);
+		const [row] = inserted;
+		if (row) return { item: toMessageItem(row), created: true };
+		return { item: await this.get(targetMessageId), created: false };
 	}
 
 	async syncSearch(messageIds: string[]): Promise<void> {
@@ -542,31 +767,65 @@ export class DrizzleMessageRepository implements IMessageRepository {
 					),
 				),
 		);
-		const listings = await this.db
-			.select({ threadMessageId: threadMessageTable.threadMessageId })
-			.from(threadMessageTable)
-			.where(eq(threadMessageTable.messageId, messageId));
-		const subtree = deleteMessageSubtree(this.db, [messageId], placed);
-		const [removed] = await this.batch([
-			subtree.removed,
-			this.db
-				.delete(threadMessageTable)
-				.where(and(eq(threadMessageTable.messageId, messageId), placed)),
-			...subtree.statements,
-		]);
-		await this.search.sync(listings.map((row) => row.threadMessageId));
-		await this.messageObjects.reclaim(removed);
-		return removed.length > 0;
+		return (await this.removeMessages([messageId], () => placed)) > 0;
+	}
+
+	async deleteManyIfPlacement(
+		mailboxId: string,
+		placements: ReadonlyArray<{ messageId: string; uid: number }>,
+	): Promise<void> {
+		const placed = this.db
+			.select({ messageId: messageTable.messageId })
+			.from(messageTable)
+			.where(
+				and(
+					eq(messageTable.mailboxId, mailboxId),
+					pairInList(
+						messageTable.messageId,
+						messageTable.uid,
+						placements.map(({ messageId, uid }) => [messageId, uid] as const),
+					),
+				),
+			);
+		await this.removeMessages(
+			placements.map((placement) => placement.messageId),
+			(column) => inArray(column, placed),
+		);
 	}
 
 	async deleteMany(messageIds: string[]): Promise<void> {
-		if (messageIds.length === 0) return;
-		const subtree = deleteMessageSubtree(this.db, messageIds);
-		const [removed] = await this.batch([
-			subtree.removed,
-			...subtree.statements,
-		]);
-		await this.messageObjects.reclaim(removed);
+		await this.removeMessages(messageIds);
+	}
+
+	private async removeMessages(
+		messageIds: readonly string[],
+		guard?: SubtreeGuard,
+	): Promise<number> {
+		let removedCount = 0;
+		for (const batch of chunk(messageIds, MAIL_DELETE_BATCH_SIZE)) {
+			const listings = await this.db
+				.select({ threadMessageId: threadMessageTable.threadMessageId })
+				.from(threadMessageTable)
+				.where(inList(threadMessageTable.messageId, batch));
+			const subtree = deleteMessageSubtree(this.db, batch, guard);
+			const [removed, removedParts] = await this.batch([
+				subtree.removed,
+				subtree.removedParts,
+				this.db
+					.delete(threadMessageTable)
+					.where(
+						and(
+							inList(threadMessageTable.messageId, batch),
+							guard?.(threadMessageTable.messageId),
+						),
+					),
+				...subtree.statements,
+			]);
+			await this.search.sync(listings.map((row) => row.threadMessageId));
+			await this.messageObjects.reclaim(removed, removedParts);
+			removedCount += removed.length;
+		}
+		return removedCount;
 	}
 
 	async listByMailbox(

package/src/schema/message-data.ts

@@ -1,7 +1,6 @@
 import * as entities from "@remit/drizzle-sqlite-schema";
 
 const bodyPartParameterTable = entities.bodyPartParameters;
-const bodyPartStorageTable = entities.bodyPartStorages;
 const bodyPartTable = entities.bodyParts;
 const bodyStructureRefetchTable = entities.bodyStructureRefetches;
 const envelopeAddressTable = entities.envelopeAddresses;
@@ -14,7 +13,6 @@ const rawMessageStorageTable = entities.rawMessageStorages;
 
 export {
 	bodyPartParameterTable,
-	bodyPartStorageTable,
 	bodyPartTable,
 	bodyStructureRefetchTable,
 	envelopeAddressTable,
@@ -33,7 +31,6 @@ export const messageDataSchema = {
 	bodyPart: bodyPartTable,
 	bodyPartParameter: bodyPartParameterTable,
 	rawMessageStorage: rawMessageStorageTable,
-	bodyPartStorage: bodyPartStorageTable,
 	bodyStructureRefetch: bodyStructureRefetchTable,
 	message: messageTable,
 	messageFlag: messageFlagTable,

package/src/schema-sets/mail.ts

@@ -4,7 +4,6 @@
 export {
 	addresses,
 	bodyPartParameters,
-	bodyPartStorages,
 	bodyParts,
 	bodyStructureRefetches,
 	calendarSuggestions,

package/src/sqlite-repositories.ts

@@ -48,7 +48,12 @@ export const composeSqliteRepositories = (
 	const messageObjects = new MessageObjects(mail.db, objects);
 	const accountSetting = new AccountSettingRepo(app.db);
 	const address = new AddressRepo(mail.db);
-	const mailbox = new MailboxRepo(mail, searchIndex, messageObjects);
+	const message = new DrizzleMessageRepository(
+		mail,
+		searchIndex,
+		messageObjects,
+	);
+	const mailbox = new MailboxRepo(mail, message);
 	const account = new AccountRepo(app.db, mailbox);
 	const calendarSuggestion = new CalendarSuggestionRepo(mail.db);
 	return {
@@ -63,7 +68,7 @@ export const composeSqliteRepositories = (
 			accountSetting,
 		),
 		mailboxLock: new MailboxLockRepo(mail.db),
-		message: new DrizzleMessageRepository(mail, searchIndex, messageObjects),
+		message,
 		messageFlag: new DrizzleMessageFlagRepository(mail.db),
 		outboxMessage: new OutboxMessageRepo(mail.db),
 		outboxAttachment: new OutboxAttachmentRepo(mail),