@remit/drizzle-service 0.0.106 → 0.0.108

package/src/repos/message.ts

@@ -8,6 +8,7 @@ import type {
 import {
 	and,
 	asc,
+	type Column,
 	eq,
 	exists,
 	gt,
@@ -19,6 +20,7 @@ import {
 	type SQL,
 } from "drizzle-orm";
 import type { SqliteBatchItem, WriteBatch } from "../batch.js";
+import { chunk, chunkRows } from "../bound-parameters.js";
 import type { RepositoryDb, SqliteStore } from "../db.js";
 import {
 	CreateFailedConflictError,
@@ -26,7 +28,11 @@ import {
 	NotFoundError,
 } from "../error.js";
 import {
+	bodyPartId as deriveBodyPartId,
+	bodyPartParameterId as deriveBodyPartParameterId,
+	envelopeAddressId as deriveEnvelopeAddressId,
 	envelopeId as deriveEnvelopeId,
+	messageReferenceId as deriveMessageReferenceId,
 	rootBodyPartId as deriveRootBodyPartId,
 } from "../id.js";
 import { decodeToken, resultList } from "../pagination.js";
@@ -45,7 +51,7 @@ import {
 } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import type { SearchIndex } from "../search/thread-message-search.js";
-import { inList } from "./in-list.js";
+import { inList, pairInList } from "./in-list.js";
 import {
 	toBodyPartItem,
 	toBodyPartParameterItem,
@@ -55,6 +61,7 @@ import {
 	toMessageReferenceItem,
 	toRawMessageStorageItem,
 } from "./mappers.js";
+import type { MessageObjects } from "./message-objects.js";
 
 type DB = RepositoryDb<Record<string, unknown>>;
 
@@ -112,59 +119,147 @@ function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
 	};
 }
 
-export type SubtreeDb = Pick<RepositoryDb<Record<string, unknown>>, "delete">;
+export type SubtreeDb = Pick<
+	RepositoryDb<Record<string, unknown>>,
+	"delete" | "select"
+>;
+
+export type SubtreeGuard = (messageId: Column) => SQL;
+
+export const MAIL_DELETE_BATCH_SIZE = 100;
 
 /**
  * The statements that delete a message and its whole per-message subtree by
  * message id: the child tables, then the message row. The caller runs them as
- * one batch so the subtree goes atomically; the last statement returns the
- * message ids it removed. `guard` rides every statement, so
- * a guard that reads the message row holds until the last statement removes it.
+ * one batch so the subtree goes atomically. `removed` and `removedParts` read
+ * the message and body-part rows the batch deletes; they go first, so they see
+ * them under the same guard. `guard` rides every statement on that
+ * statement's own `messageId` column, so a guard that reads the message row
+ * holds until the last statement removes it.
  */
 export function deleteMessageSubtree(
 	db: SubtreeDb,
-	messageIds: string[],
-	guard?: SQL,
-): SqliteBatchItem[] {
-	if (messageIds.length === 0) return [];
-	return [
+	messageIds: readonly string[],
+	guard?: SubtreeGuard,
+) {
+	const removed = db
+		.select({
+			messageId: messageTable.messageId,
+			bodyStorageKey: messageTable.bodyStorageKey,
+		})
+		.from(messageTable)
+		.where(
+			and(
+				inList(messageTable.messageId, messageIds),
+				guard?.(messageTable.messageId),
+			),
+		);
+	const removedParts = db
+		.select({
+			messageId: bodyPartTable.messageId,
+			partPath: bodyPartTable.partPath,
+		})
+		.from(bodyPartTable)
+		.where(
+			and(
+				inList(bodyPartTable.messageId, messageIds),
+				guard?.(bodyPartTable.messageId),
+			),
+		);
+	if (messageIds.length === 0) {
+		return { removed, removedParts, statements: [] };
+	}
+	const statements: SqliteBatchItem[] = [
 		db
 			.delete(messageFlagTable)
-			.where(and(inList(messageFlagTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(messageFlagTable.messageId, messageIds),
+					guard?.(messageFlagTable.messageId),
+				),
+			),
 		db
 			.delete(messageReferenceTable)
-			.where(and(inList(messageReferenceTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(messageReferenceTable.messageId, messageIds),
+					guard?.(messageReferenceTable.messageId),
+				),
+			),
 		db
 			.delete(envelopeAddressTable)
-			.where(and(inList(envelopeAddressTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(envelopeAddressTable.messageId, messageIds),
+					guard?.(envelopeAddressTable.messageId),
+				),
+			),
 		db
 			.delete(bodyPartParameterTable)
-			.where(and(inList(bodyPartParameterTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(bodyPartParameterTable.messageId, messageIds),
+					guard?.(bodyPartParameterTable.messageId),
+				),
+			),
 		db
 			.delete(bodyPartStorageTable)
-			.where(and(inList(bodyPartStorageTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(bodyPartStorageTable.messageId, messageIds),
+					guard?.(bodyPartStorageTable.messageId),
+				),
+			),
 		db
 			.delete(rawMessageStorageTable)
-			.where(and(inList(rawMessageStorageTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(rawMessageStorageTable.messageId, messageIds),
+					guard?.(rawMessageStorageTable.messageId),
+				),
+			),
 		db
 			.delete(bodyPartTable)
-			.where(and(inList(bodyPartTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(bodyPartTable.messageId, messageIds),
+					guard?.(bodyPartTable.messageId),
+				),
+			),
 		db
 			.delete(bodyStructureRefetchTable)
 			.where(
-				and(inList(bodyStructureRefetchTable.messageId, messageIds), guard),
+				and(
+					inList(bodyStructureRefetchTable.messageId, messageIds),
+					guard?.(bodyStructureRefetchTable.messageId),
+				),
 			),
 		db
 			.delete(messageSearchTextTable)
-			.where(and(inList(messageSearchTextTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(messageSearchTextTable.messageId, messageIds),
+					guard?.(messageSearchTextTable.messageId),
+				),
+			),
 		db
 			.delete(envelopeTable)
-			.where(and(inList(envelopeTable.messageId, messageIds), guard)),
+			.where(
+				and(
+					inList(envelopeTable.messageId, messageIds),
+					guard?.(envelopeTable.messageId),
+				),
+			),
 		db
 			.delete(messageTable)
-			.where(and(inList(messageTable.messageId, messageIds), guard))
-			.returning({ messageId: messageTable.messageId }),
+			.where(
+				and(
+					inList(messageTable.messageId, messageIds),
+					guard?.(messageTable.messageId),
+				),
+			),
 	];
+	return { removed, removedParts, statements };
 }
 
 /**
@@ -198,13 +293,16 @@ const placementTerms = (expected: PlacementPredicate): SQL[] => {
 export class DrizzleMessageRepository implements IMessageRepository {
 	private readonly db: DB;
 	private readonly batch: WriteBatch;
+	private readonly maxBoundParameters: number;
 
 	constructor(
 		store: SqliteStore<Record<string, unknown>>,
 		private search: SearchIndex,
+		private messageObjects: MessageObjects,
 	) {
 		this.db = store.db;
 		this.batch = store.batch;
+		this.maxBoundParameters = store.maxBoundParameters;
 	}
 
 	private rowOf(input: CreateMessageInput) {
@@ -459,16 +557,172 @@ export class DrizzleMessageRepository implements IMessageRepository {
 			});
 	}
 
-	async copySearchText(
+	async copyMessage(
 		sourceMessageId: string,
-		targetMessageId: string,
-	): Promise<void> {
-		const [source] = await this.db
-			.select({ text: messageSearchTextTable.text })
-			.from(messageSearchTextTable)
-			.where(eq(messageSearchTextTable.messageId, sourceMessageId));
-		if (!source) return;
-		await this.setSearchText(targetMessageId, source.text);
+		input: CreateMessageInput,
+	): Promise<{ item: MessageItem; created: boolean }> {
+		const targetMessageId = input.messageId;
+		const [searchTexts, envelopes, references, addresses, parts, parameters] =
+			await Promise.all([
+				this.db
+					.select({ text: messageSearchTextTable.text })
+					.from(messageSearchTextTable)
+					.where(eq(messageSearchTextTable.messageId, sourceMessageId)),
+				this.db
+					.select()
+					.from(envelopeTable)
+					.where(eq(envelopeTable.messageId, sourceMessageId)),
+				this.db
+					.select()
+					.from(messageReferenceTable)
+					.where(eq(messageReferenceTable.messageId, sourceMessageId)),
+				this.db
+					.select()
+					.from(envelopeAddressTable)
+					.where(eq(envelopeAddressTable.messageId, sourceMessageId)),
+				this.db
+					.select()
+					.from(bodyPartTable)
+					.where(eq(bodyPartTable.messageId, sourceMessageId)),
+				this.db
+					.select()
+					.from(bodyPartParameterTable)
+					.where(eq(bodyPartParameterTable.messageId, sourceMessageId)),
+			]);
+
+		const now = Date.now();
+		const envelopeId = deriveEnvelopeId(targetMessageId);
+		const copiedPartIds = new Map(
+			parts.map((part) => [
+				part.bodyPartId,
+				deriveBodyPartId(targetMessageId, part.partPath),
+			]),
+		);
+		const partPaths = new Map(
+			parts.map((part) => [part.bodyPartId, part.partPath]),
+		);
+
+		const searchTextRows = searchTexts.map((row) => ({
+			messageId: targetMessageId,
+			text: row.text,
+			createdAt: now,
+			updatedAt: now,
+		}));
+		const envelopeRows = envelopes.map((row) => ({
+			...row,
+			envelopeId,
+			messageId: targetMessageId,
+			createdAt: now,
+			updatedAt: now,
+		}));
+		const referenceRows = references.map((row) => ({
+			...row,
+			messageReferenceId: deriveMessageReferenceId(
+				targetMessageId,
+				row.referenceType,
+				row.referenceOrder,
+			),
+			messageId: targetMessageId,
+			envelopeId,
+			createdAt: now,
+			updatedAt: now,
+		}));
+		const addressRows = addresses.map((row) => ({
+			...row,
+			envelopeAddressId: deriveEnvelopeAddressId(
+				targetMessageId,
+				row.addressRole,
+				row.addressOrder,
+			),
+			messageId: targetMessageId,
+			createdAt: now,
+			updatedAt: now,
+		}));
+		const partRows = parts.map((row) => ({
+			...row,
+			bodyPartId: deriveBodyPartId(targetMessageId, row.partPath),
+			messageId: targetMessageId,
+			parentBodyPartId:
+				row.parentBodyPartId === null
+					? null
+					: (copiedPartIds.get(row.parentBodyPartId) ?? null),
+			createdAt: now,
+			updatedAt: now,
+		}));
+		const parameterRows = parameters.flatMap((row) => {
+			const partPath = partPaths.get(row.bodyPartId);
+			if (partPath === undefined) return [];
+			return [
+				{
+					...row,
+					bodyPartParameterId: deriveBodyPartParameterId(
+						targetMessageId,
+						partPath,
+						row.parameterName,
+					),
+					messageId: targetMessageId,
+					bodyPartId: deriveBodyPartId(targetMessageId, partPath),
+					createdAt: now,
+					updatedAt: now,
+				},
+			];
+		});
+
+		const insertCopy = this.db
+			.insert(messageTable)
+			.values(this.rowOf(input))
+			.onConflictDoNothing({ target: messageTable.messageId })
+			.returning();
+		const clones: SqliteBatchItem[] = [
+			...searchTextRows.map((row) =>
+				this.db
+					.insert(messageSearchTextTable)
+					.values(row)
+					.onConflictDoNothing({ target: messageSearchTextTable.messageId }),
+			),
+			...chunkRows(envelopeTable, envelopeRows, this.maxBoundParameters).map(
+				(slice) =>
+					this.db.insert(envelopeTable).values(slice).onConflictDoNothing(),
+			),
+			...chunkRows(
+				messageReferenceTable,
+				referenceRows,
+				this.maxBoundParameters,
+			).map((slice) =>
+				this.db
+					.insert(messageReferenceTable)
+					.values(slice)
+					.onConflictDoNothing(),
+			),
+			...chunkRows(
+				envelopeAddressTable,
+				addressRows,
+				this.maxBoundParameters,
+			).map((slice) =>
+				this.db
+					.insert(envelopeAddressTable)
+					.values(slice)
+					.onConflictDoNothing(),
+			),
+			...chunkRows(bodyPartTable, partRows, this.maxBoundParameters).map(
+				(slice) =>
+					this.db.insert(bodyPartTable).values(slice).onConflictDoNothing(),
+			),
+			...chunkRows(
+				bodyPartParameterTable,
+				parameterRows,
+				this.maxBoundParameters,
+			).map((slice) =>
+				this.db
+					.insert(bodyPartParameterTable)
+					.values(slice)
+					.onConflictDoNothing(),
+			),
+		];
+		const [inserted] = await this.batch([insertCopy, ...clones]);
+		const [row] = inserted;
+		if (row) return { item: toMessageItem(row), created: true };
+		return { item: await this.get(targetMessageId), created: false };
 	}
 
 	async syncSearch(messageIds: string[]): Promise<void> {
@@ -529,24 +783,65 @@ export class DrizzleMessageRepository implements IMessageRepository {
 					),
 				),
 		);
-		const listings = await this.db
-			.select({ threadMessageId: threadMessageTable.threadMessageId })
-			.from(threadMessageTable)
-			.where(eq(threadMessageTable.messageId, messageId));
-		const results = await this.batch([
-			this.db
-				.delete(threadMessageTable)
-				.where(and(eq(threadMessageTable.messageId, messageId), placed)),
-			...deleteMessageSubtree(this.db, [messageId], placed),
-		]);
-		await this.search.sync(listings.map((row) => row.threadMessageId));
-		const removed = results.at(-1);
-		return Array.isArray(removed) && removed.length > 0;
+		return (await this.removeMessages([messageId], () => placed)) > 0;
+	}
+
+	async deleteManyIfPlacement(
+		mailboxId: string,
+		placements: ReadonlyArray<{ messageId: string; uid: number }>,
+	): Promise<void> {
+		const placed = this.db
+			.select({ messageId: messageTable.messageId })
+			.from(messageTable)
+			.where(
+				and(
+					eq(messageTable.mailboxId, mailboxId),
+					pairInList(
+						messageTable.messageId,
+						messageTable.uid,
+						placements.map(({ messageId, uid }) => [messageId, uid] as const),
+					),
+				),
+			);
+		await this.removeMessages(
+			placements.map((placement) => placement.messageId),
+			(column) => inArray(column, placed),
+		);
 	}
 
 	async deleteMany(messageIds: string[]): Promise<void> {
-		if (messageIds.length === 0) return;
-		await this.batch(deleteMessageSubtree(this.db, messageIds));
+		await this.removeMessages(messageIds);
+	}
+
+	private async removeMessages(
+		messageIds: readonly string[],
+		guard?: SubtreeGuard,
+	): Promise<number> {
+		let removedCount = 0;
+		for (const batch of chunk(messageIds, MAIL_DELETE_BATCH_SIZE)) {
+			const listings = await this.db
+				.select({ threadMessageId: threadMessageTable.threadMessageId })
+				.from(threadMessageTable)
+				.where(inList(threadMessageTable.messageId, batch));
+			const subtree = deleteMessageSubtree(this.db, batch, guard);
+			const [removed, removedParts] = await this.batch([
+				subtree.removed,
+				subtree.removedParts,
+				this.db
+					.delete(threadMessageTable)
+					.where(
+						and(
+							inList(threadMessageTable.messageId, batch),
+							guard?.(threadMessageTable.messageId),
+						),
+					),
+				...subtree.statements,
+			]);
+			await this.search.sync(listings.map((row) => row.threadMessageId));
+			await this.messageObjects.reclaim(removed, removedParts);
+			removedCount += removed.length;
+		}
+		return removedCount;
 	}
 
 	async listByMailbox(

package/src/repos/thread-message-body-muted.sqlite.test.ts

@@ -16,6 +16,7 @@ import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import type { DrizzleThreadMessageRepository } from "./thread-message.js";
 
 const ACCOUNT = "acct-seam";
@@ -68,6 +69,7 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).threadMessage;
 	});
 

package/src/repos/thread-message-field-terms.sqlite.test.ts

@@ -6,6 +6,7 @@ import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import {
 	type DrizzleThreadMessageRepository,
 	THREAD_SEARCH_MAX_LIMIT,
@@ -67,6 +68,7 @@ describe("DrizzleThreadMessageRepository.listByFieldTerms (sqlite, #459)", () =>
 				search: created.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).threadMessage;
 
 		await repo.create(

package/src/repos/thread-message.sqlite.test.ts

@@ -13,6 +13,7 @@ import {
 	LIVE_TOTALS,
 	totalRows,
 } from "./category-fixture.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import {
 	type DrizzleThreadMessageRepository,
 	THREAD_SEARCH_MAX_LIMIT,
@@ -68,6 +69,7 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).threadMessage;
 	});
 

package/src/repos/thread-message.test.ts

@@ -10,6 +10,7 @@ import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import {
 	clampThreadSearchLimit,
 	type DrizzleThreadMessageRepository,
@@ -102,6 +103,7 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
 				search: harness.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).threadMessage;
 	});
 
@@ -411,6 +413,7 @@ describe("DrizzleThreadMessageRepository.searchByDate", () => {
 				search: harness.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		);
 		repo = repositories.threadMessage;
 		messages = repositories.message;
@@ -949,6 +952,7 @@ describe("DrizzleThreadMessageRepository — native text search", () => {
 				search: harness.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).threadMessage;
 	});
 
@@ -1128,6 +1132,7 @@ describe("DrizzleThreadMessageRepository — core CRUD", () => {
 				search: harness.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).threadMessage;
 	});
 

package/src/search/thread-message-search.sqlite.test.ts

@@ -6,6 +6,7 @@ import { drizzle } from "drizzle-orm/better-sqlite3";
 import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { syncStore } from "../databases.js";
 import type { Db } from "../db.js";
+import { messageObjectsInMemory } from "../repos/message-objects.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { applyMigrationSet } from "../test-shipped-sqlite-schema.js";
@@ -62,6 +63,7 @@ describe("the thread-message search index in its own database", () => {
 			search: search.db,
 		},
 		BETTER_SQLITE3_LIMITS,
+		messageObjectsInMemory(),
 	).threadMessage;
 
 	const found = async (query: string): Promise<string[]> =>

package/src/sqlite-repositories.ts

@@ -31,20 +31,29 @@ import { LabelRepo } from "./repos/label.js";
 import { DrizzleMessageRepository } from "./repos/message.js";
 import { DrizzleMessageFlagRepository } from "./repos/message-flag.js";
 import { MessageLabelRepo } from "./repos/message-label.js";
+import {
+	type MessageObjectStore,
+	MessageObjects,
+} from "./repos/message-objects.js";
 import { QuarantineRepo } from "./repos/quarantine.js";
 import { SyncWarningRepo } from "./repos/sync-warning.js";
 import { DrizzleThreadMessageRepository } from "./repos/thread-message.js";
 import { SearchIndex } from "./search/thread-message-search.js";
 
-export const composeSqliteRepositories = ({
-	app,
-	mail,
-	search,
-}: SqliteStores) => {
+export const composeSqliteRepositories = (
+	{ app, mail, search }: SqliteStores,
+	objects: MessageObjectStore,
+) => {
 	const searchIndex = new SearchIndex(mail.db, search);
+	const messageObjects = new MessageObjects(mail.db, objects);
 	const accountSetting = new AccountSettingRepo(app.db);
 	const address = new AddressRepo(mail.db);
-	const mailbox = new MailboxRepo(mail, searchIndex);
+	const message = new DrizzleMessageRepository(
+		mail,
+		searchIndex,
+		messageObjects,
+	);
+	const mailbox = new MailboxRepo(mail, message);
 	const account = new AccountRepo(app.db, mailbox);
 	const calendarSuggestion = new CalendarSuggestionRepo(mail.db);
 	return {
@@ -59,7 +68,7 @@ export const composeSqliteRepositories = ({
 			accountSetting,
 		),
 		mailboxLock: new MailboxLockRepo(mail.db),
-		message: new DrizzleMessageRepository(mail, searchIndex),
+		message,
 		messageFlag: new DrizzleMessageFlagRepository(mail.db),
 		outboxMessage: new OutboxMessageRepo(mail.db),
 		outboxAttachment: new OutboxAttachmentRepo(mail),
@@ -90,5 +99,6 @@ export type SqliteRepositories = ReturnType<typeof composeSqliteRepositories>;
 export const createSqliteRepositories = (
 	databases: SqliteDatabases,
 	limits: SqliteLimits,
+	objects: MessageObjectStore,
 ): SqliteRepositories =>
-	composeSqliteRepositories(syncStores(databases, limits));
+	composeSqliteRepositories(syncStores(databases, limits), objects);