@remit/drizzle-service 0.0.106 → 0.0.107

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.106",
+  "version": "0.0.107",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",
@@ -36,7 +36,9 @@
     "uuid": "^13.0.2",
     "@remit/drizzle-sqlite-schema": "*",
     "better-sqlite3": "^12.11.1",
-    "@types/better-sqlite3": "^7.6.13"
+    "@types/better-sqlite3": "^7.6.13",
+    "@remit/storage-service": "*",
+    "p-map": "^2.1.0"
   },
   "license": "MIT",
   "publishConfig": {

package/src/index.ts

@@ -74,6 +74,11 @@ export {
 } from "./repos/message.js";
 export { DrizzleMessageFlagRepository } from "./repos/message-flag.js";
 export { MessageLabelRepo } from "./repos/message-label.js";
+export {
+	DEFAULT_MESSAGE_OBJECT_DELETE_CONCURRENCY,
+	type MessageObjectStore,
+	messageObjectsInMemory,
+} from "./repos/message-objects.js";
 export { QuarantineRepo } from "./repos/quarantine.js";
 export { SyncWarningRepo } from "./repos/sync-warning.js";
 export { DrizzleThreadMessageRepository } from "./repos/thread-message.js";

package/src/repair/junk-only-address.sqlite.test.ts

@@ -7,6 +7,7 @@ import { drizzle } from "drizzle-orm/better-sqlite3";
 import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { AccountSettingRepo } from "../repos/i4-account-setting.js";
 import type { MailboxSpecialUseRepo } from "../repos/i4-mailbox-special-use.js";
+import { messageObjectsInMemory } from "../repos/message-objects.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { applyMigrationSet } from "../test-shipped-sqlite-schema.js";
 import {
@@ -190,6 +191,7 @@ describe("addresses standing only on mail in Junk", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).mailboxSpecialUse;
 		accountSetting = new AccountSettingRepo(db as never);
 		applyMigrationSet(sqlite, "app");

package/src/repos/account.conformance.d1.test.ts

@@ -3,6 +3,7 @@ import { NotFoundError } from "../error.js";
 import { randomId } from "../id.js";
 import { composeSqliteRepositories } from "../sqlite-repositories.js";
 import { openD1TestStores } from "../test-d1.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 let close: (() => Promise<void>) | undefined;
 
@@ -10,7 +11,8 @@ accountRepositoryConformance({
 	async createRepository() {
 		const opened = await openD1TestStores();
 		close = opened.close;
-		return composeSqliteRepositories(opened.stores).account;
+		return composeSqliteRepositories(opened.stores, messageObjectsInMemory())
+			.account;
 	},
 	teardown: async () => {
 		await close?.();

package/src/repos/account.conformance.sqlite.test.ts

@@ -5,6 +5,7 @@ import { randomId } from "../id.js";
 import { accountTable } from "../schema.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 let close: (() => Promise<void>) | undefined;
 
@@ -21,6 +22,7 @@ accountRepositoryConformance({
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).account;
 	},
 	teardown: async () => {

package/src/repos/calendar-default.suite.ts

@@ -6,6 +6,7 @@ import type { RepositoryDb } from "../db.js";
 import { composeSqliteRepositories } from "../sqlite-repositories.js";
 import type { OpenTestStores } from "../test-stores.js";
 import { CalendarCollectionRepo } from "./calendar-collection.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 const ACCOUNT_CONFIG_ID = "account-config-default";
 
@@ -19,7 +20,10 @@ export const calendarDefaultSuite = (open: OpenTestStores): void => {
 			const opened = await open();
 			db = opened.stores.app.db;
 			close = opened.close;
-			store = composeSqliteRepositories(opened.stores).calendarStore;
+			store = composeSqliteRepositories(
+				opened.stores,
+				messageObjectsInMemory(),
+			).calendarStore;
 		});
 
 		after(() => close());

package/src/repos/calendar-put.suite.ts

@@ -18,6 +18,7 @@ import {
 } from "../schema.js";
 import { composeSqliteRepositories } from "../sqlite-repositories.js";
 import type { OpenTestStores } from "../test-stores.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 const ACCOUNT_CONFIG_ID = "account-config-1";
 
@@ -74,7 +75,10 @@ export const calendarPutSuite = (open: OpenTestStores): void => {
 			const opened = await open();
 			db = opened.stores.app.db;
 			close = opened.close;
-			store = composeSqliteRepositories(opened.stores).calendarStore;
+			store = composeSqliteRepositories(
+				opened.stores,
+				messageObjectsInMemory(),
+			).calendarStore;
 			const collection = await provisionDefaultCalendar(
 				store,
 				ACCOUNT_CONFIG_ID,

package/src/repos/cascade-delete.ts

@@ -102,7 +102,7 @@ export const runDrizzleCascadeDelete = async (
 	const items: SqliteBatchItem[] = [];
 	const appItems: SqliteBatchItem[] = [];
 	const messageIds = (grouped.get("Message") ?? []).map((k) => k.messageId);
-	items.push(...deleteMessageSubtree(mail, messageIds));
+	items.push(...deleteMessageSubtree(mail, messageIds).statements);
 
 	const threadMessageIds = (grouped.get("ThreadMessage") ?? []).map(
 		(k) => k.threadMessageId,

package/src/repos/envelope.sqlite.test.ts

@@ -8,6 +8,7 @@ import {
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb, type SqliteTestDb } from "../test-db-sqlite.js";
 import type { DrizzleEnvelopeRepository } from "./envelope.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 // Envelope repo on sqlite (RFC 036 D1): the onConflictDoUpdate upsert path and
 // the transactional body-part upsert both run on better-sqlite3 unchanged.
@@ -24,6 +25,7 @@ describe("DrizzleEnvelopeRepository (sqlite)", () => {
 		repo = createSqliteRepositories(
 			{ app: db, mail: db, search: db },
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).envelope;
 	});
 

package/src/repos/envelope.test.ts

@@ -5,6 +5,7 @@ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { envelopeId as deriveEnvelopeId } from "../id.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import type { DrizzleEnvelopeRepository } from "./envelope.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import { createTestDb, type TestDb } from "./test-helpers.js";
 
 describe("DrizzleEnvelopeRepository", () => {
@@ -23,6 +24,7 @@ describe("DrizzleEnvelopeRepository", () => {
 		repo = createSqliteRepositories(
 			{ app: db, mail: db, search: db },
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).envelope;
 
 		await repo.upsertEnvelope({

package/src/repos/i4-account-config.suite.ts

@@ -7,6 +7,7 @@ import {
 } from "../sqlite-repositories.js";
 import type { OpenTestStores } from "../test-stores.js";
 import type { AccountConfigRepo } from "./i4-account-config.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 export const accountConfigRepoSuite = (open: OpenTestStores): void => {
 	describe("AccountConfigRepo", () => {
@@ -17,7 +18,10 @@ export const accountConfigRepoSuite = (open: OpenTestStores): void => {
 		before(async () => {
 			const opened = await open();
 			close = opened.close;
-			repos = composeSqliteRepositories(opened.stores);
+			repos = composeSqliteRepositories(
+				opened.stores,
+				messageObjectsInMemory(),
+			);
 			repo = repos.accountConfig;
 		});
 

package/src/repos/i4-account.test.ts

@@ -4,6 +4,7 @@ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createTestDb, randomId, type TestDb } from "../test-db.js";
 import type { AccountRepo } from "./i4-account.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import { mailboxCreated } from "./test-helpers.js";
 
 function makeAccountInput(accountConfigId: string) {
@@ -34,6 +35,7 @@ describe("AccountRepo", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).account;
 	});
 
@@ -175,6 +177,7 @@ describe("AccountRepo", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).mailbox;
 		mailboxCreated(
 			await mailboxRepo.create({

package/src/repos/i4-address-junk-move.sqlite.test.ts

@@ -6,6 +6,7 @@ import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createTestDb, type TestDb } from "../test-db.js";
 import { AddressRepo } from "./i4-address.js";
 import type { MailboxSpecialUseRepo } from "./i4-mailbox-special-use.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 const CONFIG = "cfg-1";
 
@@ -133,6 +134,7 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).mailboxSpecialUse;
 	});
 

package/src/repos/i4-mailbox-special-use.sqlite.test.ts

@@ -18,6 +18,7 @@ import { createSqliteTestDb } from "../test-db-sqlite.js";
 import type { AccountSettingRepo } from "./i4-account-setting.js";
 import type { MailboxRepo } from "./i4-mailbox.js";
 import type { MailboxSpecialUseRepo } from "./i4-mailbox-special-use.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import { mailboxCreated } from "./test-helpers.js";
 
 const makeMailboxInput = (
@@ -65,6 +66,7 @@ describe("MailboxSpecialUseRepo role lookups (sqlite)", () => {
 				search: testDb.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		);
 		repo = repos.mailboxSpecialUse;
 		mailboxes = repos.mailbox;

package/src/repos/i4-mailbox.sqlite.test.ts

@@ -13,6 +13,7 @@ import {
 	shippedTableDdl,
 } from "../test-shipped-sqlite-schema.js";
 import type { MailboxRepo } from "./i4-mailbox.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import { mailboxCreated } from "./test-helpers.js";
 
 function makeMailboxInput(accountId: string, fullPath = "INBOX") {
@@ -49,6 +50,7 @@ describe("MailboxRepo (sqlite)", () => {
 				search: testDb.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).mailbox;
 	});
 
@@ -222,6 +224,7 @@ describe("MailboxRepo (sqlite, shipped migrations)", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).mailbox;
 		close = async () => {
 			sqlite.close();

package/src/repos/i4-mailbox.test.ts

@@ -18,6 +18,7 @@ import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createTestDb, randomId, type TestDb } from "../test-db.js";
 import type { MailboxRepo } from "./i4-mailbox.js";
 import { deleteMessageSubtree } from "./message.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import { mailboxCreated } from "./test-helpers.js";
 
 /** Every state a row can carry, for a transition that decides against none. */
@@ -62,6 +63,7 @@ describe("MailboxRepo", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).mailbox;
 	});
 
@@ -554,6 +556,7 @@ describe("MailboxRepo", () => {
 					search: db,
 				},
 				BETTER_SQLITE3_LIMITS,
+				messageObjectsInMemory(),
 			).message.create({
 				messageId,
 				mailboxId,
@@ -682,7 +685,7 @@ describe("MailboxRepo", () => {
 
 			// The shape an interruption leaves: one message's rows already gone,
 			// the mailbox row still there and still `deleting`.
-			await syncBatch(db)(deleteMessageSubtree(db, [first]));
+			await syncBatch(db)(deleteMessageSubtree(db, [first]).statements);
 			await repo.transition(accountId, mailbox.mailboxId, {
 				from: EVERY_STATE,
 				to: MailboxSyncStatus.deleting,

package/src/repos/i4-mailbox.ts

@@ -46,6 +46,7 @@ import { threadMessageTable } from "../schema/thread-message.js";
 import type { SearchIndex } from "../search/thread-message-search.js";
 import { inList } from "./in-list.js";
 import { deleteMessageSubtree } from "./message.js";
+import type { MessageObjects } from "./message-objects.js";
 
 const base36Translator = shortUuid.createTranslator(
 	shortUuid.constants.uuid25Base36,
@@ -155,6 +156,7 @@ export class MailboxRepo implements IMailboxRepository {
 	constructor(
 		store: SqliteStore<Record<string, unknown>>,
 		private search: SearchIndex,
+		private messageObjects: MessageObjects,
 	) {
 		this.db = store.db;
 		this.batch = store.batch;
@@ -562,13 +564,16 @@ export class MailboxRepo implements IMailboxRepository {
 
 			// The primitive the rest of the codebase deletes mail with: nine
 			// per-message child tables. A bespoke table list would orphan them.
-			await this.batch([
-				...deleteMessageSubtree(this.db, messageIds),
+			const subtree = deleteMessageSubtree(this.db, messageIds);
+			const [removed] = await this.batch([
+				subtree.removed,
+				...subtree.statements,
 				this.db
 					.delete(threadMessageTable)
 					.where(inList(threadMessageTable.messageId, messageIds)),
 			]);
 			await this.search.sync(listings.map((row) => row.threadMessageId));
+			await this.messageObjects.reclaim(removed);
 		}
 
 		await this.db

package/src/repos/i4-outbox-attachment.test.ts

@@ -4,6 +4,7 @@ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createTestDb, randomId } from "../test-db.js";
 import type { OutboxAttachmentRepo } from "./i4-outbox-attachment.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 const CAP = { maxTotalBytes: 1000, maxCount: 3, nowSeconds: 1_000_000 };
 
@@ -36,6 +37,7 @@ before(async () => {
 			search: harness.db,
 		},
 		BETTER_SQLITE3_LIMITS,
+		messageObjectsInMemory(),
 	).outboxAttachment;
 });
 

package/src/repos/message-objects.sqlite.test.ts

@@ -0,0 +1,217 @@
+import assert from "node:assert/strict";
+import {
+	existsSync,
+	mkdirSync,
+	mkdtempSync,
+	rmSync,
+	writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { after, before, describe, test } from "node:test";
+import type { StorageService } from "@remit/storage-service";
+import { createFilesystemStorageService } from "@remit/storage-service/filesystem";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
+import {
+	envelopeId as deriveEnvelopeId,
+	rootBodyPartId as deriveRootBodyPartId,
+} from "../id.js";
+import {
+	createSqliteRepositories,
+	type SqliteRepositories,
+} from "../sqlite-repositories.js";
+import { createTestDb, randomId, type TestDb } from "../test-db.js";
+import { mailboxCreated } from "./test-helpers.js";
+
+const ACCOUNT_CONFIG_ID = "cfg-objects";
+
+describe("message objects follow the last row that references them", () => {
+	let db: TestDb;
+	let close: () => Promise<void>;
+	let directory: string;
+	let storage: StorageService;
+	let failDeletesAfter = Number.POSITIVE_INFINITY;
+	let repos: SqliteRepositories;
+
+	before(async () => {
+		({ db, close } = await createTestDb());
+		directory = mkdtempSync(join(tmpdir(), "message-objects-"));
+		storage = createFilesystemStorageService(directory);
+		repos = createSqliteRepositories(
+			{ app: db, mail: db, search: db },
+			BETTER_SQLITE3_LIMITS,
+			{
+				storage: {
+					deletePrefix: (prefix) => {
+						if (failDeletesAfter <= 0) {
+							return Promise.reject(new Error("storage went away"));
+						}
+						failDeletesAfter -= 1;
+						return storage.deletePrefix(prefix);
+					},
+				},
+				messageObjectDeleteConcurrency: 1,
+			},
+		);
+	});
+
+	after(async () => {
+		await close();
+		rmSync(directory, { recursive: true, force: true });
+	});
+
+	const folder = async (accountId: string, fullPath: string) =>
+		mailboxCreated(
+			await repos.mailbox.create({
+				accountId,
+				namespaceType: "personal",
+				namespacePrefix: "",
+				hierarchyDelimiter: "/",
+				fullPath,
+				uidValidity: 1,
+				uidNext: 1,
+				highestModseq: "0",
+				messageCount: 0,
+				unseenCount: 0,
+				deletedCount: 0,
+				totalSize: 0,
+				lastSyncUid: 0,
+				highWaterMarkUid: 0,
+				lastMessageSyncAt: 0,
+			}),
+		);
+
+	const message = async (
+		accountId: string,
+		mailboxId: string,
+		uid: number,
+		body: { from?: string } = {},
+	) => {
+		const messageId = randomId();
+		const owner = body.from ?? messageId;
+		const stored = await storage.storeMessageBody({
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			accountId,
+			messageId: owner,
+			content: Buffer.from(`body of ${owner}`),
+		});
+		await storage.storeBodyPart({
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			accountId,
+			messageId,
+			partPath: "1",
+			content: Buffer.from("part"),
+		});
+		await repos.message.create({
+			messageId,
+			mailboxId,
+			uid,
+			sequenceNumber: uid,
+			rfc822Size: 10,
+			internalDate: 1700000000000,
+			envelopeId: deriveEnvelopeId(messageId),
+			rootBodyPartId: deriveRootBodyPartId(messageId),
+			bodyStorageKey: stored.uri,
+		});
+		return { messageId, accountId, bodyStorageKey: stored.uri };
+	};
+
+	const part = (row: { accountId: string; messageId: string }) =>
+		storage.retrieveBodyPart(
+			ACCOUNT_CONFIG_ID,
+			row.accountId,
+			row.messageId,
+			"1",
+		);
+
+	test("a copy sharing the source's body keeps it until the copy goes too", async () => {
+		const accountId = randomId();
+		const inbox = await folder(accountId, "INBOX");
+		const source = await message(accountId, inbox.mailboxId, 1);
+		const copy = await message(accountId, inbox.mailboxId, 2, {
+			from: source.messageId,
+		});
+		assert.equal(copy.bodyStorageKey, source.bodyStorageKey);
+
+		await repos.message.delete(source.messageId);
+
+		assert.ok(await storage.retrieve(copy.bodyStorageKey));
+		assert.ok(await part(copy));
+
+		await repos.message.delete(copy.messageId);
+
+		assert.equal(await storage.exists(copy.bodyStorageKey), false);
+		assert.equal(await part(copy), null);
+		assert.equal(await part(source), null);
+	});
+
+	test("a placement that no longer matches keeps the row and its objects", async () => {
+		const accountId = randomId();
+		const inbox = await folder(accountId, "INBOX");
+		const kept = await message(accountId, inbox.mailboxId, 1);
+
+		assert.equal(
+			await repos.message.deleteIfPlacement(kept.messageId, { uid: 2 }),
+			false,
+		);
+		assert.ok(await storage.retrieve(kept.bodyStorageKey));
+
+		assert.equal(
+			await repos.message.deleteIfPlacement(kept.messageId, { uid: 1 }),
+			true,
+		);
+		assert.equal(await storage.exists(kept.bodyStorageKey), false);
+	});
+
+	test("a body stored under the key shape without an account config removes its message prefix", async () => {
+		const accountId = randomId();
+		const inbox = await folder(accountId, "INBOX");
+		const messageId = randomId();
+		const legacy = join(
+			directory,
+			"accounts",
+			accountId,
+			"messages",
+			messageId,
+		);
+		mkdirSync(join(legacy, "parts"), { recursive: true });
+		writeFileSync(join(legacy, "body.eml"), "legacy");
+		writeFileSync(join(legacy, "parts", "1"), "part");
+		await repos.message.create({
+			messageId,
+			mailboxId: inbox.mailboxId,
+			uid: 1,
+			sequenceNumber: 1,
+			rfc822Size: 10,
+			internalDate: 1700000000000,
+			envelopeId: deriveEnvelopeId(messageId),
+			rootBodyPartId: deriveRootBodyPartId(messageId),
+			bodyStorageKey: `file://${join(legacy, "body.eml")}`,
+		});
+
+		await repos.message.delete(messageId);
+
+		assert.equal(existsSync(legacy), false);
+	});
+
+	test("a folder delete that fails part way has already removed the batches it finished", async () => {
+		const accountId = randomId();
+		const doomed = await folder(accountId, "Doomed");
+		const rows = [];
+		for (let uid = 1; uid <= 101; uid += 1) {
+			rows.push(await message(accountId, doomed.mailboxId, uid));
+		}
+		failDeletesAfter = 100;
+
+		await assert.rejects(
+			repos.mailbox.deleteMailboxWithMail(accountId, doomed.mailboxId),
+			/storage went away/,
+		);
+		failDeletesAfter = Number.POSITIVE_INFINITY;
+
+		const gone = await Promise.all(
+			rows.map((row) => storage.exists(row.bodyStorageKey)),
+		);
+		assert.equal(gone.filter((exists) => !exists).length, 100);
+	});
+});

package/src/repos/message-objects.ts

@@ -0,0 +1,76 @@
+import {
+	createMockStorageService,
+	messageObjectPrefixOf,
+	type StorageService,
+} from "@remit/storage-service";
+import pMap from "p-map";
+import type { RepositoryDb } from "../db.js";
+import { messageTable } from "../schema/message-data.js";
+import { inList } from "./in-list.js";
+
+export const DEFAULT_MESSAGE_OBJECT_DELETE_CONCURRENCY = 8;
+
+export interface MessageObjectStore {
+	storage: Pick<StorageService, "deletePrefix">;
+	messageObjectDeleteConcurrency: number;
+}
+
+export const messageObjectsInMemory = (): MessageObjectStore => ({
+	storage: createMockStorageService(),
+	messageObjectDeleteConcurrency: 1,
+});
+
+export interface RemovedMessageRow {
+	messageId: string;
+	bodyStorageKey: string | null;
+}
+
+interface Candidate {
+	prefix: string;
+	bodyStorageKey: string;
+}
+
+const candidatesOf = (row: RemovedMessageRow): Candidate[] => {
+	const { messageId, bodyStorageKey } = row;
+	if (bodyStorageKey === null) return [];
+	const ownerPrefix = messageObjectPrefixOf(bodyStorageKey);
+	const ownPrefix = ownerPrefix.replace(/[^/]+\/$/, `${messageId}/`);
+	const owner = { prefix: ownerPrefix, bodyStorageKey };
+	if (ownPrefix === ownerPrefix) return [owner];
+	const at = bodyStorageKey.lastIndexOf(ownerPrefix);
+	const ownBodyStorageKey = `${bodyStorageKey.slice(0, at)}${ownPrefix}${bodyStorageKey.slice(at + ownerPrefix.length)}`;
+	return [owner, { prefix: ownPrefix, bodyStorageKey: ownBodyStorageKey }];
+};
+
+export class MessageObjects {
+	constructor(
+		private readonly db: Pick<RepositoryDb<Record<string, unknown>>, "select">,
+		private readonly store: MessageObjectStore,
+	) {}
+
+	async reclaim(removed: readonly RemovedMessageRow[]): Promise<void> {
+		const candidates = new Map(
+			removed.flatMap(candidatesOf).map((c) => [c.prefix, c]),
+		);
+		if (candidates.size === 0) return;
+		const keys = [
+			...new Set([...candidates.values()].map((c) => c.bodyStorageKey)),
+		];
+		const referenced = await this.db
+			.select({ bodyStorageKey: messageTable.bodyStorageKey })
+			.from(messageTable)
+			.where(inList(messageTable.bodyStorageKey, keys));
+		const stillReferenced = new Set(
+			referenced.flatMap((row) =>
+				row.bodyStorageKey === null ? [] : [row.bodyStorageKey],
+			),
+		);
+		await pMap(
+			[...candidates.values()].filter(
+				(c) => !stillReferenced.has(c.bodyStorageKey),
+			),
+			(c) => this.store.storage.deletePrefix(c.prefix),
+			{ concurrency: this.store.messageObjectDeleteConcurrency },
+		);
+	}
+}

package/src/repos/message.sqlite.test.ts

@@ -10,6 +10,7 @@ import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb, type SqliteTestDb } from "../test-db-sqlite.js";
 import type { DrizzleMessageRepository } from "./message.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 const SCHEMA = { ...messageDataSchema, threadMessages: threadMessageTable };
 
@@ -47,6 +48,7 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		repo = createSqliteRepositories(
 			{ app: db, mail: db, search: db },
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).message;
 	});
 

package/src/repos/message.test.ts

@@ -8,6 +8,7 @@ import {
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import type { DrizzleEnvelopeRepository } from "./envelope.js";
 import type { DrizzleMessageRepository } from "./message.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import { createTestDb, type TestDb } from "./test-helpers.js";
 
 describe("DrizzleMessageRepository", () => {
@@ -41,6 +42,7 @@ describe("DrizzleMessageRepository", () => {
 		const repos = createSqliteRepositories(
 			{ app: db, mail: db, search: db },
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		);
 		messageRepo = repos.message;
 		envelopeRepo = repos.envelope;

package/src/repos/message.ts

@@ -55,6 +55,7 @@ import {
 	toMessageReferenceItem,
 	toRawMessageStorageItem,
 } from "./mappers.js";
+import type { MessageObjects } from "./message-objects.js";
 
 type DB = RepositoryDb<Record<string, unknown>>;
 
@@ -112,22 +113,33 @@ function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
 	};
 }
 
-export type SubtreeDb = Pick<RepositoryDb<Record<string, unknown>>, "delete">;
+export type SubtreeDb = Pick<
+	RepositoryDb<Record<string, unknown>>,
+	"delete" | "select"
+>;
 
 /**
  * The statements that delete a message and its whole per-message subtree by
  * message id: the child tables, then the message row. The caller runs them as
- * one batch so the subtree goes atomically; the last statement returns the
- * message ids it removed. `guard` rides every statement, so
- * a guard that reads the message row holds until the last statement removes it.
+ * one batch so the subtree goes atomically. `removed` reads the rows the batch
+ * deletes; it goes first, so it sees them under the same guard. `guard` rides
+ * every statement, so a guard that reads the message row holds until the last
+ * statement removes it.
  */
 export function deleteMessageSubtree(
 	db: SubtreeDb,
 	messageIds: string[],
 	guard?: SQL,
-): SqliteBatchItem[] {
-	if (messageIds.length === 0) return [];
-	return [
+) {
+	const removed = db
+		.select({
+			messageId: messageTable.messageId,
+			bodyStorageKey: messageTable.bodyStorageKey,
+		})
+		.from(messageTable)
+		.where(and(inList(messageTable.messageId, messageIds), guard));
+	if (messageIds.length === 0) return { removed, statements: [] };
+	const statements: SqliteBatchItem[] = [
 		db
 			.delete(messageFlagTable)
 			.where(and(inList(messageFlagTable.messageId, messageIds), guard)),
@@ -162,9 +174,9 @@ export function deleteMessageSubtree(
 			.where(and(inList(envelopeTable.messageId, messageIds), guard)),
 		db
 			.delete(messageTable)
-			.where(and(inList(messageTable.messageId, messageIds), guard))
-			.returning({ messageId: messageTable.messageId }),
+			.where(and(inList(messageTable.messageId, messageIds), guard)),
 	];
+	return { removed, statements };
 }
 
 /**
@@ -202,6 +214,7 @@ export class DrizzleMessageRepository implements IMessageRepository {
 	constructor(
 		store: SqliteStore<Record<string, unknown>>,
 		private search: SearchIndex,
+		private messageObjects: MessageObjects,
 	) {
 		this.db = store.db;
 		this.batch = store.batch;
@@ -533,20 +546,27 @@ export class DrizzleMessageRepository implements IMessageRepository {
 			.select({ threadMessageId: threadMessageTable.threadMessageId })
 			.from(threadMessageTable)
 			.where(eq(threadMessageTable.messageId, messageId));
-		const results = await this.batch([
+		const subtree = deleteMessageSubtree(this.db, [messageId], placed);
+		const [removed] = await this.batch([
+			subtree.removed,
 			this.db
 				.delete(threadMessageTable)
 				.where(and(eq(threadMessageTable.messageId, messageId), placed)),
-			...deleteMessageSubtree(this.db, [messageId], placed),
+			...subtree.statements,
 		]);
 		await this.search.sync(listings.map((row) => row.threadMessageId));
-		const removed = results.at(-1);
-		return Array.isArray(removed) && removed.length > 0;
+		await this.messageObjects.reclaim(removed);
+		return removed.length > 0;
 	}
 
 	async deleteMany(messageIds: string[]): Promise<void> {
 		if (messageIds.length === 0) return;
-		await this.batch(deleteMessageSubtree(this.db, messageIds));
+		const subtree = deleteMessageSubtree(this.db, messageIds);
+		const [removed] = await this.batch([
+			subtree.removed,
+			...subtree.statements,
+		]);
+		await this.messageObjects.reclaim(removed);
 	}
 
 	async listByMailbox(

package/src/repos/thread-message-body-muted.sqlite.test.ts

@@ -16,6 +16,7 @@ import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import type { DrizzleThreadMessageRepository } from "./thread-message.js";
 
 const ACCOUNT = "acct-seam";
@@ -68,6 +69,7 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).threadMessage;
 	});
 

package/src/repos/thread-message-field-terms.sqlite.test.ts

@@ -6,6 +6,7 @@ import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import {
 	type DrizzleThreadMessageRepository,
 	THREAD_SEARCH_MAX_LIMIT,
@@ -67,6 +68,7 @@ describe("DrizzleThreadMessageRepository.listByFieldTerms (sqlite, #459)", () =>
 				search: created.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).threadMessage;
 
 		await repo.create(

package/src/repos/thread-message.sqlite.test.ts

@@ -13,6 +13,7 @@ import {
 	LIVE_TOTALS,
 	totalRows,
 } from "./category-fixture.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import {
 	type DrizzleThreadMessageRepository,
 	THREAD_SEARCH_MAX_LIMIT,
@@ -68,6 +69,7 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).threadMessage;
 	});
 

package/src/repos/thread-message.test.ts

@@ -10,6 +10,7 @@ import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import {
 	clampThreadSearchLimit,
 	type DrizzleThreadMessageRepository,
@@ -102,6 +103,7 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
 				search: harness.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).threadMessage;
 	});
 
@@ -411,6 +413,7 @@ describe("DrizzleThreadMessageRepository.searchByDate", () => {
 				search: harness.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		);
 		repo = repositories.threadMessage;
 		messages = repositories.message;
@@ -949,6 +952,7 @@ describe("DrizzleThreadMessageRepository — native text search", () => {
 				search: harness.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).threadMessage;
 	});
 
@@ -1128,6 +1132,7 @@ describe("DrizzleThreadMessageRepository — core CRUD", () => {
 				search: harness.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).threadMessage;
 	});
 

package/src/search/thread-message-search.sqlite.test.ts

@@ -6,6 +6,7 @@ import { drizzle } from "drizzle-orm/better-sqlite3";
 import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { syncStore } from "../databases.js";
 import type { Db } from "../db.js";
+import { messageObjectsInMemory } from "../repos/message-objects.js";
 import { messageSearchTextTable } from "../schema/message-data.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { applyMigrationSet } from "../test-shipped-sqlite-schema.js";
@@ -62,6 +63,7 @@ describe("the thread-message search index in its own database", () => {
 			search: search.db,
 		},
 		BETTER_SQLITE3_LIMITS,
+		messageObjectsInMemory(),
 	).threadMessage;
 
 	const found = async (query: string): Promise<string[]> =>

package/src/sqlite-repositories.ts

@@ -31,20 +31,24 @@ import { LabelRepo } from "./repos/label.js";
 import { DrizzleMessageRepository } from "./repos/message.js";
 import { DrizzleMessageFlagRepository } from "./repos/message-flag.js";
 import { MessageLabelRepo } from "./repos/message-label.js";
+import {
+	type MessageObjectStore,
+	MessageObjects,
+} from "./repos/message-objects.js";
 import { QuarantineRepo } from "./repos/quarantine.js";
 import { SyncWarningRepo } from "./repos/sync-warning.js";
 import { DrizzleThreadMessageRepository } from "./repos/thread-message.js";
 import { SearchIndex } from "./search/thread-message-search.js";
 
-export const composeSqliteRepositories = ({
-	app,
-	mail,
-	search,
-}: SqliteStores) => {
+export const composeSqliteRepositories = (
+	{ app, mail, search }: SqliteStores,
+	objects: MessageObjectStore,
+) => {
 	const searchIndex = new SearchIndex(mail.db, search);
+	const messageObjects = new MessageObjects(mail.db, objects);
 	const accountSetting = new AccountSettingRepo(app.db);
 	const address = new AddressRepo(mail.db);
-	const mailbox = new MailboxRepo(mail, searchIndex);
+	const mailbox = new MailboxRepo(mail, searchIndex, messageObjects);
 	const account = new AccountRepo(app.db, mailbox);
 	const calendarSuggestion = new CalendarSuggestionRepo(mail.db);
 	return {
@@ -59,7 +63,7 @@ export const composeSqliteRepositories = ({
 			accountSetting,
 		),
 		mailboxLock: new MailboxLockRepo(mail.db),
-		message: new DrizzleMessageRepository(mail, searchIndex),
+		message: new DrizzleMessageRepository(mail, searchIndex, messageObjects),
 		messageFlag: new DrizzleMessageFlagRepository(mail.db),
 		outboxMessage: new OutboxMessageRepo(mail.db),
 		outboxAttachment: new OutboxAttachmentRepo(mail),
@@ -90,5 +94,6 @@ export type SqliteRepositories = ReturnType<typeof composeSqliteRepositories>;
 export const createSqliteRepositories = (
 	databases: SqliteDatabases,
 	limits: SqliteLimits,
+	objects: MessageObjectStore,
 ): SqliteRepositories =>
-	composeSqliteRepositories(syncStores(databases, limits));
+	composeSqliteRepositories(syncStores(databases, limits), objects);