@remit/drizzle-service 0.0.106 → 0.0.108

package/src/repos/message-objects.sqlite.test.ts

@@ -0,0 +1,271 @@
+import assert from "node:assert/strict";
+import {
+	existsSync,
+	mkdirSync,
+	mkdtempSync,
+	rmSync,
+	writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { after, before, describe, test } from "node:test";
+import type { StorageService } from "@remit/storage-service";
+import { createFilesystemStorageService } from "@remit/storage-service/filesystem";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
+import {
+	createSqliteRepositories,
+	type SqliteRepositories,
+} from "../sqlite-repositories.js";
+import { createTestDb, randomId, type TestDb } from "../test-db.js";
+import { mailboxCreated } from "./test-helpers.js";
+
+const ACCOUNT_CONFIG_ID = "cfg-objects";
+
+const bodyPart = (partPath: string) => ({
+	partPath,
+	parentPartPath: null,
+	mediaType: "TEXT" as const,
+	mediaSubtype: "plain",
+	transferEncoding: "7BIT" as const,
+	sizeOctets: 4,
+	isMultipart: false,
+	parameters: [],
+});
+
+describe("message objects follow the last row that references them", () => {
+	let db: TestDb;
+	let close: () => Promise<void>;
+	let directory: string;
+	let storage: StorageService;
+	let failDeletesAfter = Number.POSITIVE_INFINITY;
+	let repos: SqliteRepositories;
+	let deleteCalls: number[] = [];
+
+	before(async () => {
+		({ db, close } = await createTestDb());
+		directory = mkdtempSync(join(tmpdir(), "message-objects-"));
+		storage = createFilesystemStorageService(directory);
+		const counted: Pick<StorageService, "deleteKeys"> = {
+			deleteKeys: (keys) => {
+				deleteCalls.push(keys.length);
+				if (failDeletesAfter <= 0) {
+					return Promise.reject(new Error("storage went away"));
+				}
+				failDeletesAfter -= 1;
+				return storage.deleteKeys(keys);
+			},
+		};
+		repos = createSqliteRepositories(
+			{ app: db, mail: db, search: db },
+			BETTER_SQLITE3_LIMITS,
+			{ storage: counted },
+		);
+	});
+
+	after(async () => {
+		await close();
+		rmSync(directory, { recursive: true, force: true });
+	});
+
+	const folder = async (accountId: string, fullPath: string) =>
+		mailboxCreated(
+			await repos.mailbox.create({
+				accountId,
+				namespaceType: "personal",
+				namespacePrefix: "",
+				hierarchyDelimiter: "/",
+				fullPath,
+				uidValidity: 1,
+				uidNext: 1,
+				highestModseq: "0",
+				messageCount: 0,
+				unseenCount: 0,
+				deletedCount: 0,
+				totalSize: 0,
+				lastSyncUid: 0,
+				highWaterMarkUid: 0,
+				lastMessageSyncAt: 0,
+			}),
+		);
+
+	const message = async (
+		accountId: string,
+		mailboxId: string,
+		uid: number,
+		body: { from?: string } = {},
+	) => {
+		const messageId = randomId();
+		const owner = body.from ?? messageId;
+		const stored = await storage.storeMessageBody({
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			accountId,
+			messageId: owner,
+			content: Buffer.from(`body of ${owner}`),
+		});
+		await storage.storeBodyPart({
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			accountId,
+			messageId,
+			partPath: "1",
+			content: Buffer.from("part"),
+		});
+		await repos.message.create({
+			messageId,
+			mailboxId,
+			uid,
+			sequenceNumber: uid,
+			rfc822Size: 10,
+			internalDate: 1700000000000,
+			bodyStorageKey: stored.uri,
+		});
+		await repos.envelope.upsertBodyParts(messageId, [bodyPart("1")]);
+		return { messageId, accountId, bodyStorageKey: stored.uri };
+	};
+
+	const part = (row: { accountId: string; messageId: string }) =>
+		storage.retrieveBodyPart(
+			ACCOUNT_CONFIG_ID,
+			row.accountId,
+			row.messageId,
+			"1",
+		);
+
+	test("a copy sharing the source's body keeps it until the copy goes too", async () => {
+		const accountId = randomId();
+		const inbox = await folder(accountId, "INBOX");
+		const source = await message(accountId, inbox.mailboxId, 1);
+		const copy = await message(accountId, inbox.mailboxId, 2, {
+			from: source.messageId,
+		});
+		assert.equal(copy.bodyStorageKey, source.bodyStorageKey);
+
+		await repos.message.delete(source.messageId);
+
+		assert.ok(await storage.retrieve(copy.bodyStorageKey));
+		assert.ok(await part(copy));
+
+		await repos.message.delete(copy.messageId);
+
+		assert.equal(await storage.exists(copy.bodyStorageKey), false);
+		assert.equal(await part(copy), null);
+		assert.equal(await part(source), null);
+	});
+
+	test("a placement that no longer matches keeps the row and its objects", async () => {
+		const accountId = randomId();
+		const inbox = await folder(accountId, "INBOX");
+		const kept = await message(accountId, inbox.mailboxId, 1);
+
+		assert.equal(
+			await repos.message.deleteIfPlacement(kept.messageId, { uid: 2 }),
+			false,
+		);
+		assert.ok(await storage.retrieve(kept.bodyStorageKey));
+
+		assert.equal(
+			await repos.message.deleteIfPlacement(kept.messageId, { uid: 1 }),
+			true,
+		);
+		assert.equal(await storage.exists(kept.bodyStorageKey), false);
+	});
+
+	test("a body stored under the key shape without an account config removes its message prefix", async () => {
+		const accountId = randomId();
+		const inbox = await folder(accountId, "INBOX");
+		const messageId = randomId();
+		const legacy = join(
+			directory,
+			"accounts",
+			accountId,
+			"messages",
+			messageId,
+		);
+		mkdirSync(join(legacy, "parts"), { recursive: true });
+		writeFileSync(join(legacy, "body.eml"), "legacy");
+		writeFileSync(join(legacy, "parts", "1"), "part");
+		await repos.message.create({
+			messageId,
+			mailboxId: inbox.mailboxId,
+			uid: 1,
+			sequenceNumber: 1,
+			rfc822Size: 10,
+			internalDate: 1700000000000,
+			bodyStorageKey: `file://${join(legacy, "body.eml")}`,
+		});
+		await repos.envelope.upsertBodyParts(messageId, [bodyPart("1")]);
+
+		await repos.message.delete(messageId);
+
+		assert.equal(existsSync(legacy), false);
+	});
+
+	test("a folder delete that fails part way has already removed the batches it finished", async () => {
+		const accountId = randomId();
+		const doomed = await folder(accountId, "Doomed");
+		const rows = [];
+		for (let uid = 1; uid <= 101; uid += 1) {
+			rows.push(await message(accountId, doomed.mailboxId, uid));
+		}
+		failDeletesAfter = 1;
+
+		await assert.rejects(
+			repos.mailbox.deleteMailboxWithMail(accountId, doomed.mailboxId),
+			/storage went away/,
+		);
+		failDeletesAfter = Number.POSITIVE_INFINITY;
+
+		const gone = await Promise.all(
+			rows.map((row) => storage.exists(row.bodyStorageKey)),
+		);
+		assert.equal(gone.filter((exists) => !exists).length, 100);
+	});
+
+	const gone = async (
+		rows: readonly {
+			accountId: string;
+			messageId: string;
+			bodyStorageKey: string;
+		}[],
+	) => {
+		for (const row of rows) {
+			assert.equal(await storage.exists(row.bodyStorageKey), false);
+			assert.equal(await part(row), null);
+		}
+	};
+
+	test("deleting many messages makes one storage call per batch and keeps a body a copy shares", async () => {
+		const accountId = randomId();
+		const inbox = await folder(accountId, "INBOX");
+		const rows = [];
+		for (let uid = 1; uid <= 300; uid += 1) {
+			rows.push(await message(accountId, inbox.mailboxId, uid));
+		}
+		const [shared, ...unshared] = rows;
+		const copy = await message(accountId, inbox.mailboxId, 301, {
+			from: shared.messageId,
+		});
+		deleteCalls = [];
+
+		await repos.message.deleteMany(rows.map((row) => row.messageId));
+
+		assert.equal(deleteCalls.length, 3);
+		assert.ok(await storage.retrieve(copy.bodyStorageKey));
+		assert.ok(await part(copy));
+		await gone(unshared);
+	});
+
+	test("a folder delete makes one storage call per batch and removes every object", async () => {
+		const accountId = randomId();
+		const doomed = await folder(accountId, "Archive");
+		const rows = [];
+		for (let uid = 1; uid <= 250; uid += 1) {
+			rows.push(await message(accountId, doomed.mailboxId, uid));
+		}
+		deleteCalls = [];
+
+		await repos.mailbox.deleteMailboxWithMail(accountId, doomed.mailboxId);
+
+		assert.equal(deleteCalls.length, 3);
+		await gone(rows);
+	});
+});

package/src/repos/message-objects.ts

@@ -0,0 +1,70 @@
+import {
+	createMockStorageService,
+	messageObjectKeysOf,
+	type StorageService,
+} from "@remit/storage-service";
+import type { RepositoryDb } from "../db.js";
+import { messageTable } from "../schema/message-data.js";
+import { inList } from "./in-list.js";
+
+export interface MessageObjectStore {
+	storage: Pick<StorageService, "deleteKeys">;
+}
+
+export const messageObjectsInMemory = (): MessageObjectStore => ({
+	storage: createMockStorageService(),
+});
+
+export interface RemovedMessageRow {
+	messageId: string;
+	bodyStorageKey: string | null;
+}
+
+export interface RemovedBodyPartRow {
+	messageId: string;
+	partPath: string;
+}
+
+export class MessageObjects {
+	constructor(
+		private readonly db: Pick<RepositoryDb<Record<string, unknown>>, "select">,
+		private readonly store: MessageObjectStore,
+	) {}
+
+	async reclaim(
+		removed: readonly RemovedMessageRow[],
+		removedParts: readonly RemovedBodyPartRow[],
+	): Promise<void> {
+		const partPathsOf = new Map<string, string[]>();
+		for (const { messageId, partPath } of removedParts) {
+			partPathsOf.set(messageId, [
+				...(partPathsOf.get(messageId) ?? []),
+				partPath,
+			]);
+		}
+		const own: string[] = [];
+		const sharedBy = new Map<string, string[]>();
+		for (const { messageId, bodyStorageKey } of removed) {
+			if (bodyStorageKey === null) continue;
+			const keys = messageObjectKeysOf(
+				bodyStorageKey,
+				messageId,
+				partPathsOf.get(messageId) ?? [],
+			);
+			own.push(...keys.own);
+			sharedBy.set(bodyStorageKey, keys.shared);
+		}
+		if (sharedBy.size === 0) return;
+		const referenced = await this.db
+			.select({ bodyStorageKey: messageTable.bodyStorageKey })
+			.from(messageTable)
+			.where(inList(messageTable.bodyStorageKey, [...sharedBy.keys()]));
+		for (const { bodyStorageKey } of referenced) {
+			if (bodyStorageKey !== null) sharedBy.delete(bodyStorageKey);
+		}
+		await this.store.storage.deleteKeys([
+			...own,
+			...[...sharedBy.values()].flat(),
+		]);
+	}
+}

package/src/repos/message.sqlite.test.ts

@@ -10,6 +10,7 @@ import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb, type SqliteTestDb } from "../test-db-sqlite.js";
 import type { DrizzleMessageRepository } from "./message.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 const SCHEMA = { ...messageDataSchema, threadMessages: threadMessageTable };
 
@@ -47,6 +48,7 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		repo = createSqliteRepositories(
 			{ app: db, mail: db, search: db },
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).message;
 	});
 
@@ -83,8 +85,6 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		const created = await repo.create({
 			...BASE_INPUT,
 			messageId: AUTO_MOVED_ID,
-			envelopeId: deriveEnvelopeId(AUTO_MOVED_ID),
-			rootBodyPartId: deriveRootBodyPartId(AUTO_MOVED_ID),
 			movedByRemit: true,
 			filterMove: filterMove as unknown as never,
 		});
@@ -158,8 +158,6 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 			sequenceNumber: 2,
 			rfc822Size: 1024,
 			internalDate: NOW,
-			envelopeId: deriveEnvelopeId(inboundId),
-			rootBodyPartId: deriveRootBodyPartId(inboundId),
 		});
 
 		assert.equal(created, true);
@@ -180,8 +178,6 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 				...BASE_INPUT,
 				messageId: LOCK_ID,
 				uid: 7,
-				envelopeId: deriveEnvelopeId(LOCK_ID),
-				rootBodyPartId: deriveRootBodyPartId(LOCK_ID),
 				status: "moving" as const,
 				syncStatus: "pending" as const,
 			});
@@ -253,8 +249,6 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 				...BASE_INPUT,
 				messageId: GONE_ID,
 				uid: 9,
-				envelopeId: deriveEnvelopeId(GONE_ID),
-				rootBodyPartId: deriveRootBodyPartId(GONE_ID),
 			});
 		});
 
@@ -317,19 +311,19 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		);
 	});
 
-	test("copySearchText gives a copy the source's search text", async () => {
+	test("copyMessage gives a copy the source's search text", async () => {
 		const source = "00000000-0000-0000-4444-000000000001";
 		const copy = "00000000-0000-0000-4444-000000000002";
-		for (const [messageId, uid] of [
-			[source, 201],
-			[copy, 202],
-		] as const) {
-			await repo.create({ ...BASE_INPUT, messageId, uid });
-			await repo.update(messageId, { bodyStorageKey: "file:///bodies/shared" });
-		}
+		await repo.create({ ...BASE_INPUT, messageId: source, uid: 201 });
+		await repo.update(source, { bodyStorageKey: "file:///bodies/shared" });
 		await repo.setSearchText(source, "the shared body");
 
-		await repo.copySearchText(source, copy);
+		await repo.copyMessage(source, {
+			...BASE_INPUT,
+			messageId: copy,
+			uid: 202,
+			bodyStorageKey: "file:///bodies/shared",
+		});
 
 		const missing = await repo.listMissingSearchText("", 100);
 		assert.equal(

package/src/repos/message.test.ts

@@ -6,9 +6,10 @@ import {
 	rootBodyPartId as deriveRootBodyPartId,
 } from "../id.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
+import { createTestDb, type TestDb } from "../test-db.js";
 import type { DrizzleEnvelopeRepository } from "./envelope.js";
 import type { DrizzleMessageRepository } from "./message.js";
-import { createTestDb, type TestDb } from "./test-helpers.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 describe("DrizzleMessageRepository", () => {
 	let db: TestDb;
@@ -37,10 +38,11 @@ describe("DrizzleMessageRepository", () => {
 	};
 
 	before(async () => {
-		({ db, stop } = await createTestDb());
+		({ db, close: stop } = await createTestDb());
 		const repos = createSqliteRepositories(
 			{ app: db, mail: db, search: db },
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		);
 		messageRepo = repos.message;
 		envelopeRepo = repos.envelope;
@@ -147,8 +149,6 @@ describe("DrizzleMessageRepository", () => {
 				sequenceNumber: 1,
 				rfc822Size: 512,
 				internalDate: NOW,
-				envelopeId: deriveEnvelopeId(MOVE_MESSAGE_ID),
-				rootBodyPartId: deriveRootBodyPartId(MOVE_MESSAGE_ID),
 				status: "active" as const,
 				syncStatus: "synced" as const,
 			});
@@ -262,8 +262,6 @@ describe("DrizzleMessageRepository", () => {
 				sequenceNumber: 1,
 				rfc822Size: 10,
 				internalDate: NOW,
-				envelopeId: DEL_ENVELOPE_ID,
-				rootBodyPartId: DEL_ROOT_BODY_PART_ID,
 				status: "active",
 				syncStatus: "synced",
 			});