@remit/drizzle-service 0.0.106 → 0.0.108

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.106",
+  "version": "0.0.108",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",
@@ -36,7 +36,9 @@
     "uuid": "^13.0.2",
     "@remit/drizzle-sqlite-schema": "*",
     "better-sqlite3": "^12.11.1",
-    "@types/better-sqlite3": "^7.6.13"
+    "@types/better-sqlite3": "^7.6.13",
+    "@remit/storage-service": "*",
+    "p-map": "^2.1.0"
   },
   "license": "MIT",
   "publishConfig": {

package/src/id.ts

@@ -42,3 +42,12 @@ export const envelopeAddressId = (
 	order: number,
 ): string =>
 	generateDeterministicId(`envelopeaddress:${messageId}:${role}:${order}`);
+
+export const messageReferenceId = (
+	messageId: string,
+	referenceType: string,
+	referenceOrder: number,
+): string =>
+	deterministicBase36Id(
+		`msgref:${messageId}:${referenceType}:${referenceOrder}`,
+	);

package/src/index.ts

@@ -74,6 +74,10 @@ export {
 } from "./repos/message.js";
 export { DrizzleMessageFlagRepository } from "./repos/message-flag.js";
 export { MessageLabelRepo } from "./repos/message-label.js";
+export {
+	type MessageObjectStore,
+	messageObjectsInMemory,
+} from "./repos/message-objects.js";
 export { QuarantineRepo } from "./repos/quarantine.js";
 export { SyncWarningRepo } from "./repos/sync-warning.js";
 export { DrizzleThreadMessageRepository } from "./repos/thread-message.js";

package/src/repair/junk-only-address.sqlite.test.ts

@@ -7,6 +7,7 @@ import { drizzle } from "drizzle-orm/better-sqlite3";
 import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { AccountSettingRepo } from "../repos/i4-account-setting.js";
 import type { MailboxSpecialUseRepo } from "../repos/i4-mailbox-special-use.js";
+import { messageObjectsInMemory } from "../repos/message-objects.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { applyMigrationSet } from "../test-shipped-sqlite-schema.js";
 import {
@@ -190,6 +191,7 @@ describe("addresses standing only on mail in Junk", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).mailboxSpecialUse;
 		accountSetting = new AccountSettingRepo(db as never);
 		applyMigrationSet(sqlite, "app");

package/src/repos/account.conformance.d1.test.ts

@@ -3,6 +3,7 @@ import { NotFoundError } from "../error.js";
 import { randomId } from "../id.js";
 import { composeSqliteRepositories } from "../sqlite-repositories.js";
 import { openD1TestStores } from "../test-d1.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 let close: (() => Promise<void>) | undefined;
 
@@ -10,7 +11,8 @@ accountRepositoryConformance({
 	async createRepository() {
 		const opened = await openD1TestStores();
 		close = opened.close;
-		return composeSqliteRepositories(opened.stores).account;
+		return composeSqliteRepositories(opened.stores, messageObjectsInMemory())
+			.account;
 	},
 	teardown: async () => {
 		await close?.();

package/src/repos/account.conformance.sqlite.test.ts

@@ -5,6 +5,7 @@ import { randomId } from "../id.js";
 import { accountTable } from "../schema.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 let close: (() => Promise<void>) | undefined;
 
@@ -21,6 +22,7 @@ accountRepositoryConformance({
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).account;
 	},
 	teardown: async () => {

package/src/repos/calendar-default.suite.ts

@@ -6,6 +6,7 @@ import type { RepositoryDb } from "../db.js";
 import { composeSqliteRepositories } from "../sqlite-repositories.js";
 import type { OpenTestStores } from "../test-stores.js";
 import { CalendarCollectionRepo } from "./calendar-collection.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 const ACCOUNT_CONFIG_ID = "account-config-default";
 
@@ -19,7 +20,10 @@ export const calendarDefaultSuite = (open: OpenTestStores): void => {
 			const opened = await open();
 			db = opened.stores.app.db;
 			close = opened.close;
-			store = composeSqliteRepositories(opened.stores).calendarStore;
+			store = composeSqliteRepositories(
+				opened.stores,
+				messageObjectsInMemory(),
+			).calendarStore;
 		});
 
 		after(() => close());

package/src/repos/calendar-put.suite.ts

@@ -18,6 +18,7 @@ import {
 } from "../schema.js";
 import { composeSqliteRepositories } from "../sqlite-repositories.js";
 import type { OpenTestStores } from "../test-stores.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 const ACCOUNT_CONFIG_ID = "account-config-1";
 
@@ -74,7 +75,10 @@ export const calendarPutSuite = (open: OpenTestStores): void => {
 			const opened = await open();
 			db = opened.stores.app.db;
 			close = opened.close;
-			store = composeSqliteRepositories(opened.stores).calendarStore;
+			store = composeSqliteRepositories(
+				opened.stores,
+				messageObjectsInMemory(),
+			).calendarStore;
 			const collection = await provisionDefaultCalendar(
 				store,
 				ACCOUNT_CONFIG_ID,

package/src/repos/cascade-delete.ts

@@ -102,7 +102,7 @@ export const runDrizzleCascadeDelete = async (
 	const items: SqliteBatchItem[] = [];
 	const appItems: SqliteBatchItem[] = [];
 	const messageIds = (grouped.get("Message") ?? []).map((k) => k.messageId);
-	items.push(...deleteMessageSubtree(mail, messageIds));
+	items.push(...deleteMessageSubtree(mail, messageIds).statements);
 
 	const threadMessageIds = (grouped.get("ThreadMessage") ?? []).map(
 		(k) => k.threadMessageId,

package/src/repos/envelope.sqlite.test.ts

@@ -8,6 +8,7 @@ import {
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb, type SqliteTestDb } from "../test-db-sqlite.js";
 import type { DrizzleEnvelopeRepository } from "./envelope.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 // Envelope repo on sqlite (RFC 036 D1): the onConflictDoUpdate upsert path and
 // the transactional body-part upsert both run on better-sqlite3 unchanged.
@@ -24,6 +25,7 @@ describe("DrizzleEnvelopeRepository (sqlite)", () => {
 		repo = createSqliteRepositories(
 			{ app: db, mail: db, search: db },
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).envelope;
 	});
 

package/src/repos/envelope.test.ts

@@ -5,6 +5,7 @@ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { envelopeId as deriveEnvelopeId } from "../id.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import type { DrizzleEnvelopeRepository } from "./envelope.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import { createTestDb, type TestDb } from "./test-helpers.js";
 
 describe("DrizzleEnvelopeRepository", () => {
@@ -23,6 +24,7 @@ describe("DrizzleEnvelopeRepository", () => {
 		repo = createSqliteRepositories(
 			{ app: db, mail: db, search: db },
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).envelope;
 
 		await repo.upsertEnvelope({

package/src/repos/envelope.ts

@@ -21,7 +21,7 @@ import {
 	bodyPartId as deriveBodyPartId,
 	bodyPartParameterId as deriveBodyPartParameterId,
 	envelopeId as deriveEnvelopeId,
-	deterministicBase36Id,
+	messageReferenceId as deriveMessageReferenceId,
 } from "../id.js";
 import {
 	bodyPartParameterTable,
@@ -229,8 +229,10 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
 			.where(eq(messageReferenceTable.messageId, messageId));
 		const rows = references.map((reference) => ({
 			...reference,
-			messageReferenceId: deterministicBase36Id(
-				`msgref:${messageId}:${reference.referenceType}:${reference.referenceOrder}`,
+			messageReferenceId: deriveMessageReferenceId(
+				messageId,
+				reference.referenceType,
+				reference.referenceOrder,
 			),
 			messageId,
 			envelopeId,

package/src/repos/i4-account-config.suite.ts

@@ -7,6 +7,7 @@ import {
 } from "../sqlite-repositories.js";
 import type { OpenTestStores } from "../test-stores.js";
 import type { AccountConfigRepo } from "./i4-account-config.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 export const accountConfigRepoSuite = (open: OpenTestStores): void => {
 	describe("AccountConfigRepo", () => {
@@ -17,7 +18,10 @@ export const accountConfigRepoSuite = (open: OpenTestStores): void => {
 		before(async () => {
 			const opened = await open();
 			close = opened.close;
-			repos = composeSqliteRepositories(opened.stores);
+			repos = composeSqliteRepositories(
+				opened.stores,
+				messageObjectsInMemory(),
+			);
 			repo = repos.accountConfig;
 		});
 

package/src/repos/i4-account.test.ts

@@ -4,6 +4,7 @@ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createTestDb, randomId, type TestDb } from "../test-db.js";
 import type { AccountRepo } from "./i4-account.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import { mailboxCreated } from "./test-helpers.js";
 
 function makeAccountInput(accountConfigId: string) {
@@ -34,6 +35,7 @@ describe("AccountRepo", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).account;
 	});
 
@@ -175,6 +177,7 @@ describe("AccountRepo", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).mailbox;
 		mailboxCreated(
 			await mailboxRepo.create({

package/src/repos/i4-address-junk-move.sqlite.test.ts

@@ -6,6 +6,7 @@ import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createTestDb, type TestDb } from "../test-db.js";
 import { AddressRepo } from "./i4-address.js";
 import type { MailboxSpecialUseRepo } from "./i4-mailbox-special-use.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 const CONFIG = "cfg-1";
 
@@ -133,6 +134,7 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).mailboxSpecialUse;
 	});
 

package/src/repos/i4-mailbox-special-use.sqlite.test.ts

@@ -18,6 +18,7 @@ import { createSqliteTestDb } from "../test-db-sqlite.js";
 import type { AccountSettingRepo } from "./i4-account-setting.js";
 import type { MailboxRepo } from "./i4-mailbox.js";
 import type { MailboxSpecialUseRepo } from "./i4-mailbox-special-use.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import { mailboxCreated } from "./test-helpers.js";
 
 const makeMailboxInput = (
@@ -65,6 +66,7 @@ describe("MailboxSpecialUseRepo role lookups (sqlite)", () => {
 				search: testDb.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		);
 		repo = repos.mailboxSpecialUse;
 		mailboxes = repos.mailbox;

package/src/repos/i4-mailbox.sqlite.test.ts

@@ -13,6 +13,7 @@ import {
 	shippedTableDdl,
 } from "../test-shipped-sqlite-schema.js";
 import type { MailboxRepo } from "./i4-mailbox.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import { mailboxCreated } from "./test-helpers.js";
 
 function makeMailboxInput(accountId: string, fullPath = "INBOX") {
@@ -49,6 +50,7 @@ describe("MailboxRepo (sqlite)", () => {
 				search: testDb.db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).mailbox;
 	});
 
@@ -222,6 +224,7 @@ describe("MailboxRepo (sqlite, shipped migrations)", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).mailbox;
 		close = async () => {
 			sqlite.close();

package/src/repos/i4-mailbox.test.ts

@@ -18,6 +18,7 @@ import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createTestDb, randomId, type TestDb } from "../test-db.js";
 import type { MailboxRepo } from "./i4-mailbox.js";
 import { deleteMessageSubtree } from "./message.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 import { mailboxCreated } from "./test-helpers.js";
 
 /** Every state a row can carry, for a transition that decides against none. */
@@ -62,6 +63,7 @@ describe("MailboxRepo", () => {
 				search: db,
 			},
 			BETTER_SQLITE3_LIMITS,
+			messageObjectsInMemory(),
 		).mailbox;
 	});
 
@@ -554,6 +556,7 @@ describe("MailboxRepo", () => {
 					search: db,
 				},
 				BETTER_SQLITE3_LIMITS,
+				messageObjectsInMemory(),
 			).message.create({
 				messageId,
 				mailboxId,
@@ -561,8 +564,6 @@ describe("MailboxRepo", () => {
 				sequenceNumber: uid,
 				rfc822Size: 10,
 				internalDate: 1700000000000,
-				envelopeId: deriveEnvelopeId(messageId),
-				rootBodyPartId: deriveRootBodyPartId(messageId),
 			});
 			await db.insert(threadMessageTable).values({
 				threadMessageId: randomId(),
@@ -682,7 +683,7 @@ describe("MailboxRepo", () => {
 
 			// The shape an interruption leaves: one message's rows already gone,
 			// the mailbox row still there and still `deleting`.
-			await syncBatch(db)(deleteMessageSubtree(db, [first]));
+			await syncBatch(db)(deleteMessageSubtree(db, [first]).statements);
 			await repo.transition(accountId, mailbox.mailboxId, {
 				from: EVERY_STATE,
 				to: MailboxSyncStatus.deleting,

package/src/repos/i4-mailbox.ts

@@ -42,10 +42,8 @@ import { messageFlagPushTable } from "../schema/i4-message-flag-push.js";
 import { messagePlacementMoveTable } from "../schema/i4-message-placement-move.js";
 import { messageTable } from "../schema/message-data.js";
 import { syncWarningTable } from "../schema/sync-warning.js";
-import { threadMessageTable } from "../schema/thread-message.js";
-import type { SearchIndex } from "../search/thread-message-search.js";
 import { inList } from "./in-list.js";
-import { deleteMessageSubtree } from "./message.js";
+import type { DrizzleMessageRepository } from "./message.js";
 
 const base36Translator = shortUuid.createTranslator(
 	shortUuid.constants.uuid25Base36,
@@ -60,7 +58,6 @@ type DB = RepositoryDb<Record<string, unknown>>;
  * the file's write lock, so the bound is what keeps a large folder's delete from
  * parking every other writer behind it (D8).
  */
-const MAIL_DELETE_BATCH_SIZE = 100;
 
 /**
  * The folders under `path`. A flat namespace nests nothing: with no delimiter
@@ -154,7 +151,7 @@ export class MailboxRepo implements IMailboxRepository {
 
 	constructor(
 		store: SqliteStore<Record<string, unknown>>,
-		private search: SearchIndex,
+		private messages: Pick<DrizzleMessageRepository, "deleteMany">,
 	) {
 		this.db = store.db;
 		this.batch = store.batch;
@@ -547,29 +544,11 @@ export class MailboxRepo implements IMailboxRepository {
 		// Ordered, batched and resumable rather than one transaction (D8). The
 		// caller keeps the row `deleting` until the last commit, so an interrupted
 		// run re-enters here and continues against whatever is left.
-		for (;;) {
-			const rows = await this.db
-				.select({ messageId: messageTable.messageId })
-				.from(messageTable)
-				.where(eq(messageTable.mailboxId, mailboxId))
-				.limit(MAIL_DELETE_BATCH_SIZE);
-			if (rows.length === 0) break;
-			const messageIds = rows.map((row) => row.messageId);
-			const listings = await this.db
-				.select({ threadMessageId: threadMessageTable.threadMessageId })
-				.from(threadMessageTable)
-				.where(inList(threadMessageTable.messageId, messageIds));
-
-			// The primitive the rest of the codebase deletes mail with: nine
-			// per-message child tables. A bespoke table list would orphan them.
-			await this.batch([
-				...deleteMessageSubtree(this.db, messageIds),
-				this.db
-					.delete(threadMessageTable)
-					.where(inList(threadMessageTable.messageId, messageIds)),
-			]);
-			await this.search.sync(listings.map((row) => row.threadMessageId));
-		}
+		const rows = await this.db
+			.select({ messageId: messageTable.messageId })
+			.from(messageTable)
+			.where(eq(messageTable.mailboxId, mailboxId));
+		await this.messages.deleteMany(rows.map((row) => row.messageId));
 
 		await this.db
 			.delete(mailboxSpecialUseTable)

package/src/repos/i4-outbox-attachment.test.ts

@@ -4,6 +4,7 @@ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createTestDb, randomId } from "../test-db.js";
 import type { OutboxAttachmentRepo } from "./i4-outbox-attachment.js";
+import { messageObjectsInMemory } from "./message-objects.js";
 
 const CAP = { maxTotalBytes: 1000, maxCount: 3, nowSeconds: 1_000_000 };
 
@@ -36,6 +37,7 @@ before(async () => {
 			search: harness.db,
 		},
 		BETTER_SQLITE3_LIMITS,
+		messageObjectsInMemory(),
 	).outboxAttachment;
 });