@remit/drizzle-service 0.0.103 → 0.0.104

package/src/repos/i4-account-config.test.ts

@@ -1,128 +1,4 @@
-import assert from "node:assert";
-import { after, before, describe, test } from "node:test";
-import { createSqliteRepositories } from "../sqlite-repositories.js";
-import { createTestDb, randomId, type TestDb } from "../test-db.js";
-import type { AccountConfigRepo } from "./i4-account-config.js";
+import { openSqliteTestStores } from "../test-stores.js";
+import { accountConfigRepoSuite } from "./i4-account-config.suite.js";
 
-describe("AccountConfigRepo", () => {
-	let db: TestDb;
-	let close: () => Promise<void>;
-	let repo: AccountConfigRepo;
-
-	before(async () => {
-		({ db, close } = await createTestDb());
-		repo = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).accountConfig;
-	});
-
-	after(async () => {
-		await close();
-	});
-
-	test("create and get", async () => {
-		const userId = randomId();
-		const cfg = await repo.create({ userId, name: "My Config" });
-
-		assert.ok(cfg.accountConfigId);
-		assert.equal(cfg.userId, userId);
-		assert.equal(cfg.name, "My Config");
-		assert.equal(cfg.state, "active");
-
-		const fetched = await repo.get(cfg.accountConfigId);
-		assert.equal(fetched.accountConfigId, cfg.accountConfigId);
-
-		await repo.delete(cfg.accountConfigId);
-	});
-
-	test("batchGet: WHERE id = ANY($1)", async () => {
-		const userId = randomId();
-		const c1 = await repo.create({ userId });
-		const c2 = await repo.create({ userId });
-
-		const results = await repo.get([c1.accountConfigId, c2.accountConfigId]);
-		assert.equal(results.length, 2);
-
-		await repo.deleteMany([c1.accountConfigId, c2.accountConfigId]);
-	});
-
-	test("batchGet empty array returns []", async () => {
-		const results = await repo.get([]);
-		assert.deepEqual(results, []);
-	});
-
-	test("get throws for missing config", async () => {
-		await assert.rejects(repo.get(randomId()), /not found/i);
-	});
-
-	test("describe assembles collection (accountConfig + account + address)", async () => {
-		const userId = randomId();
-		const cfg = await repo.create({ userId });
-
-		const repos = createSqliteRepositories({ app: db, mail: db, search: db });
-		const accountRepo = repos.account;
-		const addrRepo = repos.address;
-
-		await accountRepo.create({
-			accountConfigId: cfg.accountConfigId,
-			username: "u",
-			email: "u@test.com",
-			isActive: true,
-			imapHost: "imap.test.com",
-			imapPort: 993,
-			imapTls: true,
-			imapStartTls: false,
-			connectionState: "not_authenticated",
-		});
-
-		await addrRepo.createAddress({
-			addressId: randomId(),
-			accountConfigId: cfg.accountConfigId,
-			localPart: "u",
-			domain: "test.com",
-			normalizedEmail: "u@test.com",
-			normalizedCompound: "u@test.com:u",
-		});
-
-		const desc = await repo.describe(cfg.accountConfigId);
-		assert.equal(desc.accountConfig.length, 1);
-		assert.equal(desc.account.length, 1);
-		assert.equal(desc.address.length, 1);
-
-		await repo.delete(cfg.accountConfigId);
-	});
-
-	test("listAll returns all configs", async () => {
-		const userId = randomId();
-		const c1 = await repo.create({ userId });
-		const c2 = await repo.create({ userId });
-
-		const all = await repo.listAll();
-		const ids = new Set(all.map((c) => c.accountConfigId));
-		assert.ok(ids.has(c1.accountConfigId));
-		assert.ok(ids.has(c2.accountConfigId));
-
-		await repo.deleteMany([c1.accountConfigId, c2.accountConfigId]);
-	});
-
-	describe("continuation token rejection (#172)", () => {
-		for (const [label, token] of [
-			["an unparseable", "not-a-cursor"],
-			["a bare number", Buffer.from("123").toString("base64url")],
-			["a JSON array", Buffer.from("[1,2]").toString("base64url")],
-		] as const) {
-			test(`${label} token is rejected as a 400`, async () => {
-				await assert.rejects(
-					() => repo.listByUser(randomId(), { continuationToken: token }),
-					(error: unknown) => {
-						assert.equal((error as { statusCode?: number }).statusCode, 400);
-						assert.equal((error as Error).name, "BadRequestError");
-						return true;
-					},
-				);
-			});
-		}
-	});
-});
+accountConfigRepoSuite(openSqliteTestStores);

package/src/repos/i4-account-export-request.d1.test.ts

@@ -0,0 +1,4 @@
+import { openD1TestStores } from "../test-d1.js";
+import { accountExportRequestRepoSuite } from "./i4-account-export-request.suite.js";
+
+accountExportRequestRepoSuite(openD1TestStores);

package/src/repos/i4-account-export-request.suite.ts

@@ -0,0 +1,44 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import { randomId } from "../id.js";
+import type { OpenTestStores } from "../test-stores.js";
+import { AccountExportRequestRepo } from "./i4-account-export-request.js";
+
+export const accountExportRequestRepoSuite = (open: OpenTestStores): void => {
+	describe("AccountExportRequestRepo", () => {
+		let close: () => Promise<void>;
+		let repo: AccountExportRequestRepo;
+
+		before(async () => {
+			const opened = await open();
+			close = opened.close;
+			repo = new AccountExportRequestRepo(opened.stores.app.db);
+		});
+
+		after(async () => {
+			await close();
+		});
+
+		describe("continuation token rejection (#172)", () => {
+			for (const [label, token] of [
+				["an unparseable", "not-a-cursor"],
+				["a bare number", Buffer.from("123").toString("base64url")],
+				["a JSON array", Buffer.from("[1,2]").toString("base64url")],
+			] as const) {
+				test(`${label} token is rejected as a 400`, async () => {
+					await assert.rejects(
+						() =>
+							repo.listByAccountConfig(randomId(), {
+								continuationToken: token,
+							}),
+						(error: unknown) => {
+							assert.equal((error as { statusCode?: number }).statusCode, 400);
+							assert.equal((error as Error).name, "BadRequestError");
+							return true;
+						},
+					);
+				});
+			}
+		});
+	});
+};

package/src/repos/i4-account-export-request.test.ts

@@ -1,41 +1,4 @@
-import assert from "node:assert/strict";
-import { after, before, describe, test } from "node:test";
-import { createTestDb, randomId, type TestDb } from "../test-db.js";
-import { AccountExportRequestRepo } from "./i4-account-export-request.js";
+import { openSqliteTestStores } from "../test-stores.js";
+import { accountExportRequestRepoSuite } from "./i4-account-export-request.suite.js";
 
-describe("AccountExportRequestRepo", () => {
-	let db: TestDb;
-	let close: () => Promise<void>;
-	let repo: AccountExportRequestRepo;
-
-	before(async () => {
-		({ db, close } = await createTestDb());
-		repo = new AccountExportRequestRepo(db as never);
-	});
-
-	after(async () => {
-		await close();
-	});
-
-	describe("continuation token rejection (#172)", () => {
-		for (const [label, token] of [
-			["an unparseable", "not-a-cursor"],
-			["a bare number", Buffer.from("123").toString("base64url")],
-			["a JSON array", Buffer.from("[1,2]").toString("base64url")],
-		] as const) {
-			test(`${label} token is rejected as a 400`, async () => {
-				await assert.rejects(
-					() =>
-						repo.listByAccountConfig(randomId(), {
-							continuationToken: token,
-						}),
-					(error: unknown) => {
-						assert.equal((error as { statusCode?: number }).statusCode, 400);
-						assert.equal((error as Error).name, "BadRequestError");
-						return true;
-					},
-				);
-			});
-		}
-	});
-});
+accountExportRequestRepoSuite(openSqliteTestStores);

package/src/repos/i4-account-setting.d1.test.ts

@@ -0,0 +1,4 @@
+import { openD1TestStores } from "../test-d1.js";
+import { accountSettingRepoSuite } from "./i4-account-setting.suite.js";
+
+accountSettingRepoSuite(openD1TestStores);

package/src/repos/i4-account-setting.suite.ts

@@ -0,0 +1,100 @@
+import assert from "node:assert";
+import { after, before, describe, test } from "node:test";
+import { randomId } from "../id.js";
+import type { OpenTestStores } from "../test-stores.js";
+import { AccountSettingRepo } from "./i4-account-setting.js";
+
+export const accountSettingRepoSuite = (open: OpenTestStores): void => {
+	describe("AccountSettingRepo", () => {
+		let close: () => Promise<void>;
+		let repo: AccountSettingRepo;
+
+		before(async () => {
+			const opened = await open();
+			close = opened.close;
+			repo = new AccountSettingRepo(opened.stores.app.db);
+		});
+
+		after(async () => {
+			await close();
+		});
+
+		test("upsert and get", async () => {
+			const accountConfigId = randomId();
+			await repo.upsert({
+				accountConfigId,
+				name: "Theme",
+				value: { kind: "String", value: "dark" },
+			});
+
+			const setting = await repo.get(accountConfigId, "Theme");
+			assert.ok(setting);
+			assert.equal(setting.name, "Theme");
+			assert.deepEqual(setting.value, { kind: "String", value: "dark" });
+
+			await repo.delete(accountConfigId, "Theme");
+		});
+
+		test("upsert is idempotent (overwrites)", async () => {
+			const accountConfigId = randomId();
+			await repo.upsert({
+				accountConfigId,
+				name: "Density",
+				value: { kind: "String", value: "compact" },
+			});
+			await repo.upsert({
+				accountConfigId,
+				name: "Density",
+				value: { kind: "String", value: "comfortable" },
+			});
+
+			const setting = await repo.get(accountConfigId, "Density");
+			assert.deepEqual(setting?.value, {
+				kind: "String",
+				value: "comfortable",
+			});
+
+			await repo.delete(accountConfigId, "Density");
+		});
+
+		test("get returns null when setting absent", async () => {
+			const setting = await repo.get(randomId(), "Theme");
+			assert.equal(setting, null);
+		});
+
+		test("listByAccountConfig returns all settings ordered by name", async () => {
+			const accountConfigId = randomId();
+			await repo.upsert({
+				accountConfigId,
+				name: "Theme",
+				value: { kind: "String", value: "dark" },
+			});
+			await repo.upsert({
+				accountConfigId,
+				name: "Density",
+				value: { kind: "String", value: "compact" },
+			});
+
+			const settings = await repo.listByAccountConfig(accountConfigId);
+			assert.equal(settings.length, 2);
+			assert.equal(settings[0].name, "Density");
+			assert.equal(settings[1].name, "Theme");
+
+			await repo.delete(accountConfigId, "Theme");
+			await repo.delete(accountConfigId, "Density");
+		});
+
+		test("delete removes the setting", async () => {
+			const accountConfigId = randomId();
+			await repo.upsert({
+				accountConfigId,
+				name: "Theme",
+				value: { kind: "Boolean", value: true },
+			});
+			await repo.delete(accountConfigId, "Theme");
+
+			const setting = await repo.get(accountConfigId, "Theme");
+			assert.equal(setting, null);
+		});
+	});
+};

package/src/repos/i4-account-setting.test.ts

@@ -1,94 +1,4 @@
-import assert from "node:assert";
-import { after, before, describe, test } from "node:test";
-import { createTestDb, randomId, type TestDb } from "../test-db.js";
-import { AccountSettingRepo } from "./i4-account-setting.js";
+import { openSqliteTestStores } from "../test-stores.js";
+import { accountSettingRepoSuite } from "./i4-account-setting.suite.js";
 
-describe("AccountSettingRepo", () => {
-	let db: TestDb;
-	let close: () => Promise<void>;
-	let repo: AccountSettingRepo;
-
-	before(async () => {
-		({ db, close } = await createTestDb());
-		repo = new AccountSettingRepo(db as never);
-	});
-
-	after(async () => {
-		await close();
-	});
-
-	test("upsert and get", async () => {
-		const accountConfigId = randomId();
-		await repo.upsert({
-			accountConfigId,
-			name: "Theme",
-			value: { kind: "String", value: "dark" },
-		});
-
-		const setting = await repo.get(accountConfigId, "Theme");
-		assert.ok(setting);
-		assert.equal(setting.name, "Theme");
-		assert.deepEqual(setting.value, { kind: "String", value: "dark" });
-
-		await repo.delete(accountConfigId, "Theme");
-	});
-
-	test("upsert is idempotent (overwrites)", async () => {
-		const accountConfigId = randomId();
-		await repo.upsert({
-			accountConfigId,
-			name: "Density",
-			value: { kind: "String", value: "compact" },
-		});
-		await repo.upsert({
-			accountConfigId,
-			name: "Density",
-			value: { kind: "String", value: "comfortable" },
-		});
-
-		const setting = await repo.get(accountConfigId, "Density");
-		assert.deepEqual(setting?.value, { kind: "String", value: "comfortable" });
-
-		await repo.delete(accountConfigId, "Density");
-	});
-
-	test("get returns null when setting absent", async () => {
-		const setting = await repo.get(randomId(), "Theme");
-		assert.equal(setting, null);
-	});
-
-	test("listByAccountConfig returns all settings ordered by name", async () => {
-		const accountConfigId = randomId();
-		await repo.upsert({
-			accountConfigId,
-			name: "Theme",
-			value: { kind: "String", value: "dark" },
-		});
-		await repo.upsert({
-			accountConfigId,
-			name: "Density",
-			value: { kind: "String", value: "compact" },
-		});
-
-		const settings = await repo.listByAccountConfig(accountConfigId);
-		assert.equal(settings.length, 2);
-		assert.equal(settings[0].name, "Density");
-		assert.equal(settings[1].name, "Theme");
-
-		await repo.delete(accountConfigId, "Theme");
-		await repo.delete(accountConfigId, "Density");
-	});
-
-	test("delete removes the setting", async () => {
-		const accountConfigId = randomId();
-		await repo.upsert({
-			accountConfigId,
-			name: "Theme",
-			value: { kind: "Boolean", value: true },
-		});
-		await repo.delete(accountConfigId, "Theme");
-
-		const setting = await repo.get(accountConfigId, "Theme");
-		assert.equal(setting, null);
-	});
-});
+accountSettingRepoSuite(openSqliteTestStores);

package/src/repos/label.conformance.d1.test.ts

@@ -0,0 +1,20 @@
+import { labelRepositoryConformance } from "@remit/data-ports/conformance";
+import { NotFoundError } from "../error.js";
+import { randomId } from "../id.js";
+import { openD1TestStores } from "../test-d1.js";
+import { LabelRepo } from "./label.js";
+
+let close: (() => Promise<void>) | undefined;
+
+labelRepositoryConformance({
+	async createRepository() {
+		const opened = await openD1TestStores();
+		close = opened.close;
+		return new LabelRepo(opened.stores.app.db);
+	},
+	teardown: async () => {
+		await close?.();
+	},
+	makeId: () => randomId(),
+	isNotFoundError: (error) => error instanceof NotFoundError,
+});

package/src/repos/message.sqlite.test.ts

@@ -279,4 +279,58 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		await repo.delete(MESSAGE_ID);
 		await assert.rejects(() => repo.get(MESSAGE_ID));
 	});
+
+	test("listMissingSearchText names stored bodies without search text, after the cursor", async () => {
+		const id = (n: number): string => `00000000-0000-0000-3333-00000000000${n}`;
+		for (const n of [1, 2, 3, 4]) {
+			await repo.create({ ...BASE_INPUT, messageId: id(n), uid: 100 + n });
+		}
+		for (const n of [1, 2, 4]) {
+			await repo.update(id(n), { bodyStorageKey: `file:///bodies/${n}` });
+		}
+		await repo.setSearchText(id(2), "already indexed");
+
+		const ours = (rows: Array<{ messageId: string }>): string[] =>
+			rows
+				.map((row) => row.messageId)
+				.filter((messageId) => messageId.includes("-3333-"));
+
+		assert.deepEqual(ours(await repo.listMissingSearchText("", 100)), [
+			id(1),
+			id(4),
+		]);
+		assert.deepEqual(ours(await repo.listMissingSearchText(id(1), 100)), [
+			id(4),
+		]);
+		assert.deepEqual(
+			await repo
+				.listMissingSearchText(id(1), 100)
+				.then((rows) => rows.find((row) => row.messageId === id(4))),
+			{
+				messageId: id(4),
+				bodyStorageKey: "file:///bodies/4",
+			},
+		);
+	});
+
+	test("copySearchText gives a copy the source's search text", async () => {
+		const source = "00000000-0000-0000-4444-000000000001";
+		const copy = "00000000-0000-0000-4444-000000000002";
+		for (const [messageId, uid] of [
+			[source, 201],
+			[copy, 202],
+		] as const) {
+			await repo.create({ ...BASE_INPUT, messageId, uid });
+			await repo.update(messageId, { bodyStorageKey: "file:///bodies/shared" });
+		}
+		await repo.setSearchText(source, "the shared body");
+
+		await repo.copySearchText(source, copy);
+
+		const missing = await repo.listMissingSearchText("", 100);
+		assert.equal(
+			missing.some((row) => row.messageId === copy),
+			false,
+		);
+	});
 });

package/src/repos/message.ts

@@ -39,6 +39,7 @@ import {
 	envelopeTable,
 	messageFlagTable,
 	messageReferenceTable,
+	messageSearchTextTable,
 	messageTable,
 	rawMessageStorageTable,
 } from "../schema/message-data.js";
@@ -153,6 +154,9 @@ export function deleteMessageSubtree(
 			.where(
 				and(inList(bodyStructureRefetchTable.messageId, messageIds), guard),
 			),
+		db
+			.delete(messageSearchTextTable)
+			.where(and(inList(messageSearchTextTable.messageId, messageIds), guard)),
 		db
 			.delete(envelopeTable)
 			.where(and(inList(envelopeTable.messageId, messageIds), guard)),
@@ -444,6 +448,68 @@ export class DrizzleMessageRepository implements IMessageRepository {
 		return this.get(messageId);
 	}
 
+	async setSearchText(messageId: string, text: string): Promise<void> {
+		const now = Date.now();
+		await this.db
+			.insert(messageSearchTextTable)
+			.values({ messageId, text, createdAt: now, updatedAt: now })
+			.onConflictDoUpdate({
+				target: messageSearchTextTable.messageId,
+				set: { text, updatedAt: now },
+			});
+	}
+
+	async copySearchText(
+		sourceMessageId: string,
+		targetMessageId: string,
+	): Promise<void> {
+		const [source] = await this.db
+			.select({ text: messageSearchTextTable.text })
+			.from(messageSearchTextTable)
+			.where(eq(messageSearchTextTable.messageId, sourceMessageId));
+		if (!source) return;
+		await this.setSearchText(targetMessageId, source.text);
+	}
+
+	async syncSearch(messageIds: string[]): Promise<void> {
+		if (messageIds.length === 0) return;
+		const listings = await this.db
+			.select({ threadMessageId: threadMessageTable.threadMessageId })
+			.from(threadMessageTable)
+			.where(inList(threadMessageTable.messageId, messageIds));
+		await this.search.sync(listings.map((row) => row.threadMessageId));
+	}
+
+	async listMissingSearchText(
+		after: string,
+		limit: number,
+	): ReturnType<IMessageRepository["listMissingSearchText"]> {
+		const rows = await this.db
+			.select({
+				messageId: messageTable.messageId,
+				bodyStorageKey: messageTable.bodyStorageKey,
+			})
+			.from(messageTable)
+			.leftJoin(
+				messageSearchTextTable,
+				eq(messageSearchTextTable.messageId, messageTable.messageId),
+			)
+			.where(
+				and(
+					gt(messageTable.messageId, after),
+					isNotNull(messageTable.bodyStorageKey),
+					isNull(messageSearchTextTable.messageId),
+				),
+			)
+			.orderBy(asc(messageTable.messageId))
+			.limit(limit);
+		return rows.flatMap((row) =>
+			row.bodyStorageKey
+				? [{ messageId: row.messageId, bodyStorageKey: row.bodyStorageKey }]
+				: [],
+		);
+	}
+
 	async delete(messageId: string): Promise<void> {
 		await this.deleteMany([messageId]);
 	}

package/src/repos/thread-message-body-muted.sqlite.test.ts

@@ -11,16 +11,12 @@ import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import type { CreateThreadMessageInput } from "@remit/data-ports";
 import { addressTable } from "../schema/i4-address.js";
+import { messageSearchTextTable } from "../schema/message-data.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
-import { shippedIndexDdl } from "../test-shipped-sqlite-schema.js";
 import type { DrizzleThreadMessageRepository } from "./thread-message.js";
 
-// The index the mail baseline creates for this predicate, read from the one
-// committed source so the plan assertion below is about the real object.
-const mutedIndexDdl = shippedIndexDdl("address_by_normalized_email");
-
 const ACCOUNT = "acct-seam";
 const MAILBOX = "mbx-seam";
 const SCOPE = new Set([MAILBOX]);
@@ -57,10 +53,13 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
 
 	before(async () => {
 		({ db, sqlite, close } = await createSqliteTestDb(
-			{ threadMessage: threadMessageTable, address: addressTable },
+			{
+				threadMessage: threadMessageTable,
+				address: addressTable,
+				messageSearchText: messageSearchTextTable,
+			},
 			{ searchIndex: true },
 		));
-		sqlite.exec(mutedIndexDdl);
 		repo = createSqliteRepositories({
 			app: db,
 			mail: db,
@@ -86,15 +85,20 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
 					}),
 				);
 			}
-			await repo.create(
-				makeInput({
-					subject: "unrelated note",
-					fromEmail: "noise@example.com",
-					fromName: "Noise",
-					snippet: "Your parcel was left with the concierge",
-					sentDate: BASE_DATE,
-				}),
-			);
+			const parcel = makeInput({
+				subject: "unrelated note",
+				fromEmail: "noise@example.com",
+				fromName: "Noise",
+				snippet: "Your parcel was left with the concierge",
+				sentDate: BASE_DATE,
+			});
+			await db.insert(messageSearchTextTable).values({
+				messageId: parcel.messageId,
+				text: "Your parcel was left with the concierge",
+				createdAt: 0,
+				updatedAt: 0,
+			});
+			await repo.create(parcel);
 		});
 
 		test("the term reaches the body text, not just subject and From", async () => {
@@ -123,16 +127,16 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
 			assert.equal(result.items.length, 1);
 		});
 
-		test("a term under the trigram floor reads the body too", async () => {
+		test("a short word prefix reads the body too", async () => {
 			const result = await repo.searchByDate(
 				ACCOUNT,
-				{ query: "ge" },
+				{ query: "co" },
 				{ mailboxIds: SCOPE, limit: 50 },
 			);
 
 			assert.ok(
 				result.items.some((item) => item.snippet?.includes("concierge")),
-				"the folded scan matched the body preview",
+				"the prefix matched the body preview",
 			);
 		});