@remit/drizzle-service 0.0.64 → 0.0.66

package/src/repos/i4-address-junk-move.sqlite.test.ts

@@ -3,6 +3,7 @@ import { after, before, beforeEach, describe, test } from "node:test";
 import type Database from "better-sqlite3";
 import { createTestDb, type TestDb } from "../test-db.js";
 import { AddressRepo } from "./i4-address.js";
+import { MailboxSpecialUseRepo } from "./i4-mailbox-special-use.js";
 
 const CONFIG = "cfg-1";
 
@@ -11,8 +12,34 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 	let sqlite: Database.Database;
 	let close: () => Promise<void>;
 	let repo: AddressRepo;
+	let specialUse: MailboxSpecialUseRepo;
 
-	const mailbox = (mailboxId: string, specialUse: string | null): void => {
+	const account = (accountId: string): void => {
+		sqlite
+			.prepare(
+				`INSERT OR IGNORE INTO account (
+					account_id, account_config_id, username, email, imap_host,
+					imap_port, imap_tls, imap_start_tls, smtp_port, is_active,
+					connection_state, created_at, updated_at
+				) VALUES (?, ?, 'user', 'user@example.com', 'imap.example.com',
+					993, 1, 0, 465, 1, 'disconnected', 0, 0)`,
+			)
+			.run(accountId, CONFIG);
+	};
+
+	/**
+	 * A folder the server designated, written the way mailbox sync writes one:
+	 * the denormalized column AND the entry row. The paths are deliberately not
+	 * English — the role has to be read off the designation, not guessed from
+	 * the name.
+	 */
+	const mailbox = (
+		mailboxId: string,
+		designation: string | null,
+		folder: { accountId?: string; fullPath?: string } = {},
+	): void => {
+		const accountId = folder.accountId ?? "acc";
+		account(accountId);
 		sqlite
 			.prepare(
 				`INSERT INTO mailbox (
@@ -21,9 +48,22 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 					unseen_count, deleted_count, total_size, last_sync_uid,
 					high_water_mark_uid, last_message_sync_at, special_use,
 					created_at, updated_at
-				) VALUES (?, 'acc', '', '/', ?, 1, 1, '0', 0, 0, 0, 0, 0, 0, 0, ?, 0, 0)`,
+				) VALUES (?, ?, '', '/', ?, 1, 1, '0', 0, 0, 0, 0, 0, 0, 0, ?, 0, 0)`,
+			)
+			.run(
+				mailboxId,
+				accountId,
+				folder.fullPath ?? mailboxId,
+				designation ? JSON.stringify([designation]) : null,
+			);
+		if (!designation) return;
+		sqlite
+			.prepare(
+				`INSERT INTO mailbox_special_use_entry (
+					mailbox_special_use_id, mailbox_id, special_use
+				) VALUES (?, ?, ?)`,
 			)
-			.run(mailboxId, mailboxId, specialUse);
+			.run(`${mailboxId}-${designation}`, mailboxId, designation);
 	};
 
 	const message = (messageId: string, mailboxId: string): void => {
@@ -84,19 +124,38 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 	before(async () => {
 		({ db, sqlite, close } = await createTestDb());
 		repo = new AddressRepo(db as never);
+		specialUse = new MailboxSpecialUseRepo(db as never);
 	});
 
+	/**
+	 * The roles the caller resolves once and hands down, covering every account
+	 * under the config — which is the scope the predicate reads, because the
+	 * address book is keyed by config.
+	 */
+	const reconcile = async (messageId: string): Promise<void> =>
+		repo.reconcileJunkOnlyForMessage(
+			messageId,
+			await specialUse.resolveJunkRolesForConfig(CONFIG),
+		);
+
 	after(async () => {
 		await close();
 	});
 
 	beforeEach(() => {
-		for (const table of ["address", "envelope_address", "message", "mailbox"]) {
+		for (const table of [
+			"address",
+			"envelope_address",
+			"message",
+			"mailbox",
+			"mailbox_special_use_entry",
+			"account",
+		]) {
 			sqlite.exec(`DELETE FROM ${table}`);
 		}
-		mailbox("inbox", null);
-		mailbox("junk", '["Junk"]');
-		mailbox("trash", '["Trash"]');
+		mailbox("inbox", null, { fullPath: "INBOX" });
+		mailbox("junk", "Junk", { fullPath: "Ongewenst" });
+		mailbox("trash", "Trash", { fullPath: "Prullenbak" });
 	});
 
 	test("a message moved into Junk stops the sender being suggested", async () => {
@@ -106,7 +165,7 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 		assert.deepEqual(await suggested("spammer"), ["spammer"]);
 
 		moveTo("msg", "junk");
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 
 		assert.equal(await withheld("spammer"), true);
 		assert.deepEqual(await suggested("spammer"), []);
@@ -119,7 +178,7 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 		await repo.incrementOutboundCount(CONFIG, "client", Date.now());
 
 		moveTo("msg", "junk");
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 
 		assert.equal(await withheld("client"), false);
 	});
@@ -132,7 +191,7 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 		sighting("colleague", "real");
 
 		moveTo("spam", "junk");
-		await repo.reconcileJunkOnlyForMessage("spam");
+		await reconcile("spam");
 
 		assert.equal(await withheld("colleague"), false);
 	});
@@ -152,7 +211,7 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 		assert.deepEqual(await suggested("misfiled"), []);
 
 		moveTo("msg", "inbox");
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 
 		assert.deepEqual(await suggested("misfiled"), ["misfiled"]);
 	});
@@ -171,7 +230,7 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 		sighting("spammer", "msg");
 
 		moveTo("msg", "trash");
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 
 		assert.equal(await withheld("spammer"), true);
 	});
@@ -183,20 +242,65 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 		sighting("bystander", "other");
 
 		moveTo("msg", "junk");
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 
 		assert.equal(await withheld("bystander"), false);
 	});
 
+	test("a sender met only in Junk on two accounts of one config stays withheld", async () => {
+		mailbox("junk-b", "Junk", {
+			accountId: "acc-b",
+			fullPath: "Indésirables",
+		});
+		message("ma", "junk");
+		message("mb", "junk-b");
+		await harvest("spammer");
+		sighting("spammer", "ma");
+		sighting("spammer", "mb");
+
+		await reconcile("ma");
+		assert.equal(await withheld("spammer"), true);
+
+		await reconcile("mb");
+
+		assert.equal(await withheld("spammer"), true);
+	});
+
+	/**
+	 * The name hints are English only, so an account whose server flags nothing
+	 * and whose Junk folder is called `Ongewenst` resolves no Junk at all. That
+	 * silence must lift no mark: reading it as "every sighting is live mail"
+	 * would hand the spammer back on the next move.
+	 */
+	test("a move never lifts a mark when no folder holds Junk", async () => {
+		sqlite.exec("DELETE FROM mailbox_special_use_entry");
+		sqlite.exec("UPDATE mailbox SET special_use = NULL");
+		message("msg", "inbox");
+		await repo.upsertJunkAddress({
+			addressId: "spammer",
+			accountConfigId: CONFIG,
+			displayName: "Name",
+			localPart: "spammer",
+			domain: "example.com",
+			normalizedEmail: "spammer@example.com",
+			normalizedCompound: "name spammer@example.com",
+		});
+		sighting("spammer", "msg");
+
+		await reconcile("msg");
+
+		assert.equal(await withheld("spammer"), true);
+	});
+
 	test("a second reconcile of the same move writes nothing new", async () => {
 		message("msg", "inbox");
 		await harvest("spammer");
 		sighting("spammer", "msg");
 		moveTo("msg", "junk");
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 		const first = await repo.getAddress(CONFIG, "spammer");
 
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 
 		assert.deepEqual(await repo.getAddress(CONFIG, "spammer"), first);
 	});

package/src/repos/i4-address.ts

@@ -10,6 +10,7 @@ import type {
 	UpdateAddressInput,
 } from "@remit/data-ports";
 import { BadRequestError } from "@remit/data-ports/errors";
+import type { JunkRoleMailboxes } from "@remit/data-ports/folder-role";
 import { shouldPromoteWellknown } from "@remit/data-ports/wellknown";
 import {
 	and,
@@ -26,6 +27,7 @@ import { NotFoundError } from "../error.js";
 import { envelopeAddressId } from "../id.js";
 import { decodeToken, resultList } from "../pagination.js";
 import {
+	type BoundSql,
 	JUNK_ONLY_FLAG,
 	restoreSql,
 	withholdSql,
@@ -331,15 +333,21 @@ export class AddressRepo implements IAddressRepository {
 		return rowToAddress(row);
 	}
 
-	async reconcileJunkOnlyForMessage(messageId: string): Promise<void> {
-		const scope = ` AND address.address_id IN (
+	async reconcileJunkOnlyForMessage(
+		messageId: string,
+		roles: JunkRoleMailboxes,
+	): Promise<void> {
+		const scope: BoundSql = {
+			sql: ` AND address.address_id IN (
 			SELECT address_id FROM envelope_address WHERE message_id = ?
-		)`;
+		)`,
+			params: [messageId],
+		};
 		const now = Date.now();
-		await this.db.run(
-			boundToDrizzle(withholdSql(scope), [now, JUNK_MOVE, now, messageId]),
-		);
-		await this.db.run(boundToDrizzle(restoreSql(scope), [now, messageId]));
+		const withhold = withholdSql(roles, now, JUNK_MOVE, scope);
+		await this.db.run(boundToDrizzle(withhold.sql, withhold.params));
+		const restore = restoreSql(roles, now, scope);
+		await this.db.run(boundToDrizzle(restore.sql, restore.params));
 	}
 
 	async getAddress(

package/src/repos/i4-config-import.ts

@@ -0,0 +1,94 @@
+import type {
+	ConfigImportItem,
+	CreateConfigImportInput,
+	IConfigImportRepository,
+	UpdateConfigImportInput,
+} from "@remit/data-ports";
+import { desc, eq } from "drizzle-orm";
+import type { Db } from "../db.js";
+import { NotFoundError } from "../error.js";
+import { randomId } from "../id.js";
+import { configImportTable } from "../schema/i4-config-import.js";
+
+type DB = Db<Record<string, unknown>>;
+
+function rowToConfigImport(
+	row: typeof configImportTable.$inferSelect,
+): ConfigImportItem {
+	return {
+		importId: row.importId,
+		accountConfigId: row.accountConfigId,
+		schemaVersion: row.schemaVersion,
+		state: row.state as ConfigImportItem["state"],
+		document: row.document as ConfigImportItem["document"],
+		unresolvedRefs: row.unresolvedRefs as ConfigImportItem["unresolvedRefs"],
+		createdAt: row.createdAt,
+		completedAt: row.completedAt,
+		updatedAt: row.updatedAt,
+	};
+}
+
+export class ConfigImportRepo implements IConfigImportRepository {
+	constructor(private db: DB) {}
+
+	async create(input: CreateConfigImportInput): Promise<ConfigImportItem> {
+		const now = Date.now();
+		const [row] = await this.db
+			.insert(configImportTable)
+			.values({
+				importId: randomId(),
+				accountConfigId: input.accountConfigId,
+				schemaVersion: input.schemaVersion,
+				state: input.state ?? "Pending",
+				document: input.document as never,
+				unresolvedRefs: input.unresolvedRefs as never,
+				createdAt: now,
+				completedAt: input.completedAt ?? 0,
+				updatedAt: now,
+			})
+			.returning();
+		return rowToConfigImport(row);
+	}
+
+	async get(importId: string): Promise<ConfigImportItem> {
+		const [row] = await this.db
+			.select()
+			.from(configImportTable)
+			.where(eq(configImportTable.importId, importId));
+		if (!row) throw new NotFoundError(`ConfigImport not found: ${importId}`);
+		return rowToConfigImport(row);
+	}
+
+	async update(
+		importId: string,
+		input: UpdateConfigImportInput,
+	): Promise<ConfigImportItem> {
+		const updates: Partial<typeof configImportTable.$inferInsert> = {
+			updatedAt: Date.now(),
+		};
+		if (input.state !== undefined) updates.state = input.state;
+		if (input.unresolvedRefs !== undefined)
+			updates.unresolvedRefs = input.unresolvedRefs as never;
+		if (input.completedAt !== undefined)
+			updates.completedAt = input.completedAt;
+
+		const [row] = await this.db
+			.update(configImportTable)
+			.set(updates)
+			.where(eq(configImportTable.importId, importId))
+			.returning();
+		if (!row) throw new NotFoundError(`ConfigImport not found: ${importId}`);
+		return rowToConfigImport(row);
+	}
+
+	async listByAccountConfig(
+		accountConfigId: string,
+	): Promise<ConfigImportItem[]> {
+		const rows = await this.db
+			.select()
+			.from(configImportTable)
+			.where(eq(configImportTable.accountConfigId, accountConfigId))
+			.orderBy(desc(configImportTable.createdAt));
+		return rows.map(rowToConfigImport);
+	}
+}

package/src/repos/i4-mailbox-special-use.ts

@@ -6,6 +6,7 @@ import type {
 import {
 	type CanonicalMailboxRoleValue,
 	composeFolderRoleAppointmentName,
+	type JunkRoleMailboxes,
 	type RoleMailboxCandidate,
 	type RoleResolution,
 	resolveMailboxForRole,
@@ -22,6 +23,23 @@ import { AccountSettingRepo } from "./i4-account-setting.js";
 
 type DB = Db<Record<string, unknown>>;
 
+const JUNK_ROLES: readonly CanonicalMailboxRoleValue[] = [
+	CanonicalMailboxRole.Junk,
+	CanonicalMailboxRole.Trash,
+];
+
+const appointmentKey = (
+	accountId: string,
+	role: CanonicalMailboxRoleValue,
+): string => `${accountId}\u0000${role}`;
+
+// The only place this repository names an appointment setting, so that no read
+// here can reach the display-only label row sitting beside it (#887).
+const appointmentSettingName = (
+	accountId: string,
+	role: CanonicalMailboxRoleValue,
+): string => composeFolderRoleAppointmentName(accountId, role);
+
 interface RoleCandidate extends RoleMailboxCandidate {
 	fullPath: string;
 }
@@ -173,6 +191,148 @@ export class MailboxSpecialUseRepo implements IMailboxSpecialUseRepository {
 		return this.findMailboxForRole(accountId, CanonicalMailboxRole.Junk);
 	}
 
+	/**
+	 * Junk and Trash for every account under one config, in a fixed number of
+	 * reads however many accounts the config holds — this feeds a predicate the
+	 * per-message reconcile runs inside the sync loop.
+	 */
+	async resolveJunkRolesForConfig(
+		accountConfigId: string,
+	): Promise<JunkRoleMailboxes> {
+		const accounts = await this.db
+			.select({ accountId: accountTable.accountId })
+			.from(accountTable)
+			.where(eq(accountTable.accountConfigId, accountConfigId));
+		return this.resolveJunkRoles(accounts.map((row) => row.accountId));
+	}
+
+	/**
+	 * The same answer for every account the instance holds. A mailbox id belongs
+	 * to exactly one account, so a union across accounts is no less selective
+	 * than asking each of them separately — which is what lets the boot sweep
+	 * run one pass over the address table instead of one per config.
+	 */
+	async resolveJunkRolesForInstance(): Promise<JunkRoleMailboxes> {
+		const accounts = await this.db
+			.select({ accountId: accountTable.accountId })
+			.from(accountTable);
+		return this.resolveJunkRoles(accounts.map((row) => row.accountId));
+	}
+
+	private async resolveJunkRoles(
+		accountIds: readonly string[],
+	): Promise<JunkRoleMailboxes> {
+		if (accountIds.length === 0) {
+			return { junkMailboxIds: [], trashMailboxIds: [] };
+		}
+		const [candidates, appointments] = await Promise.all([
+			this.roleCandidatesFor(accountIds),
+			this.appointedMailboxIds(accountIds, JUNK_ROLES),
+		]);
+
+		const junkMailboxIds: string[] = [];
+		const trashMailboxIds: string[] = [];
+		for (const accountId of accountIds) {
+			const mailboxes = candidates.get(accountId) ?? [];
+			const junk = resolveMailboxForRole(
+				CanonicalMailboxRole.Junk,
+				mailboxes,
+				appointments.get(appointmentKey(accountId, CanonicalMailboxRole.Junk)),
+			);
+			if (junk) junkMailboxIds.push(junk.mailboxId);
+			const trash = resolveMailboxForRole(
+				CanonicalMailboxRole.Trash,
+				mailboxes,
+				appointments.get(appointmentKey(accountId, CanonicalMailboxRole.Trash)),
+			);
+			if (trash) trashMailboxIds.push(trash.mailboxId);
+		}
+		return { junkMailboxIds, trashMailboxIds };
+	}
+
+	/**
+	 * Each account's appointment for each role, in two reads. Same precedence
+	 * input as `appointedMailboxId`, batched: an account row missing is a caller
+	 * racing a delete, and leaves the account with no appointment rather than
+	 * failing the lookup.
+	 */
+	private async appointedMailboxIds(
+		accountIds: readonly string[],
+		roles: readonly CanonicalMailboxRoleValue[],
+	): Promise<Map<string, string>> {
+		const accounts = await this.db
+			.select({
+				accountId: accountTable.accountId,
+				accountConfigId: accountTable.accountConfigId,
+			})
+			.from(accountTable)
+			.where(inArray(accountTable.accountId, [...accountIds]));
+		if (accounts.length === 0) return new Map();
+
+		const wanted = new Map<string, string>();
+		for (const account of accounts) {
+			for (const role of roles) {
+				wanted.set(
+					appointmentSettingName(account.accountId, role),
+					appointmentKey(account.accountId, role),
+				);
+			}
+		}
+
+		const settings = await this.accountSetting.getMany(
+			accounts.map((account) => account.accountConfigId),
+			[...wanted.keys()],
+		);
+
+		const appointed = new Map<string, string>();
+		for (const setting of settings) {
+			const key = wanted.get(setting.name);
+			if (!key) continue;
+			if (setting.value.kind !== "String") continue;
+			appointed.set(key, setting.value.value);
+		}
+		return appointed;
+	}
+
+	private async roleCandidatesFor(
+		accountIds: readonly string[],
+	): Promise<Map<string, RoleCandidate[]>> {
+		const rows = await this.db
+			.select()
+			.from(mailboxTable)
+			.where(inArray(mailboxTable.accountId, [...accountIds]));
+		if (rows.length === 0) return new Map();
+
+		const entries = await this.db
+			.select()
+			.from(mailboxSpecialUseTable)
+			.where(
+				inArray(
+					mailboxSpecialUseTable.mailboxId,
+					rows.map((row) => row.mailboxId),
+				),
+			);
+		const byMailbox = new Map<string, string[]>();
+		for (const entry of entries) {
+			const designations = byMailbox.get(entry.mailboxId) ?? [];
+			designations.push(entry.specialUse);
+			byMailbox.set(entry.mailboxId, designations);
+		}
+
+		const byAccount = new Map<string, RoleCandidate[]>();
+		for (const row of rows) {
+			const candidates = byAccount.get(row.accountId) ?? [];
+			candidates.push({
+				mailboxId: row.mailboxId,
+				fullPath: row.fullPath,
+				hierarchyDelimiter: row.hierarchyDelimiter,
+				specialUse: byMailbox.get(row.mailboxId) ?? [],
+			});
+			byAccount.set(row.accountId, candidates);
+		}
+		return byAccount;
+	}
+
 	/**
 	 * The one read behind every `find<Role>Mailbox`: the account's mailboxes and
 	 * its appointment for this role, handed to the shared precedence rule. Read
@@ -211,7 +371,7 @@ export class MailboxSpecialUseRepo implements IMailboxSpecialUseRepository {
 
 		const setting = await this.accountSetting.get(
 			account.accountConfigId,
-			composeFolderRoleAppointmentName(accountId, role),
+			appointmentSettingName(accountId, role),
 		);
 		if (!setting || setting.value.kind !== "String") return undefined;
 		return setting.value.value;

package/src/schema/i4-config-import.ts

@@ -0,0 +1,3 @@
+import * as entities from "@remit/drizzle-sqlite-schema";
+
+export const configImportTable = entities.configImports;

package/src/schema.ts

@@ -17,6 +17,7 @@ export * from "./schema/i4-account-config.js";
 export * from "./schema/i4-account-export-request.js";
 export * from "./schema/i4-account-setting.js";
 export * from "./schema/i4-address.js";
+export * from "./schema/i4-config-import.js";
 export * from "./schema/i4-mailbox.js";
 export * from "./schema/i4-mailbox-lock.js";
 export * from "./schema/i4-message-flag-push.js";