@remit/drizzle-service 0.0.63 → 0.0.65

package/src/repair/junk-only-address.ts

@@ -1,8 +1,4 @@
-import {
-	JUNK_FOLDER_NAMES,
-	TRASH_FOLDER_NAMES,
-} from "@remit/data-ports/mailbox-role";
-import { MailboxSpecialUse } from "@remit/domain-enums";
+import type { JunkRoleMailboxes } from "@remit/data-ports/folder-role";
 
 export interface JunkOnlyRepairClient {
 	all(sql: string, params: readonly unknown[]): Promise<unknown[]>;
@@ -19,6 +15,12 @@ export interface JunkOnlyReport {
 	readonly restored: number;
 }
 
+/** A statement or fragment with the values its `?` placeholders take, in order. */
+export interface BoundSql {
+	readonly sql: string;
+	readonly params: readonly unknown[];
+}
+
 export const JUNK_ONLY_FLAG = "junkOnly";
 
 const STORED_FLAGS = "coalesce(nullif(address.flags, ''), '{}')";
@@ -26,41 +28,23 @@ const STORED_FLAGS = "coalesce(nullif(address.flags, ''), '{}')";
 const flagIsSet = (name: string): string =>
 	`coalesce(json_extract(${STORED_FLAGS}, '$.${name}.value'), 0) = 1`;
 
-const quoted = (names: readonly string[]): string =>
-	names.map((name) => `'${name}'`).join(", ");
-
-const MAILBOX_LEAF = `lower(substr(
-		mailbox.full_path,
-		length(rtrim(
-			mailbox.full_path,
-			replace(mailbox.full_path, mailbox.hierarchy_delimiter, '')
-		)) + 1
-	))`;
-
-const mailboxCarriesRole = (
-	specialUse: string,
-	names: readonly string[],
-): string => `(
-	exists (
-		SELECT 1 FROM mailbox_special_use_entry entry
-		WHERE entry.mailbox_id = message.mailbox_id
-		  AND entry.special_use = '${specialUse}'
-	)
-	OR exists (
-		SELECT 1 FROM mailbox
-		WHERE mailbox.mailbox_id = message.mailbox_id
-		  AND (
-			mailbox.special_use LIKE '%"${specialUse}"%'
-			OR ${MAILBOX_LEAF} IN (${quoted(names)})
-		  )
-	)
-)`;
+const EMPTY: BoundSql = { sql: "", params: [] };
 
-const IN_JUNK = mailboxCarriesRole(MailboxSpecialUse.Junk, JUNK_FOLDER_NAMES);
-const IN_TRASH = mailboxCarriesRole(
-	MailboxSpecialUse.Trash,
-	TRASH_FOLDER_NAMES,
-);
+const MATCHES_NOTHING: BoundSql = { sql: "0 = 1", params: [] };
+
+/**
+ * Whether the message sits in one of these mailboxes. An empty list is
+ * false — no mailbox holds the role, so no message is in one. That reading is
+ * right for Trash and wrong for Junk, which is why an unresolvable Junk folder
+ * is handled before this is ever reached.
+ */
+const inMailboxes = (mailboxIds: readonly string[]): BoundSql =>
+	mailboxIds.length === 0
+		? { sql: "(0 = 1)", params: [] }
+		: {
+				sql: `(message.mailbox_id IN (${mailboxIds.map(() => "?").join(", ")}))`,
+				params: mailboxIds,
+			};
 
 const sightingWhere = (extra: string): string => `exists (
 	SELECT 1 FROM envelope_address
@@ -68,47 +52,95 @@ const sightingWhere = (extra: string): string => `exists (
 	WHERE envelope_address.address_id = address.address_id${extra}
 )`;
 
-const SIGHTING_IN_JUNK = sightingWhere(` AND ${IN_JUNK}`);
-const SIGHTING_IN_LIVE_MAIL = sightingWhere(
-	` AND NOT ${IN_JUNK} AND NOT ${IN_TRASH}`,
-);
-
-export const ACCOUNT_HAS_CORRESPONDED = `(
+const ACCOUNT_HAS_CORRESPONDED = `(
 	address.outbound_count > 0
 	OR address.reply_count > 0
 	OR ${flagIsSet("vip")}
 	OR ${flagIsSet("trusted")}
 )`;
 
-export const WITHHOLDABLE = `NOT ${flagIsSet(JUNK_ONLY_FLAG)}
+/**
+ * No account in scope has a Junk folder, so nothing is known about any
+ * sighting: the mark cannot be earned. Silence is not the same as "every
+ * message is live mail" — read that way, one move would restore every address
+ * the sweep had withheld.
+ */
+const withholdable = (roles: JunkRoleMailboxes): BoundSql => {
+	if (roles.junkMailboxIds.length === 0) return MATCHES_NOTHING;
+	const inJunk = inMailboxes(roles.junkMailboxIds);
+	const inTrash = inMailboxes(roles.trashMailboxIds);
+	return {
+		sql: `NOT ${flagIsSet(JUNK_ONLY_FLAG)}
 	AND NOT ${ACCOUNT_HAS_CORRESPONDED}
-	AND ${SIGHTING_IN_JUNK}
-	AND NOT ${SIGHTING_IN_LIVE_MAIL}`;
+	AND ${sightingWhere(` AND ${inJunk.sql}`)}
+	AND NOT ${sightingWhere(` AND NOT ${inJunk.sql} AND NOT ${inTrash.sql}`)}`,
+		params: [...inJunk.params, ...inJunk.params, ...inTrash.params],
+	};
+};
 
-export const RESTORABLE = `${flagIsSet(JUNK_ONLY_FLAG)}
-	AND (${ACCOUNT_HAS_CORRESPONDED} OR ${SIGHTING_IN_LIVE_MAIL})`;
+/**
+ * The same silence lifts no mark either — "stands on live mail" is
+ * unanswerable without knowing which folder is Junk. Standing the account
+ * gave the address itself still lifts it: that evidence needs no folder.
+ */
+const restorable = (roles: JunkRoleMailboxes): BoundSql => {
+	if (roles.junkMailboxIds.length === 0) {
+		return {
+			sql: `${flagIsSet(JUNK_ONLY_FLAG)}
+	AND ${ACCOUNT_HAS_CORRESPONDED}`,
+			params: [],
+		};
+	}
+	const inJunk = inMailboxes(roles.junkMailboxIds);
+	const inTrash = inMailboxes(roles.trashMailboxIds);
+	return {
+		sql: `${flagIsSet(JUNK_ONLY_FLAG)}
+	AND (${ACCOUNT_HAS_CORRESPONDED}
+		OR ${sightingWhere(` AND NOT ${inJunk.sql} AND NOT ${inTrash.sql}`)})`,
+		params: [...inJunk.params, ...inTrash.params],
+	};
+};
 
-export const withholdSql = (scope = ""): string =>
-	`UPDATE address
+export const withholdSql = (
+	roles: JunkRoleMailboxes,
+	now: number,
+	setBy: string,
+	scope: BoundSql = EMPTY,
+): BoundSql => {
+	const predicate = withholdable(roles);
+	return {
+		sql: `UPDATE address
 	 SET flags = json_set(${STORED_FLAGS}, '$.${JUNK_ONLY_FLAG}',
 		   json_object('value', json('true'), 'setAt', CAST(? AS INTEGER), 'setBy', ?)),
 		 updated_at = ?
-	 WHERE ${WITHHOLDABLE}${scope}`;
+	 WHERE ${predicate.sql}${scope.sql}`,
+		params: [now, setBy, now, ...predicate.params, ...scope.params],
+	};
+};
 
-export const restoreSql = (scope = ""): string =>
-	`UPDATE address
+export const restoreSql = (
+	roles: JunkRoleMailboxes,
+	now: number,
+	scope: BoundSql = EMPTY,
+): BoundSql => {
+	const predicate = restorable(roles);
+	return {
+		sql: `UPDATE address
 	 SET flags = json_remove(${STORED_FLAGS}, '$.${JUNK_ONLY_FLAG}'), updated_at = ?
-	 WHERE ${RESTORABLE}${scope}`;
+	 WHERE ${predicate.sql}${scope.sql}`,
+		params: [now, ...predicate.params, ...scope.params],
+	};
+};
 
 const REPAIR_SET_BY = "junk-only-repair";
 
 const countWhere = async (
 	client: JunkOnlyRepairClient,
-	predicate: string,
+	predicate: BoundSql,
 ): Promise<number> => {
 	const [row] = (await client.all(
-		`SELECT count(*) AS row_count FROM address WHERE ${predicate}`,
-		[],
+		`SELECT count(*) AS row_count FROM address WHERE ${predicate.sql}`,
+		predicate.params,
 	)) as { row_count: number }[];
 	return row?.row_count ?? 0;
 };
@@ -116,23 +148,39 @@ const countWhere = async (
 export const sweepJunkOnlyAddresses = async (
 	client: JunkOnlyRepairClient,
 	mode: JunkOnlyRepairMode,
+	roles: JunkRoleMailboxes,
 	now: number = Date.now(),
 ): Promise<JunkOnlyReport> => {
-	const withholdable = await countWhere(client, WITHHOLDABLE);
-	const restorable = await countWhere(client, RESTORABLE);
+	const withholdableCount = await countWhere(client, withholdable(roles));
+	const restorableCount = await countWhere(client, restorable(roles));
 
 	if (mode === "check") {
-		return { mode, withholdable, withheld: 0, restorable, restored: 0 };
+		return {
+			mode,
+			withholdable: withholdableCount,
+			withheld: 0,
+			restorable: restorableCount,
+			restored: 0,
+		};
 	}
 
+	const withhold = withholdSql(roles, now, REPAIR_SET_BY);
 	const withheld =
-		withholdable === 0
+		withholdableCount === 0
 			? 0
-			: await client.run(withholdSql(), [now, REPAIR_SET_BY, now]);
-
-	const restored = restorable === 0 ? 0 : await client.run(restoreSql(), [now]);
-
-	return { mode, withholdable, withheld, restorable, restored };
+			: await client.run(withhold.sql, withhold.params);
+
+	const restore = restoreSql(roles, now);
+	const restored =
+		restorableCount === 0 ? 0 : await client.run(restore.sql, restore.params);
+
+	return {
+		mode,
+		withholdable: withholdableCount,
+		withheld,
+		restorable: restorableCount,
+		restored,
+	};
 };
 
 export const formatJunkOnlyReport = (report: JunkOnlyReport): string[] => {

package/src/repos/i4-account-setting.ts

@@ -5,7 +5,7 @@ import type {
 	IAccountSettingRepository,
 	UpsertAccountSettingInput,
 } from "@remit/data-ports";
-import { eq } from "drizzle-orm";
+import { and, eq, inArray } from "drizzle-orm";
 import type { Db } from "../db.js";
 import { accountSettingTable } from "../schema/i4-account-setting.js";
 
@@ -48,6 +48,28 @@ export class AccountSettingRepo implements IAccountSettingRepository {
 		return row ? rowToAccountSetting(row) : null;
 	}
 
+	/**
+	 * The named settings of several configs in one read, for callers that would
+	 * otherwise `get` once per account. Missing rows are simply absent from the
+	 * result rather than an error.
+	 */
+	async getMany(
+		accountConfigIds: readonly string[],
+		names: readonly string[],
+	): Promise<AccountSettingItem[]> {
+		if (accountConfigIds.length === 0 || names.length === 0) return [];
+		const rows = await this.db
+			.select()
+			.from(accountSettingTable)
+			.where(
+				and(
+					inArray(accountSettingTable.accountConfigId, [...accountConfigIds]),
+					inArray(accountSettingTable.name, [...names]),
+				),
+			);
+		return rows.map(rowToAccountSetting);
+	}
+
 	async listByAccountConfig(
 		accountConfigId: string,
 	): Promise<AccountSettingItem[]> {

package/src/repos/i4-address-junk-move.sqlite.test.ts

@@ -3,6 +3,7 @@ import { after, before, beforeEach, describe, test } from "node:test";
 import type Database from "better-sqlite3";
 import { createTestDb, type TestDb } from "../test-db.js";
 import { AddressRepo } from "./i4-address.js";
+import { MailboxSpecialUseRepo } from "./i4-mailbox-special-use.js";
 
 const CONFIG = "cfg-1";
 
@@ -11,8 +12,34 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 	let sqlite: Database.Database;
 	let close: () => Promise<void>;
 	let repo: AddressRepo;
+	let specialUse: MailboxSpecialUseRepo;
 
-	const mailbox = (mailboxId: string, specialUse: string | null): void => {
+	const account = (accountId: string): void => {
+		sqlite
+			.prepare(
+				`INSERT OR IGNORE INTO account (
+					account_id, account_config_id, username, email, imap_host,
+					imap_port, imap_tls, imap_start_tls, smtp_port, is_active,
+					connection_state, created_at, updated_at
+				) VALUES (?, ?, 'user', 'user@example.com', 'imap.example.com',
+					993, 1, 0, 465, 1, 'disconnected', 0, 0)`,
+			)
+			.run(accountId, CONFIG);
+	};
+
+	/**
+	 * A folder the server designated, written the way mailbox sync writes one:
+	 * the denormalized column AND the entry row. The paths are deliberately not
+	 * English — the role has to be read off the designation, not guessed from
+	 * the name.
+	 */
+	const mailbox = (
+		mailboxId: string,
+		designation: string | null,
+		folder: { accountId?: string; fullPath?: string } = {},
+	): void => {
+		const accountId = folder.accountId ?? "acc";
+		account(accountId);
 		sqlite
 			.prepare(
 				`INSERT INTO mailbox (
@@ -21,9 +48,22 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 					unseen_count, deleted_count, total_size, last_sync_uid,
 					high_water_mark_uid, last_message_sync_at, special_use,
 					created_at, updated_at
-				) VALUES (?, 'acc', '', '/', ?, 1, 1, '0', 0, 0, 0, 0, 0, 0, 0, ?, 0, 0)`,
+				) VALUES (?, ?, '', '/', ?, 1, 1, '0', 0, 0, 0, 0, 0, 0, 0, ?, 0, 0)`,
+			)
+			.run(
+				mailboxId,
+				accountId,
+				folder.fullPath ?? mailboxId,
+				designation ? JSON.stringify([designation]) : null,
+			);
+		if (!designation) return;
+		sqlite
+			.prepare(
+				`INSERT INTO mailbox_special_use_entry (
+					mailbox_special_use_id, mailbox_id, special_use
+				) VALUES (?, ?, ?)`,
 			)
-			.run(mailboxId, mailboxId, specialUse);
+			.run(`${mailboxId}-${designation}`, mailboxId, designation);
 	};
 
 	const message = (messageId: string, mailboxId: string): void => {
@@ -84,19 +124,38 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 	before(async () => {
 		({ db, sqlite, close } = await createTestDb());
 		repo = new AddressRepo(db as never);
+		specialUse = new MailboxSpecialUseRepo(db as never);
 	});
 
+	/**
+	 * The roles the caller resolves once and hands down, covering every account
+	 * under the config — which is the scope the predicate reads, because the
+	 * address book is keyed by config.
+	 */
+	const reconcile = async (messageId: string): Promise<void> =>
+		repo.reconcileJunkOnlyForMessage(
+			messageId,
+			await specialUse.resolveJunkRolesForConfig(CONFIG),
+		);
+
 	after(async () => {
 		await close();
 	});
 
 	beforeEach(() => {
-		for (const table of ["address", "envelope_address", "message", "mailbox"]) {
+		for (const table of [
+			"address",
+			"envelope_address",
+			"message",
+			"mailbox",
+			"mailbox_special_use_entry",
+			"account",
+		]) {
 			sqlite.exec(`DELETE FROM ${table}`);
 		}
-		mailbox("inbox", null);
-		mailbox("junk", '["Junk"]');
-		mailbox("trash", '["Trash"]');
+		mailbox("inbox", null, { fullPath: "INBOX" });
+		mailbox("junk", "Junk", { fullPath: "Ongewenst" });
+		mailbox("trash", "Trash", { fullPath: "Prullenbak" });
 	});
 
 	test("a message moved into Junk stops the sender being suggested", async () => {
@@ -106,7 +165,7 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 		assert.deepEqual(await suggested("spammer"), ["spammer"]);
 
 		moveTo("msg", "junk");
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 
 		assert.equal(await withheld("spammer"), true);
 		assert.deepEqual(await suggested("spammer"), []);
@@ -119,7 +178,7 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 		await repo.incrementOutboundCount(CONFIG, "client", Date.now());
 
 		moveTo("msg", "junk");
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 
 		assert.equal(await withheld("client"), false);
 	});
@@ -132,7 +191,7 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 		sighting("colleague", "real");
 
 		moveTo("spam", "junk");
-		await repo.reconcileJunkOnlyForMessage("spam");
+		await reconcile("spam");
 
 		assert.equal(await withheld("colleague"), false);
 	});
@@ -152,7 +211,7 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 		assert.deepEqual(await suggested("misfiled"), []);
 
 		moveTo("msg", "inbox");
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 
 		assert.deepEqual(await suggested("misfiled"), ["misfiled"]);
 	});
@@ -171,7 +230,7 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 		sighting("spammer", "msg");
 
 		moveTo("msg", "trash");
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 
 		assert.equal(await withheld("spammer"), true);
 	});
@@ -183,20 +242,65 @@ describe("reconciling one message's addresses at the moment it moves", () => {
 		sighting("bystander", "other");
 
 		moveTo("msg", "junk");
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 
 		assert.equal(await withheld("bystander"), false);
 	});
 
+	test("a sender met only in Junk on two accounts of one config stays withheld", async () => {
+		mailbox("junk-b", "Junk", {
+			accountId: "acc-b",
+			fullPath: "Indésirables",
+		});
+		message("ma", "junk");
+		message("mb", "junk-b");
+		await harvest("spammer");
+		sighting("spammer", "ma");
+		sighting("spammer", "mb");
+
+		await reconcile("ma");
+		assert.equal(await withheld("spammer"), true);
+
+		await reconcile("mb");
+
+		assert.equal(await withheld("spammer"), true);
+	});
+
+	/**
+	 * The name hints are English only, so an account whose server flags nothing
+	 * and whose Junk folder is called `Ongewenst` resolves no Junk at all. That
+	 * silence must lift no mark: reading it as "every sighting is live mail"
+	 * would hand the spammer back on the next move.
+	 */
+	test("a move never lifts a mark when no folder holds Junk", async () => {
+		sqlite.exec("DELETE FROM mailbox_special_use_entry");
+		sqlite.exec("UPDATE mailbox SET special_use = NULL");
+		message("msg", "inbox");
+		await repo.upsertJunkAddress({
+			addressId: "spammer",
+			accountConfigId: CONFIG,
+			displayName: "Name",
+			localPart: "spammer",
+			domain: "example.com",
+			normalizedEmail: "spammer@example.com",
+			normalizedCompound: "name spammer@example.com",
+		});
+		sighting("spammer", "msg");
+
+		await reconcile("msg");
+
+		assert.equal(await withheld("spammer"), true);
+	});
+
 	test("a second reconcile of the same move writes nothing new", async () => {
 		message("msg", "inbox");
 		await harvest("spammer");
 		sighting("spammer", "msg");
 		moveTo("msg", "junk");
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 		const first = await repo.getAddress(CONFIG, "spammer");
 
-		await repo.reconcileJunkOnlyForMessage("msg");
+		await reconcile("msg");
 
 		assert.deepEqual(await repo.getAddress(CONFIG, "spammer"), first);
 	});

package/src/repos/i4-address.ts

@@ -10,6 +10,7 @@ import type {
 	UpdateAddressInput,
 } from "@remit/data-ports";
 import { BadRequestError } from "@remit/data-ports/errors";
+import type { JunkRoleMailboxes } from "@remit/data-ports/folder-role";
 import { shouldPromoteWellknown } from "@remit/data-ports/wellknown";
 import {
 	and,
@@ -26,6 +27,7 @@ import { NotFoundError } from "../error.js";
 import { envelopeAddressId } from "../id.js";
 import { decodeToken, resultList } from "../pagination.js";
 import {
+	type BoundSql,
 	JUNK_ONLY_FLAG,
 	restoreSql,
 	withholdSql,
@@ -331,15 +333,21 @@ export class AddressRepo implements IAddressRepository {
 		return rowToAddress(row);
 	}
 
-	async reconcileJunkOnlyForMessage(messageId: string): Promise<void> {
-		const scope = ` AND address.address_id IN (
+	async reconcileJunkOnlyForMessage(
+		messageId: string,
+		roles: JunkRoleMailboxes,
+	): Promise<void> {
+		const scope: BoundSql = {
+			sql: ` AND address.address_id IN (
 			SELECT address_id FROM envelope_address WHERE message_id = ?
-		)`;
+		)`,
+			params: [messageId],
+		};
 		const now = Date.now();
-		await this.db.run(
-			boundToDrizzle(withholdSql(scope), [now, JUNK_MOVE, now, messageId]),
-		);
-		await this.db.run(boundToDrizzle(restoreSql(scope), [now, messageId]));
+		const withhold = withholdSql(roles, now, JUNK_MOVE, scope);
+		await this.db.run(boundToDrizzle(withhold.sql, withhold.params));
+		const restore = restoreSql(roles, now, scope);
+		await this.db.run(boundToDrizzle(restore.sql, restore.params));
 	}
 
 	async getAddress(