@remit/drizzle-service 0.0.49 → 0.0.50

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.49",
+  "version": "0.0.50",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/repair/junk-only-address.sqlite.test.ts

@@ -0,0 +1,535 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { after, before, beforeEach, describe, test } from "node:test";
+import Database from "better-sqlite3";
+import { shippedTableDdl } from "../test-shipped-sqlite-schema.js";
+import {
+	type JunkOnlyRepairClient,
+	sweepJunkOnlyAddresses,
+} from "./junk-only-address.js";
+
+const DDL_TAG = "0000_happy_roland_deschain";
+
+const clientOver = (sqlite: Database.Database): JunkOnlyRepairClient => ({
+	all: async (sql, params) => sqlite.prepare(sql).all(...params),
+	run: async (sql, params) => sqlite.prepare(sql).run(...params).changes,
+});
+
+interface AddressRow {
+	flags: string;
+	updated_at: number;
+}
+
+describe("addresses standing only on mail in Junk", () => {
+	let sqlite: Database.Database;
+
+	const mailbox = (mailboxId: string, specialUse: string | null): void => {
+		sqlite
+			.prepare(
+				`INSERT INTO mailbox (
+					mailbox_id, account_id, namespace_prefix, hierarchy_delimiter,
+					full_path, uid_validity, uid_next, highest_modseq, message_count,
+					unseen_count, deleted_count, total_size, last_sync_uid,
+					high_water_mark_uid, last_message_sync_at, special_use,
+					created_at, updated_at
+				) VALUES (?, 'acc', '', '/', ?, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, ?, 0, 0)`,
+			)
+			.run(mailboxId, mailboxId, specialUse);
+	};
+
+	const specialUseEntry = (mailboxId: string, specialUse: string): void => {
+		sqlite
+			.prepare(
+				`INSERT INTO mailbox_special_use_entry (
+					mailbox_special_use_id, mailbox_id, special_use
+				) VALUES (?, ?, ?)`,
+			)
+			.run(`${mailboxId}-${specialUse}`, mailboxId, specialUse);
+	};
+
+	const message = (messageId: string, mailboxId: string): void => {
+		sqlite
+			.prepare(
+				`INSERT INTO message (
+					message_id, mailbox_id, uid, sequence_number, rfc822_size,
+					internal_date, envelope_id, root_body_part_id, created_at, updated_at
+				) VALUES (?, ?, 1, 1, 10, 0, 'env', 'body', 0, 0)`,
+			)
+			.run(messageId, mailboxId);
+	};
+
+	const address = (
+		addressId: string,
+		counters: { outbound?: number; reply?: number } = {},
+		flags = "{}",
+	): void => {
+		sqlite
+			.prepare(
+				`INSERT INTO address (
+					address_id, account_config_id, display_name, local_part, domain,
+					normalized_email, normalized_compound, flags, inbound_count,
+					outbound_count, reply_count, last_inbound_at, last_outbound_at,
+					last_reply_at, created_at, updated_at
+				) VALUES (?, 'cfg-1', 'Name', ?, 'example.com', ?, ?, ?, 0, ?, ?, 0,
+					NULL, 0, 0, 0)`,
+			)
+			.run(
+				addressId,
+				addressId,
+				`${addressId}@example.com`,
+				`name ${addressId}@example.com`,
+				flags,
+				counters.outbound ?? 0,
+				counters.reply ?? 0,
+			);
+	};
+
+	const sighting = (addressId: string, messageId: string): void => {
+		sqlite
+			.prepare(
+				`INSERT INTO envelope_address (
+					envelope_address_id, message_id, address_id, display_name,
+					normalized_email, address_role, address_order, created_at, updated_at
+				) VALUES (?, ?, ?, 'Name', ?, 'From', 0, 0, 0)`,
+			)
+			.run(
+				`${addressId}-${messageId}`,
+				messageId,
+				addressId,
+				`${addressId}@example.com`,
+			);
+	};
+
+	const read = (addressId: string): AddressRow =>
+		sqlite
+			.prepare("SELECT flags, updated_at FROM address WHERE address_id = ?")
+			.get(addressId) as AddressRow;
+
+	const withheld = (addressId: string): boolean =>
+		JSON.parse(read(addressId).flags).junkOnly?.value === true;
+
+	const addressCount = (): number =>
+		(
+			sqlite.prepare("SELECT count(*) AS n FROM address").get() as {
+				n: number;
+			}
+		).n;
+
+	before(() => {
+		sqlite = new Database(":memory:");
+		for (const table of [
+			"address",
+			"envelope_address",
+			"message",
+			"mailbox",
+			"mailbox_special_use_entry",
+		]) {
+			sqlite.exec(shippedTableDdl(DDL_TAG, table));
+		}
+		sqlite.exec(
+			readFileSync(
+				new URL(
+					"../../../../npm-scripts/sqlite-address-sightings-index.sql",
+					import.meta.url,
+				),
+				"utf8",
+			),
+		);
+	});
+
+	after(() => {
+		sqlite.close();
+	});
+
+	beforeEach(() => {
+		for (const table of [
+			"address",
+			"envelope_address",
+			"message",
+			"mailbox",
+			"mailbox_special_use_entry",
+		]) {
+			sqlite.exec(`DELETE FROM ${table}`);
+		}
+		mailbox("junk", '["Junk"]');
+		specialUseEntry("junk", "Junk");
+		mailbox("inbox", null);
+		mailbox("trash", '["Trash"]');
+		specialUseEntry("trash", "Trash");
+		message("spam-1", "junk");
+		message("spam-2", "junk");
+		message("mail-1", "inbox");
+		message("bin-1", "trash");
+	});
+
+	test("withholds an address seen only on mail in Junk", async () => {
+		address("spammer");
+		sighting("spammer", "spam-1");
+		sighting("spammer", "spam-2");
+
+		const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(report.withholdable, 1);
+		assert.equal(report.withheld, 1);
+		assert.equal(withheld("spammer"), true);
+		// `int64` on the wire, so the timestamp must not land as a float.
+		assert.match(read("spammer").flags, /"setAt":\d+,/);
+	});
+
+	test("keeps an address with one sighting outside Junk", async () => {
+		address("colleague");
+		sighting("colleague", "spam-1");
+		sighting("colleague", "mail-1");
+
+		const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(report.withholdable, 0);
+		assert.equal(withheld("colleague"), false);
+	});
+
+	test("withholds an address whose only message moved into Junk", async () => {
+		address("newsletter");
+		sighting("newsletter", "mail-1");
+		sqlite
+			.prepare("UPDATE message SET mailbox_id = 'junk' WHERE message_id = ?")
+			.run("mail-1");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("newsletter"), true);
+	});
+
+	test("restores an address whose message moved out of Junk", async () => {
+		address("misfiled");
+		sighting("misfiled", "spam-1");
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+		assert.equal(withheld("misfiled"), true);
+
+		sqlite
+			.prepare("UPDATE message SET mailbox_id = 'inbox' WHERE message_id = ?")
+			.run("spam-1");
+		const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(report.restorable, 1);
+		assert.equal(report.restored, 1);
+		assert.equal(withheld("misfiled"), false);
+	});
+
+	test("never withholds an address the account has written to", async () => {
+		address("client", { outbound: 1 });
+		sighting("client", "spam-1");
+
+		const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(report.withholdable, 0);
+		assert.equal(withheld("client"), false);
+	});
+
+	test("never withholds an address the account has replied to", async () => {
+		address("friend", { reply: 2 });
+		sighting("friend", "spam-1");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("friend"), false);
+	});
+
+	test("never withholds a VIP", async () => {
+		address("boss", {}, '{"vip":{"value":true,"setAt":1}}');
+		sighting("boss", "spam-1");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("boss"), false);
+	});
+
+	test("restores a withheld address once the account writes to it", async () => {
+		address("reformed", {}, '{"junkOnly":{"value":true,"setAt":1}}');
+		sighting("reformed", "spam-1");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+		assert.equal(withheld("reformed"), true);
+
+		sqlite
+			.prepare("UPDATE address SET outbound_count = 1 WHERE address_id = ?")
+			.run("reformed");
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("reformed"), false);
+	});
+
+	test("leaves mail in Trash feeding the address book", async () => {
+		address("ex-colleague");
+		sighting("ex-colleague", "bin-1");
+
+		const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(report.withholdable, 0);
+		assert.equal(withheld("ex-colleague"), false);
+	});
+
+	test("one deleted message does not keep a spammer suggestible", async () => {
+		address("spammer");
+		sighting("spammer", "spam-1");
+		sighting("spammer", "bin-1");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("spammer"), true);
+	});
+
+	test("Trash reads the same whichever move happened first", async () => {
+		address("junk-then-bin");
+		address("bin-then-junk");
+		sighting("junk-then-bin", "spam-1");
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+		sighting("junk-then-bin", "bin-1");
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		sighting("bin-then-junk", "bin-1");
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+		sighting("bin-then-junk", "spam-1");
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("junk-then-bin"), true);
+		assert.equal(withheld("bin-then-junk"), true);
+	});
+
+	test("a cold sweep agrees with the moves that built the state", async () => {
+		address("spammer");
+		sighting("spammer", "spam-1");
+		sighting("spammer", "bin-1");
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		sqlite
+			.prepare("UPDATE address SET flags = '{}' WHERE address_id = ?")
+			.run("spammer");
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("spammer"), true);
+	});
+
+	test("deleting every spam message leaves the mark standing", async () => {
+		address("spammer");
+		sighting("spammer", "spam-1");
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		sqlite
+			.prepare("UPDATE message SET mailbox_id = 'trash' WHERE message_id = ?")
+			.run("spam-1");
+		const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(report.restorable, 0);
+		assert.equal(withheld("spammer"), true);
+	});
+
+	test("deleting the spam does not put its sender back", async () => {
+		address("spammer");
+		sighting("spammer", "spam-1");
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		sqlite
+			.prepare("UPDATE message SET mailbox_id = 'trash' WHERE message_id = ?")
+			.run("spam-1");
+		const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(report.restorable, 0);
+		assert.equal(withheld("spammer"), true);
+	});
+
+	test("purging the spam does not put its sender back", async () => {
+		address("spammer");
+		sighting("spammer", "spam-1");
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		sqlite.prepare("DELETE FROM message WHERE message_id = ?").run("spam-1");
+		const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(report.restorable, 0);
+		assert.equal(withheld("spammer"), true);
+	});
+
+	test("withholds a sender the account blocked or muted", async () => {
+		address("reported", {}, '{"blocked":{"value":true,"setAt":1}}');
+		address("hushed", {}, '{"muted":{"value":true,"setAt":1}}');
+		sighting("reported", "spam-1");
+		sighting("hushed", "spam-1");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("reported"), true);
+		assert.equal(withheld("hushed"), true);
+	});
+
+	test("blocking a withheld sender never lifts the mark", async () => {
+		address("spammer");
+		sighting("spammer", "spam-1");
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		sqlite
+			.prepare(
+				`UPDATE address SET flags = json_set(flags, '$.blocked',
+					json_object('value', json('true'), 'setAt', 1))
+				 WHERE address_id = ?`,
+			)
+			.run("spammer");
+		const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(report.restorable, 0);
+		assert.equal(withheld("spammer"), true);
+	});
+
+	test("reads a Junk folder a server does not designate", async () => {
+		mailbox("named-spam", null);
+		message("spam-5", "named-spam");
+		address("by-name");
+		sighting("by-name", "spam-5");
+		sqlite
+			.prepare("UPDATE mailbox SET full_path = 'Spam' WHERE mailbox_id = ?")
+			.run("named-spam");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("by-name"), true);
+	});
+
+	test("reads a Junk folder nested under any prefix", async () => {
+		mailbox("nested-spam", null);
+		message("spam-6", "nested-spam");
+		address("nested");
+		sighting("nested", "spam-6");
+		sqlite
+			.prepare(
+				"UPDATE mailbox SET full_path = 'INBOX/Spam' WHERE mailbox_id = ?",
+			)
+			.run("nested-spam");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("nested"), true);
+	});
+
+	test("reads a Junk folder under a delimiter that is not a slash", async () => {
+		mailbox("dotted-spam", null);
+		message("spam-7", "dotted-spam");
+		address("dotted");
+		sighting("dotted", "spam-7");
+		sqlite
+			.prepare(
+				`UPDATE mailbox SET full_path = 'Mail.Junk E-mail',
+					hierarchy_delimiter = '.' WHERE mailbox_id = ?`,
+			)
+			.run("dotted-spam");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("dotted"), true);
+	});
+
+	test("never reads a prefix as the folder it names", async () => {
+		mailbox("spam-parent", null);
+		message("mail-2", "spam-parent");
+		address("under-spam");
+		sighting("under-spam", "mail-2");
+		sqlite
+			.prepare(
+				"UPDATE mailbox SET full_path = 'Spam/Receipts' WHERE mailbox_id = ?",
+			)
+			.run("spam-parent");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("under-spam"), false);
+	});
+
+	test("reads the designation from either place it is stored", async () => {
+		mailbox("column-only", '["Junk"]');
+		mailbox("entry-only", null);
+		specialUseEntry("entry-only", "Junk");
+		message("spam-3", "column-only");
+		message("spam-4", "entry-only");
+		address("by-column");
+		address("by-entry");
+		sighting("by-column", "spam-3");
+		sighting("by-entry", "spam-4");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("by-column"), true);
+		assert.equal(withheld("by-entry"), true);
+	});
+
+	test("leaves an address no message has ever carried alone", async () => {
+		address("orphan");
+
+		const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(report.withholdable, 0);
+		assert.equal(withheld("orphan"), false);
+	});
+
+	test("removes no row", async () => {
+		address("spammer");
+		address("colleague");
+		address("client", { outbound: 3 });
+		sighting("spammer", "spam-1");
+		sighting("colleague", "mail-1");
+		sighting("client", "spam-2");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(addressCount(), 3);
+	});
+
+	test("keeps the rest of an address's flags", async () => {
+		address("noisy", {}, '{"muted":{"value":true,"setAt":7}}');
+		sighting("noisy", "spam-1");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.deepEqual(JSON.parse(read("noisy").flags).muted, {
+			value: true,
+			setAt: 7,
+		});
+	});
+
+	test("a second run writes nothing", async () => {
+		address("spammer");
+		sighting("spammer", "spam-1");
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+		const first = read("spammer");
+
+		const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(report.withholdable, 0);
+		assert.equal(report.restorable, 0);
+		assert.equal(report.withheld, 0);
+		assert.equal(report.restored, 0);
+		assert.deepEqual(read("spammer"), first);
+	});
+
+	test("check mode reports what it would do and writes nothing", async () => {
+		address("spammer");
+		sighting("spammer", "spam-1");
+		address("misfiled", {}, '{"junkOnly":{"value":true,"setAt":1}}');
+		sighting("misfiled", "mail-1");
+
+		const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "check");
+
+		assert.equal(report.withholdable, 1);
+		assert.equal(report.restorable, 1);
+		assert.equal(report.withheld, 0);
+		assert.equal(report.restored, 0);
+		assert.equal(withheld("spammer"), false);
+		assert.equal(withheld("misfiled"), true);
+	});
+
+	test("survives a row whose flags were never populated", async () => {
+		address("legacy", {}, "");
+		sighting("legacy", "spam-1");
+
+		await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
+
+		assert.equal(withheld("legacy"), true);
+	});
+});

package/src/repair/junk-only-address.ts

@@ -0,0 +1,158 @@
+import {
+	JUNK_FOLDER_NAMES,
+	TRASH_FOLDER_NAMES,
+} from "@remit/data-ports/mailbox-role";
+import { MailboxSpecialUse } from "@remit/domain-enums";
+
+export interface JunkOnlyRepairClient {
+	all(sql: string, params: readonly unknown[]): Promise<unknown[]>;
+	run(sql: string, params: readonly unknown[]): Promise<number>;
+}
+
+export type JunkOnlyRepairMode = "check" | "repair";
+
+export interface JunkOnlyReport {
+	readonly mode: JunkOnlyRepairMode;
+	readonly withholdable: number;
+	readonly withheld: number;
+	readonly restorable: number;
+	readonly restored: number;
+}
+
+export const JUNK_ONLY_FLAG = "junkOnly";
+
+const STORED_FLAGS = "coalesce(nullif(address.flags, ''), '{}')";
+
+const flagIsSet = (name: string): string =>
+	`coalesce(json_extract(${STORED_FLAGS}, '$.${name}.value'), 0) = 1`;
+
+const quoted = (names: readonly string[]): string =>
+	names.map((name) => `'${name}'`).join(", ");
+
+const MAILBOX_LEAF = `lower(substr(
+		mailbox.full_path,
+		length(rtrim(
+			mailbox.full_path,
+			replace(mailbox.full_path, mailbox.hierarchy_delimiter, '')
+		)) + 1
+	))`;
+
+const mailboxCarriesRole = (
+	specialUse: string,
+	names: readonly string[],
+): string => `(
+	exists (
+		SELECT 1 FROM mailbox_special_use_entry entry
+		WHERE entry.mailbox_id = message.mailbox_id
+		  AND entry.special_use = '${specialUse}'
+	)
+	OR exists (
+		SELECT 1 FROM mailbox
+		WHERE mailbox.mailbox_id = message.mailbox_id
+		  AND (
+			mailbox.special_use LIKE '%"${specialUse}"%'
+			OR ${MAILBOX_LEAF} IN (${quoted(names)})
+		  )
+	)
+)`;
+
+const IN_JUNK = mailboxCarriesRole(MailboxSpecialUse.Junk, JUNK_FOLDER_NAMES);
+const IN_TRASH = mailboxCarriesRole(
+	MailboxSpecialUse.Trash,
+	TRASH_FOLDER_NAMES,
+);
+
+const sightingWhere = (extra: string): string => `exists (
+	SELECT 1 FROM envelope_address
+	JOIN message ON message.message_id = envelope_address.message_id
+	WHERE envelope_address.address_id = address.address_id${extra}
+)`;
+
+const SIGHTING_IN_JUNK = sightingWhere(` AND ${IN_JUNK}`);
+const SIGHTING_IN_LIVE_MAIL = sightingWhere(
+	` AND NOT ${IN_JUNK} AND NOT ${IN_TRASH}`,
+);
+
+export const ACCOUNT_HAS_CORRESPONDED = `(
+	address.outbound_count > 0
+	OR address.reply_count > 0
+	OR ${flagIsSet("vip")}
+	OR ${flagIsSet("trusted")}
+)`;
+
+export const WITHHOLDABLE = `NOT ${flagIsSet(JUNK_ONLY_FLAG)}
+	AND NOT ${ACCOUNT_HAS_CORRESPONDED}
+	AND ${SIGHTING_IN_JUNK}
+	AND NOT ${SIGHTING_IN_LIVE_MAIL}`;
+
+export const RESTORABLE = `${flagIsSet(JUNK_ONLY_FLAG)}
+	AND (${ACCOUNT_HAS_CORRESPONDED} OR ${SIGHTING_IN_LIVE_MAIL})`;
+
+export const withholdSql = (scope = ""): string =>
+	`UPDATE address
+	 SET flags = json_set(${STORED_FLAGS}, '$.${JUNK_ONLY_FLAG}',
+		   json_object('value', json('true'), 'setAt', CAST(? AS INTEGER), 'setBy', ?)),
+		 updated_at = ?
+	 WHERE ${WITHHOLDABLE}${scope}`;
+
+export const restoreSql = (scope = ""): string =>
+	`UPDATE address
+	 SET flags = json_remove(${STORED_FLAGS}, '$.${JUNK_ONLY_FLAG}'), updated_at = ?
+	 WHERE ${RESTORABLE}${scope}`;
+
+const REPAIR_SET_BY = "junk-only-repair";
+
+const countWhere = async (
+	client: JunkOnlyRepairClient,
+	predicate: string,
+): Promise<number> => {
+	const [row] = (await client.all(
+		`SELECT count(*) AS row_count FROM address WHERE ${predicate}`,
+		[],
+	)) as { row_count: number }[];
+	return row?.row_count ?? 0;
+};
+
+export const sweepJunkOnlyAddresses = async (
+	client: JunkOnlyRepairClient,
+	mode: JunkOnlyRepairMode,
+	now: number = Date.now(),
+): Promise<JunkOnlyReport> => {
+	const withholdable = await countWhere(client, WITHHOLDABLE);
+	const restorable = await countWhere(client, RESTORABLE);
+
+	if (mode === "check") {
+		return { mode, withholdable, withheld: 0, restorable, restored: 0 };
+	}
+
+	const withheld =
+		withholdable === 0
+			? 0
+			: await client.run(withholdSql(), [now, REPAIR_SET_BY, now]);
+
+	const restored = restorable === 0 ? 0 : await client.run(restoreSql(), [now]);
+
+	return { mode, withholdable, withheld, restorable, restored };
+};
+
+export const formatJunkOnlyReport = (report: JunkOnlyReport): string[] => {
+	if (report.withholdable === 0 && report.restorable === 0) {
+		return ["no address stands only on mail in Junk"];
+	}
+	const lines: string[] = [];
+	if (report.withholdable > 0) {
+		lines.push(
+			report.mode === "check"
+				? `${report.withholdable} address(es) stand only on mail in Junk, would be withheld from autocomplete`
+				: `${report.withheld} of ${report.withholdable} address(es) standing only on mail in Junk withheld from autocomplete`,
+		);
+	}
+	if (report.restorable > 0) {
+		lines.push(
+			report.mode === "check"
+				? `${report.restorable} withheld address(es) now stand on live mail, would be restored`
+				: `${report.restored} of ${report.restorable} withheld address(es) restored to autocomplete`,
+		);
+	}
+	return lines;
+};

package/src/repos/address-search-predicates.ts

@@ -80,6 +80,21 @@ const flagValue = (name: string): SQL<number> =>
 export const addressPreference = (): SQL<number> =>
 	sql<number>`(2 * ${flagValue("vip")} + ${flagValue("trusted")})`;
 
+const accountHasCorresponded = (): SQL<number> =>
+	sql<number>`(${addressTable.outboundCount} + ${addressTable.replyCount}
+		+ ${flagValue("vip")} + ${flagValue("trusted")})`;
+
+const accountHasFlagged = (): SQL<number> =>
+	sql<number>`(${flagValue("blocked")} + ${flagValue("muted")})`;
+
+export const addressListable = (term: string | undefined): SQL => {
+	const shown = sql`${flagValue("junkOnly")} = 0
+		or ${accountHasCorresponded()} > 0
+		or ${accountHasFlagged()} > 0`;
+	if (!term) return sql`(${shown})`;
+	return sql`(${shown} or ${addressTable.normalizedEmail} = ${term.toLowerCase()})`;
+};
+
 export const addressCorrespondence = (): SQL<number> =>
 	sql<number>`(${addressTable.replyCount} + ${addressTable.inboundCount} + ${addressTable.outboundCount})`;
 

package/src/repos/i4-address-junk-move.sqlite.test.ts

@@ -0,0 +1,203 @@
+import assert from "node:assert/strict";
+import { after, before, beforeEach, describe, test } from "node:test";
+import type Database from "better-sqlite3";
+import { createTestDb, type TestDb } from "../test-db.js";
+import { AddressRepo } from "./i4-address.js";
+
+const CONFIG = "cfg-1";
+
+describe("reconciling one message's addresses at the moment it moves", () => {
+	let db: TestDb;
+	let sqlite: Database.Database;
+	let close: () => Promise<void>;
+	let repo: AddressRepo;
+
+	const mailbox = (mailboxId: string, specialUse: string | null): void => {
+		sqlite
+			.prepare(
+				`INSERT INTO mailbox (
+					mailbox_id, account_id, namespace_prefix, hierarchy_delimiter,
+					full_path, uid_validity, uid_next, highest_modseq, message_count,
+					unseen_count, deleted_count, total_size, last_sync_uid,
+					high_water_mark_uid, last_message_sync_at, special_use,
+					created_at, updated_at
+				) VALUES (?, 'acc', '', '/', ?, 1, 1, '0', 0, 0, 0, 0, 0, 0, 0, ?, 0, 0)`,
+			)
+			.run(mailboxId, mailboxId, specialUse);
+	};
+
+	const message = (messageId: string, mailboxId: string): void => {
+		sqlite
+			.prepare(
+				`INSERT INTO message (
+					message_id, mailbox_id, uid, sequence_number, rfc822_size,
+					internal_date, envelope_id, root_body_part_id, created_at, updated_at
+				) VALUES (?, ?, 1, 1, 10, 0, 'env', 'body', 0, 0)`,
+			)
+			.run(messageId, mailboxId);
+	};
+
+	const sighting = (addressId: string, messageId: string): void => {
+		sqlite
+			.prepare(
+				`INSERT INTO envelope_address (
+					envelope_address_id, message_id, address_id, display_name,
+					normalized_email, address_role, address_order, created_at, updated_at
+				) VALUES (?, ?, ?, 'Name', ?, 'From', 0, 0, 0)`,
+			)
+			.run(
+				`${addressId}-${messageId}`,
+				messageId,
+				addressId,
+				`${addressId}@example.com`,
+			);
+	};
+
+	const harvest = async (addressId: string) =>
+		repo.upsertCorrespondentAddress({
+			addressId,
+			accountConfigId: CONFIG,
+			displayName: "Name",
+			localPart: addressId,
+			domain: "example.com",
+			normalizedEmail: `${addressId}@example.com`,
+			normalizedCompound: `name ${addressId}@example.com`,
+		});
+
+	const moveTo = (messageId: string, mailboxId: string): void => {
+		sqlite
+			.prepare("UPDATE message SET mailbox_id = ? WHERE message_id = ?")
+			.run(mailboxId, messageId);
+	};
+
+	const withheld = async (addressId: string): Promise<boolean> =>
+		(await repo.getAddress(CONFIG, addressId)).flags?.junkOnly?.value === true;
+
+	const suggested = async (term: string): Promise<string[]> =>
+		(
+			await repo.listByAccountConfig({
+				accountConfigId: CONFIG,
+				search: term,
+			})
+		).items.map((item) => item.addressId);
+
+	before(async () => {
+		({ db, sqlite, close } = await createTestDb());
+		repo = new AddressRepo(db as never);
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	beforeEach(() => {
+		for (const table of ["address", "envelope_address", "message", "mailbox"]) {
+			sqlite.exec(`DELETE FROM ${table}`);
+		}
+		mailbox("inbox", null);
+		mailbox("junk", '["Junk"]');
+		mailbox("trash", '["Trash"]');
+	});
+
+	test("a message moved into Junk stops the sender being suggested", async () => {
+		message("msg", "inbox");
+		await harvest("spammer");
+		sighting("spammer", "msg");
+		assert.deepEqual(await suggested("spammer"), ["spammer"]);
+
+		moveTo("msg", "junk");
+		await repo.reconcileJunkOnlyForMessage("msg");
+
+		assert.equal(await withheld("spammer"), true);
+		assert.deepEqual(await suggested("spammer"), []);
+	});
+
+	test("a sender the account has written to survives the move", async () => {
+		message("msg", "inbox");
+		await harvest("client");
+		sighting("client", "msg");
+		await repo.incrementOutboundCount(CONFIG, "client", Date.now());
+
+		moveTo("msg", "junk");
+		await repo.reconcileJunkOnlyForMessage("msg");
+
+		assert.equal(await withheld("client"), false);
+	});
+
+	test("a sender still on live mail survives the move", async () => {
+		message("spam", "inbox");
+		message("real", "inbox");
+		await harvest("colleague");
+		sighting("colleague", "spam");
+		sighting("colleague", "real");
+
+		moveTo("spam", "junk");
+		await repo.reconcileJunkOnlyForMessage("spam");
+
+		assert.equal(await withheld("colleague"), false);
+	});
+
+	test("a message rescued out of Junk offers the sender again", async () => {
+		message("msg", "junk");
+		await repo.upsertJunkAddress({
+			addressId: "misfiled",
+			accountConfigId: CONFIG,
+			displayName: "Name",
+			localPart: "misfiled",
+			domain: "example.com",
+			normalizedEmail: "misfiled@example.com",
+			normalizedCompound: "name misfiled@example.com",
+		});
+		sighting("misfiled", "msg");
+		assert.deepEqual(await suggested("misfiled"), []);
+
+		moveTo("msg", "inbox");
+		await repo.reconcileJunkOnlyForMessage("msg");
+
+		assert.deepEqual(await suggested("misfiled"), ["misfiled"]);
+	});
+
+	test("a spam message moved to Trash keeps the sender withheld", async () => {
+		message("msg", "junk");
+		await repo.upsertJunkAddress({
+			addressId: "spammer",
+			accountConfigId: CONFIG,
+			displayName: "Name",
+			localPart: "spammer",
+			domain: "example.com",
+			normalizedEmail: "spammer@example.com",
+			normalizedCompound: "name spammer@example.com",
+		});
+		sighting("spammer", "msg");
+
+		moveTo("msg", "trash");
+		await repo.reconcileJunkOnlyForMessage("msg");
+
+		assert.equal(await withheld("spammer"), true);
+	});
+
+	test("touches no address the message does not carry", async () => {
+		message("msg", "inbox");
+		message("other", "junk");
+		await harvest("bystander");
+		sighting("bystander", "other");
+
+		moveTo("msg", "junk");
+		await repo.reconcileJunkOnlyForMessage("msg");
+
+		assert.equal(await withheld("bystander"), false);
+	});
+
+	test("a second reconcile of the same move writes nothing new", async () => {
+		message("msg", "inbox");
+		await harvest("spammer");
+		sighting("spammer", "msg");
+		moveTo("msg", "junk");
+		await repo.reconcileJunkOnlyForMessage("msg");
+		const first = await repo.getAddress(CONFIG, "spammer");
+
+		await repo.reconcileJunkOnlyForMessage("msg");
+
+		assert.deepEqual(await repo.getAddress(CONFIG, "spammer"), first);
+	});
+});

package/src/repos/i4-address.test.ts

@@ -990,6 +990,223 @@ describe("AddressRepo", () => {
 		await repo.deleteAddress(configB, b.addressId);
 	});
 
+	describe("addresses met only in Junk (#822)", () => {
+		const junkInput = (accountConfigId: string, email: string) => ({
+			...makeAddressInput(accountConfigId, email),
+			displayName: "Pharma Deals",
+			normalizedCompound: `pharma deals ${email}`,
+		});
+
+		test("a suggestion list never offers a withheld address", async () => {
+			const accountConfigId = randomId();
+			const withheld = await repo.upsertJunkAddress(
+				junkInput(accountConfigId, "sales@pharma.example"),
+			);
+			const ordinary = await repo.upsertCorrespondentAddress(
+				makeAddressInput(accountConfigId, "colleague@pharma.example"),
+			);
+
+			const page = await repo.listByAccountConfig({
+				accountConfigId,
+				search: "pharma",
+			});
+
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[ordinary.addressId],
+			);
+
+			await repo.deleteManyAddresses(accountConfigId, [
+				withheld.addressId,
+				ordinary.addressId,
+			]);
+		});
+
+		test("an exact address still resolves a withheld row", async () => {
+			const accountConfigId = randomId();
+			const withheld = await repo.upsertJunkAddress(
+				junkInput(accountConfigId, "sales@pharma.example"),
+			);
+
+			const page = await repo.listByAccountConfig({
+				accountConfigId,
+				search: "sales@pharma.example",
+			});
+
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[withheld.addressId],
+			);
+
+			await repo.deleteAddress(accountConfigId, withheld.addressId);
+		});
+
+		test("a bare listing never offers a withheld address", async () => {
+			const accountConfigId = randomId();
+			const withheld = await repo.upsertJunkAddress(
+				junkInput(accountConfigId, "sales@pharma.example"),
+			);
+			const ordinary = await repo.upsertCorrespondentAddress(
+				makeAddressInput(accountConfigId, "colleague@pharma.example"),
+			);
+
+			const page = await repo.listByAccountConfig({ accountConfigId });
+
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[ordinary.addressId],
+			);
+
+			await repo.deleteManyAddresses(accountConfigId, [
+				withheld.addressId,
+				ordinary.addressId,
+			]);
+		});
+
+		test("the row is still there to resolve the message that carried it", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "sales@pharma.example");
+			const withheld = await repo.upsertJunkAddress(input);
+
+			const fetched = await repo.getAddress(accountConfigId, input.addressId);
+			assert.equal(fetched.normalizedEmail, "sales@pharma.example");
+			assert.equal(fetched.flags?.junkOnly?.value, true);
+
+			await repo.deleteAddress(accountConfigId, withheld.addressId);
+		});
+
+		test("one sighting on live mail restores it", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "misfiled@pharma.example");
+			await repo.upsertJunkAddress(input);
+
+			const harvested = await repo.upsertCorrespondentAddress(input);
+
+			assert.equal(harvested.flags?.junkOnly, undefined);
+			const page = await repo.listByAccountConfig({
+				accountConfigId,
+				search: "misfiled",
+			});
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[input.addressId],
+			);
+
+			await repo.deleteAddress(accountConfigId, input.addressId);
+		});
+
+		test("a sighting in Junk leaves an address the account knows alone", async () => {
+			const accountConfigId = randomId();
+			const known = makeAddressInput(accountConfigId, "friend@pharma.example");
+			await repo.upsertCorrespondentAddress({
+				...known,
+				displayName: "Real Friend",
+			});
+
+			const after = await repo.upsertJunkAddress({
+				...known,
+				displayName: "Pharma Deals",
+				normalizedCompound: "pharma deals friend@pharma.example",
+			});
+
+			assert.equal(after.displayName, "Real Friend");
+			assert.equal(after.flags?.junkOnly, undefined);
+
+			await repo.deleteAddress(accountConfigId, known.addressId);
+		});
+
+		test("a sighting in Trash decides nothing either way", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "discarded@pharma.example");
+			await repo.upsertJunkAddress(input);
+
+			const after = await repo.upsertAddress(input);
+
+			assert.equal(after.flags?.junkOnly?.value, true);
+
+			await repo.deleteAddress(accountConfigId, input.addressId);
+		});
+
+		test("a sender the account blocked stays findable, and marked", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "reported@pharma.example");
+			await repo.upsertJunkAddress(input);
+			await repo.mergeFlags(accountConfigId, input.addressId, {
+				blocked: { value: true, setAt: 1 },
+			});
+
+			const page = await repo.listByAccountConfig({
+				accountConfigId,
+				search: "pharma",
+			});
+
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[input.addressId],
+			);
+			assert.equal(page.items[0].flags?.junkOnly?.value, true);
+
+			await repo.deleteAddress(accountConfigId, input.addressId);
+		});
+
+		test("the account's own mail to a sender offers them again", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "supplier@pharma.example");
+			await repo.upsertJunkAddress(input);
+			await repo.incrementOutboundCount(accountConfigId, input.addressId, 1);
+
+			const page = await repo.listByAccountConfig({
+				accountConfigId,
+				search: "pharma",
+			});
+
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[input.addressId],
+			);
+
+			await repo.deleteAddress(accountConfigId, input.addressId);
+		});
+
+		test("clearing the mark by hand offers the address again", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "rescued@pharma.example");
+			await repo.upsertJunkAddress(input);
+
+			await repo.mergeFlags(accountConfigId, input.addressId, {
+				junkOnly: null,
+			});
+
+			const page = await repo.listByAccountConfig({
+				accountConfigId,
+				search: "pharma",
+			});
+
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[input.addressId],
+			);
+
+			await repo.deleteAddress(accountConfigId, input.addressId);
+		});
+
+		test("clearing the mark keeps the rest of the flags", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "noisy@pharma.example");
+			await repo.upsertJunkAddress(input);
+			await repo.mergeFlags(accountConfigId, input.addressId, {
+				muted: { value: true, setAt: 7 },
+			});
+
+			const harvested = await repo.upsertCorrespondentAddress(input);
+
+			assert.equal(harvested.flags?.junkOnly, undefined);
+			assert.equal(harvested.flags?.muted?.value, true);
+
+			await repo.deleteAddress(accountConfigId, input.addressId);
+		});
+	});
+
 	describe("continuation token rejection (#172)", () => {
 		for (const [label, token] of [
 			["an unparseable", "not-a-cursor"],

package/src/repos/i4-address.ts

@@ -24,10 +24,16 @@ import type { Db } from "../db.js";
 import { NotFoundError } from "../error.js";
 import { envelopeAddressId as deriveEnvelopeAddressId } from "../id.js";
 import { decodeToken, resultList } from "../pagination.js";
+import {
+	JUNK_ONLY_FLAG,
+	restoreSql,
+	withholdSql,
+} from "../repair/junk-only-address.js";
 import { addressTable } from "../schema/i4-address.js";
 import { envelopeAddressTable } from "../schema/message-data.js";
 import {
 	addressCorrespondence,
+	addressListable,
 	addressMatchRank,
 	addressPreference,
 	addressRecency,
@@ -151,6 +157,23 @@ function rowToEnvelopeAddress(
 
 const VIP_SUGGESTIONS_DEFAULT_LIMIT = 10;
 
+const JUNK_HARVEST = "junk-harvest";
+const JUNK_MOVE = "junk-move";
+
+const withoutJunkOnlyFlagSql = (): SQL<string> =>
+	sql<string>`json_remove(coalesce(nullif(${addressTable.flags}, ''), '{}'), ${`$.${JUNK_ONLY_FLAG}`})`;
+
+const boundToDrizzle = (query: string, params: readonly unknown[]): SQL => {
+	const chunks = query.split("?");
+	const head = sql.raw(chunks[0]);
+	return chunks
+		.slice(1)
+		.reduce(
+			(acc, chunk, index) => sql`${acc}${params[index]}${sql.raw(chunk)}`,
+			sql`${head}`,
+		);
+};
+
 export class AddressRepo implements IAddressRepository {
 	constructor(private db: DB) {}
 
@@ -220,6 +243,87 @@ export class AddressRepo implements IAddressRepository {
 		return rowToAddress(row);
 	}
 
+	async upsertCorrespondentAddress(
+		input: CreateAddressInput,
+	): Promise<AddressItem> {
+		const now = Date.now();
+		const [row] = await this.db
+			.insert(addressTable)
+			.values({
+				addressId: input.addressId,
+				accountConfigId: input.accountConfigId,
+				displayName: input.displayName,
+				localPart: input.localPart,
+				domain: input.domain,
+				normalizedEmail: input.normalizedEmail,
+				normalizedCompound: input.normalizedCompound,
+				flags: input.flags ?? {},
+				inboundCount: input.inboundCount ?? 0,
+				outboundCount: input.outboundCount ?? 0,
+				replyCount: input.replyCount ?? 0,
+				lastInboundAt: input.lastInboundAt ?? 0,
+				lastOutboundAt: input.lastOutboundAt,
+				lastReplyAt: input.lastReplyAt ?? 0,
+				createdAt: now,
+				updatedAt: now,
+			})
+			.onConflictDoUpdate({
+				target: addressTable.addressId,
+				set: input.displayName
+					? {
+							displayName: input.displayName,
+							normalizedCompound: input.normalizedCompound,
+							flags: withoutJunkOnlyFlagSql(),
+							updatedAt: now,
+						}
+					: { flags: withoutJunkOnlyFlagSql(), updatedAt: now },
+			})
+			.returning();
+		return rowToAddress(row);
+	}
+
+	async upsertJunkAddress(input: CreateAddressInput): Promise<AddressItem> {
+		const now = Date.now();
+		const [row] = await this.db
+			.insert(addressTable)
+			.values({
+				addressId: input.addressId,
+				accountConfigId: input.accountConfigId,
+				displayName: input.displayName,
+				localPart: input.localPart,
+				domain: input.domain,
+				normalizedEmail: input.normalizedEmail,
+				normalizedCompound: input.normalizedCompound,
+				flags: {
+					...(input.flags ?? {}),
+					[JUNK_ONLY_FLAG]: { value: true, setAt: now, setBy: JUNK_HARVEST },
+				},
+				inboundCount: input.inboundCount ?? 0,
+				outboundCount: input.outboundCount ?? 0,
+				replyCount: input.replyCount ?? 0,
+				lastInboundAt: input.lastInboundAt ?? 0,
+				lastOutboundAt: input.lastOutboundAt,
+				lastReplyAt: input.lastReplyAt ?? 0,
+				createdAt: now,
+				updatedAt: now,
+			})
+			.onConflictDoNothing()
+			.returning();
+		if (!row) return this.getAddress(input.accountConfigId, input.addressId);
+		return rowToAddress(row);
+	}
+
+	async reconcileJunkOnlyForMessage(messageId: string): Promise<void> {
+		const scope = ` AND address.address_id IN (
+			SELECT address_id FROM envelope_address WHERE message_id = ?
+		)`;
+		const now = Date.now();
+		await this.db.run(
+			boundToDrizzle(withholdSql(scope), [now, JUNK_MOVE, now, messageId]),
+		);
+		await this.db.run(boundToDrizzle(restoreSql(scope), [now, messageId]));
+	}
+
 	async getAddress(
 		accountConfigId: string,
 		addressId: string,
@@ -573,6 +677,7 @@ export class AddressRepo implements IAddressRepository {
 				and(
 					eq(addressTable.accountConfigId, accountConfigId),
 					search ? addressSearchMatch(search) : undefined,
+					addressListable(search),
 					position ? after(order, position) : undefined,
 				),
 			)

package/src/repos/i4-mailbox-special-use.ts

@@ -4,6 +4,7 @@ import type {
 	MailboxSpecialUseValue,
 } from "@remit/data-ports";
 import { resolveMailboxByLeafName } from "@remit/data-ports/mailbox-name";
+import { JUNK_FOLDER_NAMES } from "@remit/data-ports/mailbox-role";
 import { eq } from "drizzle-orm";
 import type { Db } from "../db.js";
 import { randomId } from "../id.js";
@@ -11,14 +12,6 @@ import { mailboxSpecialUseTable, mailboxTable } from "../schema/i4-mailbox.js";
 
 type DB = Db<Record<string, unknown>>;
 
-const JUNK_FOLDER_NAMES = [
-	"junk",
-	"spam",
-	"junk e-mail",
-	"junk email",
-	"bulk mail",
-];
-
 function rowToSpecialUse(
 	row: typeof mailboxSpecialUseTable.$inferSelect,
 ): MailboxSpecialUseItem {