@remit/drizzle-service 0.0.49 → 0.0.51

package/src/repos/address-search-predicates.ts

@@ -80,6 +80,21 @@ const flagValue = (name: string): SQL<number> =>
 export const addressPreference = (): SQL<number> =>
 	sql<number>`(2 * ${flagValue("vip")} + ${flagValue("trusted")})`;
 
+const accountHasCorresponded = (): SQL<number> =>
+	sql<number>`(${addressTable.outboundCount} + ${addressTable.replyCount}
+		+ ${flagValue("vip")} + ${flagValue("trusted")})`;
+
+const accountHasFlagged = (): SQL<number> =>
+	sql<number>`(${flagValue("blocked")} + ${flagValue("muted")})`;
+
+export const addressListable = (term: string | undefined): SQL => {
+	const shown = sql`${flagValue("junkOnly")} = 0
+		or ${accountHasCorresponded()} > 0
+		or ${accountHasFlagged()} > 0`;
+	if (!term) return sql`(${shown})`;
+	return sql`(${shown} or ${addressTable.normalizedEmail} = ${term.toLowerCase()})`;
+};
+
 export const addressCorrespondence = (): SQL<number> =>
 	sql<number>`(${addressTable.replyCount} + ${addressTable.inboundCount} + ${addressTable.outboundCount})`;
 

package/src/repos/i4-address-junk-move.sqlite.test.ts

@@ -0,0 +1,203 @@
+import assert from "node:assert/strict";
+import { after, before, beforeEach, describe, test } from "node:test";
+import type Database from "better-sqlite3";
+import { createTestDb, type TestDb } from "../test-db.js";
+import { AddressRepo } from "./i4-address.js";
+
+const CONFIG = "cfg-1";
+
+describe("reconciling one message's addresses at the moment it moves", () => {
+	let db: TestDb;
+	let sqlite: Database.Database;
+	let close: () => Promise<void>;
+	let repo: AddressRepo;
+
+	const mailbox = (mailboxId: string, specialUse: string | null): void => {
+		sqlite
+			.prepare(
+				`INSERT INTO mailbox (
+					mailbox_id, account_id, namespace_prefix, hierarchy_delimiter,
+					full_path, uid_validity, uid_next, highest_modseq, message_count,
+					unseen_count, deleted_count, total_size, last_sync_uid,
+					high_water_mark_uid, last_message_sync_at, special_use,
+					created_at, updated_at
+				) VALUES (?, 'acc', '', '/', ?, 1, 1, '0', 0, 0, 0, 0, 0, 0, 0, ?, 0, 0)`,
+			)
+			.run(mailboxId, mailboxId, specialUse);
+	};
+
+	const message = (messageId: string, mailboxId: string): void => {
+		sqlite
+			.prepare(
+				`INSERT INTO message (
+					message_id, mailbox_id, uid, sequence_number, rfc822_size,
+					internal_date, envelope_id, root_body_part_id, created_at, updated_at
+				) VALUES (?, ?, 1, 1, 10, 0, 'env', 'body', 0, 0)`,
+			)
+			.run(messageId, mailboxId);
+	};
+
+	const sighting = (addressId: string, messageId: string): void => {
+		sqlite
+			.prepare(
+				`INSERT INTO envelope_address (
+					envelope_address_id, message_id, address_id, display_name,
+					normalized_email, address_role, address_order, created_at, updated_at
+				) VALUES (?, ?, ?, 'Name', ?, 'From', 0, 0, 0)`,
+			)
+			.run(
+				`${addressId}-${messageId}`,
+				messageId,
+				addressId,
+				`${addressId}@example.com`,
+			);
+	};
+
+	const harvest = async (addressId: string) =>
+		repo.upsertCorrespondentAddress({
+			addressId,
+			accountConfigId: CONFIG,
+			displayName: "Name",
+			localPart: addressId,
+			domain: "example.com",
+			normalizedEmail: `${addressId}@example.com`,
+			normalizedCompound: `name ${addressId}@example.com`,
+		});
+
+	const moveTo = (messageId: string, mailboxId: string): void => {
+		sqlite
+			.prepare("UPDATE message SET mailbox_id = ? WHERE message_id = ?")
+			.run(mailboxId, messageId);
+	};
+
+	const withheld = async (addressId: string): Promise<boolean> =>
+		(await repo.getAddress(CONFIG, addressId)).flags?.junkOnly?.value === true;
+
+	const suggested = async (term: string): Promise<string[]> =>
+		(
+			await repo.listByAccountConfig({
+				accountConfigId: CONFIG,
+				search: term,
+			})
+		).items.map((item) => item.addressId);
+
+	before(async () => {
+		({ db, sqlite, close } = await createTestDb());
+		repo = new AddressRepo(db as never);
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	beforeEach(() => {
+		for (const table of ["address", "envelope_address", "message", "mailbox"]) {
+			sqlite.exec(`DELETE FROM ${table}`);
+		}
+		mailbox("inbox", null);
+		mailbox("junk", '["Junk"]');
+		mailbox("trash", '["Trash"]');
+	});
+
+	test("a message moved into Junk stops the sender being suggested", async () => {
+		message("msg", "inbox");
+		await harvest("spammer");
+		sighting("spammer", "msg");
+		assert.deepEqual(await suggested("spammer"), ["spammer"]);
+
+		moveTo("msg", "junk");
+		await repo.reconcileJunkOnlyForMessage("msg");
+
+		assert.equal(await withheld("spammer"), true);
+		assert.deepEqual(await suggested("spammer"), []);
+	});
+
+	test("a sender the account has written to survives the move", async () => {
+		message("msg", "inbox");
+		await harvest("client");
+		sighting("client", "msg");
+		await repo.incrementOutboundCount(CONFIG, "client", Date.now());
+
+		moveTo("msg", "junk");
+		await repo.reconcileJunkOnlyForMessage("msg");
+
+		assert.equal(await withheld("client"), false);
+	});
+
+	test("a sender still on live mail survives the move", async () => {
+		message("spam", "inbox");
+		message("real", "inbox");
+		await harvest("colleague");
+		sighting("colleague", "spam");
+		sighting("colleague", "real");
+
+		moveTo("spam", "junk");
+		await repo.reconcileJunkOnlyForMessage("spam");
+
+		assert.equal(await withheld("colleague"), false);
+	});
+
+	test("a message rescued out of Junk offers the sender again", async () => {
+		message("msg", "junk");
+		await repo.upsertJunkAddress({
+			addressId: "misfiled",
+			accountConfigId: CONFIG,
+			displayName: "Name",
+			localPart: "misfiled",
+			domain: "example.com",
+			normalizedEmail: "misfiled@example.com",
+			normalizedCompound: "name misfiled@example.com",
+		});
+		sighting("misfiled", "msg");
+		assert.deepEqual(await suggested("misfiled"), []);
+
+		moveTo("msg", "inbox");
+		await repo.reconcileJunkOnlyForMessage("msg");
+
+		assert.deepEqual(await suggested("misfiled"), ["misfiled"]);
+	});
+
+	test("a spam message moved to Trash keeps the sender withheld", async () => {
+		message("msg", "junk");
+		await repo.upsertJunkAddress({
+			addressId: "spammer",
+			accountConfigId: CONFIG,
+			displayName: "Name",
+			localPart: "spammer",
+			domain: "example.com",
+			normalizedEmail: "spammer@example.com",
+			normalizedCompound: "name spammer@example.com",
+		});
+		sighting("spammer", "msg");
+
+		moveTo("msg", "trash");
+		await repo.reconcileJunkOnlyForMessage("msg");
+
+		assert.equal(await withheld("spammer"), true);
+	});
+
+	test("touches no address the message does not carry", async () => {
+		message("msg", "inbox");
+		message("other", "junk");
+		await harvest("bystander");
+		sighting("bystander", "other");
+
+		moveTo("msg", "junk");
+		await repo.reconcileJunkOnlyForMessage("msg");
+
+		assert.equal(await withheld("bystander"), false);
+	});
+
+	test("a second reconcile of the same move writes nothing new", async () => {
+		message("msg", "inbox");
+		await harvest("spammer");
+		sighting("spammer", "msg");
+		moveTo("msg", "junk");
+		await repo.reconcileJunkOnlyForMessage("msg");
+		const first = await repo.getAddress(CONFIG, "spammer");
+
+		await repo.reconcileJunkOnlyForMessage("msg");
+
+		assert.deepEqual(await repo.getAddress(CONFIG, "spammer"), first);
+	});
+});

package/src/repos/i4-address.test.ts

@@ -990,6 +990,223 @@ describe("AddressRepo", () => {
 		await repo.deleteAddress(configB, b.addressId);
 	});
 
+	describe("addresses met only in Junk (#822)", () => {
+		const junkInput = (accountConfigId: string, email: string) => ({
+			...makeAddressInput(accountConfigId, email),
+			displayName: "Pharma Deals",
+			normalizedCompound: `pharma deals ${email}`,
+		});
+
+		test("a suggestion list never offers a withheld address", async () => {
+			const accountConfigId = randomId();
+			const withheld = await repo.upsertJunkAddress(
+				junkInput(accountConfigId, "sales@pharma.example"),
+			);
+			const ordinary = await repo.upsertCorrespondentAddress(
+				makeAddressInput(accountConfigId, "colleague@pharma.example"),
+			);
+
+			const page = await repo.listByAccountConfig({
+				accountConfigId,
+				search: "pharma",
+			});
+
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[ordinary.addressId],
+			);
+
+			await repo.deleteManyAddresses(accountConfigId, [
+				withheld.addressId,
+				ordinary.addressId,
+			]);
+		});
+
+		test("an exact address still resolves a withheld row", async () => {
+			const accountConfigId = randomId();
+			const withheld = await repo.upsertJunkAddress(
+				junkInput(accountConfigId, "sales@pharma.example"),
+			);
+
+			const page = await repo.listByAccountConfig({
+				accountConfigId,
+				search: "sales@pharma.example",
+			});
+
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[withheld.addressId],
+			);
+
+			await repo.deleteAddress(accountConfigId, withheld.addressId);
+		});
+
+		test("a bare listing never offers a withheld address", async () => {
+			const accountConfigId = randomId();
+			const withheld = await repo.upsertJunkAddress(
+				junkInput(accountConfigId, "sales@pharma.example"),
+			);
+			const ordinary = await repo.upsertCorrespondentAddress(
+				makeAddressInput(accountConfigId, "colleague@pharma.example"),
+			);
+
+			const page = await repo.listByAccountConfig({ accountConfigId });
+
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[ordinary.addressId],
+			);
+
+			await repo.deleteManyAddresses(accountConfigId, [
+				withheld.addressId,
+				ordinary.addressId,
+			]);
+		});
+
+		test("the row is still there to resolve the message that carried it", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "sales@pharma.example");
+			const withheld = await repo.upsertJunkAddress(input);
+
+			const fetched = await repo.getAddress(accountConfigId, input.addressId);
+			assert.equal(fetched.normalizedEmail, "sales@pharma.example");
+			assert.equal(fetched.flags?.junkOnly?.value, true);
+
+			await repo.deleteAddress(accountConfigId, withheld.addressId);
+		});
+
+		test("one sighting on live mail restores it", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "misfiled@pharma.example");
+			await repo.upsertJunkAddress(input);
+
+			const harvested = await repo.upsertCorrespondentAddress(input);
+
+			assert.equal(harvested.flags?.junkOnly, undefined);
+			const page = await repo.listByAccountConfig({
+				accountConfigId,
+				search: "misfiled",
+			});
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[input.addressId],
+			);
+
+			await repo.deleteAddress(accountConfigId, input.addressId);
+		});
+
+		test("a sighting in Junk leaves an address the account knows alone", async () => {
+			const accountConfigId = randomId();
+			const known = makeAddressInput(accountConfigId, "friend@pharma.example");
+			await repo.upsertCorrespondentAddress({
+				...known,
+				displayName: "Real Friend",
+			});
+
+			const after = await repo.upsertJunkAddress({
+				...known,
+				displayName: "Pharma Deals",
+				normalizedCompound: "pharma deals friend@pharma.example",
+			});
+
+			assert.equal(after.displayName, "Real Friend");
+			assert.equal(after.flags?.junkOnly, undefined);
+
+			await repo.deleteAddress(accountConfigId, known.addressId);
+		});
+
+		test("a sighting in Trash decides nothing either way", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "discarded@pharma.example");
+			await repo.upsertJunkAddress(input);
+
+			const after = await repo.upsertAddress(input);
+
+			assert.equal(after.flags?.junkOnly?.value, true);
+
+			await repo.deleteAddress(accountConfigId, input.addressId);
+		});
+
+		test("a sender the account blocked stays findable, and marked", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "reported@pharma.example");
+			await repo.upsertJunkAddress(input);
+			await repo.mergeFlags(accountConfigId, input.addressId, {
+				blocked: { value: true, setAt: 1 },
+			});
+
+			const page = await repo.listByAccountConfig({
+				accountConfigId,
+				search: "pharma",
+			});
+
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[input.addressId],
+			);
+			assert.equal(page.items[0].flags?.junkOnly?.value, true);
+
+			await repo.deleteAddress(accountConfigId, input.addressId);
+		});
+
+		test("the account's own mail to a sender offers them again", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "supplier@pharma.example");
+			await repo.upsertJunkAddress(input);
+			await repo.incrementOutboundCount(accountConfigId, input.addressId, 1);
+
+			const page = await repo.listByAccountConfig({
+				accountConfigId,
+				search: "pharma",
+			});
+
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[input.addressId],
+			);
+
+			await repo.deleteAddress(accountConfigId, input.addressId);
+		});
+
+		test("clearing the mark by hand offers the address again", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "rescued@pharma.example");
+			await repo.upsertJunkAddress(input);
+
+			await repo.mergeFlags(accountConfigId, input.addressId, {
+				junkOnly: null,
+			});
+
+			const page = await repo.listByAccountConfig({
+				accountConfigId,
+				search: "pharma",
+			});
+
+			assert.deepEqual(
+				page.items.map((a) => a.addressId),
+				[input.addressId],
+			);
+
+			await repo.deleteAddress(accountConfigId, input.addressId);
+		});
+
+		test("clearing the mark keeps the rest of the flags", async () => {
+			const accountConfigId = randomId();
+			const input = junkInput(accountConfigId, "noisy@pharma.example");
+			await repo.upsertJunkAddress(input);
+			await repo.mergeFlags(accountConfigId, input.addressId, {
+				muted: { value: true, setAt: 7 },
+			});
+
+			const harvested = await repo.upsertCorrespondentAddress(input);
+
+			assert.equal(harvested.flags?.junkOnly, undefined);
+			assert.equal(harvested.flags?.muted?.value, true);
+
+			await repo.deleteAddress(accountConfigId, input.addressId);
+		});
+	});
+
 	describe("continuation token rejection (#172)", () => {
 		for (const [label, token] of [
 			["an unparseable", "not-a-cursor"],

package/src/repos/i4-address.ts

@@ -24,10 +24,16 @@ import type { Db } from "../db.js";
 import { NotFoundError } from "../error.js";
 import { envelopeAddressId as deriveEnvelopeAddressId } from "../id.js";
 import { decodeToken, resultList } from "../pagination.js";
+import {
+	JUNK_ONLY_FLAG,
+	restoreSql,
+	withholdSql,
+} from "../repair/junk-only-address.js";
 import { addressTable } from "../schema/i4-address.js";
 import { envelopeAddressTable } from "../schema/message-data.js";
 import {
 	addressCorrespondence,
+	addressListable,
 	addressMatchRank,
 	addressPreference,
 	addressRecency,
@@ -151,6 +157,23 @@ function rowToEnvelopeAddress(
 
 const VIP_SUGGESTIONS_DEFAULT_LIMIT = 10;
 
+const JUNK_HARVEST = "junk-harvest";
+const JUNK_MOVE = "junk-move";
+
+const withoutJunkOnlyFlagSql = (): SQL<string> =>
+	sql<string>`json_remove(coalesce(nullif(${addressTable.flags}, ''), '{}'), ${`$.${JUNK_ONLY_FLAG}`})`;
+
+const boundToDrizzle = (query: string, params: readonly unknown[]): SQL => {
+	const chunks = query.split("?");
+	const head = sql.raw(chunks[0]);
+	return chunks
+		.slice(1)
+		.reduce(
+			(acc, chunk, index) => sql`${acc}${params[index]}${sql.raw(chunk)}`,
+			sql`${head}`,
+		);
+};
+
 export class AddressRepo implements IAddressRepository {
 	constructor(private db: DB) {}
 
@@ -220,6 +243,87 @@ export class AddressRepo implements IAddressRepository {
 		return rowToAddress(row);
 	}
 
+	async upsertCorrespondentAddress(
+		input: CreateAddressInput,
+	): Promise<AddressItem> {
+		const now = Date.now();
+		const [row] = await this.db
+			.insert(addressTable)
+			.values({
+				addressId: input.addressId,
+				accountConfigId: input.accountConfigId,
+				displayName: input.displayName,
+				localPart: input.localPart,
+				domain: input.domain,
+				normalizedEmail: input.normalizedEmail,
+				normalizedCompound: input.normalizedCompound,
+				flags: input.flags ?? {},
+				inboundCount: input.inboundCount ?? 0,
+				outboundCount: input.outboundCount ?? 0,
+				replyCount: input.replyCount ?? 0,
+				lastInboundAt: input.lastInboundAt ?? 0,
+				lastOutboundAt: input.lastOutboundAt,
+				lastReplyAt: input.lastReplyAt ?? 0,
+				createdAt: now,
+				updatedAt: now,
+			})
+			.onConflictDoUpdate({
+				target: addressTable.addressId,
+				set: input.displayName
+					? {
+							displayName: input.displayName,
+							normalizedCompound: input.normalizedCompound,
+							flags: withoutJunkOnlyFlagSql(),
+							updatedAt: now,
+						}
+					: { flags: withoutJunkOnlyFlagSql(), updatedAt: now },
+			})
+			.returning();
+		return rowToAddress(row);
+	}
+
+	async upsertJunkAddress(input: CreateAddressInput): Promise<AddressItem> {
+		const now = Date.now();
+		const [row] = await this.db
+			.insert(addressTable)
+			.values({
+				addressId: input.addressId,
+				accountConfigId: input.accountConfigId,
+				displayName: input.displayName,
+				localPart: input.localPart,
+				domain: input.domain,
+				normalizedEmail: input.normalizedEmail,
+				normalizedCompound: input.normalizedCompound,
+				flags: {
+					...(input.flags ?? {}),
+					[JUNK_ONLY_FLAG]: { value: true, setAt: now, setBy: JUNK_HARVEST },
+				},
+				inboundCount: input.inboundCount ?? 0,
+				outboundCount: input.outboundCount ?? 0,
+				replyCount: input.replyCount ?? 0,
+				lastInboundAt: input.lastInboundAt ?? 0,
+				lastOutboundAt: input.lastOutboundAt,
+				lastReplyAt: input.lastReplyAt ?? 0,
+				createdAt: now,
+				updatedAt: now,
+			})
+			.onConflictDoNothing()
+			.returning();
+		if (!row) return this.getAddress(input.accountConfigId, input.addressId);
+		return rowToAddress(row);
+	}
+
+	async reconcileJunkOnlyForMessage(messageId: string): Promise<void> {
+		const scope = ` AND address.address_id IN (
+			SELECT address_id FROM envelope_address WHERE message_id = ?
+		)`;
+		const now = Date.now();
+		await this.db.run(
+			boundToDrizzle(withholdSql(scope), [now, JUNK_MOVE, now, messageId]),
+		);
+		await this.db.run(boundToDrizzle(restoreSql(scope), [now, messageId]));
+	}
+
 	async getAddress(
 		accountConfigId: string,
 		addressId: string,
@@ -573,6 +677,7 @@ export class AddressRepo implements IAddressRepository {
 				and(
 					eq(addressTable.accountConfigId, accountConfigId),
 					search ? addressSearchMatch(search) : undefined,
+					addressListable(search),
 					position ? after(order, position) : undefined,
 				),
 			)