@remit/drizzle-service 0.0.56 → 0.0.58

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.56",
+  "version": "0.0.58",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/repos/address-search-predicates.ts

@@ -1,4 +1,4 @@
-import { or, type SQL, sql } from "drizzle-orm";
+import { eq, or, type SQL, sql } from "drizzle-orm";
 import { addressTable } from "../schema/i4-address.js";
 
 // The address search seam. A term is matched as a substring of the display name,
@@ -49,10 +49,12 @@ export const addressSearchMatch = (term: string): SQL => {
  * spells out ranks above every prefix of it: a display name is free text the
  * sender picks, and a domain somebody else registered is a prefix away from the
  * address the reader typed, so neither may take the suggestion slot from the
- * address that matches whole. Below that, a match at the start of a column
- * outranks one in the middle, and within each the whole address, the local part
- * and the display name outrank the domain they share. A mid-string match still
- * comes back — this only decides the order.
+ * address that matches whole. A term that spells out a domain whole ranks next,
+ * so `ischen.nl` reaches the domain it names before `ischen.nl.co`, which only
+ * starts with it (#829). Below that, a match at the start of a column outranks
+ * one in the middle, and within each the whole address, the local part and the
+ * display name outrank the domain they share. A mid-string match still comes
+ * back — this only decides the order.
  */
 export const addressMatchRank = (term: string | undefined): SQL<number> => {
 	// Not the bare literal `0`: SQLite reads an integer literal in ORDER BY as a
@@ -60,7 +62,10 @@ export const addressMatchRank = (term: string | undefined): SQL<number> => {
 	if (!term) return sql<number>`cast(0 as integer)`;
 	const { leading, anywhere } = patterns(term);
 	const tiers = [
-		sql`${addressTable.normalizedEmail} = ${term.toLowerCase()}`,
+		eq(addressTable.normalizedEmail, term.toLowerCase()),
+		// `domain` is stored as the envelope spelled it, so `Ischen.NL` only meets
+		// a folded term through `lower()`; `like` folds ASCII on its own.
+		sql`lower(${addressTable.domain}) = ${term.toLowerCase()}`,
 		...SEARCH_COLUMNS.map((column) => like(column, leading)),
 		...SEARCH_COLUMNS.map((column) => like(column, anywhere)),
 	];

package/src/repos/i4-address-merge-flags.sqlite.test.ts

@@ -0,0 +1,97 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import type { AddressFlags } from "@remit/data-ports";
+import { createTestDb, randomId, type TestDb } from "../test-db.js";
+import { serializeSqliteWrites } from "../tx.js";
+import { AddressRepo } from "./i4-address.js";
+
+const FLAG_KEYS = [
+	"trusted",
+	"blocked",
+	"muted",
+	"vip",
+	"wellknown",
+	"junkOnly",
+	"autoArchive",
+	"unsubscribed",
+] as const;
+
+const CONCURRENT_CALLS = 50;
+
+describe("concurrent flag merges on one address", () => {
+	let db: TestDb;
+	let close: () => Promise<void>;
+	let repo: AddressRepo;
+
+	before(async () => {
+		({ db, close } = await createTestDb());
+		repo = new AddressRepo(serializeSqliteWrites(db) as never);
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	const address = async () => {
+		const accountConfigId = randomId();
+		return repo.createAddress({
+			addressId: randomId(),
+			accountConfigId,
+			localPart: "sender",
+			domain: "example.com",
+			normalizedEmail: "sender@example.com",
+			normalizedCompound: "sender@example.com:sender",
+		});
+	};
+
+	test("fifty concurrent merges all survive", async () => {
+		const addr = await address();
+		const writtenAt = new Map<string, Set<number>>();
+
+		await Promise.all(
+			Array.from({ length: CONCURRENT_CALLS }, (_, index) => {
+				const key = FLAG_KEYS[index % FLAG_KEYS.length];
+				const setAt = 1_000 + index;
+				const stamps = writtenAt.get(key) ?? new Set<number>();
+				stamps.add(setAt);
+				writtenAt.set(key, stamps);
+				return repo.mergeFlags(addr.accountConfigId, addr.addressId, {
+					[key]: { value: true, setAt },
+				});
+			}),
+		);
+
+		const merged = await repo.getAddress(addr.accountConfigId, addr.addressId);
+		const flags: AddressFlags = merged.flags ?? {};
+		for (const key of FLAG_KEYS) {
+			const flag = flags[key];
+			assert.ok(flag, `${key} was lost by a concurrent merge`);
+			assert.ok(
+				writtenAt.get(key)?.has(flag.setAt),
+				`${key} holds a value no merge wrote`,
+			);
+		}
+
+		await repo.deleteAddress(addr.accountConfigId, addr.addressId);
+	});
+
+	test("a concurrent merge does not resurrect a deleted flag", async () => {
+		const addr = await address();
+		await repo.mergeFlags(addr.accountConfigId, addr.addressId, {
+			muted: { value: true, setAt: 1 },
+		});
+
+		await Promise.all([
+			repo.mergeFlags(addr.accountConfigId, addr.addressId, { muted: null }),
+			repo.mergeFlags(addr.accountConfigId, addr.addressId, {
+				vip: { value: true, setAt: 2 },
+			}),
+		]);
+
+		const merged = await repo.getAddress(addr.accountConfigId, addr.addressId);
+		assert.equal(merged.flags?.muted, undefined, "muted must stay deleted");
+		assert.equal(merged.flags?.vip?.value, true, "vip must survive the delete");
+
+		await repo.deleteAddress(addr.accountConfigId, addr.addressId);
+	});
+});

package/src/repos/i4-address.test.ts

@@ -474,6 +474,85 @@ describe("AddressRepo", () => {
 		]);
 	});
 
+	test("listByAccountConfig ranks the domain a bare term names above one that only starts with it", async () => {
+		const accountConfigId = randomId();
+		const lookalike = await repo.createAddress({
+			...makeAddressInput(accountConfigId, "matthijs@ischen.nl.co"),
+			displayName: "Ischen Support",
+			normalizedCompound: "ischen support matthijs@ischen.nl.co",
+			inboundCount: 900,
+		});
+		// The envelope's own spelling: `domain` is stored unfolded.
+		const own = await repo.createAddress({
+			...makeAddressInput(accountConfigId, "matthijs@ischen.nl"),
+			domain: "Ischen.NL",
+			displayName: "Matthijs van Henten",
+			normalizedCompound: "matthijs van henten matthijs@ischen.nl",
+			inboundCount: 1,
+		});
+
+		const found = await repo.listByAccountConfig({
+			accountConfigId,
+			search: "ischen.nl",
+		});
+		assert.deepEqual(
+			found.items.map((a) => a.normalizedEmail),
+			[own.normalizedEmail, lookalike.normalizedEmail],
+			"the domain the term names leads one that registered a suffix of it",
+		);
+
+		const onlySuggestion = await repo.listByAccountConfig({
+			accountConfigId,
+			search: "ischen.nl",
+			limit: 1,
+		});
+		assert.deepEqual(
+			onlySuggestion.items.map((a) => a.normalizedEmail),
+			[own.normalizedEmail],
+			"volume on the lookalike must not take the single suggestion slot",
+		);
+
+		await repo.deleteManyAddresses(accountConfigId, [
+			lookalike.addressId,
+			own.addressId,
+		]);
+	});
+
+	test("listByAccountConfig keeps an address the term spells out whole above a row that only owns the domain", async () => {
+		const accountConfigId = randomId();
+		// A harvested header that carried the bare domain where an address belongs.
+		const bare = await repo.createAddress({
+			addressId: randomId(),
+			accountConfigId,
+			localPart: "ischen.nl",
+			domain: "",
+			normalizedEmail: "ischen.nl",
+			normalizedCompound: "ischen.nl",
+			displayName: "Ischen",
+		});
+		const domainOwner = await repo.createAddress({
+			...makeAddressInput(accountConfigId, "matthijs@ischen.nl"),
+			displayName: "Matthijs van Henten",
+			normalizedCompound: "matthijs van henten matthijs@ischen.nl",
+			inboundCount: 900,
+		});
+
+		const found = await repo.listByAccountConfig({
+			accountConfigId,
+			search: "ischen.nl",
+		});
+		assert.deepEqual(
+			found.items.map((a) => a.normalizedEmail),
+			[bare.normalizedEmail, domainOwner.normalizedEmail],
+			"the address that matches whole stays above the domain tier below it",
+		);
+
+		await repo.deleteManyAddresses(accountConfigId, [
+			bare.addressId,
+			domainOwner.addressId,
+		]);
+	});
+
 	test("listByAccountConfig keeps a leading display-name match above a domain that only contains the term", async () => {
 		const accountConfigId = randomId();
 		const shop = await repo.createAddress({

package/src/repos/i4-address.ts

@@ -32,6 +32,7 @@ import {
 } from "../repair/junk-only-address.js";
 import { addressTable } from "../schema/i4-address.js";
 import { envelopeAddressTable } from "../schema/message-data.js";
+import { runInTransaction } from "../tx.js";
 import {
 	addressCorrespondence,
 	addressListable,
@@ -49,6 +50,23 @@ type AddressUpdate = Partial<{
 		| SQL;
 }>;
 
+const MERGE_FLAGS_ATTEMPTS = 8;
+
+/**
+ * The stored flags exactly as they sit in the column, so a merge can guard its
+ * write on the bytes it read rather than on a re-serialization of them.
+ */
+const storedFlagsSql = sql<string>`cast(${addressTable.flags} as text)`;
+
+/** A row harvested before the column existed carries `''`, not `'{}'`. */
+const parseStoredFlags = (stored: string): AddressFlags =>
+	stored === "" ? {} : (JSON.parse(stored) as AddressFlags);
+
+type MergeAttempt =
+	| { outcome: "merged"; address: AddressItem }
+	| { outcome: "missing" }
+	| { outcome: "contended" };
+
 /**
  * The stored `"<display name> <email>"` compound, folded in JavaScript exactly
  * as message sync folds it — SQL `lower()` stops at ASCII, and the search reads
@@ -400,36 +418,77 @@ export class AddressRepo implements IAddressRepository {
 		return rowToAddress(row);
 	}
 
+	/**
+	 * Read the flags, fold `merge` over them, and write the result back only if
+	 * the column still holds the bytes the read returned. Read and write share
+	 * one serialized unit, so the only writer that can slip between them is the
+	 * junk-only reconcile, which runs its raw statement outside the write queue;
+	 * that write fails the guard and the merge folds again over the winner
+	 * instead of overwriting it. The unit never throws, so a reconcile statement
+	 * landing inside its savepoint is never rolled back with it.
+	 */
+	private async mergeStoredFlags(
+		accountConfigId: string,
+		addressId: string,
+		merge: (current: AddressFlags) => AddressFlags,
+		extra: AddressUpdate = {},
+	): Promise<AddressItem> {
+		const key = and(
+			eq(addressTable.accountConfigId, accountConfigId),
+			eq(addressTable.addressId, addressId),
+		);
+		for (let attempt = 0; attempt < MERGE_FLAGS_ATTEMPTS; attempt++) {
+			const result = await runInTransaction(
+				this.db,
+				async (tx): Promise<MergeAttempt> => {
+					const [current] = await tx
+						.select({ stored: storedFlagsSql })
+						.from(addressTable)
+						.where(key);
+					if (!current) return { outcome: "missing" };
+					const [row] = await tx
+						.update(addressTable)
+						.set({
+							...extra,
+							flags: merge(parseStoredFlags(current.stored)) as never,
+							updatedAt: Date.now(),
+						})
+						.where(and(key, eq(storedFlagsSql, current.stored)))
+						.returning();
+					return row
+						? { outcome: "merged", address: rowToAddress(row) }
+						: { outcome: "contended" };
+				},
+			);
+			if (result.outcome === "merged") return result.address;
+			if (result.outcome === "missing")
+				throw new NotFoundError(`Address not found: ${addressId}`);
+		}
+		throw new Error(
+			`Address flags stayed contended after ${MERGE_FLAGS_ATTEMPTS} attempts: ${addressId}`,
+		);
+	}
+
 	async mergeFlags(
 		accountConfigId: string,
 		addressId: string,
 		patch: FlagsMergePatch,
 	): Promise<AddressItem> {
-		const current = await this.getAddress(accountConfigId, addressId);
-		const next: AddressFlags = { ...(current.flags ?? {}) };
-		for (const [key, value] of Object.entries(patch) as [
-			keyof AddressFlags,
-			AddressFlags[keyof AddressFlags] | null | undefined,
-		][]) {
-			if (value === undefined) continue;
-			if (value === null) {
-				delete next[key];
-				continue;
+		return this.mergeStoredFlags(accountConfigId, addressId, (current) => {
+			const next: AddressFlags = { ...current };
+			for (const [key, value] of Object.entries(patch) as [
+				keyof AddressFlags,
+				AddressFlags[keyof AddressFlags] | null | undefined,
+			][]) {
+				if (value === undefined) continue;
+				if (value === null) {
+					delete next[key];
+					continue;
+				}
+				(next[key] as AddressFlags[keyof AddressFlags]) = value;
 			}
-			(next[key] as AddressFlags[keyof AddressFlags]) = value;
-		}
-		const [row] = await this.db
-			.update(addressTable)
-			.set({ flags: next as never, updatedAt: Date.now() })
-			.where(
-				and(
-					eq(addressTable.accountConfigId, accountConfigId),
-					eq(addressTable.addressId, addressId),
-				),
-			)
-			.returning();
-		if (!row) throw new NotFoundError(`Address not found: ${addressId}`);
-		return rowToAddress(row);
+			return next;
+		});
 	}
 
 	async promoteWellknownByUser(
@@ -437,23 +496,10 @@ export class AddressRepo implements IAddressRepository {
 		addressId: string,
 		now: number,
 	): Promise<AddressItem> {
-		const current = await this.getAddress(accountConfigId, addressId);
-		const next: AddressFlags = {
-			...(current.flags ?? {}),
+		return this.mergeStoredFlags(accountConfigId, addressId, (current) => ({
+			...current,
 			wellknown: { value: true, setAt: now, setBy: "user-junk-rescue" },
-		};
-		const [row] = await this.db
-			.update(addressTable)
-			.set({ flags: next as never, updatedAt: Date.now() })
-			.where(
-				and(
-					eq(addressTable.accountConfigId, accountConfigId),
-					eq(addressTable.addressId, addressId),
-				),
-			)
-			.returning();
-		if (!row) throw new NotFoundError(`Address not found: ${addressId}`);
-		return rowToAddress(row);
+		}));
 	}
 
 	async demoteSenderTrust(
@@ -461,25 +507,12 @@ export class AddressRepo implements IAddressRepository {
 		addressId: string,
 		_now: number,
 	): Promise<AddressItem> {
-		const current = await this.getAddress(accountConfigId, addressId);
-		const { wellknown: _w, vip: _v, ...rest } = current.flags ?? {};
-		const [row] = await this.db
-			.update(addressTable)
-			.set({
-				flags: rest as never,
-				inboundCount: 0,
-				replyCount: 0,
-				updatedAt: Date.now(),
-			})
-			.where(
-				and(
-					eq(addressTable.accountConfigId, accountConfigId),
-					eq(addressTable.addressId, addressId),
-				),
-			)
-			.returning();
-		if (!row) throw new NotFoundError(`Address not found: ${addressId}`);
-		return rowToAddress(row);
+		return this.mergeStoredFlags(
+			accountConfigId,
+			addressId,
+			({ wellknown: _w, vip: _v, ...rest }) => rest,
+			{ inboundCount: 0, replyCount: 0 },
+		);
 	}
 
 	async deleteAddress(