@remit/drizzle-service 0.0.57 → 0.0.59

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.57",
+  "version": "0.0.59",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/repos/i4-address-merge-flags.sqlite.test.ts

@@ -0,0 +1,97 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import type { AddressFlags } from "@remit/data-ports";
+import { createTestDb, randomId, type TestDb } from "../test-db.js";
+import { serializeSqliteWrites } from "../tx.js";
+import { AddressRepo } from "./i4-address.js";
+
+const FLAG_KEYS = [
+	"trusted",
+	"blocked",
+	"muted",
+	"vip",
+	"wellknown",
+	"junkOnly",
+	"autoArchive",
+	"unsubscribed",
+] as const;
+
+const CONCURRENT_CALLS = 50;
+
+describe("concurrent flag merges on one address", () => {
+	let db: TestDb;
+	let close: () => Promise<void>;
+	let repo: AddressRepo;
+
+	before(async () => {
+		({ db, close } = await createTestDb());
+		repo = new AddressRepo(serializeSqliteWrites(db) as never);
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	const address = async () => {
+		const accountConfigId = randomId();
+		return repo.createAddress({
+			addressId: randomId(),
+			accountConfigId,
+			localPart: "sender",
+			domain: "example.com",
+			normalizedEmail: "sender@example.com",
+			normalizedCompound: "sender@example.com:sender",
+		});
+	};
+
+	test("fifty concurrent merges all survive", async () => {
+		const addr = await address();
+		const writtenAt = new Map<string, Set<number>>();
+
+		await Promise.all(
+			Array.from({ length: CONCURRENT_CALLS }, (_, index) => {
+				const key = FLAG_KEYS[index % FLAG_KEYS.length];
+				const setAt = 1_000 + index;
+				const stamps = writtenAt.get(key) ?? new Set<number>();
+				stamps.add(setAt);
+				writtenAt.set(key, stamps);
+				return repo.mergeFlags(addr.accountConfigId, addr.addressId, {
+					[key]: { value: true, setAt },
+				});
+			}),
+		);
+
+		const merged = await repo.getAddress(addr.accountConfigId, addr.addressId);
+		const flags: AddressFlags = merged.flags ?? {};
+		for (const key of FLAG_KEYS) {
+			const flag = flags[key];
+			assert.ok(flag, `${key} was lost by a concurrent merge`);
+			assert.ok(
+				writtenAt.get(key)?.has(flag.setAt),
+				`${key} holds a value no merge wrote`,
+			);
+		}
+
+		await repo.deleteAddress(addr.accountConfigId, addr.addressId);
+	});
+
+	test("a concurrent merge does not resurrect a deleted flag", async () => {
+		const addr = await address();
+		await repo.mergeFlags(addr.accountConfigId, addr.addressId, {
+			muted: { value: true, setAt: 1 },
+		});
+
+		await Promise.all([
+			repo.mergeFlags(addr.accountConfigId, addr.addressId, { muted: null }),
+			repo.mergeFlags(addr.accountConfigId, addr.addressId, {
+				vip: { value: true, setAt: 2 },
+			}),
+		]);
+
+		const merged = await repo.getAddress(addr.accountConfigId, addr.addressId);
+		assert.equal(merged.flags?.muted, undefined, "muted must stay deleted");
+		assert.equal(merged.flags?.vip?.value, true, "vip must survive the delete");
+
+		await repo.deleteAddress(addr.accountConfigId, addr.addressId);
+	});
+});

package/src/repos/i4-address.test.ts

@@ -148,7 +148,6 @@ describe("AddressRepo", () => {
 		});
 
 		const ea = await repo.createEnvelopeAddress({
-			envelopeAddressId: envId,
 			messageId,
 			addressId,
 			normalizedEmail: "test@example.com",
@@ -170,7 +169,6 @@ describe("AddressRepo", () => {
 	test("upsertEnvelopeAddress is idempotent", async () => {
 		const messageId = randomUUID();
 		const input = {
-			envelopeAddressId: envelopeAddressId(messageId, "to", 1),
 			messageId,
 			addressId: randomUUID(),
 			normalizedEmail: "x@example.com",
@@ -179,15 +177,18 @@ describe("AddressRepo", () => {
 		};
 		const first = await repo.upsertEnvelopeAddress(input);
 		const second = await repo.upsertEnvelopeAddress(input);
-		assert.equal(first.envelopeAddressId, second.envelopeAddressId);
+		assert.equal(
+			first.envelopeAddressId,
+			envelopeAddressId(messageId, "to", 1),
+		);
+		assert.equal(second.envelopeAddressId, first.envelopeAddressId);
 
-		await repo.deleteEnvelopeAddress(input.envelopeAddressId);
+		await repo.deleteEnvelopeAddress(first.envelopeAddressId);
 	});
 
 	test("deleteManyEnvelopeAddresses removes in batch", async () => {
 		const messageId = randomUUID();
 		const ea1 = {
-			envelopeAddressId: envelopeAddressId(messageId, "from", 0),
 			messageId,
 			addressId: randomUUID(),
 			normalizedEmail: "a@x.com",
@@ -196,22 +197,21 @@ describe("AddressRepo", () => {
 		};
 		const ea2 = {
 			...ea1,
-			envelopeAddressId: envelopeAddressId(messageId, "to", 1),
 			addressRole: "to" as const,
 			addressOrder: 1,
 		};
 
-		await repo.createEnvelopeAddress(ea1);
-		await repo.createEnvelopeAddress(ea2);
+		const created1 = await repo.createEnvelopeAddress(ea1);
+		const created2 = await repo.createEnvelopeAddress(ea2);
 
 		await repo.deleteManyEnvelopeAddresses([
-			ea1.envelopeAddressId,
-			ea2.envelopeAddressId,
+			created1.envelopeAddressId,
+			created2.envelopeAddressId,
 		]);
 
 		const results = await repo.getEnvelopeAddress([
-			ea1.envelopeAddressId,
-			ea2.envelopeAddressId,
+			created1.envelopeAddressId,
+			created2.envelopeAddressId,
 		]);
 		assert.equal(results.length, 0);
 	});

package/src/repos/i4-address.ts

@@ -23,7 +23,7 @@ import {
 } from "drizzle-orm";
 import type { Db } from "../db.js";
 import { NotFoundError } from "../error.js";
-import { envelopeAddressId as deriveEnvelopeAddressId } from "../id.js";
+import { envelopeAddressId } from "../id.js";
 import { decodeToken, resultList } from "../pagination.js";
 import {
 	JUNK_ONLY_FLAG,
@@ -32,6 +32,7 @@ import {
 } from "../repair/junk-only-address.js";
 import { addressTable } from "../schema/i4-address.js";
 import { envelopeAddressTable } from "../schema/message-data.js";
+import { runInTransaction } from "../tx.js";
 import {
 	addressCorrespondence,
 	addressListable,
@@ -49,6 +50,23 @@ type AddressUpdate = Partial<{
 		| SQL;
 }>;
 
+const MERGE_FLAGS_ATTEMPTS = 8;
+
+/**
+ * The stored flags exactly as they sit in the column, so a merge can guard its
+ * write on the bytes it read rather than on a re-serialization of them.
+ */
+const storedFlagsSql = sql<string>`cast(${addressTable.flags} as text)`;
+
+/** A row harvested before the column existed carries `''`, not `'{}'`. */
+const parseStoredFlags = (stored: string): AddressFlags =>
+	stored === "" ? {} : (JSON.parse(stored) as AddressFlags);
+
+type MergeAttempt =
+	| { outcome: "merged"; address: AddressItem }
+	| { outcome: "missing" }
+	| { outcome: "contended" };
+
 /**
  * The stored `"<display name> <email>"` compound, folded in JavaScript exactly
  * as message sync folds it — SQL `lower()` stops at ASCII, and the search reads
@@ -400,36 +418,77 @@ export class AddressRepo implements IAddressRepository {
 		return rowToAddress(row);
 	}
 
+	/**
+	 * Read the flags, fold `merge` over them, and write the result back only if
+	 * the column still holds the bytes the read returned. Read and write share
+	 * one serialized unit, so the only writer that can slip between them is the
+	 * junk-only reconcile, which runs its raw statement outside the write queue;
+	 * that write fails the guard and the merge folds again over the winner
+	 * instead of overwriting it. The unit never throws, so a reconcile statement
+	 * landing inside its savepoint is never rolled back with it.
+	 */
+	private async mergeStoredFlags(
+		accountConfigId: string,
+		addressId: string,
+		merge: (current: AddressFlags) => AddressFlags,
+		extra: AddressUpdate = {},
+	): Promise<AddressItem> {
+		const key = and(
+			eq(addressTable.accountConfigId, accountConfigId),
+			eq(addressTable.addressId, addressId),
+		);
+		for (let attempt = 0; attempt < MERGE_FLAGS_ATTEMPTS; attempt++) {
+			const result = await runInTransaction(
+				this.db,
+				async (tx): Promise<MergeAttempt> => {
+					const [current] = await tx
+						.select({ stored: storedFlagsSql })
+						.from(addressTable)
+						.where(key);
+					if (!current) return { outcome: "missing" };
+					const [row] = await tx
+						.update(addressTable)
+						.set({
+							...extra,
+							flags: merge(parseStoredFlags(current.stored)) as never,
+							updatedAt: Date.now(),
+						})
+						.where(and(key, eq(storedFlagsSql, current.stored)))
+						.returning();
+					return row
+						? { outcome: "merged", address: rowToAddress(row) }
+						: { outcome: "contended" };
+				},
+			);
+			if (result.outcome === "merged") return result.address;
+			if (result.outcome === "missing")
+				throw new NotFoundError(`Address not found: ${addressId}`);
+		}
+		throw new Error(
+			`Address flags stayed contended after ${MERGE_FLAGS_ATTEMPTS} attempts: ${addressId}`,
+		);
+	}
+
 	async mergeFlags(
 		accountConfigId: string,
 		addressId: string,
 		patch: FlagsMergePatch,
 	): Promise<AddressItem> {
-		const current = await this.getAddress(accountConfigId, addressId);
-		const next: AddressFlags = { ...(current.flags ?? {}) };
-		for (const [key, value] of Object.entries(patch) as [
-			keyof AddressFlags,
-			AddressFlags[keyof AddressFlags] | null | undefined,
-		][]) {
-			if (value === undefined) continue;
-			if (value === null) {
-				delete next[key];
-				continue;
+		return this.mergeStoredFlags(accountConfigId, addressId, (current) => {
+			const next: AddressFlags = { ...current };
+			for (const [key, value] of Object.entries(patch) as [
+				keyof AddressFlags,
+				AddressFlags[keyof AddressFlags] | null | undefined,
+			][]) {
+				if (value === undefined) continue;
+				if (value === null) {
+					delete next[key];
+					continue;
+				}
+				(next[key] as AddressFlags[keyof AddressFlags]) = value;
 			}
-			(next[key] as AddressFlags[keyof AddressFlags]) = value;
-		}
-		const [row] = await this.db
-			.update(addressTable)
-			.set({ flags: next as never, updatedAt: Date.now() })
-			.where(
-				and(
-					eq(addressTable.accountConfigId, accountConfigId),
-					eq(addressTable.addressId, addressId),
-				),
-			)
-			.returning();
-		if (!row) throw new NotFoundError(`Address not found: ${addressId}`);
-		return rowToAddress(row);
+			return next;
+		});
 	}
 
 	async promoteWellknownByUser(
@@ -437,23 +496,10 @@ export class AddressRepo implements IAddressRepository {
 		addressId: string,
 		now: number,
 	): Promise<AddressItem> {
-		const current = await this.getAddress(accountConfigId, addressId);
-		const next: AddressFlags = {
-			...(current.flags ?? {}),
+		return this.mergeStoredFlags(accountConfigId, addressId, (current) => ({
+			...current,
 			wellknown: { value: true, setAt: now, setBy: "user-junk-rescue" },
-		};
-		const [row] = await this.db
-			.update(addressTable)
-			.set({ flags: next as never, updatedAt: Date.now() })
-			.where(
-				and(
-					eq(addressTable.accountConfigId, accountConfigId),
-					eq(addressTable.addressId, addressId),
-				),
-			)
-			.returning();
-		if (!row) throw new NotFoundError(`Address not found: ${addressId}`);
-		return rowToAddress(row);
+		}));
 	}
 
 	async demoteSenderTrust(
@@ -461,25 +507,12 @@ export class AddressRepo implements IAddressRepository {
 		addressId: string,
 		_now: number,
 	): Promise<AddressItem> {
-		const current = await this.getAddress(accountConfigId, addressId);
-		const { wellknown: _w, vip: _v, ...rest } = current.flags ?? {};
-		const [row] = await this.db
-			.update(addressTable)
-			.set({
-				flags: rest as never,
-				inboundCount: 0,
-				replyCount: 0,
-				updatedAt: Date.now(),
-			})
-			.where(
-				and(
-					eq(addressTable.accountConfigId, accountConfigId),
-					eq(addressTable.addressId, addressId),
-				),
-			)
-			.returning();
-		if (!row) throw new NotFoundError(`Address not found: ${addressId}`);
-		return rowToAddress(row);
+		return this.mergeStoredFlags(
+			accountConfigId,
+			addressId,
+			({ wellknown: _w, vip: _v, ...rest }) => rest,
+			{ inboundCount: 0, replyCount: 0 },
+		);
 	}
 
 	async deleteAddress(
@@ -711,15 +744,14 @@ export class AddressRepo implements IAddressRepository {
 		input: CreateEnvelopeAddressInput,
 	): Promise<EnvelopeAddressItem> {
 		const now = Date.now();
-		const envelopeAddressId = deriveEnvelopeAddressId(
-			input.messageId,
-			input.addressRole,
-			input.addressOrder,
-		);
 		const [row] = await this.db
 			.insert(envelopeAddressTable)
 			.values({
-				envelopeAddressId,
+				envelopeAddressId: envelopeAddressId(
+					input.messageId,
+					input.addressRole,
+					input.addressOrder,
+				),
 				messageId: input.messageId,
 				addressId: input.addressId,
 				displayName: input.displayName,
@@ -737,7 +769,7 @@ export class AddressRepo implements IAddressRepository {
 		input: CreateEnvelopeAddressInput,
 	): Promise<EnvelopeAddressItem> {
 		const now = Date.now();
-		const envelopeAddressId = deriveEnvelopeAddressId(
+		const rowId = envelopeAddressId(
 			input.messageId,
 			input.addressRole,
 			input.addressOrder,
@@ -745,7 +777,7 @@ export class AddressRepo implements IAddressRepository {
 		const [row] = await this.db
 			.insert(envelopeAddressTable)
 			.values({
-				envelopeAddressId,
+				envelopeAddressId: rowId,
 				messageId: input.messageId,
 				addressId: input.addressId,
 				displayName: input.displayName,
@@ -758,7 +790,7 @@ export class AddressRepo implements IAddressRepository {
 			.onConflictDoNothing()
 			.returning();
 		if (!row) {
-			return this.getEnvelopeAddress(envelopeAddressId);
+			return this.getEnvelopeAddress(rowId);
 		}
 		return rowToEnvelopeAddress(row);
 	}