@remit/drizzle-service 0.0.79 → 0.0.81

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.79",
+  "version": "0.0.81",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/repos/message-placement-backfill.sqlite.test.ts

@@ -0,0 +1,190 @@
+/**
+ * The two data migrations this release carries, against a database written by
+ * the build before it.
+ *
+ * Both exist because a predicate changed meaning under rows already on disk.
+ * `carriesForeignUid` reads `originalUid` without consulting `status`, so a
+ * settled row that a pre-#1217 build left one on answers "this uid belongs to
+ * another folder" forever — and Empty Trash refuses to remove it after the
+ * server copy is gone (reader#1230). And the give-up signal moved from
+ * `active` + `failed` to its own value, so an abandoned delete written before
+ * the upgrade would go quiet: the row stays where it was handed back and the
+ * user is told nothing.
+ *
+ * Applied through the shipped journal rather than a schema pushed from the
+ * table objects, because a data migration that never runs is exactly the
+ * failure being pinned.
+ */
+
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import Database from "better-sqlite3";
+import {
+	applyMigration,
+	migrationJournal,
+} from "../test-shipped-sqlite-schema.js";
+
+const STALE_ORIGINAL_UID = "msg-stale-original-uid";
+const IN_FLIGHT = "msg-in-flight";
+const GAVE_UP = "msg-gave-up";
+const RETRYING_DELETE = "msg-retrying-delete";
+
+/** The migration that clears a settled row's stale `original_uid`. */
+const ORIGINAL_UID_BACKFILL = 24;
+
+/** The migration that carries an abandoned delete onto the give-up value. */
+const GIVE_UP_BACKFILL = 25;
+
+interface MessageRow {
+	message_id: string;
+	status: string;
+	sync_status: string;
+	abandoned_mutation: string;
+	original_uid: number | null;
+	original_mailbox_id: string | null;
+}
+
+describe("the placement backfills carry a pre-upgrade database forward", () => {
+	let sqlite: Database.Database;
+
+	const rowFor = (messageId: string): MessageRow =>
+		sqlite
+			.prepare("SELECT * FROM message WHERE message_id = ?")
+			.get(messageId) as MessageRow;
+
+	before(() => {
+		sqlite = new Database(":memory:");
+		const entries = migrationJournal();
+
+		// Everything the previous release shipped, and nothing this one adds.
+		for (const entry of entries.filter(
+			(candidate) => candidate.idx < ORIGINAL_UID_BACKFILL,
+		)) {
+			applyMigration(sqlite, entry.tag);
+		}
+
+		const insert = sqlite.prepare(
+			`INSERT INTO message (
+				message_id, mailbox_id, uid, sequence_number, rfc822_size,
+				internal_date, envelope_id, root_body_part_id,
+				status, sync_status, original_mailbox_id, original_uid,
+				created_at, updated_at
+			) VALUES (?, ?, ?, 1, 100, 0, ?, ?, ?, ?, ?, ?, 0, 0)`,
+		);
+
+		// Settled, with the `original_uid` a pre-#1217 `updateUid` left behind —
+		// and Trash handing back the source's own number, which on a young
+		// account is the common case rather than the exotic one.
+		insert.run(
+			STALE_ORIGINAL_UID,
+			"mbx-trash",
+			10,
+			`env-${STALE_ORIGINAL_UID}`,
+			`bp-${STALE_ORIGINAL_UID}`,
+			"active",
+			"synced",
+			"mbx-inbox",
+			10,
+		);
+
+		// Genuinely mid-move: this row's `original_uid` is not stale and must
+		// survive, or the guard that stops a move binding somebody else's uid
+		// stops working on every row the upgrade touched.
+		insert.run(
+			IN_FLIGHT,
+			"mbx-archive",
+			42,
+			`env-${IN_FLIGHT}`,
+			`bp-${IN_FLIGHT}`,
+			"moving",
+			"pending",
+			"mbx-inbox",
+			42,
+		);
+
+		// The pair a give-up wrote before this release. Two writers produced it —
+		// `abandonDelete` and the paused-cursor hand-back for an unproven MOVE —
+		// through the identical `restoreSourcePlacement` call, so the row cannot
+		// say which.
+		insert.run(
+			GAVE_UP,
+			"mbx-inbox",
+			7,
+			`env-${GAVE_UP}`,
+			`bp-${GAVE_UP}`,
+			"active",
+			"failed",
+			"mbx-inbox",
+			null,
+		);
+
+		// A transient attempt with a redelivery behind it. It writes the same
+		// `failed`, and keeping `deleting` beside it is the only thing that tells
+		// the two apart — so the backfill must not touch it.
+		insert.run(
+			RETRYING_DELETE,
+			"mbx-trash",
+			8,
+			`env-${RETRYING_DELETE}`,
+			`bp-${RETRYING_DELETE}`,
+			"deleting",
+			"failed",
+			"mbx-inbox",
+			8,
+		);
+
+		for (const entry of entries.filter(
+			(candidate) => candidate.idx >= ORIGINAL_UID_BACKFILL,
+		)) {
+			applyMigration(sqlite, entry.tag);
+		}
+	});
+
+	after(() => {
+		sqlite.close();
+	});
+
+	test("the journal ships both backfills", () => {
+		const idxs = migrationJournal().map((entry) => entry.idx);
+		assert.ok(idxs.includes(ORIGINAL_UID_BACKFILL));
+		assert.ok(idxs.includes(GIVE_UP_BACKFILL));
+	});
+
+	test("a settled row loses the stale uid that made it look foreign", () => {
+		const row = rowFor(STALE_ORIGINAL_UID);
+		assert.equal(row.original_uid, null);
+		assert.equal(
+			row.original_mailbox_id,
+			"mbx-inbox",
+			"Undo still restores to the folder the message came from",
+		);
+	});
+
+	test("a row still mid-move keeps the uid its guard depends on", () => {
+		assert.equal(rowFor(IN_FLIGHT).original_uid, 42);
+	});
+
+	test("a give-up keeps its chip, on the value the client now reads", () => {
+		const row = rowFor(GAVE_UP);
+		assert.equal(row.sync_status, "abandoned");
+		assert.equal(row.status, "active");
+	});
+
+	/**
+	 * The two pre-upgrade writers are indistinguishable in the row, so the
+	 * backfill has to pick one label for both. It picks the survivable mistake:
+	 * a wrongly-labelled move offers a folder picker the user can dismiss, while
+	 * a wrongly-labelled delete offers a button that destroys the message — the
+	 * #1229 defect, on a press the user believed was a repair.
+	 */
+	test("and is never labelled a delete, which would offer to destroy it", () => {
+		assert.equal(rowFor(GAVE_UP).abandoned_mutation, "move");
+	});
+
+	test("a delete mid-retry is left exactly as it was", () => {
+		const row = rowFor(RETRYING_DELETE);
+		assert.equal(row.sync_status, "failed");
+		assert.equal(row.abandoned_mutation, "none");
+		assert.equal(row.status, "deleting");
+	});
+});

package/src/repos/message.sqlite.test.ts

@@ -171,6 +171,84 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		assert.equal(item.status, "active");
 	});
 
+	// The lock of imap-mutations R3. There is no version column: the predicate
+	// is over the placement fields themselves, so an unrelated write to any
+	// other column cannot make a transition lose, and a placement the caller
+	// did not read cannot make it win.
+	describe("transitionPlacement", () => {
+		const LOCK_ID = "00000000-0000-0000-2222-00000000000a";
+		const OTHER_MAILBOX = "00000000-0000-0000-2222-00000000000b";
+
+		before(async () => {
+			await repo.create({
+				...BASE_INPUT,
+				messageId: LOCK_ID,
+				uid: 7,
+				envelopeId: deriveEnvelopeId(LOCK_ID),
+				rootBodyPartId: deriveRootBodyPartId(LOCK_ID),
+				status: "moving" as const,
+				syncStatus: "pending" as const,
+			});
+		});
+
+		test("writes the row when every named field still matches", async () => {
+			const written = await repo.transitionPlacement(
+				LOCK_ID,
+				{ status: "moving", mailboxId: MAILBOX_ID, uid: 7 },
+				{ status: "active", syncStatus: "synced", mailboxId: OTHER_MAILBOX },
+			);
+
+			assert.equal(written?.status, "active");
+			assert.equal(written?.syncStatus, "synced");
+			assert.equal(written?.mailboxId, OTHER_MAILBOX);
+		});
+
+		test("answers undefined and writes nothing when the row has moved on", async () => {
+			const lost = await repo.transitionPlacement(
+				LOCK_ID,
+				{ status: "moving" },
+				{ status: "deleting" },
+			);
+
+			assert.equal(lost, undefined);
+			assert.equal(
+				(await repo.get(LOCK_ID)).status,
+				"active",
+				"the loser leaves the winner's placement exactly as it found it",
+			);
+		});
+
+		test("a set of from-states matches any of them", async () => {
+			const written = await repo.transitionPlacement(
+				LOCK_ID,
+				{ status: ["moving", "active"] },
+				{ syncStatus: "abandoned" },
+			);
+
+			assert.equal(written?.syncStatus, "abandoned");
+		});
+
+		test("a field the caller did not read constrains nothing", async () => {
+			const written = await repo.transitionPlacement(
+				LOCK_ID,
+				{ uid: 7 },
+				{ syncStatus: "synced" },
+			);
+
+			assert.equal(written?.syncStatus, "synced");
+		});
+
+		test("a row that no longer exists loses like any other predicate", async () => {
+			const lost = await repo.transitionPlacement(
+				"00000000-0000-0000-2222-0000000000ff",
+				{ status: "active" },
+				{ status: "deleting" },
+			);
+
+			assert.equal(lost, undefined);
+		});
+	});
+
 	test("delete removes the message and appends a removal outbox row", async () => {
 		await repo.delete(MESSAGE_ID);
 		await assert.rejects(() => repo.get(MESSAGE_ID));

package/src/repos/message.test.ts

@@ -195,13 +195,18 @@ describe("DrizzleMessageRepository", () => {
 		});
 
 		test("returns the moved message with new mailbox and pending status", async () => {
-			const moved = await messageRepo.updateForMove(MOVE_MESSAGE_ID, {
-				mailboxId: DEST_MAILBOX_ID,
-				status: "moving",
-				syncStatus: "pending",
-				originalMailboxId: SOURCE_MAILBOX_ID,
-				originalUid: 7,
-			});
+			const moved = await messageRepo.transitionPlacement(
+				MOVE_MESSAGE_ID,
+				{ status: "active", mailboxId: SOURCE_MAILBOX_ID, uid: 7 },
+				{
+					mailboxId: DEST_MAILBOX_ID,
+					status: "moving",
+					syncStatus: "pending",
+					originalMailboxId: SOURCE_MAILBOX_ID,
+					originalUid: 7,
+				},
+			);
+			assert.ok(moved);
 			assert.equal(moved.mailboxId, DEST_MAILBOX_ID);
 			assert.equal(moved.status, "moving");
 			assert.equal(moved.syncStatus, "pending");
@@ -277,16 +282,25 @@ describe("DrizzleMessageRepository", () => {
 			assert.equal(rows[0].processedAt, null, "event starts undrained");
 		});
 
-		test("updateForMove throws NotFoundError for unknown messageId", async () => {
-			await assert.rejects(
-				() =>
-					messageRepo.updateForMove("00000000-0000-0000-9999-000000000001", {
-						mailboxId: DEST_MAILBOX_ID,
-					}),
-				(err: Error) => {
-					assert.equal(err.name, "NotFoundError");
-					return true;
-				},
+		test("updateUid drops the source uid and keeps the folder Undo restores to", async () => {
+			// `originalUid` says the row's `uid` was recorded under
+			// `originalMailboxId`. The confirmation above makes that untrue, and a
+			// leftover that happens to equal the destination's own uid — two
+			// independent per-folder counters, so a routine collision — reads as an
+			// unsettled placement for the rest of the row's life (#1217).
+			const read = await messageRepo.get(MOVE_MESSAGE_ID);
+			assert.equal(read.originalUid, undefined);
+			assert.equal(read.originalMailboxId, SOURCE_MAILBOX_ID);
+		});
+
+		test("the optimistic move answers undefined for an unknown messageId", async () => {
+			assert.equal(
+				await messageRepo.transitionPlacement(
+					"00000000-0000-0000-9999-000000000001",
+					{ status: "active" },
+					{ mailboxId: DEST_MAILBOX_ID },
+				),
+				undefined,
 			);
 		});
 

package/src/repos/message.ts

@@ -4,8 +4,9 @@ import type {
 	IMessageRepository,
 	MessageDescription,
 	MessageItem,
+	PlacementPredicate,
 } from "@remit/data-ports";
-import { and, asc, eq, gt, inArray, or } from "drizzle-orm";
+import { and, asc, eq, gt, inArray, or, type SQL } from "drizzle-orm";
 
 import type { Db } from "../db.js";
 import {
@@ -57,6 +58,7 @@ function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
 		envelopeId: row.envelopeId,
 		rootBodyPartId: row.rootBodyPartId,
 		status: row.status,
+		abandonedMutation: row.abandonedMutation,
 		syncStatus: row.syncStatus,
 		category: row.category,
 		classificationState: row.classificationState,
@@ -169,6 +171,34 @@ export async function deleteMessageSubtree(
 	);
 }
 
+/**
+ * The WHERE terms of a placement transition (imap-mutations R3). A field the
+ * caller left out is a field it did not read, so it constrains nothing; a field
+ * given as a list matches any of the states that share one column value.
+ */
+const isList = <T>(value: T | readonly T[]): value is readonly T[] =>
+	Array.isArray(value);
+
+const oneOf = <T>(value: T | readonly T[]): T[] =>
+	isList(value) ? [...value] : [value];
+
+const placementTerms = (expected: PlacementPredicate): SQL[] => {
+	const terms: SQL[] = [];
+	if (expected.status !== undefined) {
+		terms.push(inArray(messageTable.status, oneOf(expected.status)));
+	}
+	if (expected.syncStatus !== undefined) {
+		terms.push(inArray(messageTable.syncStatus, oneOf(expected.syncStatus)));
+	}
+	if (expected.mailboxId !== undefined) {
+		terms.push(eq(messageTable.mailboxId, expected.mailboxId));
+	}
+	if (expected.uid !== undefined) {
+		terms.push(eq(messageTable.uid, expected.uid));
+	}
+	return terms;
+};
+
 export class DrizzleMessageRepository implements IMessageRepository {
 	constructor(private db: DB) {}
 
@@ -359,10 +389,6 @@ export class DrizzleMessageRepository implements IMessageRepository {
 			...(input.bodyStorageKey !== undefined
 				? { bodyStorageKey: input.bodyStorageKey }
 				: {}),
-			...(input.status !== undefined ? { status: input.status } : {}),
-			...(input.syncStatus !== undefined
-				? { syncStatus: input.syncStatus }
-				: {}),
 			...(input.category !== undefined ? { category: input.category } : {}),
 			...(input.classificationState !== undefined
 				? { classificationState: input.classificationState }
@@ -455,17 +481,6 @@ export class DrizzleMessageRepository implements IMessageRepository {
 		return this.get(messageId);
 	}
 
-	async clearOriginalMailboxId(
-		messageId: string,
-	): ReturnType<IMessageRepository["clearOriginalMailboxId"]> {
-		const now = Date.now();
-		await this.db
-			.update(messageTable)
-			.set({ originalMailboxId: null, originalUid: null, updatedAt: now })
-			.where(eq(messageTable.messageId, messageId));
-		return this.get(messageId);
-	}
-
 	async delete(messageId: string): Promise<void> {
 		await this.deleteMany([messageId]);
 	}
@@ -532,22 +547,24 @@ export class DrizzleMessageRepository implements IMessageRepository {
 		return rows.map(toMessageItem);
 	}
 
-	async updateForMove(
+	async transitionPlacement(
 		messageId: string,
-		input: Parameters<IMessageRepository["updateForMove"]>[1],
-	): ReturnType<IMessageRepository["updateForMove"]> {
+		expected: PlacementPredicate,
+		next: Parameters<IMessageRepository["transitionPlacement"]>[2],
+	): ReturnType<IMessageRepository["transitionPlacement"]> {
 		const setValues = {
-			...(input.mailboxId !== undefined ? { mailboxId: input.mailboxId } : {}),
-			...(input.uid !== undefined ? { uid: input.uid } : {}),
-			...(input.status !== undefined ? { status: input.status } : {}),
-			...(input.syncStatus !== undefined
-				? { syncStatus: input.syncStatus }
+			...(next.mailboxId !== undefined ? { mailboxId: next.mailboxId } : {}),
+			...(next.uid !== undefined ? { uid: next.uid } : {}),
+			...(next.status !== undefined ? { status: next.status } : {}),
+			...(next.syncStatus !== undefined ? { syncStatus: next.syncStatus } : {}),
+			...(next.abandonedMutation !== undefined
+				? { abandonedMutation: next.abandonedMutation }
 				: {}),
-			...(input.originalMailboxId !== undefined
-				? { originalMailboxId: input.originalMailboxId }
+			...(next.originalMailboxId !== undefined
+				? { originalMailboxId: next.originalMailboxId }
 				: {}),
-			...(input.originalUid !== undefined
-				? { originalUid: input.originalUid }
+			...(next.originalUid !== undefined
+				? { originalUid: next.originalUid }
 				: {}),
 			updatedAt: Date.now(),
 		};
@@ -555,11 +572,11 @@ export class DrizzleMessageRepository implements IMessageRepository {
 		const rows = await this.db
 			.update(messageTable)
 			.set(setValues)
-			.where(eq(messageTable.messageId, messageId))
+			.where(
+				and(eq(messageTable.messageId, messageId), ...placementTerms(expected)),
+			)
 			.returning();
-		if (rows.length === 0) {
-			throw new NotFoundError(`Message not found: ${messageId}`);
-		}
+		if (rows.length === 0) return undefined;
 		return toMessageItem(rows[0]);
 	}
 
@@ -574,14 +591,28 @@ export class DrizzleMessageRepository implements IMessageRepository {
 		// re-index would skip them on content hash. Enqueue a move re-index event
 		// in the same transaction as the update; the search-index worker drains it
 		// with force, refreshing the stored mailbox metadata.
+		//
+		// `originalUid` goes with the settle. It exists to say the row's `uid`
+		// was recorded under `originalMailboxId`, and that stops being true here;
+		// left behind it makes the settled row indistinguishable from an
+		// in-flight one whenever the destination's counter happens to hand back
+		// the source's number, which on a young account is most of them (#1217).
+		// `originalMailboxId` stays: Undo restores the message to it.
 		const rows = await runInTransaction(this.db, async (tx) => {
 			const updated = await tx
 				.update(messageTable)
 				.set({
 					uid: newUid,
 					mailboxId: newMailboxId,
+					originalUid: null,
 					status: "active",
 					syncStatus: "synced",
+					// The settle clears the epitaph as well as the pair. `syncStatus`
+					// alone already gates every reader of it, so this is tidiness
+					// rather than correctness — but a row that has just settled has
+					// nothing it gave up on, and leaving a value there invites a
+					// reader that forgets the gate.
+					abandonedMutation: "none",
 					updatedAt: Date.now(),
 				})
 				.where(eq(messageTable.messageId, messageId))