@remit/mailbox-service 0.0.90 → 0.0.92

package/src/placement-settled.test.ts

@@ -9,10 +9,10 @@ import { carriesForeignUid, placementBindingOf } from "./placement-settled.js";
  * only end-to-end specs covered it, so a change made for the client's benefit
  * could have moved it without anything going red.
  *
- * Note what these cases say about `syncStatus: failed`: it answers `abandoned`
- * here for a row that may well be mid-retry. That is correct for a guard whose
- * consequence is a temporary refusal, and it is exactly why no user-facing
- * statement may be derived from that field alone.
+ * Note what these cases say about `syncStatus`: `failed` is a transient attempt
+ * with a redelivery behind it, so a foreign-uid row carrying it is `in_flight`
+ * and gets the settle ceiling. Only `abandoned` — the give-up value of the
+ * placement state model (imap-mutations R3) — refuses outright.
  */
 const row = (over: Partial<MessageItem>): MessageItem =>
 	({
@@ -38,10 +38,40 @@ describe("placementBindingOf", () => {
 		assert.equal(placementBindingOf(row({})), "in_flight");
 	});
 
-	test("`failed` on a foreign-uid row is refused as abandoned", () => {
+	/**
+	 * A give-up cannot reach this pair. `abandoned` is written only alongside
+	 * `active` or `deleted`, on a row already put back on a placement the server
+	 * confirmed, so the binding it produces is `consistent` — there is no
+	 * give-up member here to return, and no dependent mutation is refused on
+	 * account of one.
+	 */
+	test("a give-up never presents as a foreign-uid row at all", () => {
+		assert.equal(
+			placementBindingOf(
+				row({
+					status: MessageStatus.active,
+					syncStatus: MessageSyncStatus.abandoned,
+					mailboxId: "mbx-src",
+				}),
+			),
+			"consistent",
+		);
+	});
+
+	test("`failed` on a foreign-uid row is mid-retry, so it waits rather than refusing", () => {
 		assert.equal(
 			placementBindingOf(row({ syncStatus: MessageSyncStatus.failed })),
-			"abandoned",
+			"in_flight",
+		);
+	});
+
+	// R3's state table calls `deleting` in flight exactly as `moving` is: a
+	// delete points the row at Trash with the source's uid, so a dependent
+	// mutation reading it resolves the same mismatched pair.
+	test("a delete in flight is unsettled, not settled", () => {
+		assert.equal(
+			placementBindingOf(row({ status: MessageStatus.deleting })),
+			"in_flight",
 		);
 	});
 

package/src/placement-settled.ts

@@ -1,5 +1,5 @@
 import type { MessageItem } from "@remit/data-ports";
-import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
+import { MessageStatus } from "@remit/domain-enums";
 
 /**
  * Whether the message row's placement is still an unconfirmed local write
@@ -12,18 +12,26 @@ import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
  *
  * `syncStatus` is not the signal. An ordinary freshly-synced inbound row is
  * `pending` forever (nothing on the inbound path promotes it), so keying off it
- * defers every outbound mutation in the product. `status` is set to `moving`
- * only by an actual move and cleared only once that move settles.
+ * defers every outbound mutation in the product. `status` is what an actual
+ * mutation writes and what settling clears.
+ *
+ * Both in-flight states count, per the R3 state table: a delete points the row
+ * at Trash with the source's uid exactly as a move points it at the
+ * destination, so a dependent mutation that reads a `deleting` row resolves the
+ * same mismatched pair. Reading `moving` alone let a move enqueued behind a
+ * delete bind to the row mid-delete and be dropped when the delete settled.
  */
 export const isPlacementUnsettled = (
 	message: Pick<MessageItem, "status">,
-): boolean => message.status === MessageStatus.moving;
+): boolean =>
+	message.status === MessageStatus.moving ||
+	message.status === MessageStatus.deleting;
 
 /**
  * Whether the row's `uid` belongs to a folder other than the one `mailboxId`
  * names — the state that makes the pair a lie rather than merely incomplete.
  *
- * `updateForMove` is the only writer of this shape: it points the row at the
+ * The optimistic half of a move is the only writer of this shape: it points the row at the
  * destination and records the pre-move pair, leaving `uid` untouched until the
  * server confirms and `updateUid` replaces it. A row with no `originalUid` was
  * never moved, so its uid names nothing but itself — a freshly copied row is
@@ -57,32 +65,27 @@ export const bindsForeignUid = (
 ): boolean => isPlacementUnsettled(message) && carriesForeignUid(message);
 
 /**
- * `in_flight` — a mover is still working on the row, so waiting resolves it.
- * `abandoned` — the last attempt failed (`syncStatus: failed`) and the row is
- * still `moving`: either a redelivery is pending, or the record dead-lettered
- * and nothing is coming for it. The two are indistinguishable from the row
- * (`message-settlement.ts`), and the pair is a lie either way, so neither is
- * worth spending the settle ceiling on. A mover that exhausts its budget
- * settles the row against IMAP instead of leaving this state behind (#1005).
+ * `in_flight` — a mutation is still working on the row, so waiting resolves it.
+ * That includes a row mid-retry: `message-move`, `message-delete` and
+ * `message-copy` write `syncStatus: failed` on an ordinary transient attempt
+ * and re-throw for redelivery, so a failed attempt is a mutation about to
+ * succeed and the settle ceiling is exactly what it is for (imap-mutations R3).
+ *
+ * There is no give-up member, because a give-up cannot reach here: `abandoned`
+ * is written only alongside `status: active` (a hand-back) or `deleted` (a copy
+ * that never existed), and the row it leaves names a placement the server
+ * confirmed — so it binds `consistent` and no dependent mutation is refused on
+ * account of it. A mutation that exhausts its budget settles the row against
+ * IMAP rather than leaving a mismatched pair behind (#1005).
  */
-export type PlacementBinding = "consistent" | "in_flight" | "abandoned";
+export type PlacementBinding = "consistent" | "in_flight";
 
 export const placementBindingOf = (
 	message: Pick<
 		MessageItem,
-		| "status"
-		| "syncStatus"
-		| "mailboxId"
-		| "uid"
-		| "originalMailboxId"
-		| "originalUid"
+		"status" | "mailboxId" | "uid" | "originalMailboxId" | "originalUid"
 	>,
-): PlacementBinding => {
-	if (!bindsForeignUid(message)) return "consistent";
-	return message.syncStatus === MessageSyncStatus.failed
-		? "abandoned"
-		: "in_flight";
-};
+): PlacementBinding => (bindsForeignUid(message) ? "in_flight" : "consistent");
 
 export interface PlacementSettleOptions {
 	timeoutMs: number;

package/src/placement-transition-race.test.ts

@@ -0,0 +1,482 @@
+/**
+ * The three optimistic writers, against a row that moves under them
+ * (docs/architecture/imap-mutations.md R3).
+ *
+ * Each reads a message, does several lookups, then writes where it decided to
+ * put it. Between the read and the write, PLACEMENT_MOVE_PUSH can settle the
+ * same row: that lane rides a standard queue, which the account's FIFO group
+ * does not order against. A blind write then stamps `moving` and the OLD uid
+ * over a folder and uid the mail server has just confirmed — a pair
+ * `carriesForeignUid` reads as consistent, so nothing downstream catches it —
+ * and the event that follows sends the worker after whatever now holds the old
+ * number.
+ *
+ * The repository honours the predicate here the way the real one does, so a
+ * losing write returns `undefined` and lands nothing at all.
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	IAddressRepository,
+	IMailboxRepository,
+	IMailboxSpecialUseRepository,
+	IMessagePlacementMoveRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+	PlacementPredicate,
+	PlacementTransitionInput,
+} from "@remit/data-ports";
+import { MessagePlacementUnsettledError } from "@remit/data-ports/errors";
+import { MessageMoveService } from "./message-move.js";
+import { PlacementMoveService } from "./placement-move.js";
+import { NO_JUNK_ROLES } from "./test-helpers/folder-roles.js";
+
+const ACCOUNT = "acc-1";
+const ACCOUNT_CONFIG = "cfg-1";
+const INBOX = "mbx-inbox";
+const ARCHIVE = "mbx-archive";
+const TRASH = "mbx-trash";
+const JUNK = "mbx-junk";
+const MESSAGE_ID = "msg-1";
+
+/** What the row is when each writer reads it. */
+const READ_UID = 42;
+
+/** What the placement push settles it to while the writer is still deciding. */
+const SETTLED_UID = 907;
+
+const mailboxes = [
+	{ mailboxId: INBOX, fullPath: "INBOX", accountId: ACCOUNT, uidValidity: 1 },
+	{
+		mailboxId: ARCHIVE,
+		fullPath: "Archive",
+		accountId: ACCOUNT,
+		uidValidity: 1,
+	},
+	{ mailboxId: TRASH, fullPath: "Trash", accountId: ACCOUNT, uidValidity: 1 },
+	{ mailboxId: JUNK, fullPath: "Junk", accountId: ACCOUNT, uidValidity: 1 },
+];
+
+interface CapturedEvent {
+	type: string;
+	messageId?: string;
+	uid?: number;
+}
+
+interface Row {
+	messageId: string;
+	accountConfigId: string;
+	mailboxId: string;
+	uid: number;
+	status: string;
+	syncStatus: string;
+	abandonedMutation: string;
+	originalMailboxId?: string;
+	originalUid?: number;
+}
+
+const freshRow = (): Row => ({
+	messageId: MESSAGE_ID,
+	accountConfigId: ACCOUNT_CONFIG,
+	mailboxId: INBOX,
+	uid: READ_UID,
+	status: "active",
+	syncStatus: "synced",
+	abandonedMutation: "none",
+});
+
+interface World {
+	row: Row;
+	events: CapturedEvent[];
+	writes: PlacementTransitionInput[];
+	/** Destination folders the listing row was actually moved to. */
+	threadMoves: string[];
+	/** Stand in for the push settling the row onto Archive with Archive's uid. */
+	settleElsewhere: () => void;
+	messageService: IMessageRepository;
+	threadMessageService: IThreadMessageRepository;
+	mailboxService: IMailboxRepository;
+	mailboxSpecialUseService: IMailboxSpecialUseRepository;
+	addressService: IAddressRepository;
+}
+
+const buildWorld = (onLookup?: (world: World) => void): World => {
+	const row = freshRow();
+	const events: CapturedEvent[] = [];
+	const writes: PlacementTransitionInput[] = [];
+
+	const threadMoves: string[] = [];
+	const world = { row, events, writes, threadMoves } as World;
+
+	world.settleElsewhere = () => {
+		row.mailboxId = ARCHIVE;
+		row.uid = SETTLED_UID;
+		row.status = "active";
+		row.syncStatus = "synced";
+		row.originalUid = undefined;
+	};
+
+	world.messageService = {
+		// A snapshot, the way a repository read is one. Handing back the live
+		// object would let the caller's `message` follow the concurrent settle,
+		// and the predicate it builds from it would match by accident.
+		get: async (ids: string | string[]) =>
+			Array.isArray(ids) ? [{ ...row }] : ({ ...row } as unknown),
+		update: async () => row,
+		transitionPlacement: async (
+			_messageId: string,
+			expected: PlacementPredicate,
+			next: PlacementTransitionInput,
+		) => {
+			if (expected.status !== undefined && expected.status !== row.status) {
+				return undefined;
+			}
+			if (
+				expected.mailboxId !== undefined &&
+				expected.mailboxId !== row.mailboxId
+			) {
+				return undefined;
+			}
+			if (expected.uid !== undefined && expected.uid !== row.uid) {
+				return undefined;
+			}
+			writes.push(next);
+			Object.assign(row, next);
+			return row;
+		},
+	} as unknown as IMessageRepository;
+
+	world.threadMessageService = {
+		getByMessageId: async () => ({
+			accountConfigId: ACCOUNT_CONFIG,
+			threadMessageId: "tm-1",
+			messageId: MESSAGE_ID,
+			mailboxId: row.mailboxId,
+			sentDate: 1,
+			isRead: false,
+			isDeleted: false,
+			hasStars: false,
+			hasAttachment: false,
+		}),
+		findAllByMessageId: async () => [],
+		update: async (
+			_accountConfigId: string,
+			_threadMessageId: string,
+			patch: { mailboxId?: string },
+		) => {
+			if (patch.mailboxId) threadMoves.push(patch.mailboxId);
+		},
+	} as unknown as IThreadMessageRepository;
+
+	// Every writer resolves folders between reading the row and writing it, so
+	// this is where the concurrent settle is made to land.
+	world.mailboxService = {
+		get: async (_accountId: string, ids: string | string[]) => {
+			onLookup?.(world);
+			return Array.isArray(ids)
+				? mailboxes.filter((mailbox) => ids.includes(mailbox.mailboxId))
+				: mailboxes.find((mailbox) => mailbox.mailboxId === ids);
+		},
+	} as unknown as IMailboxRepository;
+
+	world.mailboxSpecialUseService = {
+		findTrashMailbox: async () => mailboxes[2],
+		resolveTrashRole: async () => ({ kind: "flagged", mailbox: mailboxes[2] }),
+		resolveJunkRolesForConfig: async () => NO_JUNK_ROLES,
+	} as unknown as IMailboxSpecialUseRepository;
+
+	world.addressService = {
+		reconcileJunkOnlyForMessage: async () => {},
+	} as unknown as IAddressRepository;
+
+	return world;
+};
+
+const buildMoveService = (world: World): MessageMoveService => {
+	const service = new MessageMoveService({
+		messageService: world.messageService,
+		mailboxService: world.mailboxService,
+		mailboxSpecialUseService: world.mailboxSpecialUseService,
+		threadMessageService: world.threadMessageService,
+		addressService: world.addressService,
+		sqsQueueUrl: "http://localhost:9324/000000000000/remit-messages.fifo",
+		moveSettleTimeoutMs: 100,
+		moveSettlePollMs: 10,
+	});
+	(
+		service as unknown as {
+			enqueueEvent: (e: CapturedEvent) => Promise<void>;
+			enqueueEventsBatch: (e: CapturedEvent[]) => Promise<void>;
+		}
+	).enqueueEvent = async (event) => {
+		world.events.push(event);
+	};
+	(
+		service as unknown as {
+			enqueueEventsBatch: (e: CapturedEvent[]) => Promise<void>;
+		}
+	).enqueueEventsBatch = async (batch) => {
+		world.events.push(...batch);
+	};
+	return service;
+};
+
+describe("a user move writes only against the placement it read (R3)", () => {
+	it("moves the message when nothing else touched the row", async () => {
+		const world = buildWorld();
+		const service = buildMoveService(world);
+
+		await service.moveMessage(ACCOUNT_CONFIG, MESSAGE_ID, ARCHIVE, ACCOUNT);
+
+		assert.equal(world.events.length, 1);
+		assert.equal(world.row.status, "moving");
+		assert.equal(world.row.mailboxId, ARCHIVE);
+	});
+
+	it("refuses, and writes nothing, when the row settles elsewhere mid-decision", async () => {
+		let settled = false;
+		const world = buildWorld((w) => {
+			if (settled) return;
+			settled = true;
+			w.settleElsewhere();
+		});
+		const service = buildMoveService(world);
+
+		await assert.rejects(
+			() => service.moveMessage(ACCOUNT_CONFIG, MESSAGE_ID, JUNK, ACCOUNT),
+			(error: unknown) =>
+				error instanceof MessagePlacementUnsettledError &&
+				error.statusCode === 409,
+		);
+
+		assert.deepEqual(world.writes, [], "no placement was written");
+		assert.deepEqual(world.events, [], "and nothing was enqueued");
+		assert.equal(
+			world.row.mailboxId,
+			ARCHIVE,
+			"the winner's placement stands exactly as it left it",
+		);
+		assert.equal(world.row.uid, SETTLED_UID);
+		assert.equal(world.row.status, "active");
+	});
+});
+
+describe("a delete writes only against the placement it read (R3)", () => {
+	it("marks the message when nothing else touched the row", async () => {
+		const world = buildWorld();
+		const service = buildMoveService(world);
+
+		await service.deleteMessages(ACCOUNT_CONFIG, [MESSAGE_ID], ACCOUNT);
+
+		assert.equal(world.events.length, 1);
+		assert.equal(world.row.status, "moving");
+		assert.equal(world.row.mailboxId, TRASH);
+	});
+
+	it("skips the message, and enqueues nothing for it, when the row settles elsewhere", async () => {
+		let settled = false;
+		const world = buildWorld((w) => {
+			if (settled) return;
+			settled = true;
+			w.settleElsewhere();
+		});
+		const service = buildMoveService(world);
+
+		const { refusedMessageIds } = await service.deleteMessages(
+			ACCOUNT_CONFIG,
+			[MESSAGE_ID],
+			ACCOUNT,
+		);
+
+		assert.deepEqual(
+			refusedMessageIds,
+			[MESSAGE_ID],
+			"the caller is told which rows this delete never claimed",
+		);
+		assert.deepEqual(world.writes, [], "no placement was written");
+		assert.deepEqual(
+			world.events,
+			[],
+			"and no delete was enqueued against a uid the source no longer holds",
+		);
+		assert.equal(world.row.mailboxId, ARCHIVE);
+		assert.equal(world.row.uid, SETTLED_UID);
+	});
+});
+
+// The folder lookup is scoped to one account. The daily brief lists every
+// account's mail, so a selection made there carries rows this lookup cannot
+// resolve — and dropping them silently answered "deleted" over messages still
+// sitting where they were.
+describe("a delete reports a row whose folder it could not resolve", () => {
+	it("refuses it rather than counting it as deleted", async () => {
+		const world = buildWorld();
+		world.row.mailboxId = "mbx-another-account";
+		const service = buildMoveService(world);
+
+		const { refusedMessageIds } = await service.deleteMessages(
+			ACCOUNT_CONFIG,
+			[MESSAGE_ID],
+			ACCOUNT,
+		);
+
+		assert.deepEqual(refusedMessageIds, [MESSAGE_ID]);
+		assert.deepEqual(world.writes, [], "nothing was written");
+		assert.deepEqual(world.events, [], "and nothing was enqueued");
+	});
+});
+
+describe("a permanent delete writes only against the placement it read (R3)", () => {
+	// The row is already in Trash, so the batch takes the expunge path rather
+	// than the move-to-Trash one — a different writer, and the fourth of the
+	// four. Its event names one uid in one folder, so a row that has moved since
+	// the read must not be marked for it: a restore out of Trash settling in
+	// between was overwritten with `deleting`, and the expunge went after the
+	// uid Trash no longer holds.
+	const inTrash = (world: World): void => {
+		world.row.mailboxId = TRASH;
+	};
+
+	it("marks the message when nothing else touched the row", async () => {
+		const world = buildWorld();
+		inTrash(world);
+		const service = buildMoveService(world);
+
+		const { refusedMessageIds } = await service.deleteMessages(
+			ACCOUNT_CONFIG,
+			[MESSAGE_ID],
+			ACCOUNT,
+			{ permanent: true },
+		);
+
+		assert.deepEqual(refusedMessageIds, []);
+		assert.equal(world.row.status, "deleting");
+		assert.equal(world.events.length, 1);
+	});
+
+	it("reports the message, and enqueues nothing, when the row settles elsewhere", async () => {
+		let settled = false;
+		const world = buildWorld((w) => {
+			if (settled) return;
+			settled = true;
+			w.settleElsewhere();
+		});
+		inTrash(world);
+		const service = buildMoveService(world);
+
+		const { refusedMessageIds } = await service.deleteMessages(
+			ACCOUNT_CONFIG,
+			[MESSAGE_ID],
+			ACCOUNT,
+			{ permanent: true },
+		);
+
+		// Reported, never silent: the client removed the row optimistically, and
+		// one that reappears with nothing said is a failure the user cannot act
+		// on — the same class as a button that does nothing (#1229).
+		assert.deepEqual(refusedMessageIds, [MESSAGE_ID]);
+		assert.deepEqual(world.writes, [], "no placement was written");
+		assert.deepEqual(world.events, [], "and no expunge was enqueued");
+		assert.equal(world.row.mailboxId, ARCHIVE);
+		assert.equal(world.row.status, "active");
+	});
+});
+
+describe("a classification filing writes only against the placement it read (R3)", () => {
+	const buildPlacementService = (world: World, onPut?: () => void) => {
+		const markers = new Map<string, unknown>();
+		const pushed: string[] = [];
+		const markerService = {
+			find: async () => markers.get(MESSAGE_ID) ?? null,
+			// The last step between reading the row and writing it, so this is
+			// where a concurrent settle is staged.
+			put: async (input: Record<string, unknown>) => {
+				onPut?.();
+				const marker = { ...input, state: "pending", createdAt: Date.now() };
+				markers.set(MESSAGE_ID, marker);
+				return marker;
+			},
+			updateState: async () => markers.get(MESSAGE_ID),
+			delete: async () => {
+				markers.delete(MESSAGE_ID);
+			},
+		} as unknown as IMessagePlacementMoveRepository;
+
+		const service = new PlacementMoveService({
+			messageService: world.messageService,
+			threadMessageService: world.threadMessageService,
+			markerService,
+			addressService: world.addressService,
+			mailboxSpecialUseService: world.mailboxSpecialUseService,
+			sqsQueueUrl: "http://localhost:9324/000000000000/remit-message-mgmt",
+			moveSettleTimeoutMs: 100,
+			moveSettlePollMs: 10,
+		});
+		(service as unknown as { enqueuePush: () => Promise<void> }).enqueuePush =
+			async () => {
+				pushed.push(MESSAGE_ID);
+			};
+		return { service, markers, pushed };
+	};
+
+	it("files the message when nothing else touched the row", async () => {
+		const world = buildWorld();
+		const { service, markers, pushed } = buildPlacementService(world);
+
+		await service.moveMessage(ACCOUNT_CONFIG, MESSAGE_ID, JUNK, ACCOUNT);
+
+		assert.equal(world.row.mailboxId, JUNK);
+		assert.deepEqual(
+			world.threadMoves,
+			[JUNK],
+			"and the listing row follows the message it describes",
+		);
+		assert.deepEqual(pushed, [MESSAGE_ID]);
+		assert.ok(markers.get(MESSAGE_ID));
+	});
+
+	// The Message row is written first, so a loser leaves nothing behind. Moving
+	// the listing row ahead of the transition made the loser diverge instead: the
+	// listing showed the classification folder while the Message row sat where
+	// the winner put it, and nothing routine reconciles that.
+	it("leaves the listing row alone too when the row settles elsewhere", async () => {
+		const world = buildWorld();
+		const { service, markers, pushed } = buildPlacementService(world, () =>
+			world.settleElsewhere(),
+		);
+
+		await service.moveMessage(ACCOUNT_CONFIG, MESSAGE_ID, JUNK, ACCOUNT);
+
+		assert.deepEqual(world.writes, [], "no placement was written");
+		assert.deepEqual(
+			world.threadMoves,
+			[],
+			"and the listing row never claimed a folder the message is not in",
+		);
+		assert.deepEqual(pushed, [], "and no push was enqueued");
+		assert.equal(
+			markers.get(MESSAGE_ID),
+			undefined,
+			"the marker goes with it: nothing owes IMAP this move",
+		);
+		assert.equal(world.row.mailboxId, ARCHIVE);
+	});
+
+	// A permanent delete has already claimed the row. Waiting the ceiling out and
+	// then throwing drove an ordinary sequence toward the DLQ; there is nothing
+	// left to file, so this skips.
+	it("skips a message a delete has claimed, rather than burning the ceiling", async () => {
+		const world = buildWorld();
+		world.row.status = "deleting";
+		const { service, markers, pushed } = buildPlacementService(world);
+
+		const startedAt = Date.now();
+		await service.moveMessage(ACCOUNT_CONFIG, MESSAGE_ID, JUNK, ACCOUNT);
+
+		assert.ok(Date.now() - startedAt < 100, "and never entered the wait");
+		assert.deepEqual(world.writes, []);
+		assert.deepEqual(pushed, []);
+		assert.equal(markers.get(MESSAGE_ID), undefined);
+	});
+});