@remit/mailbox-service 0.0.111 → 0.0.113

package/src/placement-move-unsettled.test.ts

@@ -1,275 +0,0 @@
-/**
- * Issue #496's rule applied to the placement move's own re-entry: a second
- * verdict, for a different destination, arriving while the first move is still
- * unconfirmed. The row then names the first destination but still carries the
- * uid of the folder before it, so binding a fresh marker to that pair sends the
- * reconciler to move the destination folder's OWN message at that uid.
- */
-
-import assert from "node:assert/strict";
-import { afterEach, beforeEach, describe, it, mock } from "node:test";
-import { SQSClient } from "@aws-sdk/client-sqs";
-import type {
-	IAddressRepository,
-	IMessagePlacementMoveRepository,
-	IMessageRepository,
-	IThreadMessageRepository,
-	MessageItem,
-	MessagePlacementMoveItem,
-	PlacementPredicate,
-	PlacementTransitionInput,
-	PutMessagePlacementMoveInput,
-} from "@remit/data-ports";
-import { PlacementMoveService } from "./placement-move.js";
-import { noFolderRoles } from "./test-helpers/folder-roles.js";
-
-const ACCOUNT_ID = "acc-pm";
-const ACCOUNT_CONFIG_ID = "cfg-pm";
-const MESSAGE_ID = "msg-pm";
-const INBOX_ID = "mbx-inbox";
-const ARCHIVE_ID = "mbx-archive";
-const JUNK_ID = "mbx-junk";
-const INBOX_UID = 42;
-
-interface Harness {
-	service: PlacementMoveService;
-	row: MessageItem;
-	puts: PutMessagePlacementMoveInput[];
-	marker: MessagePlacementMoveItem | null;
-	reconciled: string[];
-}
-
-const buildHarness = (): Harness => {
-	const row = {
-		messageId: MESSAGE_ID,
-		accountConfigId: ACCOUNT_CONFIG_ID,
-		mailboxId: INBOX_ID,
-		uid: INBOX_UID,
-		status: "active",
-		syncStatus: "synced",
-	} as unknown as MessageItem;
-
-	const puts: PutMessagePlacementMoveInput[] = [];
-	const reconciled: string[] = [];
-	const harness = { row, puts, marker: null, reconciled } as Harness;
-
-	const addressService = {
-		reconcileJunkOnlyForMessage: async (messageId: string) => {
-			reconciled.push(`${messageId}@${row.mailboxId}`);
-		},
-	} as unknown as IAddressRepository;
-
-	const messageService = {
-		get: async () => row,
-		transitionPlacement: async (
-			_messageId: string,
-			expected: PlacementPredicate,
-			next: PlacementTransitionInput,
-		) => {
-			if (expected.status !== undefined && expected.status !== row.status) {
-				return undefined;
-			}
-			if (expected.uid !== undefined && expected.uid !== row.uid) {
-				return undefined;
-			}
-			if (
-				expected.mailboxId !== undefined &&
-				expected.mailboxId !== row.mailboxId
-			) {
-				return undefined;
-			}
-			Object.assign(row, next);
-			return row;
-		},
-	} as unknown as IMessageRepository;
-
-	const threadMessageService = {
-		getByMessageId: async () => ({
-			accountConfigId: ACCOUNT_CONFIG_ID,
-			threadMessageId: "tm-pm",
-			sentDate: 1,
-			mailboxId: row.mailboxId,
-			isRead: false,
-			isDeleted: false,
-			hasStars: false,
-			hasAttachment: false,
-		}),
-		update: async () => {},
-	} as unknown as IThreadMessageRepository;
-
-	const markerService = {
-		put: async (input: PutMessagePlacementMoveInput) => {
-			puts.push(input);
-			harness.marker = {
-				...input,
-				state: "pending",
-				createdAt: Date.now(),
-			} as unknown as MessagePlacementMoveItem;
-			return harness.marker;
-		},
-		find: async () => harness.marker,
-		updateState: async (
-			_messageId: string,
-			state: MessagePlacementMoveItem["state"],
-		) => {
-			if (!harness.marker) throw new Error("no marker");
-			harness.marker.state = state;
-			return harness.marker;
-		},
-		delete: async () => {
-			harness.marker = null;
-		},
-	} as unknown as IMessagePlacementMoveRepository;
-
-	harness.service = new PlacementMoveService({
-		messageService,
-		threadMessageService,
-		markerService,
-		addressService,
-		mailboxSpecialUseService: noFolderRoles,
-		sqsQueueUrl: "https://sqs.eu-west-1.amazonaws.com/000/message-mgmt",
-		moveSettleTimeoutMs: 200,
-		moveSettlePollMs: 10,
-	});
-
-	return harness;
-};
-
-describe("PlacementMoveService — a second destination while the first move is unsettled (#496)", () => {
-	let harness: Harness;
-
-	beforeEach(() => {
-		mock.method(SQSClient.prototype, "send", async () => ({}));
-		harness = buildHarness();
-	});
-
-	afterEach(() => mock.restoreAll());
-
-	it("binds to the settled row once the earlier move confirms", async () => {
-		await harness.service.moveMessage(
-			ACCOUNT_CONFIG_ID,
-			MESSAGE_ID,
-			ARCHIVE_ID,
-			ACCOUNT_ID,
-		);
-		assert.equal(harness.row.status, "moving");
-		assert.equal(harness.row.uid, INBOX_UID);
-
-		// What the reconciler writes when the IMAP move lands: Archive's own
-		// COPYUID, and the status back to active.
-		setTimeout(() => {
-			Object.assign(harness.row, { uid: 907, status: "active" });
-			harness.marker = null;
-		}, 30);
-
-		await harness.service.moveMessage(
-			ACCOUNT_CONFIG_ID,
-			MESSAGE_ID,
-			JUNK_ID,
-			ACCOUNT_ID,
-		);
-
-		assert.equal(harness.puts.length, 2);
-		assert.equal(
-			harness.puts[1]?.sourceMailboxId,
-			ARCHIVE_ID,
-			"the source is where the confirmed move left the message",
-		);
-		assert.equal(harness.row.originalUid, 907, "and its confirmed uid");
-	});
-
-	it("does not apply the move when the earlier one never settles", async () => {
-		await harness.service.moveMessage(
-			ACCOUNT_CONFIG_ID,
-			MESSAGE_ID,
-			ARCHIVE_ID,
-			ACCOUNT_ID,
-		);
-
-		await assert.rejects(
-			() =>
-				harness.service.moveMessage(
-					ACCOUNT_CONFIG_ID,
-					MESSAGE_ID,
-					JUNK_ID,
-					ACCOUNT_ID,
-				),
-			/has not settled/,
-		);
-
-		assert.equal(harness.puts.length, 1, "no second marker was written");
-		assert.equal(
-			harness.puts[0]?.sourceMailboxId,
-			INBOX_ID,
-			"the surviving marker still names the folder the message is actually in",
-		);
-		assert.equal(harness.marker?.destinationMailboxId, ARCHIVE_ID);
-	});
-
-	it("drives a surviving pending marker for the same destination forward", async () => {
-		await harness.service.moveMessage(
-			ACCOUNT_CONFIG_ID,
-			MESSAGE_ID,
-			ARCHIVE_ID,
-			ACCOUNT_ID,
-		);
-		// The enqueue failed on the earlier call, so the marker never advanced.
-		if (harness.marker) harness.marker.state = "pending";
-
-		await harness.service.moveMessage(
-			ACCOUNT_CONFIG_ID,
-			MESSAGE_ID,
-			ARCHIVE_ID,
-			ACCOUNT_ID,
-		);
-
-		assert.equal(harness.puts.length, 1, "the local move is never re-derived");
-		assert.equal(harness.marker?.state, "queued");
-	});
-
-	it("moves a settled row normally", async () => {
-		await harness.service.moveMessage(
-			ACCOUNT_CONFIG_ID,
-			MESSAGE_ID,
-			ARCHIVE_ID,
-			ACCOUNT_ID,
-		);
-		Object.assign(harness.row, { uid: 907, status: "active" });
-		harness.marker = null;
-
-		await harness.service.moveMessage(
-			ACCOUNT_CONFIG_ID,
-			MESSAGE_ID,
-			JUNK_ID,
-			ACCOUNT_ID,
-		);
-
-		assert.equal(harness.puts.length, 2);
-		assert.equal(harness.puts[1]?.sourceMailboxId, ARCHIVE_ID);
-		assert.equal(harness.puts[1]?.destinationMailboxId, JUNK_ID);
-	});
-
-	it("marks the sender when the classifier demotes the message", async () => {
-		await harness.service.moveMessage(
-			ACCOUNT_CONFIG_ID,
-			MESSAGE_ID,
-			JUNK_ID,
-			ACCOUNT_ID,
-		);
-
-		assert.deepEqual(harness.reconciled, [`${MESSAGE_ID}@${JUNK_ID}`]);
-	});
-
-	it("asks nothing of a destination the message is already in", async () => {
-		Object.assign(harness.row, { mailboxId: JUNK_ID });
-
-		await harness.service.moveMessage(
-			ACCOUNT_CONFIG_ID,
-			MESSAGE_ID,
-			JUNK_ID,
-			ACCOUNT_ID,
-		);
-
-		assert.deepEqual(harness.reconciled, []);
-	});
-});

package/src/placement-move.ts

@@ -1,336 +0,0 @@
-import { randomUUID } from "node:crypto";
-import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
-import type {
-	IAddressRepository,
-	IMailboxSpecialUseRepository,
-	IMessagePlacementMoveRepository,
-	IMessageRepository,
-	IThreadMessageRepository,
-} from "@remit/data-ports";
-import {
-	MessageMutation,
-	MessageStatus,
-	MessageSyncStatus,
-} from "@remit/domain-enums";
-import { createQueueProducer } from "@remit/sqs-client/producer";
-import {
-	isPlacementUnsettled,
-	waitForPlacementToSettle,
-} from "./placement-settled.js";
-
-/**
- * Event the reconciler (imap-worker `handlePlacementMovePush`) drains. Carries
- * ONLY our own message id — never a UID (epic #1281 invariant 1). The UID is
- * resolved fresh from the Message row at push time, so the event stays valid
- * across any amount of queue delay or an unrelated UIDVALIDITY rebuild
- * (#1272).
- */
-export interface PlacementMovePushEvent {
-	type: "PLACEMENT_MOVE_PUSH";
-	eventId: string;
-	timestamp: number;
-	accountId: string;
-	accountConfigId: string;
-	messageId: string;
-}
-
-export interface PlacementMoveLogger {
-	info(obj: Record<string, unknown>, msg: string): void;
-	error(obj: Record<string, unknown>, msg: string): void;
-}
-
-const noopLogger: PlacementMoveLogger = {
-	info: () => {},
-	error: () => {},
-};
-
-export interface PlacementMoveConfig {
-	messageService: IMessageRepository;
-	threadMessageService: IThreadMessageRepository;
-	markerService: IMessagePlacementMoveRepository;
-	addressService: IAddressRepository;
-	/**
-	 * Required, not optional: this mover carries automatic spam placement, and
-	 * one built without the folder-role seam would silently leave the sender of
-	 * every auto-filed spam suggestible in the compose picker.
-	 */
-	mailboxSpecialUseService: IMailboxSpecialUseRepository;
-	sqsQueueUrl: string;
-	sqsEndpoint?: string;
-	logger?: PlacementMoveLogger;
-	moveSettleTimeoutMs?: number;
-	moveSettlePollMs?: number;
-}
-
-const DEFAULT_MOVE_SETTLE_TIMEOUT_MS = 5_000;
-const DEFAULT_MOVE_SETTLE_POLL_MS = 250;
-
-/**
- * Local-first mover for a classification-driven placement move (issue #1271,
- * epic #1281). Distinct from {@link MessageMoveService} (user-initiated
- * moves/deletes/copies), which is untouched by this fix — the epic's UID
- * staleness invariant (1) only applies to the NEW reconciler this class
- * feeds; generalizing it to every move type is out of scope here.
- *
- * `moveMessage` matches {@link MessageMoveService.moveMessage}'s signature so
- * `BodySyncService`'s existing rescue/demote call site and its tests need no
- * shape changes beyond the dependency's name.
- *
- * Ordering is the fix: every step below (marker, ThreadMessage, Message row,
- * SQS enqueue) completes — or the whole call rejects — BEFORE the caller
- * (`BodySyncService.storeStreamedBody`) writes `bodyStorageKey`. A rejection
- * here therefore always happens before the body-sync skip-guard can trip, so
- * a retry reprocesses the message from scratch. Nothing here is swallowed —
- * let it crash, same as any other infra failure (an SQS enqueue failure is a
- * serious operational failure, not a routine blip: it propagates so a
- * repeated failure reaches the alarmed message-mgmt DLQ, never absorbed).
- *
- * Recovery is driven by the marker's explicit state engine (`pending ->
- * queued -> processing -> processed`, `MessagePlacementMoveState`), not by
- * comparing the message's current location against the requested
- * destination — that comparison alone cannot tell "already confirmed" apart
- * from "local move landed but the enqueue never did" (the defect this issue
- * originally fixes). A retry always re-fetches the marker and drives it
- * forward from whichever state it is actually in.
- */
-export class PlacementMoveService {
-	private messageService: IMessageRepository;
-	private threadMessageService: IThreadMessageRepository;
-	private markerService: IMessagePlacementMoveRepository;
-	private addressService: IAddressRepository;
-	private mailboxSpecialUseService: IMailboxSpecialUseRepository;
-	private sqs: SQSClient;
-	private queueUrl: string;
-	private log: PlacementMoveLogger;
-	private moveSettleTimeoutMs: number;
-	private moveSettlePollMs: number;
-
-	constructor(config: PlacementMoveConfig) {
-		this.messageService = config.messageService;
-		this.threadMessageService = config.threadMessageService;
-		this.markerService = config.markerService;
-		this.addressService = config.addressService;
-		this.mailboxSpecialUseService = config.mailboxSpecialUseService;
-		this.queueUrl = config.sqsQueueUrl;
-		this.log = config.logger ?? noopLogger;
-		this.moveSettleTimeoutMs =
-			config.moveSettleTimeoutMs ?? DEFAULT_MOVE_SETTLE_TIMEOUT_MS;
-		this.moveSettlePollMs =
-			config.moveSettlePollMs ?? DEFAULT_MOVE_SETTLE_POLL_MS;
-		this.sqs = createQueueProducer({
-			queueUrl: config.sqsQueueUrl,
-			endpoint: config.sqsEndpoint,
-		});
-	}
-
-	moveMessage = async (
-		accountConfigId: string,
-		messageId: string,
-		destinationMailboxId: string,
-		accountId: string,
-	): Promise<void> => {
-		let message = await this.messageService.get(messageId);
-
-		// Recovery: a marker already exists for THIS message and destination —
-		// from an earlier (possibly partially-failed) call. The marker's STATE,
-		// not a comparison against the message's current location, decides what
-		// happens next (PR #1289 review finding 1 + design amendment: an
-		// explicit pending -> queued -> processing -> processed engine).
-		const existingMarker = await this.markerService.find(messageId);
-		if (
-			existingMarker &&
-			existingMarker.destinationMailboxId === destinationMailboxId
-		) {
-			if (existingMarker.state === "pending") {
-				// Local move + marker already committed; only the enqueue is
-				// missing (or a further retry after the enqueue itself failed
-				// again). Drive the SAME marker forward — never re-derive the
-				// local move, which already happened.
-				await this.enqueuePush({ accountId, accountConfigId, messageId });
-				await this.markerService.updateState(messageId, "queued");
-				this.log.info(
-					{ messageId, accountId, destinationMailboxId },
-					"Pending marker survived an earlier enqueue failure; re-enqueued and advanced to queued",
-				);
-			}
-			// queued / processing / processed already has its own driver in
-			// flight (the SQS message, or the reconciler currently running) —
-			// nothing to do here either way.
-			return;
-		}
-
-		// A different destination, decided while an earlier move is still
-		// unconfirmed (issue #496): the row names that move's destination but its
-		// `uid` still belongs to the folder before it, so a marker bound to the
-		// pair as it stands sends the reconciler to move the destination folder's
-		// OWN message at that uid. Wait for the earlier move to confirm and bind
-		// to the settled row (docs/architecture/imap-mutations.md R2). Blocking is
-		// cheap — a move settles in well under a second — and on timeout the
-		// dependent write is simply not made.
-		// A delete in flight is not a move to wait out: the user has asked for the
-		// message to go, and a classification filing behind it has nothing left to
-		// file. Skipping is the answer, and it is a clean one — throwing would
-		// burn the ceiling and then drive an ordinary sequence toward the DLQ.
-		if (message.status === MessageStatus.deleting) {
-			this.log.info(
-				{ messageId, accountId, destinationMailboxId },
-				"Placement move not applied: this message is being deleted",
-			);
-			return;
-		}
-
-		if (isPlacementUnsettled(message)) {
-			message = await waitForPlacementToSettle(this.messageService, messageId, {
-				timeoutMs: this.moveSettleTimeoutMs,
-				pollMs: this.moveSettlePollMs,
-			});
-			if (message.status === MessageStatus.deleting) {
-				this.log.info(
-					{ messageId, accountId, destinationMailboxId },
-					"Placement move not applied: a delete claimed this message while the earlier move settled",
-				);
-				return;
-			}
-			if (isPlacementUnsettled(message)) {
-				throw new Error(
-					`Placement move for ${messageId} to ${destinationMailboxId} not applied: an earlier move has not settled`,
-				);
-			}
-		}
-
-		const sourceMailboxId = message.mailboxId;
-
-		// Genuine no-op: nothing pending for this destination, and the message
-		// is already there (a duplicate verdict recomputed after a confirmed
-		// move).
-		if (sourceMailboxId === destinationMailboxId) return;
-
-		// Fresh move. Strictly sequential — nothing here is parallel: the local
-		// move (marker + ThreadMessage + Message row) fully commits BEFORE the
-		// queue kick runs. `put` is idempotent and always resets `state` back to
-		// `pending`, since a fresh decision starts a new lifecycle regardless of
-		// what a stale row (different destination) held.
-		await this.markerService.put({
-			messageId,
-			accountId,
-			accountConfigId,
-			sourceMailboxId,
-			destinationMailboxId,
-		});
-
-		// The Message row first, the listing row after it — the order the user
-		// move and the delete already use. A transition off the row the wait above
-		// settled on (imap-mutations R3): this lane rides a standard queue that
-		// the account's FIFO group does not order, so a user's own move or delete
-		// can claim the row between that read and this write, and writing anyway
-		// would stamp Remit's classification over a folder the user just chose,
-		// with the old uid attached.
-		//
-		// Writing the listing row first meant a loser left the two disagreeing —
-		// the listing showing the classification folder while the Message row sat
-		// where the winner put it — and nothing routine reconciles that.
-		//
-		// The marker goes with the loss. It is the record that a move owes IMAP a
-		// push, and this move is not happening, so leaving it would send the
-		// reconciler after a placement nobody wrote and would inflate the folder
-		// counts it predicts.
-		const filed = await this.messageService.transitionPlacement(
-			messageId,
-			{
-				status: MessageStatus.active,
-				mailboxId: sourceMailboxId,
-				uid: message.uid,
-			},
-			{
-				mailboxId: destinationMailboxId,
-				status: MessageStatus.moving,
-				syncStatus: MessageSyncStatus.pending,
-				abandonedMutation: MessageMutation.none,
-				originalMailboxId: sourceMailboxId,
-				originalUid: message.uid,
-			},
-		);
-		if (!filed) {
-			await this.markerService.delete(messageId);
-			this.log.info(
-				{ messageId, accountId, sourceMailboxId, destinationMailboxId },
-				"Placement move not applied: something else claimed this message's placement first",
-			);
-			return;
-		}
-
-		await this.updateThreadMessageMailbox(
-			accountConfigId,
-			messageId,
-			destinationMailboxId,
-		);
-
-		await this.addressService.reconcileJunkOnlyForMessage(
-			messageId,
-			await this.mailboxSpecialUseService.resolveJunkRolesForConfig(
-				accountConfigId,
-			),
-		);
-
-		// The queue kick is a serious operational step, not a routine one — a
-		// failure here MUST propagate (never swallowed): the marker stays
-		// `pending`, the surrounding body-sync call fails, SQS redelivers, and
-		// a repeated failure reaches the alarmed message-mgmt DLQ. The state
-		// engine makes this *recoverable* (the recovery branch above drives a
-		// surviving `pending` marker forward on the next call) — it must never
-		// make it *quiet*.
-		await this.enqueuePush({ accountId, accountConfigId, messageId });
-		await this.markerService.updateState(messageId, "queued");
-
-		this.log.info(
-			{ messageId, accountId, sourceMailboxId, destinationMailboxId },
-			"Placement move applied locally; pending marker queued for IMAP push",
-		);
-	};
-
-	private updateThreadMessageMailbox = async (
-		accountConfigId: string,
-		messageId: string,
-		destinationMailboxId: string,
-	): Promise<void> => {
-		const threadMessage = await this.threadMessageService.getByMessageId(
-			accountConfigId,
-			messageId,
-		);
-
-		await this.threadMessageService.update(
-			threadMessage.accountConfigId,
-			threadMessage.threadMessageId,
-			{ mailboxId: destinationMailboxId },
-			{
-				composites: {
-					sentDate: threadMessage.sentDate,
-					mailboxId: threadMessage.mailboxId,
-					isRead: threadMessage.isRead,
-					isDeleted: threadMessage.isDeleted,
-					hasStars: threadMessage.hasStars,
-					hasAttachment: threadMessage.hasAttachment,
-				},
-			},
-		);
-	};
-
-	private enqueuePush = async (params: {
-		accountId: string;
-		accountConfigId: string;
-		messageId: string;
-	}): Promise<void> => {
-		const event: PlacementMovePushEvent = {
-			type: "PLACEMENT_MOVE_PUSH",
-			eventId: randomUUID(),
-			timestamp: Date.now(),
-			...params,
-		};
-		await this.sqs.send(
-			new SendMessageCommand({
-				QueueUrl: this.queueUrl,
-				MessageBody: JSON.stringify(event),
-			}),
-		);
-	};
-}