@remit/mailbox-service 0.0.48 → 0.0.50

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.48",
+  "version": "0.0.50",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/index.ts

@@ -217,6 +217,7 @@ export {
 	type ResolveExhaustedPlacementMoveResult,
 	resolveExhaustedPlacementMoveFailure,
 } from "./placement-move-terminal.js";
+export { isPlacementUnsettled } from "./placement-settled.js";
 export {
 	type QuarantineContext,
 	QuarantinedUids,

package/src/message-sync-changedsince.test.ts

@@ -14,7 +14,11 @@ import type {
 	UpdateMailboxInput,
 	UpdateThreadMessageInput,
 } from "@remit/data-ports";
-import { MailboxCursorState, MessageSystemFlag } from "@remit/domain-enums";
+import {
+	MailboxCursorState,
+	MessageKeywordFlag,
+	MessageSystemFlag,
+} from "@remit/domain-enums";
 import type { ManagedConnectionFactory } from "./connection-factory.js";
 import type { FlagPushService } from "./flag-push.js";
 import { FlagQueueService } from "./flag-queue.js";
@@ -112,6 +116,8 @@ interface Harness {
 	/** The canonical flag record after the round. */
 	flagStore: Set<string>;
 	messageFlagService: IMessageFlagRepository;
+	/** Method names invoked on the top-level address repository. */
+	addressCalls: string[];
 }
 
 const buildHarness = (options: HarnessOptions): Harness => {
@@ -241,12 +247,29 @@ const buildHarness = (options: HarnessOptions): Harness => {
 		},
 	} as unknown as IMessageFlagRepository;
 
+	// The existing-message path (applyChange -> applyServerFlags) never
+	// touches this repository. Recording every call, rather than leaving the
+	// stub as `{}`, is what makes "$Junk stays message state" an assertion
+	// instead of an assumption a future change could silently break.
+	const addressCalls: string[] = [];
+	const addressRepository = new Proxy(
+		{},
+		{
+			get:
+				(_target, prop: string) =>
+				(..._args: unknown[]) => {
+					addressCalls.push(prop);
+					throw new Error(`unexpected IAddressRepository.${prop} call`);
+				},
+		},
+	) as IAddressRepository;
+
 	const service = new MessageSyncService(
 		connectionFactory,
 		mailboxService,
 		{} as IMessageRepository,
 		{} as IEnvelopeRepository,
-		{} as IAddressRepository,
+		addressRepository,
 		threadMessageService,
 		logger,
 		unitOfWork,
@@ -263,6 +286,7 @@ const buildHarness = (options: HarnessOptions): Harness => {
 		errors,
 		flagStore,
 		messageFlagService,
+		addressCalls,
 	};
 };
 
@@ -603,6 +627,71 @@ describe("MessageSyncService CHANGEDSINCE path", () => {
 	});
 });
 
+// $Junk/$NotJunk mirroring (report-spam epic, message-state slice). This
+// mailbox is never remit-only — Apple Mail or another IMAP client is
+// routinely connected too — so a keyword another client set is message
+// state only. It must never become a standing rule about the sender.
+describe("MessageSyncService mirrors $Junk/$NotJunk keywords", () => {
+	it("mirrors a server-side $Junk into the canonical flag record", async () => {
+		const harness = buildHarness({
+			mailbox: mailbox(),
+			supportsCondstore: true,
+			changed: [serverMessage({ flags: [MessageKeywordFlag.Junk] })],
+			storedRows: [storedRow()],
+			storedFlags: [],
+		});
+
+		await syncOnce(harness);
+
+		assert.ok(harness.flagStore.has(MessageKeywordFlag.Junk));
+	});
+
+	it("mirrors a server-side $NotJunk into the canonical flag record", async () => {
+		const harness = buildHarness({
+			mailbox: mailbox(),
+			supportsCondstore: true,
+			changed: [serverMessage({ flags: [MessageKeywordFlag.NotJunk] })],
+			storedRows: [storedRow()],
+			storedFlags: [],
+		});
+
+		await syncOnce(harness);
+
+		assert.ok(harness.flagStore.has(MessageKeywordFlag.NotJunk));
+	});
+
+	it("leaves $Junk alone while its local flip is still owed to IMAP", async () => {
+		const harness = buildHarness({
+			mailbox: mailbox(),
+			supportsCondstore: true,
+			// Server has not seen the local flip yet.
+			changed: [serverMessage({ flags: [] })],
+			storedRows: [storedRow()],
+			storedFlags: [MessageKeywordFlag.Junk],
+			pendingFlags: new Set([MessageKeywordFlag.Junk]),
+		});
+
+		await syncOnce(harness);
+
+		assert.ok(harness.flagStore.has(MessageKeywordFlag.Junk));
+	});
+
+	it("mirroring $Junk leaves the sender's Address record completely untouched", async () => {
+		const harness = buildHarness({
+			mailbox: mailbox(),
+			supportsCondstore: true,
+			changed: [serverMessage({ flags: [MessageKeywordFlag.Junk] })],
+			storedRows: [storedRow()],
+			storedFlags: [],
+		});
+
+		await syncOnce(harness);
+
+		assert.ok(harness.flagStore.has(MessageKeywordFlag.Junk));
+		assert.deepEqual(harness.addressCalls, []);
+	});
+});
+
 describe("MessageSyncService without CONDSTORE", () => {
 	it("falls back to full enumeration and never issues CHANGEDSINCE", async () => {
 		const harness = buildHarness({

package/src/message-sync.ts

@@ -23,6 +23,7 @@ import {
 import {
 	AddressRole,
 	MailboxCursorState,
+	MessageKeywordFlag,
 	MessageSystemFlag,
 	StarColor,
 } from "@remit/domain-enums";
@@ -1025,8 +1026,9 @@ export class MessageSyncService {
 	}
 
 	/**
-	 * Bring a stored row's read and star state in line with the server's
-	 * flags.
+	 * Bring a stored row's read and star state, and the $Junk/$NotJunk
+	 * keywords on the canonical MessageFlag record, in line with the
+	 * server's flags.
 	 *
 	 * A field with a pending outbound push is left alone: the user flipped it
 	 * locally, IMAP has not been told yet, and the server's answer is
@@ -1073,6 +1075,17 @@ export class MessageSyncService {
 			}
 		}
 
+		await this.mirrorKeywordFlag(
+			existing.messageId,
+			MessageKeywordFlag.Junk,
+			flags,
+		);
+		await this.mirrorKeywordFlag(
+			existing.messageId,
+			MessageKeywordFlag.NotJunk,
+			flags,
+		);
+
 		if (Object.keys(updates).length === 0) return;
 
 		// MessageFlag is the canonical flag record — `FlagQueueService` reads it
@@ -1157,6 +1170,32 @@ export class MessageSyncService {
 		return marker !== null;
 	}
 
+	/**
+	 * Mirror one RFC 5788 keyword ($Junk / $NotJunk) from the server into the
+	 * canonical MessageFlag record, guarded the same way as the system flags:
+	 * a pending local push wins over the server's known-stale answer.
+	 *
+	 * This mailbox is never remit-only — Apple Mail or another IMAP client is
+	 * routinely connected too — so the keyword is message state, nothing
+	 * more. It never reaches the Address model: reading another client's
+	 * keyword as a standing rule about the sender is the exact fight this
+	 * mirror must not pick.
+	 */
+	private async mirrorKeywordFlag(
+		messageId: string,
+		flagName: string,
+		serverFlags: string[],
+	): Promise<void> {
+		if (!this.messageFlagService) return;
+		if (await this.hasPendingPush(messageId, flagName)) return;
+
+		const present = serverFlags.includes(flagName);
+		const hasFlag = await this.messageFlagService.hasFlag(messageId, flagName);
+		if (hasFlag === present) return;
+
+		await this.setMessageFlag(messageId, flagName, present);
+	}
+
 	/**
 	 * Fetch a batch of messages using the managed connection.
 	 * Assumes mailbox is already open from fetchUidsToSync.

package/src/placement-move-unsettled.test.ts

@@ -0,0 +1,225 @@
+/**
+ * Issue #496's rule applied to the placement move's own re-entry: a second
+ * verdict, for a different destination, arriving while the first move is still
+ * unconfirmed. The row then names the first destination but still carries the
+ * uid of the folder before it, so binding a fresh marker to that pair sends the
+ * reconciler to move the destination folder's OWN message at that uid.
+ */
+
+import assert from "node:assert/strict";
+import { afterEach, beforeEach, describe, it, mock } from "node:test";
+import { SQSClient } from "@aws-sdk/client-sqs";
+import type {
+	IMessagePlacementMoveRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+	MessageItem,
+	MessagePlacementMoveItem,
+	PutMessagePlacementMoveInput,
+	UpdateMessageMoveInput,
+} from "@remit/data-ports";
+import { PlacementMoveService } from "./placement-move.js";
+
+const ACCOUNT_ID = "acc-pm";
+const ACCOUNT_CONFIG_ID = "cfg-pm";
+const MESSAGE_ID = "msg-pm";
+const INBOX_ID = "mbx-inbox";
+const ARCHIVE_ID = "mbx-archive";
+const JUNK_ID = "mbx-junk";
+const INBOX_UID = 42;
+
+interface Harness {
+	service: PlacementMoveService;
+	row: MessageItem;
+	puts: PutMessagePlacementMoveInput[];
+	marker: MessagePlacementMoveItem | null;
+}
+
+const buildHarness = (): Harness => {
+	const row = {
+		messageId: MESSAGE_ID,
+		accountConfigId: ACCOUNT_CONFIG_ID,
+		mailboxId: INBOX_ID,
+		uid: INBOX_UID,
+		status: "active",
+		syncStatus: "synced",
+	} as unknown as MessageItem;
+
+	const puts: PutMessagePlacementMoveInput[] = [];
+	const harness = { row, puts, marker: null } as Harness;
+
+	const messageService = {
+		get: async () => row,
+		updateForMove: async (
+			_messageId: string,
+			input: UpdateMessageMoveInput,
+		) => {
+			Object.assign(row, input);
+			return row;
+		},
+	} as unknown as IMessageRepository;
+
+	const threadMessageService = {
+		getByMessageId: async () => ({
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			threadMessageId: "tm-pm",
+			sentDate: 1,
+			mailboxId: row.mailboxId,
+			isRead: false,
+			isDeleted: false,
+			hasStars: false,
+			hasAttachment: false,
+		}),
+		update: async () => {},
+	} as unknown as IThreadMessageRepository;
+
+	const markerService = {
+		put: async (input: PutMessagePlacementMoveInput) => {
+			puts.push(input);
+			harness.marker = {
+				...input,
+				state: "pending",
+				createdAt: Date.now(),
+			} as unknown as MessagePlacementMoveItem;
+			return harness.marker;
+		},
+		find: async () => harness.marker,
+		updateState: async (
+			_messageId: string,
+			state: MessagePlacementMoveItem["state"],
+		) => {
+			if (!harness.marker) throw new Error("no marker");
+			harness.marker.state = state;
+			return harness.marker;
+		},
+		delete: async () => {
+			harness.marker = null;
+		},
+	} as unknown as IMessagePlacementMoveRepository;
+
+	harness.service = new PlacementMoveService({
+		messageService,
+		threadMessageService,
+		markerService,
+		sqsQueueUrl: "https://sqs.eu-west-1.amazonaws.com/000/message-mgmt",
+		moveSettleTimeoutMs: 200,
+		moveSettlePollMs: 10,
+	});
+
+	return harness;
+};
+
+describe("PlacementMoveService — a second destination while the first move is unsettled (#496)", () => {
+	let harness: Harness;
+
+	beforeEach(() => {
+		mock.method(SQSClient.prototype, "send", async () => ({}));
+		harness = buildHarness();
+	});
+
+	afterEach(() => mock.restoreAll());
+
+	it("binds to the settled row once the earlier move confirms", async () => {
+		await harness.service.moveMessage(
+			ACCOUNT_CONFIG_ID,
+			MESSAGE_ID,
+			ARCHIVE_ID,
+			ACCOUNT_ID,
+		);
+		assert.equal(harness.row.status, "moving");
+		assert.equal(harness.row.uid, INBOX_UID);
+
+		// What the reconciler writes when the IMAP move lands: Archive's own
+		// COPYUID, and the status back to active.
+		setTimeout(() => {
+			Object.assign(harness.row, { uid: 907, status: "active" });
+			harness.marker = null;
+		}, 30);
+
+		await harness.service.moveMessage(
+			ACCOUNT_CONFIG_ID,
+			MESSAGE_ID,
+			JUNK_ID,
+			ACCOUNT_ID,
+		);
+
+		assert.equal(harness.puts.length, 2);
+		assert.equal(
+			harness.puts[1]?.sourceMailboxId,
+			ARCHIVE_ID,
+			"the source is where the confirmed move left the message",
+		);
+		assert.equal(harness.row.originalUid, 907, "and its confirmed uid");
+	});
+
+	it("does not apply the move when the earlier one never settles", async () => {
+		await harness.service.moveMessage(
+			ACCOUNT_CONFIG_ID,
+			MESSAGE_ID,
+			ARCHIVE_ID,
+			ACCOUNT_ID,
+		);
+
+		await assert.rejects(
+			() =>
+				harness.service.moveMessage(
+					ACCOUNT_CONFIG_ID,
+					MESSAGE_ID,
+					JUNK_ID,
+					ACCOUNT_ID,
+				),
+			/has not settled/,
+		);
+
+		assert.equal(harness.puts.length, 1, "no second marker was written");
+		assert.equal(
+			harness.puts[0]?.sourceMailboxId,
+			INBOX_ID,
+			"the surviving marker still names the folder the message is actually in",
+		);
+		assert.equal(harness.marker?.destinationMailboxId, ARCHIVE_ID);
+	});
+
+	it("drives a surviving pending marker for the same destination forward", async () => {
+		await harness.service.moveMessage(
+			ACCOUNT_CONFIG_ID,
+			MESSAGE_ID,
+			ARCHIVE_ID,
+			ACCOUNT_ID,
+		);
+		// The enqueue failed on the earlier call, so the marker never advanced.
+		if (harness.marker) harness.marker.state = "pending";
+
+		await harness.service.moveMessage(
+			ACCOUNT_CONFIG_ID,
+			MESSAGE_ID,
+			ARCHIVE_ID,
+			ACCOUNT_ID,
+		);
+
+		assert.equal(harness.puts.length, 1, "the local move is never re-derived");
+		assert.equal(harness.marker?.state, "queued");
+	});
+
+	it("moves a settled row normally", async () => {
+		await harness.service.moveMessage(
+			ACCOUNT_CONFIG_ID,
+			MESSAGE_ID,
+			ARCHIVE_ID,
+			ACCOUNT_ID,
+		);
+		Object.assign(harness.row, { uid: 907, status: "active" });
+		harness.marker = null;
+
+		await harness.service.moveMessage(
+			ACCOUNT_CONFIG_ID,
+			MESSAGE_ID,
+			JUNK_ID,
+			ACCOUNT_ID,
+		);
+
+		assert.equal(harness.puts.length, 2);
+		assert.equal(harness.puts[1]?.sourceMailboxId, ARCHIVE_ID);
+		assert.equal(harness.puts[1]?.destinationMailboxId, JUNK_ID);
+	});
+});

package/src/placement-move.ts

@@ -7,6 +7,10 @@ import type {
 } from "@remit/data-ports";
 import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
 import { createQueueProducer } from "@remit/sqs-client/producer";
+import {
+	isPlacementUnsettled,
+	waitForPlacementToSettle,
+} from "./placement-settled.js";
 
 /**
  * Event the reconciler (imap-worker `handlePlacementMovePush`) drains. Carries
@@ -41,8 +45,13 @@ export interface PlacementMoveConfig {
 	sqsQueueUrl: string;
 	sqsEndpoint?: string;
 	logger?: PlacementMoveLogger;
+	moveSettleTimeoutMs?: number;
+	moveSettlePollMs?: number;
 }
 
+const DEFAULT_MOVE_SETTLE_TIMEOUT_MS = 5_000;
+const DEFAULT_MOVE_SETTLE_POLL_MS = 250;
+
 /**
  * Local-first mover for a classification-driven placement move (issue #1271,
  * epic #1281). Distinct from {@link MessageMoveService} (user-initiated
@@ -78,6 +87,8 @@ export class PlacementMoveService {
 	private sqs: SQSClient;
 	private queueUrl: string;
 	private log: PlacementMoveLogger;
+	private moveSettleTimeoutMs: number;
+	private moveSettlePollMs: number;
 
 	constructor(config: PlacementMoveConfig) {
 		this.messageService = config.messageService;
@@ -85,6 +96,10 @@ export class PlacementMoveService {
 		this.markerService = config.markerService;
 		this.queueUrl = config.sqsQueueUrl;
 		this.log = config.logger ?? noopLogger;
+		this.moveSettleTimeoutMs =
+			config.moveSettleTimeoutMs ?? DEFAULT_MOVE_SETTLE_TIMEOUT_MS;
+		this.moveSettlePollMs =
+			config.moveSettlePollMs ?? DEFAULT_MOVE_SETTLE_POLL_MS;
 		this.sqs = createQueueProducer({
 			queueUrl: config.sqsQueueUrl,
 			endpoint: config.sqsEndpoint,
@@ -97,8 +112,7 @@ export class PlacementMoveService {
 		destinationMailboxId: string,
 		accountId: string,
 	): Promise<void> => {
-		const message = await this.messageService.get(messageId);
-		const sourceMailboxId = message.mailboxId;
+		let message = await this.messageService.get(messageId);
 
 		// Recovery: a marker already exists for THIS message and destination —
 		// from an earlier (possibly partially-failed) call. The marker's STATE,
@@ -128,6 +142,28 @@ export class PlacementMoveService {
 			return;
 		}
 
+		// A different destination, decided while an earlier move is still
+		// unconfirmed (issue #496): the row names that move's destination but its
+		// `uid` still belongs to the folder before it, so a marker bound to the
+		// pair as it stands sends the reconciler to move the destination folder's
+		// OWN message at that uid. Wait for the earlier move to confirm and bind
+		// to the settled row (docs/architecture/imap-mutations.md R2). Blocking is
+		// cheap — a move settles in well under a second — and on timeout the
+		// dependent write is simply not made.
+		if (isPlacementUnsettled(message)) {
+			message = await waitForPlacementToSettle(this.messageService, messageId, {
+				timeoutMs: this.moveSettleTimeoutMs,
+				pollMs: this.moveSettlePollMs,
+			});
+			if (isPlacementUnsettled(message)) {
+				throw new Error(
+					`Placement move for ${messageId} to ${destinationMailboxId} not applied: an earlier move has not settled`,
+				);
+			}
+		}
+
+		const sourceMailboxId = message.mailboxId;
+
 		// Genuine no-op: nothing pending for this destination, and the message
 		// is already there (a duplicate verdict recomputed after a confirmed
 		// move).

package/src/placement-settled.ts

@@ -0,0 +1,47 @@
+import type { MessageItem } from "@remit/data-ports";
+import { MessageStatus } from "@remit/domain-enums";
+
+/**
+ * Whether the message row's placement is still an unconfirmed local write
+ * (issue #496). While a move is in flight the row carries the destination in
+ * `mailboxId` but the SOURCE folder's `uid` — the pair is only made consistent
+ * when the IMAP move confirms and `updateUid` writes the destination's COPYUID.
+ * Any dependent mutation that resolves a folder and a uid from such a row
+ * therefore addresses the destination's OWN message at that uid: a different
+ * message, on a mailbox Remit is never the only client of.
+ *
+ * `syncStatus` is not the signal. An ordinary freshly-synced inbound row is
+ * `pending` forever (nothing on the inbound path promotes it), so keying off it
+ * defers every outbound mutation in the product. `status` is set to `moving`
+ * only by an actual move and cleared only once that move settles.
+ */
+export const isPlacementUnsettled = (
+	message: Pick<MessageItem, "status">,
+): boolean => message.status === MessageStatus.moving;
+
+export interface PlacementSettleOptions {
+	timeoutMs: number;
+	pollMs: number;
+}
+
+/**
+ * Block a dependent mutation until the row's placement settles, then hand back
+ * the confirmed row — the wait half of docs/architecture/imap-mutations.md R2.
+ * A move ordinarily settles in well under a second, so blocking is the cheap
+ * option the doc's default guidance names. The row comes back still unsettled
+ * when the deadline passes; the caller decides what a dependency that never
+ * settled means for it.
+ */
+export const waitForPlacementToSettle = async (
+	messageService: { get(messageId: string): Promise<MessageItem> },
+	messageId: string,
+	{ timeoutMs, pollMs }: PlacementSettleOptions,
+): Promise<MessageItem> => {
+	const deadline = Date.now() + timeoutMs;
+	let message = await messageService.get(messageId);
+	while (isPlacementUnsettled(message) && Date.now() < deadline) {
+		await new Promise((resolve) => setTimeout(resolve, pollMs));
+		message = await messageService.get(messageId);
+	}
+	return message;
+};