@remit/imap-worker 0.0.36 → 0.0.38

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.36",
+  "version": "0.0.38",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -32,8 +32,8 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1055.0",
-    "nodemailer": "^9.0.3",
     "@remit/data-ports": "*",
+    "@remit/smtp-service": "*",
     "@remit/logger-lambda": "*",
     "@remit/mailbox-service": "*",
     "@remit/search-index-worker": "*",
@@ -47,8 +47,7 @@
     "@aws-sdk/client-ssm": "*",
     "expect-env": "*",
     "p-map": "*",
-    "@types/aws-lambda": "*",
-    "@types/nodemailer": "*"
+    "@types/aws-lambda": "*"
   },
   "license": "MIT",
   "publishConfig": {

package/src/handlers/append-sent-message.ts

@@ -2,11 +2,13 @@ import { getClient } from "@remit/backend/client";
 import type {
 	IMailboxRepository,
 	IMailboxSpecialUseRepository,
-	OutboxMessageItem,
 } from "@remit/data-ports";
 import { MailboxSpecialUse } from "@remit/domain-enums";
 import type { Logger } from "@remit/logger-lambda";
-import nodemailer from "nodemailer";
+import {
+	buildMailMessage,
+	renderRawMessage,
+} from "@remit/smtp-service/message-builder";
 import { isAccountDeleted } from "../account-check.js";
 import { createConnectionScopeWithCredentials } from "../connection-scope.js";
 import type { AppendSentMessageEvent } from "../events.js";
@@ -46,35 +48,6 @@ const findSentMailbox = async (
 	return null;
 };
 
-const buildRawMessage = async (outbox: OutboxMessageItem): Promise<Buffer> => {
-	const from = outbox.fromName
-		? `${outbox.fromName} <${outbox.fromAddress}>`
-		: outbox.fromAddress;
-
-	const transport = nodemailer.createTransport({ streamTransport: true });
-
-	const info = await transport.sendMail({
-		from,
-		to: outbox.toAddresses.join(", "),
-		cc: outbox.ccAddresses?.join(", "),
-		bcc: outbox.bccAddresses?.join(", "),
-		replyTo: outbox.replyToAddress,
-		subject: outbox.subject,
-		text: outbox.textBody,
-		html: outbox.htmlBody,
-		messageId: `<${outbox.messageIdValue}>`,
-		inReplyTo: outbox.inReplyTo ? `<${outbox.inReplyTo}>` : undefined,
-		references: outbox.references?.map((r) => `<${r}>`).join(" "),
-		date: outbox.sentAt ? new Date(outbox.sentAt) : new Date(),
-	});
-
-	const chunks: Buffer[] = [];
-	for await (const chunk of info.message as AsyncIterable<Buffer>) {
-		chunks.push(chunk);
-	}
-	return Buffer.concat(chunks);
-};
-
 export interface AppendSentMessageDeps {
 	getClient: typeof getClient;
 	buildLifecycleDeps: typeof buildLifecycleDeps;
@@ -150,7 +123,7 @@ export const handleAppendSentMessage = async (
 			await scope
 				.getConnection()
 				.then(async (connection) => {
-					const rawMessage = await buildRawMessage(outbox);
+					const rawMessage = await renderRawMessage(buildMailMessage(outbox));
 
 					const result = await connection.append(
 						sentMailbox.fullPath,

package/src/handlers/flag-push-placement-race.test.ts

@@ -0,0 +1,360 @@
+/**
+ * Issue #496: auto-mark-read and the placement move are enqueued from the same
+ * body-sync pass onto two different queues, so the flag push can reach the
+ * server first. It must not, because the row it would resolve against names the
+ * destination mailbox while still carrying the source folder's uid — a STORE
+ * there lands on the destination's OWN message at that uid.
+ *
+ * These tests drive the REAL `PlacementMoveService` and `FlagQueueService` to
+ * produce the row and the marker, rather than hand-writing the shape the
+ * handler guards on. A change that stops `PlacementMoveService` marking the row
+ * `moving` reopens the defect while a hand-written fixture stays green.
+ */
+
+import assert from "node:assert/strict";
+import { afterEach, beforeEach, describe, it, mock } from "node:test";
+import { SQSClient } from "@aws-sdk/client-sqs";
+import { getClient, type RemitClient, setClient } from "@remit/backend/client";
+import type {
+	IMessageFlagPushRepository,
+	IMessageFlagRepository,
+	IMessagePlacementMoveRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+	MessageFlagPushItem,
+	MessageItem,
+	MessagePlacementMoveItem,
+	UpdateMessageMoveInput,
+} from "@remit/data-ports";
+import type { Logger } from "@remit/logger-lambda";
+import {
+	FlagPushService,
+	FlagQueueService,
+	PlacementMoveService,
+} from "@remit/mailbox-service";
+import type { FlagPushEvent } from "../events.js";
+import { handleFlagPush } from "./flag-push.js";
+
+const ACCOUNT_ID = "acc-496";
+const ACCOUNT_CONFIG_ID = "cfg-496";
+const MESSAGE_ID = "msg-496";
+const INBOX_ID = "mbx-inbox";
+const ARCHIVE_ID = "mbx-archive";
+/** The newsletter's uid in INBOX — and, on a real server, some unrelated message's uid in Archive. */
+const INBOX_UID = 42;
+const SEEN = "\\Seen";
+
+const silentLogger = (() => {
+	const noop = () => {};
+	const log = {
+		info: noop,
+		warn: noop,
+		error: noop,
+		debug: noop,
+		fatal: noop,
+		trace: noop,
+		child: () => log,
+	} as unknown as Logger;
+	return log;
+})();
+
+interface Harness {
+	row: MessageItem;
+	flagMarkers: Map<string, MessageFlagPushItem>;
+	placementMarkers: Map<string, MessagePlacementMoveItem>;
+	placementMoveService: PlacementMoveService;
+	flagQueueService: FlagQueueService;
+}
+
+const buildHarness = (): Harness => {
+	const row = {
+		messageId: MESSAGE_ID,
+		accountConfigId: ACCOUNT_CONFIG_ID,
+		mailboxId: INBOX_ID,
+		uid: INBOX_UID,
+		status: "active",
+		syncStatus: "synced",
+	} as unknown as MessageItem;
+
+	const flagMarkers = new Map<string, MessageFlagPushItem>();
+	const placementMarkers = new Map<string, MessagePlacementMoveItem>();
+	const localFlags = new Set<string>();
+
+	const messageService = {
+		get: async (messageId: string) => {
+			assert.equal(messageId, MESSAGE_ID);
+			return row;
+		},
+		updateForMove: async (
+			_messageId: string,
+			input: UpdateMessageMoveInput,
+		) => {
+			Object.assign(row, input);
+			return row;
+		},
+	} as unknown as IMessageRepository;
+
+	const threadMessageService = {
+		getByMessageId: async () => ({
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			threadMessageId: "tm-496",
+			sentDate: 1,
+			mailboxId: row.mailboxId,
+			isRead: false,
+			isDeleted: false,
+			hasStars: false,
+			hasAttachment: false,
+		}),
+		findAllByMessageId: async () => [],
+		update: async () => {},
+	} as unknown as IThreadMessageRepository;
+
+	const placementMarkerService = {
+		put: async (input: { messageId: string }) => {
+			const marker = {
+				...input,
+				state: "pending",
+				createdAt: Date.now(),
+			} as unknown as MessagePlacementMoveItem;
+			placementMarkers.set(input.messageId, marker);
+			return marker;
+		},
+		find: async (messageId: string) => placementMarkers.get(messageId) ?? null,
+		updateState: async (
+			messageId: string,
+			state: MessagePlacementMoveItem["state"],
+		) => {
+			const marker = placementMarkers.get(messageId);
+			if (!marker) throw new Error(`no placement marker for ${messageId}`);
+			marker.state = state;
+			return marker;
+		},
+		delete: async (messageId: string) => {
+			placementMarkers.delete(messageId);
+		},
+	} as unknown as IMessagePlacementMoveRepository;
+
+	const flagMarkerService = {
+		put: async (input: { messageId: string; flagName: string }) => {
+			const marker = {
+				...input,
+				state: "pending",
+				createdAt: Date.now(),
+			} as unknown as MessageFlagPushItem;
+			flagMarkers.set(`${input.messageId}:${input.flagName}`, marker);
+			return marker;
+		},
+		find: async (messageId: string, flagName: string) =>
+			flagMarkers.get(`${messageId}:${flagName}`) ?? null,
+		updateState: async (
+			messageId: string,
+			flagName: string,
+			state: MessageFlagPushItem["state"],
+		) => {
+			const marker = flagMarkers.get(`${messageId}:${flagName}`);
+			if (!marker) throw new Error(`no flag marker for ${messageId}`);
+			marker.state = state;
+			return marker;
+		},
+		delete: async (messageId: string, flagName: string) => {
+			flagMarkers.delete(`${messageId}:${flagName}`);
+		},
+	} as unknown as IMessageFlagPushRepository;
+
+	const messageFlagService = {
+		hasFlag: async (_messageId: string, flagName: string) =>
+			localFlags.has(flagName),
+		addFlag: async (_messageId: string, flagName: string) => {
+			localFlags.add(flagName);
+			return {} as never;
+		},
+		removeFlag: async (_messageId: string, flagName: string) => {
+			localFlags.delete(flagName);
+		},
+	} as unknown as IMessageFlagRepository;
+
+	const placementMoveService = new PlacementMoveService({
+		messageService,
+		threadMessageService,
+		markerService: placementMarkerService,
+		sqsQueueUrl: "https://sqs.eu-west-1.amazonaws.com/000/message-mgmt",
+	});
+
+	const flagQueueService = new FlagQueueService({
+		messageFlagService,
+		messageService,
+		threadMessageService,
+		flagPushService: new FlagPushService({
+			markerService: flagMarkerService,
+			sqsQueueUrl: "https://sqs.eu-west-1.amazonaws.com/000/messages",
+		}),
+	});
+
+	return {
+		row,
+		flagMarkers,
+		placementMarkers,
+		placementMoveService,
+		flagQueueService,
+	};
+};
+
+const event: FlagPushEvent = {
+	type: "FLAG_PUSH",
+	accountId: ACCOUNT_ID,
+	accountConfigId: ACCOUNT_CONFIG_ID,
+	messageId: MESSAGE_ID,
+	flagName: SEEN,
+} as FlagPushEvent;
+
+describe("auto-mark-read racing the placement move (#496)", () => {
+	let harness: Harness;
+
+	beforeEach(async () => {
+		mock.method(SQSClient.prototype, "send", async () => ({}));
+		harness = buildHarness();
+
+		setClient({
+			account: { get: async () => undefined },
+			message: { get: async () => undefined },
+			mailbox: { get: async () => undefined },
+			flagPush: {
+				find: async () => undefined,
+				updateState: async () => undefined,
+				delete: async () => undefined,
+			},
+		} as unknown as RemitClient);
+	});
+
+	afterEach(() => mock.restoreAll());
+
+	it("leaves the row naming the destination while it still carries the source uid", async () => {
+		await harness.placementMoveService.moveMessage(
+			ACCOUNT_CONFIG_ID,
+			MESSAGE_ID,
+			ARCHIVE_ID,
+			ACCOUNT_ID,
+		);
+
+		assert.equal(harness.row.mailboxId, ARCHIVE_ID);
+		assert.equal(
+			harness.row.uid,
+			INBOX_UID,
+			"the uid still belongs to INBOX — this pair is what any dependent push would resolve",
+		);
+		assert.equal(harness.row.status, "moving");
+	});
+
+	it("never pushes the flag while the move that row is waiting on has not settled", async () => {
+		await harness.placementMoveService.moveMessage(
+			ACCOUNT_CONFIG_ID,
+			MESSAGE_ID,
+			ARCHIVE_ID,
+			ACCOUNT_ID,
+		);
+		await harness.flagQueueService.markAsRead(
+			ACCOUNT_CONFIG_ID,
+			MESSAGE_ID,
+			ACCOUNT_ID,
+		);
+
+		const marker = harness.flagMarkers.get(`${MESSAGE_ID}:${SEEN}`);
+		assert.ok(marker, "the auto-read wrote a pending flag-push marker");
+		assert.equal(
+			(marker as unknown as { mailboxId: string }).mailboxId,
+			ARCHIVE_ID,
+			"the marker names the destination the unsettled move wrote",
+		);
+
+		const client = await getClient();
+		mock.method(client.account, "get", async () => ({
+			accountId: ACCOUNT_ID,
+			accountConfigId: ACCOUNT_CONFIG_ID,
+		}));
+		mock.method(client.message, "get", async () => [harness.row]);
+		const mailboxGet = mock.method(client.mailbox, "get", async () => ({
+			mailboxId: ARCHIVE_ID,
+			fullPath: "Archive",
+		}));
+		mock.method(client.flagPush, "find", async () => marker);
+		const updateState = mock.method(
+			client.flagPush,
+			"updateState",
+			async () => {},
+		);
+		const deleteMarker = mock.method(client.flagPush, "delete", async () => {});
+
+		// Captured rather than awaited bare: unguarded, the handler walks on to
+		// borrow a connection and fails there, which would mask the assertion
+		// below that says why it should never have got that far.
+		const fault = await handleFlagPush(event, silentLogger, 1).then(
+			() => undefined,
+			(error: unknown) => error,
+		);
+
+		assert.equal(
+			mailboxGet.mock.calls.length,
+			0,
+			"returns before resolving Archive — nothing reaches the server against uid 42",
+		);
+		assert.equal(fault, undefined, "the deferral acks, it does not fault");
+		assert.equal(
+			deleteMarker.mock.calls.length,
+			0,
+			"the read intent survives the deferral",
+		);
+		assert.deepEqual(updateState.mock.calls[0]?.arguments, [
+			MESSAGE_ID,
+			SEEN,
+			"pending",
+		]);
+	});
+
+	it("pushes once the move has settled and the row's uid matches its mailbox", async () => {
+		await harness.placementMoveService.moveMessage(
+			ACCOUNT_CONFIG_ID,
+			MESSAGE_ID,
+			ARCHIVE_ID,
+			ACCOUNT_ID,
+		);
+		await harness.flagQueueService.markAsRead(
+			ACCOUNT_CONFIG_ID,
+			MESSAGE_ID,
+			ACCOUNT_ID,
+		);
+
+		// What `updateUid` writes when the IMAP move confirms: the destination's
+		// own COPYUID, and the status back to active.
+		Object.assign(harness.row, {
+			uid: 907,
+			mailboxId: ARCHIVE_ID,
+			status: "active",
+			syncStatus: "synced",
+		});
+
+		const client = await getClient();
+		mock.method(client.account, "get", async () => ({
+			accountId: ACCOUNT_ID,
+			accountConfigId: ACCOUNT_CONFIG_ID,
+		}));
+		mock.method(client.message, "get", async () => [harness.row]);
+		const mailboxGet = mock.method(client.mailbox, "get", async () => {
+			throw Object.assign(new Error("stop before connecting"), {
+				name: "NotFoundError",
+			});
+		});
+		mock.method(client.flagPush, "find", async () =>
+			harness.flagMarkers.get(`${MESSAGE_ID}:${SEEN}`),
+		);
+		mock.method(client.flagPush, "updateState", async () => {});
+		mock.method(client.flagPush, "delete", async () => {});
+
+		await handleFlagPush(event, silentLogger, 1);
+
+		assert.equal(
+			mailboxGet.mock.calls.length,
+			1,
+			"the deferral is not permanent — a settled row resolves its mailbox and proceeds",
+		);
+	});
+});

package/src/handlers/flag-push.ts

@@ -1,10 +1,10 @@
 import { getClient } from "@remit/backend/client";
-import { MessageStatus } from "@remit/domain-enums";
 import type { Logger } from "@remit/logger-lambda";
 import { recordImapFailure } from "@remit/logger-lambda";
 import {
 	guardConnectionCursor,
 	isCursorRebuildNeeded,
+	isPlacementUnsettled,
 	MailboxCursorPausedError,
 	resolveExhaustedFlagPushFailure,
 } from "@remit/mailbox-service";
@@ -132,16 +132,14 @@ export const handleFlagPush = async (
 		return;
 	}
 
-	// The message carries a move still in flight — its `mailboxId`/`uid` are
-	// the local optimistic write `MessageMoveService` made before enqueueing
-	// the IMAP MOVE, not yet confirmed by the server, so a STORE resolved
-	// against this row right now would land on the wrong UID or the wrong
-	// folder (or both). `syncStatus` is NOT the right signal here: an ordinary
-	// freshly-synced inbound row is `pending` too (nothing on the inbound path
-	// ever promotes it to `synced`), so keying off it would defer every
-	// outbound flag push in the product forever. `status === moving` is set
-	// only by an actual move/delete-to-trash and cleared only once it settles.
-	if (message.status === MessageStatus.moving) {
+	// The message carries a move still in flight — `MessageMoveService`'s or
+	// `PlacementMoveService`'s local optimistic write, not yet confirmed by the
+	// server. A STORE resolved against this row addresses the destination
+	// folder at the SOURCE folder's uid: a different message (issue #496). This
+	// is the wait half of the wait-or-reconcile decision
+	// (docs/architecture/imap-mutations.md R2) — the push blocks until the move
+	// it depends on settles.
+	if (isPlacementUnsettled(message)) {
 		const deferredForMs = Date.now() - marker.createdAt;
 
 		if (deferredForMs > FLAG_PUSH_DEFER_MAX_MS) {