@remit/imap-worker 0.0.90 → 0.0.91

package/src/handlers/message-copy.ts

@@ -1,569 +0,0 @@
-import { getClient } from "@remit/backend/client";
-import {
-	MessageMutation,
-	MessageStatus,
-	MessageSyncStatus,
-} from "@remit/domain-enums";
-import type { Logger } from "@remit/logger";
-import {
-	guardConnectionCursor,
-	type IImapConnection,
-	isCursorRebuildNeeded,
-	MailboxCursorPausedError,
-} from "@remit/mailbox-service";
-import { attemptBudget } from "@remit/sqs-client/attempt-budget";
-import { isAccountDeleted } from "../account-check.js";
-import { createConnectionScopeWithCredentials } from "../connection-scope.js";
-import { createEmitEvent } from "../emit.js";
-import type { MessageCopyEvent } from "../events.js";
-import { findMailboxRow } from "../mailbox-row.js";
-import type { WorkerQueues } from "../queues.js";
-import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
-import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
-import {
-	emitMoveResync,
-	searchMailboxForHighestMessageIdUid,
-} from "./message-move.js";
-import { probeableMessageIdHeader } from "./probe-message-id.js";
-
-export const getMessageCopyMaxAttempts = (
-	processEnv: NodeJS.ProcessEnv = process.env,
-): number => attemptBudget("MESSAGE_COPY_MAX_ATTEMPTS", 3, processEnv);
-
-export const MESSAGE_COPY_MAX_ATTEMPTS = getMessageCopyMaxAttempts();
-
-/**
- * Where the destination says this copy is. `unprobeable` is the row that
- * carries no usable Message-ID header (drafts, automated senders and bounces
- * do arrive without one): the server was never
- * asked, so its silence says nothing.
- */
-export type CopyProbe =
-	| { kind: "confirmed"; uid: number }
-	| { kind: "absent" }
-	| { kind: "unprobeable" };
-
-/**
- * Handle MESSAGE_COPY events.
- * Executes IMAP COPY command and updates local state with new UID.
- *
- * IMAP COPY is not idempotent and the message queue is FIFO per account, so
- * this handler never manufactures a retry to resolve an ambiguous answer: a
- * second COPY duplicates the mail, and a record that keeps coming back blocks
- * every later move, copy and delete for that account (issues #287, #289,
- * #290). Ambiguity is resolved in place, by asking the destination.
- *
- * A fault the server threw is still retried up to
- * {@link MESSAGE_COPY_MAX_ATTEMPTS}, because it may be a dropped connection.
- * Two guards make that safe: a redelivery asks the destination BEFORE issuing
- * COPY again, and the last attempt settles the row instead of dead-lettering
- * it. The copy row is a reconcile-model dependent (`copySettledMessage` writes
- * its ThreadMessage row at `uid: 0` before the server has confirmed anything,
- * docs/architecture/imap-mutations.md R2) and these terminal settles are its
- * reconciliation path: every verdict the server gives an answer for settles the
- * row out of `uid: 0`/`moving` (#1097). A server that cannot be reached at all
- * gives no verdict — the record dead-letters with the row untouched, the same
- * caveat `resolveExhaustedMessageMoveFailure` carries, because absence is only
- * ever concluded from an answer.
- */
-export const handleMessageCopy = async (
-	event: MessageCopyEvent,
-	log: Logger,
-	queues: WorkerQueues,
-	receiveCount = 1,
-	createConnectionScope = createConnectionScopeWithCredentials,
-): Promise<void> => {
-	const emit = createEmitEvent(queues);
-	const {
-		account: accountService,
-		message: messageService,
-		threadMessage: threadMessageService,
-		mailbox: mailboxService,
-		secrets,
-	} = await getClient();
-
-	const {
-		accountId,
-		sourceMessageId,
-		newMessageId,
-		sourceMailboxId,
-		destinationMailboxId,
-		uid,
-	} = event;
-
-	log.info(
-		{
-			event: event.type,
-			accountId,
-			sourceMessageId,
-			newMessageId,
-			from: sourceMailboxId,
-			to: destinationMailboxId,
-		},
-		"Handling event",
-	);
-
-	const account = await accountService.get(accountId);
-	if (!account) {
-		throw new Error(`Account ${accountId} not found`);
-	}
-
-	if (isAccountDeleted(account, log)) {
-		return;
-	}
-
-	const [copyRow] = await messageService.get([newMessageId]);
-
-	// The copy row is already gone — a reconciliation path settled it. Nothing
-	// left to bind a UID to; ack.
-	if (!copyRow) {
-		log.warn(
-			{ accountId, sourceMessageId, newMessageId },
-			"Skipping MESSAGE_COPY: copy row no longer exists",
-		);
-		return;
-	}
-
-	// This copy already settled — `updateUid` cleared `status: moving` when the
-	// server confirmed it. A redelivery reaching here would COPY the message a
-	// second time, and COPY has no source-side effect to make that a no-op.
-	//
-	// `moving` specifically, not the shared unsettled predicate: the question is
-	// whether THIS copy is still outstanding, and a `deleting` row is somebody
-	// else's work — reading it as unsettled is what would let the second COPY
-	// through.
-	const copyPlacement = {
-		status: copyRow.status,
-		mailboxId: copyRow.mailboxId,
-		uid: copyRow.uid,
-	};
-	if (copyRow.status !== MessageStatus.moving) {
-		log.info(
-			{ accountId, newMessageId, uid: copyRow.uid, status: copyRow.status },
-			"Skipping MESSAGE_COPY: the copy already settled against confirmed IMAP state",
-		);
-		return;
-	}
-
-	// `failed` is an earlier attempt's mark, and the re-drive's: either may
-	// have issued a COPY whose answer was lost.
-	const earlierAttempt =
-		receiveCount > 1 || copyRow.syncStatus === MessageSyncStatus.failed;
-
-	await withOAuthLifecycle(
-		buildLifecycleDeps(secrets, accountService),
-		account,
-		log,
-		async (credentials) => {
-			// The source folder can be deleted between enqueue and this sync, leaving
-			// a queued event pointing at a gone row. The lookup then throws
-			// NotFoundError forever, and on the account's per-group FIFO that head
-			// message stalls the whole pipeline (issues #287, #289, #290). A deleted
-			// source mailbox makes the copy moot: ack with a WARN.
-			const source = await findMailboxRow(
-				mailboxService,
-				accountId,
-				sourceMailboxId,
-			);
-			if (source.kind === "gone") {
-				log.warn(
-					{ accountId, sourceMessageId, mailboxId: sourceMailboxId },
-					"Skipping MESSAGE_COPY: source mailbox no longer exists (deleted)",
-				);
-				return;
-			}
-			const mailbox = source.mailbox;
-			const sourceMailboxPath = mailbox.fullPath;
-
-			const destination = await findMailboxRow(
-				mailboxService,
-				accountId,
-				destinationMailboxId,
-			);
-			if (destination.kind === "gone") {
-				const reconciled = await messageService.deleteIfPlacement(
-					newMessageId,
-					copyPlacement,
-				);
-				log.warn(
-					{
-						metric: "message_copy_destination_gone",
-						accountId,
-						sourceMessageId,
-						newMessageId,
-						mailboxId: destinationMailboxId,
-						reconciled,
-					},
-					"Skipping MESSAGE_COPY: destination mailbox no longer exists (deleted); optimistic copy row reconciled away",
-				);
-				return;
-			}
-			const destinationMailboxPath = destination.mailbox.fullPath;
-
-			// The copy row carries the source's Message-ID header, so the
-			// destination can be asked where this copy is. Always on the UNGUARDED
-			// handle: a `guardConnectionCursor` wrap is bound to the SOURCE mailbox
-			// snapshot alone, and opening the destination through it trips that
-			// mailbox into a rebuild.
-			const probeDestination = async (
-				connection: IImapConnection,
-			): Promise<CopyProbe> => {
-				const messageIdHeader = await probeableMessageIdHeader(
-					threadMessageService,
-					account.accountConfigId,
-					newMessageId,
-				);
-				if (!messageIdHeader) return { kind: "unprobeable" };
-				const probedUid = await searchMailboxForHighestMessageIdUid(
-					connection,
-					destinationMailboxPath,
-					messageIdHeader,
-				);
-				return probedUid === null
-					? { kind: "absent" }
-					: { kind: "confirmed", uid: probedUid };
-			};
-
-			const settleCopied = async (newUid: number): Promise<void> => {
-				// `updateUid` writes the confirmed uid, `active` and `synced` in one
-				// statement, so the second write this used to make said nothing the
-				// first had not — and said it without a predicate.
-				await messageService.updateUid(
-					newMessageId,
-					newUid,
-					destinationMailboxId,
-				);
-
-				const threadMessage = await threadMessageService.findByMessageId(
-					account.accountConfigId,
-					newMessageId,
-				);
-				if (threadMessage) {
-					await threadMessageService.update(
-						threadMessage.accountConfigId,
-						threadMessage.threadMessageId,
-						{ uid: newUid },
-						{
-							composites: {
-								sentDate: threadMessage.sentDate,
-								mailboxId: threadMessage.mailboxId,
-								isRead: threadMessage.isRead,
-								isDeleted: threadMessage.isDeleted,
-								hasStars: threadMessage.hasStars,
-								hasAttachment: threadMessage.hasAttachment,
-							},
-						},
-					);
-				}
-
-				log.info(
-					{
-						sourceMessageId,
-						newMessageId,
-						oldUid: uid,
-						newUid,
-						destination: destinationMailboxPath,
-					},
-					"Message copied successfully",
-				);
-			};
-
-			// The destination answered that it does not hold this message, so the
-			// copy never landed and the optimistic row records something the server
-			// does not have. Deleting it (with its ThreadMessage rows) is the exact
-			// reality, the same terminal outcome `resolveExhaustedMessageMoveFailure`
-			// calls `reconciled`. Routine: metric, no alarm.
-			const settleNeverLanded = async (reason: string): Promise<void> => {
-				const reconciled = await messageService.deleteIfPlacement(
-					newMessageId,
-					copyPlacement,
-				);
-				log.info(
-					{
-						metric: "message_copy_not_landed",
-						accountId,
-						sourceMessageId,
-						newMessageId,
-						destination: destinationMailboxPath,
-						reconciled,
-						reason,
-					},
-					"Copy absent from its destination; optimistic copy row reconciled away",
-				);
-			};
-
-			// The server never answered where the copy is, so nothing here may
-			// delete a row that might describe real mail. The row is settled out of
-			// `moving` instead — `status: deleted` is the state this handler already
-			// writes for a copy that will not happen, and it keeps `holdsCopyOf`
-			// answering true so a sighting at the destination cannot drag the SOURCE
-			// row out of the folder it was copied from.
-			//
-			// The cost, paid deliberately: if that COPY did land, this row is the
-			// only one the copy will ever have — `holdsCopyOf` tests row existence
-			// alone (`message-sync.ts`), so no sync creates a second one — and
-			// `deleted` hides it. Real mail then sits at the destination on the
-			// server and invisible in the product until an operator acts on the
-			// alert. `settleNeverLanded` has the opposite property: it deletes only
-			// what the server denied holding, and a later sync re-projects anything
-			// that turns out to be there. Silence buys the safer half of one pair
-			// and the worse half of the other.
-			const settleBroken = async (reason: string): Promise<void> => {
-				// A transition, not a plain update (imap-mutations R3): the copy row
-				// is only this handler's to settle while its own placement is still
-				// unsettled. The row it leaves is `deleted`, so no listing carries it
-				// and the marker is read by nothing — it is written anyway, because a
-				// give-up that does not name itself is the gap #1229 came from.
-				await messageService.transitionPlacement(
-					newMessageId,
-					{ status: MessageStatus.moving },
-					{
-						status: MessageStatus.deleted,
-						syncStatus: MessageSyncStatus.abandoned,
-						abandonedMutation: MessageMutation.copy,
-					},
-				);
-				log.error(
-					{
-						alert: "message_copy_unconfirmed",
-						accountId,
-						sourceMessageId,
-						newMessageId,
-						uid,
-						destination: destinationMailboxPath,
-						reason,
-					},
-					"Copy could not be bound to a destination UID; row settled failed for operator investigation",
-				);
-			};
-
-			const scope = createConnectionScope(account, credentials);
-
-			// A paused cursor settles rather than acks: acking leaves the optimistic
-			// row at `uid: 0`/`moving` with nothing to re-enqueue it and no folder
-			// set for the cursor rebuild to find it in (issue #1203). Like every
-			// other verdict on this row, it reconciles rather than waits (R2,
-			// docs/architecture/imap-mutations.md) — the settle below IS the
-			// reconciliation path, since the destination is not paused and no
-			// rebuild is coming for it.
-			//
-			// What may be concluded from the pause depends on what was issued, and
-			// `receiveCount` alone does not say: `commandIssued` is what this round
-			// did, a redelivery is what an earlier one may have done and lost with
-			// its connection. Where neither says the server may have been asked, the
-			// row records something the server was never told about and reconciling
-			// it away is exact. Otherwise this row may be the only record real mail
-			// will ever have, and deleting it would let the next destination sync
-			// repoint the SOURCE row out of the folder the server still holds it in.
-			// The pause is on the source and says nothing about the destination,
-			// which is probed on the unguarded handle anyway, so the destination is
-			// asked before anything is deleted and silence settles broken.
-			//
-			// A settle is a settle, so both folders re-read their counts from IMAP,
-			// the same resync the move and delete handlers run: the destination is
-			// not paused, so nothing else is coming to re-project it.
-			const settlePausedCopy = async (
-				commandIssued: boolean,
-			): Promise<void> => {
-				const mayHaveIssued = commandIssued || earlierAttempt;
-
-				if (!mayHaveIssued) {
-					await settleNeverLanded(
-						"source mailbox cursor paused before the copy",
-					);
-				} else {
-					const probe = await probeDestination(await scope.getConnection());
-					if (probe.kind === "confirmed") {
-						await settleCopied(probe.uid);
-					} else if (probe.kind === "absent") {
-						await settleNeverLanded(
-							"redelivered onto a paused source cursor, absent from destination",
-						);
-					} else {
-						await settleBroken(
-							"redelivered onto a paused source cursor with no Message-ID to ask the destination with",
-						);
-					}
-				}
-
-				await emitMoveResync(emit, {
-					accountId,
-					sourceMailboxId,
-					destinationMailboxId,
-				});
-			};
-
-			// Cheap frugal skip (epic #1281 invariant 6): a mailbox already known
-			// paused never even opens a connection on a first delivery. Optimization
-			// only — the guardConnectionCursor openBox wrap below is the structural
-			// guarantee.
-			if (isCursorRebuildNeeded(mailbox.cursorState)) {
-				log.info(
-					{ accountId, sourceMessageId, mailboxId: sourceMailboxId },
-					"Mailbox cursor not normal; pausing outbound copy this round",
-				);
-				await settlePausedCopy(false).finally(() => scope.disconnect());
-				return;
-			}
-
-			// Flipped the instant the COPY leaves, so every settle downstream of it
-			// knows the server was asked. IMAP COPY is not idempotent, so this is
-			// the flag that must never be inferred.
-			let commandIssued = false;
-
-			await scope
-				.getConnection()
-				.then(async (rawConnection) => {
-					// Guard at the openBox choke point (epic #1281 invariants 3 & 5):
-					// a fresh mismatch trips the mailbox and throws once the SELECT
-					// reveals it.
-					const connection = guardConnectionCursor(
-						rawConnection,
-						{ mailboxService },
-						accountId,
-						mailbox,
-					);
-
-					// A redelivery means the previous attempt threw, which does not
-					// mean its COPY failed — the tagged OK can be lost with the
-					// connection. Ask before copying again, or the retry is what
-					// duplicates the mail. `absent` is an answer the server gave, so
-					// copying again is safe; `unprobeable` is silence, and copying on
-					// silence is the duplicate this guard exists to prevent.
-					if (earlierAttempt) {
-						const earlier = await probeDestination(rawConnection);
-						if (earlier.kind === "confirmed") {
-							await settleCopied(earlier.uid);
-							return;
-						}
-						if (earlier.kind === "unprobeable") {
-							await settleBroken(
-								"redelivered with no Message-ID to ask the destination with",
-							);
-							return;
-						}
-					}
-
-					// Open source mailbox (read-only is fine for COPY)
-					await connection.openBox(sourceMailboxPath, true);
-
-					// Execute IMAP COPY
-					commandIssued = true;
-					const result = await connection.copyMessages(
-						[uid],
-						destinationMailboxPath,
-					);
-
-					const copiedUid = result.uidMap.get(uid);
-					if (copiedUid) {
-						await settleCopied(copiedUid);
-						return;
-					}
-
-					// A server without UIDPLUS answers a perfectly successful COPY
-					// with no COPYUID entry, so an empty map is UNCONFIRMED, never
-					// evidence the server matched nothing (#1097). The destination is
-					// asked before any verdict, as `message-move.ts` does (#912, #979).
-					const probe = await probeDestination(rawConnection);
-					if (probe.kind === "confirmed") {
-						await settleCopied(probe.uid);
-						return;
-					}
-					if (probe.kind === "absent") {
-						await settleNeverLanded(
-							"no COPYUID entry, absent from destination",
-						);
-						return;
-					}
-					await settleBroken(
-						"no COPYUID entry and no Message-ID to probe with",
-					);
-				})
-				.catch(async (error: unknown) => {
-					if (error instanceof MailboxCursorPausedError) {
-						log.info(
-							{
-								accountId,
-								sourceMessageId,
-								mailboxId: sourceMailboxId,
-								cursorState: error.state,
-							},
-							"Mailbox cursor not normal; pausing outbound copy this round",
-						);
-						await settlePausedCopy(commandIssued);
-						return;
-					}
-
-					const errorMessage =
-						error instanceof Error ? error.message : String(error);
-
-					// Handle TRYCREATE - destination doesn't exist
-					if (errorMessage.includes("TRYCREATE")) {
-						log.info(
-							{ destinationMailboxPath },
-							"Destination mailbox doesn't exist, creating",
-						);
-						const connection = await scope.getConnection();
-						await connection.createMailbox(destinationMailboxPath);
-						// Re-throw to let the event be retried against the folder just
-						// created. Kept unconditional past the attempt budget, as the
-						// move path does: the destination now exists, so the record is
-						// worth redriving from the DLQ.
-						throw error;
-					}
-
-					// Handle source message not found on IMAP - already deleted (fail the copy)
-					if (
-						errorMessage.includes("not found") ||
-						errorMessage.includes("NONEXISTENT")
-					) {
-						log.info(
-							{ sourceMessageId, uid },
-							"Source message not found on IMAP, marking copy as failed",
-						);
-						await messageService.transitionPlacement(
-							newMessageId,
-							{ status: MessageStatus.moving },
-							{
-								status: MessageStatus.deleted,
-								syncStatus: MessageSyncStatus.abandoned,
-								abandonedMutation: MessageMutation.copy,
-							},
-						);
-						return;
-					}
-
-					if (receiveCount < MESSAGE_COPY_MAX_ATTEMPTS) {
-						// Transient copy failure — expected (connections drop). Queue
-						// redelivery retries, and the probe above keeps the retry from
-						// copying the message twice.
-						await messageService.transitionPlacement(
-							newMessageId,
-							{ status: MessageStatus.moving },
-							{ syncStatus: MessageSyncStatus.failed },
-						);
-						throw error;
-					}
-
-					// Redelivery budget exhausted. Dead-lettering here would leave the
-					// row at `uid: 0`/`moving` with nothing left to settle it and the
-					// account's FIFO group still holding the record, so the row is
-					// resolved into one terminal outcome instead — never by inferring
-					// the server's state from our own failures. A server that cannot be
-					// reached reaches no verdict: the probe throws and the record
-					// dead-letters with the row untouched.
-					const probe = await probeDestination(await scope.getConnection());
-					if (probe.kind === "confirmed") {
-						await settleCopied(probe.uid);
-						return;
-					}
-					if (probe.kind === "absent") {
-						await settleNeverLanded(`retry exhausted: ${errorMessage}`);
-						return;
-					}
-					await settleBroken(`retry exhausted: ${errorMessage}`);
-				})
-				.finally(() => scope.disconnect());
-		},
-	);
-};