@remit/imap-worker 0.0.90 → 0.0.92

package/src/handlers/sync-message-body.ts

@@ -1,366 +0,0 @@
-import { getClient } from "@remit/backend/client";
-import { isNotFoundError } from "@remit/data-ports/errors";
-import { SYNC_WARNING_SURFACE_ATTEMPTS } from "@remit/data-ports/sync-warning-state";
-import type { Logger } from "@remit/logger";
-import { recordImapFailure } from "@remit/logger-lambda/metrics";
-import {
-	BodySyncService,
-	guardConnectionCursor,
-	isCursorRebuildNeeded,
-	isFolderMutationInFlight,
-	MailboxCursorPausedError,
-	PlacementMoveService,
-	QuarantineService,
-	resolveExhaustedBodySyncFailures,
-	resolveFolderPlacement,
-	SyncWarningService,
-} from "@remit/mailbox-service";
-import { buildFilterConfig } from "@remit/mailbox-service/filter-config";
-import { attemptBudget } from "@remit/sqs-client/attempt-budget";
-import { env } from "expect-env";
-import { isAccountDeleted } from "../account-check.js";
-import { isBodySyncEnabled } from "../body-sync-gate.js";
-import {
-	borrowWarmConnection,
-	createConnectionScopeWithCredentials,
-} from "../connection-scope.js";
-import type { SyncMessageBodyEvent } from "../events.js";
-import type { WorkerQueues } from "../queues.js";
-import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
-import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
-import { workerVersion } from "../worker-version.js";
-
-const bodySyncEnabledParameterName = env.BODY_SYNC_ENABLED_PARAMETER_NAME;
-
-export const getBodySyncMaxAttempts = (
-	processEnv: NodeJS.ProcessEnv = process.env,
-): number =>
-	attemptBudget(
-		"BODY_SYNC_MAX_ATTEMPTS",
-		SYNC_WARNING_SURFACE_ATTEMPTS,
-		processEnv,
-	);
-
-export const BODY_SYNC_MAX_ATTEMPTS = getBodySyncMaxAttempts();
-
-/**
- * The ordered message ids to sync, plus a uid lookup when the event carried the
- * preferred `messages` shape. The uid map lets the body-sync service skip the
- * per-message DDB get; legacy events (ids only) leave it undefined and the
- * service resolves uids itself.
- */
-export interface ResolvedBatch {
-	messageIds: string[];
-	uidByMessageId?: Map<string, number>;
-	force: boolean;
-}
-
-export const resolveBatch = (event: SyncMessageBodyEvent): ResolvedBatch => {
-	const force = event.force === true;
-	if (event.messages !== undefined) {
-		return {
-			messageIds: event.messages.map((m) => m.messageId),
-			uidByMessageId: new Map(
-				event.messages.map((m): [string, number] => [m.messageId, m.uid]),
-			),
-			force,
-		};
-	}
-	return { messageIds: event.messageIds, force };
-};
-
-/**
- * Build the loud, structured error thrown when a batch still has failed
- * messages and SQS redelivery budget left. Throwing (rather than swallowing
- * into a fresh re-enqueue) is what lets a genuine processing failure reach
- * the body-dlq once `BODY_SYNC_MAX_ATTEMPTS` is exhausted — the SQS-level
- * redrive owns retry scheduling now, not this handler.
- */
-export const buildRetryableFailureError = (
-	failedMessageIds: string[],
-	receiveCount: number,
-): Error =>
-	new Error(
-		`Body sync failed for ${failedMessageIds.length} message(s) ` +
-			`(attempt ${receiveCount}/${BODY_SYNC_MAX_ATTEMPTS}): ${failedMessageIds.join(", ")}`,
-	);
-
-export const syncMessageBody = async (
-	event: SyncMessageBodyEvent,
-	log: Logger,
-	queues: WorkerQueues,
-	receiveCount = 1,
-	createConnectionScope = createConnectionScopeWithCredentials,
-): Promise<void> => {
-	const { accountId, mailboxId } = event;
-
-	// Prefer the messageId+uid pairs when present (one ranged FETCH, no
-	// per-message UID lookup); fall back to the legacy id-only list otherwise.
-	const { messageIds, force } = resolveBatch(event);
-
-	log.info(
-		{
-			event: event.type,
-			accountId,
-			mailboxId,
-			messageCount: messageIds.length,
-			hasUids: event.messages !== undefined,
-			force,
-			receiveCount,
-		},
-		"Handling event",
-	);
-
-	// Pause gate stays first: ack-and-skip before touching any account/IMAP state.
-	if (!(await isBodySyncEnabled(bodySyncEnabledParameterName, log))) {
-		log.info(
-			{
-				event: event.type,
-				accountId,
-				mailboxId,
-				messageCount: messageIds.length,
-			},
-			"Body sync paused via SSM toggle, acking and skipping",
-		);
-		return;
-	}
-
-	const {
-		account: accountService,
-		mailbox: mailboxService,
-		mailboxSpecialUse: mailboxSpecialUseService,
-		message: messageService,
-		threadMessage: threadMessageService,
-		address: addressService,
-		envelope: envelopeService,
-		placementMove: markerService,
-		filter: filterService,
-		messageLabel: messageLabelService,
-		storage,
-		secrets,
-		mailboxSpecialUse: mailboxSpecialUseRepository,
-		quarantine: quarantineRepository,
-		syncWarning: syncWarningRepository,
-		flagQueue: flagQueueService,
-		calendarSuggestion: calendarSuggestionService,
-		calendarStore,
-	} = await getClient();
-
-	const account = await accountService.get(accountId);
-	if (!account) {
-		throw new Error(`Account ${accountId} not found`);
-	}
-
-	if (isAccountDeleted(account, log)) {
-		return;
-	}
-
-	// PLACEMENT_MOVE_PUSH rides messageMgmtQueue (issue #1271) — same queue
-	// the general MessageMoveService move/delete/copy machinery uses, not a
-	// dedicated one.
-	const placementMoveService = new PlacementMoveService({
-		messageService,
-		threadMessageService,
-		markerService,
-		addressService,
-		mailboxSpecialUseService,
-		mailboxService,
-		queue: queues.messageManagement,
-	});
-
-	await withOAuthLifecycle(
-		buildLifecycleDeps(secrets, accountService),
-		account,
-		log,
-		async (credentials) => {
-			// A SYNC_MESSAGE_BODY trigger can outlive the mailbox it targets: deleting
-			// a folder that held mail leaves already-queued body events pointing at a
-			// row that is now gone, and the lookup then throws NotFoundError forever.
-			// Since the body queue carries MessageGroupId=accountId, that head message
-			// stalls the account's whole body pipeline (issues #287, #289, #290). A
-			// deleted folder is an expected terminal outcome: ack with a WARN.
-			const mailbox = await mailboxService
-				.get(accountId, mailboxId)
-				.catch((error: unknown) => {
-					if (isNotFoundError(error)) return null;
-					throw error;
-				});
-			// A folder still `pending` is terminal for the same reason: the batch
-			// was cut before the folder reached the server, so there is nothing
-			// there to fetch from and no retry that changes it.
-			if (!mailbox || isFolderMutationInFlight(mailbox)) {
-				log.warn(
-					{ accountId, mailboxId, eventId: event.eventId },
-					"Skipping SYNC_MESSAGE_BODY: a folder mutation is in flight",
-				);
-				return;
-			}
-
-			// Cheap frugal skip (epic #1281 invariant 6): a mailbox already known
-			// paused never even borrows a connection. This is an optimization only
-			// — guardConnectionCursor below is the structural guarantee, so a
-			// handler that forgot this check still cannot reach a stale UID.
-			if (isCursorRebuildNeeded(mailbox.cursorState)) {
-				log.info(
-					{ accountId, mailboxId, cursorState: mailbox.cursorState },
-					"Mailbox cursor not normal; pausing outbound body sync this round",
-				);
-				return;
-			}
-
-			// Warm reuse: borrow a live IMAP connection from the module-scoped pool
-			// (keyed by accountId) instead of dialing a fresh one per invocation. A
-			// warm container skips TCP+TLS+LOGIN+SELECT; a dead pooled connection is
-			// liveness-checked and replaced inside borrowWarmConnection.
-			const borrowed = borrowWarmConnection(accountId, () =>
-				createConnectionScope(account, credentials),
-			);
-
-			// A confident, actionable placement verdict moves mail directly on
-			// body-sync. Safety lives in the verdict itself (only confident,
-			// INBOX/Junk-only) and the movedByRemit loop guard.
-			const placementConfig = {
-				mailboxSpecialUseService,
-				placementMoveService,
-			};
-
-			// A matched filter's actions (label upsert, exclusive move) apply on
-			// the same body-sync pass, reusing the placement mover for the move
-			// (RFC 034 Decision 3.1).
-			const filterConfig = buildFilterConfig({
-				filterService,
-				messageLabelService,
-				mailboxService,
-				placementMoveService,
-			});
-
-			const bodySyncService = new BodySyncService(
-				messageService,
-				storage,
-				threadMessageService,
-				addressService,
-				envelopeService,
-				new SyncWarningService(syncWarningRepository),
-				log,
-				placementConfig,
-				filterConfig,
-				{
-					quarantineService: new QuarantineService(
-						quarantineRepository,
-						mailboxSpecialUseRepository,
-						workerVersion(),
-						log,
-					),
-					mailboxId,
-					uidValidity: mailbox.uidValidity ?? 0,
-					attempts: receiveCount,
-				},
-				{ flagQueueService },
-				// An invitation becomes a card the first time the message is seen,
-				// alongside the filters that run on the same pass (issue #1033).
-				{ calendarSuggestionService, calendarStore, filterService },
-			);
-
-			// Guard at the openBox choke point (epic #1281 invariants 3 & 5). The
-			// served UIDVALIDITY is only knowable by opening the box, which only a
-			// genuine fetch needs — BodySyncService.syncBodies skips connecting
-			// entirely when every message already has a stored body (nothing would
-			// touch a stored UID either way) — so the check fires lazily, the first
-			// time `openBox` is actually called, instead of forcing an extra open
-			// on every event.
-			const getConnectionChecked = async () =>
-				guardConnectionCursor(
-					await borrowed.getConnection(),
-					{ mailboxService },
-					accountId,
-					mailbox,
-				);
-
-			// Both the sync attempt AND the retry-exhaustion resolution below run
-			// against the SAME borrowed connection, so the mailbox stays open
-			// across the two — the connection is only released once both are done.
-			const folder = {
-				mailboxId: mailbox.mailboxId,
-				fullPath: mailbox.fullPath,
-				uidValidity: mailbox.uidValidity,
-				placement: await resolveFolderPlacement(
-					mailboxSpecialUseService,
-					accountId,
-					mailbox.mailboxId,
-				),
-			};
-
-			await (async () => {
-				const result = await bodySyncService.syncBodies(
-					messageIds,
-					accountId,
-					account.accountConfigId,
-					folder,
-					getConnectionChecked,
-					force,
-				);
-
-				if (result.failedMessageIds.length === 0) {
-					return;
-				}
-
-				if (receiveCount < BODY_SYNC_MAX_ATTEMPTS) {
-					// Redelivery budget remains: let SQS retry the whole record
-					// naturally. Messages that already synced skip via the
-					// already-stored guard on the next attempt, so only the failed
-					// ones actually redo work. This — not a manual re-enqueue — is
-					// what lets a genuine processing failure ever reach the body-dlq.
-					throw buildRetryableFailureError(
-						result.failedMessageIds,
-						receiveCount,
-					);
-				}
-
-				// Redelivery budget exhausted: a persistent per-message failure is
-				// never a steady state (epic #1281 invariant 3). Resolve every
-				// failed id into exactly one of the two terminal outcomes instead of
-				// letting the record dead-letter with no diagnosis.
-				const { brokenMessageIds } = await resolveExhaustedBodySyncFailures(
-					{
-						messageService,
-						storageService: storage,
-						log,
-					},
-					{
-						accountId,
-						accountConfigId: account.accountConfigId,
-						mailboxId,
-						mailboxPath: mailbox.fullPath,
-						failedMessageIds: result.failedMessageIds,
-						getConnection: getConnectionChecked,
-					},
-				);
-
-				// A broken body is terminal on a record the handler acks, so the
-				// handler-outcome series never sees it.
-				if (brokenMessageIds.length > 0) {
-					recordImapFailure(
-						"SYNC_MESSAGE_BODY_BROKEN",
-						"other",
-						brokenMessageIds.length,
-					);
-				}
-			})()
-				.catch((error: unknown) => {
-					// Expected pause (epic #1281 invariant 3), not a fault: ack and
-					// skip rather than propagating into SQS retry/DLQ. The read stays
-					// served from whatever is already stored; the fetch resumes once
-					// the mailbox returns to normal.
-					if (error instanceof MailboxCursorPausedError) {
-						log.info(
-							{ accountId, mailboxId, cursorState: error.state },
-							"UIDVALIDITY changed; mailbox cursor tripped, pausing outbound body sync",
-						);
-						return;
-					}
-					throw error;
-				})
-				.finally(() => borrowed.release());
-		},
-	);
-};

package/src/handlers/sync-messages.sqlite.test.ts

@@ -1,195 +0,0 @@
-import assert from "node:assert/strict";
-import { after, before, describe, it } from "node:test";
-import { createEmitEvent } from "../emit.js";
-import type { SyncMessagesEvent } from "../events.js";
-import { openStore, type World } from "../test-helpers/test-world.js";
-import { drainPendingFlagPushes, syncMessages } from "./sync-messages.js";
-
-let store: Awaited<ReturnType<typeof openStore>>;
-
-before(async () => {
-	store = await openStore();
-});
-
-after(() => store.close());
-
-const syncEvent = (w: World, mailboxId: string): SyncMessagesEvent => ({
-	type: "SYNC_MESSAGES",
-	accountId: w.account.accountId,
-	mailboxId,
-	eventId: `evt-${mailboxId}`,
-	timestamp: Date.now(),
-});
-
-const sync = async (w: World, mailboxId: string): Promise<number> => {
-	await syncMessages(
-		syncEvent(w, mailboxId),
-		w.log,
-		w.queues,
-		w.connectionFactory,
-	);
-	return w.imap.connections;
-};
-
-describe("which SYNC_MESSAGES events reach the server", () => {
-	it("a live mailbox does", async () => {
-		const w = await store.world();
-		const inbox = await w.folder("INBOX");
-
-		assert.equal(await sync(w, inbox.mailboxId), 1);
-	});
-
-	// Issue #287: a NotFoundError at the head of the account's FIFO group
-	// stalled every later event for the account.
-	it("a mailbox whose row is gone is acked without connecting", async () => {
-		const w = await store.world();
-		const old = await w.folder("Old");
-		await w.client.mailbox.delete(w.account.accountId, old.mailboxId);
-
-		assert.equal(await sync(w, old.mailboxId), 0);
-	});
-
-	it("a mailbox being deleted is acked without connecting", async () => {
-		const w = await store.world();
-		const old = await w.folder("Old");
-		await w.client.mailboxQueue.deleteMailbox(
-			old.mailboxId,
-			w.account.accountId,
-		);
-
-		assert.equal(await sync(w, old.mailboxId), 0);
-	});
-
-	it("a mailbox not yet created on the server is acked without connecting", async () => {
-		const w = await store.world();
-		const created = await w.client.mailboxQueue.createMailbox(
-			{
-				accountId: w.account.accountId,
-				namespacePrefix: "",
-				hierarchyDelimiter: "/",
-				fullPath: "Receipts",
-				uidValidity: 0,
-				uidNext: 0,
-				highestModseq: "0",
-				messageCount: 0,
-				unseenCount: 0,
-				deletedCount: 0,
-				totalSize: 0,
-				lastSyncUid: 0,
-				highWaterMarkUid: 0,
-				lastMessageSyncAt: 0,
-			},
-			w.account.accountId,
-		);
-		if (created.outcome !== "Created") throw new Error("path taken");
-
-		assert.equal(await sync(w, created.mailbox.mailboxId), 0);
-	});
-});
-
-// Issue #339: a delete asked for while the round is in flight lands inside
-// the guard, and the round then fails deeper on a folder that is gone.
-describe("a folder deleted while its sync is in flight", () => {
-	it("resolves the event and records no error phase on the account", async () => {
-		const w = await store.world();
-		const old = await w.folder("Old");
-		const connectionFactory = () => {
-			const factory = w.connectionFactory();
-			const connection = factory.getConnection();
-			const openBox = connection.openBox;
-			let first = true;
-			connection.openBox = async (path, readOnly) => {
-				if (first) {
-					first = false;
-					await w.client.mailboxQueue.deleteMailbox(
-						old.mailboxId,
-						w.account.accountId,
-					);
-					w.imap.removeFolder("Old");
-				}
-				return openBox(path, readOnly);
-			};
-			return { getConnection: () => connection, close: factory.close };
-		};
-
-		await syncMessages(
-			syncEvent(w, old.mailboxId),
-			w.log,
-			w.queues,
-			connectionFactory,
-		);
-
-		const account = await w.client.account.get(w.account.accountId);
-		assert.notEqual(account.syncPhase, "error");
-	});
-});
-
-describe("drainPendingFlagPushes (issue #1273)", () => {
-	const marker = (w: World, mailboxId: string, messageId: string) =>
-		w.client.flagPush.put({
-			messageId,
-			flagName: "\\Seen",
-			accountId: w.account.accountId,
-			accountConfigId: w.account.accountConfigId,
-			mailboxId,
-			operation: "add",
-		});
-
-	it("re-arms the markers stuck before their wake-up and leaves the ones a driver owns", async () => {
-		const w = await store.world();
-		const inbox = await w.folder("INBOX");
-		await marker(w, inbox.mailboxId, "msg-pending");
-		await marker(w, inbox.mailboxId, "msg-queued");
-		await w.client.flagPush.updateState("msg-queued", "\\Seen", "queued");
-		await marker(w, inbox.mailboxId, "msg-processing");
-		await w.client.flagPush.updateState(
-			"msg-processing",
-			"\\Seen",
-			"processing",
-		);
-
-		await drainPendingFlagPushes(
-			w.client.flagPush,
-			w.account,
-			inbox.mailboxId,
-			w.log,
-			createEmitEvent(w.queues),
-		);
-
-		assert.deepEqual(
-			w
-				.queued()
-				.filter((body) => body.type === "FLAG_PUSH")
-				.map((body) => body.messageId),
-			["msg-pending"],
-		);
-	});
-
-	it("keeps re-arming the others when one re-arm fails, and says so loudly", async () => {
-		const w = await store.world();
-		const inbox = await w.folder("INBOX");
-		await marker(w, inbox.mailboxId, "will-fail");
-		await marker(w, inbox.mailboxId, "will-succeed");
-		const armed: string[] = [];
-
-		await drainPendingFlagPushes(
-			w.client.flagPush,
-			w.account,
-			inbox.mailboxId,
-			w.log,
-			async (event) => {
-				const { messageId } = event as unknown as { messageId: string };
-				if (messageId === "will-fail") throw new Error("queue down");
-				armed.push(messageId);
-			},
-		);
-
-		assert.deepEqual(armed, ["will-succeed"]);
-		assert.equal(
-			w.logs.filter(
-				(line) => line.fields.alert === "flag_push_drain_rearm_failed",
-			).length,
-			1,
-		);
-	});
-});