@remit/imap-worker 0.0.1

package/src/handlers/delete-account-objects.ts

@@ -0,0 +1,116 @@
+import {
+	DeleteObjectsCommand,
+	ListObjectsV2Command,
+	S3Client,
+} from "@aws-sdk/client-s3";
+import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
+import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
+import type { Logger } from "@remit/logger-lambda";
+import { resolveSqsCredentials } from "@remit/sqs-client";
+import { env } from "expect-env";
+
+export interface DeleteAccountObjectsEvent {
+	type: "DELETE_ACCOUNT_OBJECTS";
+	accountConfigId: string;
+	continuationToken?: string;
+}
+
+const s3 = new S3Client({});
+
+const sqsQueueUrl = env.SQS_QUEUE_URL_MESSAGE_MGMT;
+const isLocal = sqsQueueUrl.startsWith("http://localhost");
+
+const sqs = new SQSClient({
+	endpoint: isLocal ? new URL(sqsQueueUrl).origin : undefined,
+	...(isLocal && { protocol: AwsQueryProtocol }),
+	credentials: resolveSqsCredentials(),
+});
+
+const BATCH_SIZE = 1_000;
+const MIN_REMAINING_MS = 30_000;
+
+export const handleDeleteAccountObjects = async (
+	event: DeleteAccountObjectsEvent,
+	log: Logger,
+	getRemainingTimeMs?: () => number,
+): Promise<void> => {
+	const BUCKET_NAME = env.S3_BUCKET_NAME;
+	const { accountConfigId, continuationToken } = event;
+	const prefix = `accounts/${accountConfigId}/`;
+
+	log.info(
+		{ accountConfigId, prefix, hasContinuation: !!continuationToken },
+		"Deleting account objects from S3",
+	);
+
+	let currentToken = continuationToken;
+	let totalDeleted = 0;
+
+	// eslint-disable-next-line no-constant-condition
+	while (true) {
+		// Check remaining time before starting a new page
+		if (getRemainingTimeMs && getRemainingTimeMs() < MIN_REMAINING_MS) {
+			log.info(
+				{ accountConfigId, totalDeleted, continuationToken: currentToken },
+				"Near timeout, re-enqueuing with continuation token",
+			);
+			await reenqueue(accountConfigId, currentToken);
+			return;
+		}
+
+		const listResult = await s3.send(
+			new ListObjectsV2Command({
+				Bucket: BUCKET_NAME,
+				Prefix: prefix,
+				MaxKeys: BATCH_SIZE,
+				ContinuationToken: currentToken,
+			}),
+		);
+
+		const keys = (listResult.Contents ?? [])
+			.map((obj) => obj.Key)
+			.filter((k): k is string => k !== undefined);
+
+		if (keys.length > 0) {
+			await s3.send(
+				new DeleteObjectsCommand({
+					Bucket: BUCKET_NAME,
+					Delete: {
+						Objects: keys.map((Key) => ({ Key })),
+						Quiet: true,
+					},
+				}),
+			);
+			totalDeleted += keys.length;
+		}
+
+		if (!listResult.IsTruncated) {
+			break;
+		}
+
+		currentToken = listResult.NextContinuationToken;
+	}
+
+	log.info(
+		{ accountConfigId, totalDeleted },
+		"Finished deleting account objects",
+	);
+};
+
+const reenqueue = async (
+	accountConfigId: string,
+	continuationToken: string | undefined,
+): Promise<void> => {
+	const event: DeleteAccountObjectsEvent = {
+		type: "DELETE_ACCOUNT_OBJECTS",
+		accountConfigId,
+		continuationToken,
+	};
+
+	await sqs.send(
+		new SendMessageCommand({
+			QueueUrl: sqsQueueUrl,
+			MessageBody: JSON.stringify(event),
+		}),
+	);
+};

package/src/handlers/empty-trash.ts

@@ -0,0 +1,136 @@
+import { getClient } from "@remit/backend/client";
+import type { Logger } from "@remit/logger-lambda";
+import {
+	guardConnectionCursor,
+	isCursorRebuildNeeded,
+	MailboxCursorPausedError,
+} from "@remit/mailbox-service";
+import { isAccountDeleted } from "../account-check.js";
+import { createConnectionScopeWithCredentials } from "../connection-scope.js";
+import type { EmptyTrashEvent } from "../events.js";
+import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
+import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
+
+/**
+ * Handle EMPTY_TRASH events.
+ * Permanently deletes all messages in the Trash mailbox.
+ */
+export const handleEmptyTrash = async (
+	event: EmptyTrashEvent,
+	log: Logger,
+): Promise<void> => {
+	const {
+		account: accountService,
+		message: messageService,
+		threadMessage: threadMessageService,
+		mailbox: mailboxService,
+		secrets,
+	} = await getClient();
+
+	const { accountId, trashMailboxId, trashMailboxPath } = event;
+
+	log.info(
+		{ event: event.type, accountId, trashMailboxPath },
+		"Handling event",
+	);
+
+	const account = await accountService.get(accountId);
+	if (!account) {
+		throw new Error(`Account ${accountId} not found`);
+	}
+
+	if (isAccountDeleted(account, log)) {
+		return;
+	}
+
+	await withOAuthLifecycle(
+		buildLifecycleDeps(secrets, accountService),
+		account,
+		log,
+		async (credentials) => {
+			const mailbox = await mailboxService.get(accountId, trashMailboxId);
+
+			// Cheap frugal skip (epic #1281 invariant 6): a mailbox already known
+			// paused never even opens a connection. Optimization only — the
+			// guardConnectionCursor openBox wrap below is the structural guarantee.
+			if (isCursorRebuildNeeded(mailbox.cursorState)) {
+				log.info(
+					{ accountId, mailboxId: trashMailboxId },
+					"Mailbox cursor not normal; pausing empty-trash this round",
+				);
+				return;
+			}
+
+			const scope = createConnectionScopeWithCredentials(account, credentials);
+
+			await scope
+				.getConnection()
+				.then(async (rawConnection) => {
+					// Guard at the openBox choke point (epic #1281 invariants 3 & 5):
+					// a fresh mismatch trips the mailbox and throws once the SELECT
+					// reveals it. Local rows stay marked for deletion and are picked
+					// up once the mailbox returns to normal.
+					const connection = guardConnectionCursor(
+						rawConnection,
+						{ mailboxService },
+						accountId,
+						mailbox,
+					);
+					await connection.openBox(trashMailboxPath, false);
+
+					// Search for all messages in Trash
+					const uids = await connection.search(["ALL"]);
+
+					if (uids.length > 0) {
+						// Delete all messages on IMAP
+						await connection.deleteMessages(uids);
+						log.info(
+							{ count: uids.length },
+							"Deleted messages from IMAP trash",
+						);
+					}
+
+					// Delete all local messages in trash
+					const localMessages =
+						await messageService.listAllByMailbox(trashMailboxId);
+
+					for (const message of localMessages) {
+						// Delete the Message entity
+						await messageService.delete(message.messageId);
+
+						// Delete the ThreadMessage entity
+						const threadMessage = await threadMessageService.findByMessageId(
+							account.accountConfigId,
+							message.messageId,
+						);
+						if (threadMessage) {
+							await threadMessageService.delete(
+								threadMessage.accountConfigId,
+								threadMessage.threadMessageId,
+							);
+						}
+					}
+
+					log.info(
+						{ accountId, deletedCount: localMessages.length },
+						"Trash emptied successfully",
+					);
+				})
+				.catch((error: unknown) => {
+					if (error instanceof MailboxCursorPausedError) {
+						log.info(
+							{
+								accountId,
+								mailboxId: trashMailboxId,
+								cursorState: error.state,
+							},
+							"Mailbox cursor not normal; pausing empty-trash this round",
+						);
+						return;
+					}
+					throw error;
+				})
+				.finally(() => scope.disconnect());
+		},
+	);
+};

package/src/handlers/flag-push.test.ts

@@ -0,0 +1,25 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { FLAG_PUSH_MAX_ATTEMPTS, getFlagPushMaxAttempts } from "./flag-push.js";
+
+describe("getFlagPushMaxAttempts — env-derived threshold (mirrors #1270's getBodySyncMaxAttempts / #1289's getPlacementMoveMaxAttempts)", () => {
+	it("parses the CDK-injected env var", () => {
+		assert.equal(getFlagPushMaxAttempts({ FLAG_PUSH_MAX_ATTEMPTS: "3" }), 3);
+		assert.equal(getFlagPushMaxAttempts({ FLAG_PUSH_MAX_ATTEMPTS: "5" }), 5);
+	});
+
+	it("defaults to 3 when unset", () => {
+		assert.equal(getFlagPushMaxAttempts({}), 3);
+	});
+
+	it("defaults to 3 on a non-numeric or non-positive value", () => {
+		assert.equal(getFlagPushMaxAttempts({ FLAG_PUSH_MAX_ATTEMPTS: "nope" }), 3);
+		assert.equal(getFlagPushMaxAttempts({ FLAG_PUSH_MAX_ATTEMPTS: "0" }), 3);
+		assert.equal(getFlagPushMaxAttempts({ FLAG_PUSH_MAX_ATTEMPTS: "-1" }), 3);
+	});
+
+	it("the module-level constant reflects the actual process env at load time", () => {
+		assert.equal(typeof FLAG_PUSH_MAX_ATTEMPTS, "number");
+		assert.ok(FLAG_PUSH_MAX_ATTEMPTS > 0);
+	});
+});

package/src/handlers/flag-push.ts

@@ -0,0 +1,224 @@
+import { getClient } from "@remit/backend/client";
+import type { Logger } from "@remit/logger-lambda";
+import { MetricUnit, metrics } from "@remit/logger-lambda";
+import {
+	guardConnectionCursor,
+	isCursorRebuildNeeded,
+	MailboxCursorPausedError,
+	resolveExhaustedFlagPushFailure,
+} from "@remit/mailbox-service";
+import { isAccountDeleted } from "../account-check.js";
+import { createConnectionScopeWithCredentials } from "../connection-scope.js";
+import type { FlagPushEvent } from "../events.js";
+import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
+import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
+
+/**
+ * Fallback when `FLAG_PUSH_MAX_ATTEMPTS` is unset (local dev, unit tests).
+ * Matches the shared `MAX_RECEIVE_COUNT` every queue's redrive policy uses
+ * (`infra/stacks/dev/stacks/remit-queue-stack.ts`), same pattern as
+ * `BODY_SYNC_MAX_ATTEMPTS` (#1270) / `PLACEMENT_MOVE_MAX_ATTEMPTS` (#1289).
+ */
+const DEFAULT_FLAG_PUSH_MAX_ATTEMPTS = 3;
+
+export const getFlagPushMaxAttempts = (
+	processEnv: NodeJS.ProcessEnv = process.env,
+): number => {
+	const raw = processEnv.FLAG_PUSH_MAX_ATTEMPTS;
+	if (!raw) return DEFAULT_FLAG_PUSH_MAX_ATTEMPTS;
+	const parsed = Number.parseInt(raw, 10);
+	return Number.isFinite(parsed) && parsed > 0
+		? parsed
+		: DEFAULT_FLAG_PUSH_MAX_ATTEMPTS;
+};
+
+export const FLAG_PUSH_MAX_ATTEMPTS = getFlagPushMaxAttempts();
+
+/**
+ * Handle FLAG_PUSH events (issue #1273, epic #1281). Drains ONE pending
+ * flag-push marker: resolves the message's UID and CURRENT mailbox fresh
+ * from the Message row (never a value captured at enqueue — invariant 1),
+ * pushes the IMAP STORE (add or remove, per the marker's `operation`), and
+ * clears the marker ONLY on confirmed success.
+ *
+ * Precedence (epic invariant 2):
+ * - While pending, resync never reverts the flag — nothing in this handler
+ *   (or anywhere else) reads flags FROM IMAP back into `MessageFlag`/
+ *   `ThreadMessage` for an existing row; the only flag-state writes happen
+ *   here (confirmed push) or in `FlagQueueService` (the user's own local
+ *   flip, which already applied before this marker existed).
+ * - A later flip of the SAME field already replaced this marker (`put`) by
+ *   the time this event is processed, OR advanced it past `pending` — either
+ *   way `markerService.find` returns the CURRENT marker, so this handler
+ *   always drives the freshest intent, never a stale one.
+ * - An external delete supersedes the marker entirely — handled by
+ *   `resolveExhaustedFlagPushFailure`'s `reconciled` outcome.
+ *
+ * Cursor-guarded (#1272, epic #1281 invariant 5): the connection is wrapped
+ * via `guardConnectionCursor` around the mailbox's current `openBox` choke
+ * point, so no stored UID touches the server while the mailbox's axis is
+ * being rebuilt. A trip pauses the push (routine, no alarm) — the marker
+ * stays durable and pushes again on the next event or sync tick.
+ */
+export const handleFlagPush = async (
+	event: FlagPushEvent,
+	log: Logger,
+	receiveCount = 1,
+): Promise<void> => {
+	const {
+		account: accountService,
+		mailbox: mailboxService,
+		message: messageService,
+		threadMessage: threadMessageService,
+		flagPush: markerService,
+		secrets,
+	} = await getClient();
+
+	const { accountId, accountConfigId, messageId, flagName } = event;
+
+	const marker = await markerService.find(messageId, flagName);
+	if (!marker) {
+		log.info(
+			{ messageId, flagName, accountId },
+			"No pending flag-push marker (already confirmed or superseded); nothing to push",
+		);
+		return;
+	}
+
+	const account = await accountService.get(accountId);
+	if (!account) {
+		throw new Error(`Account ${accountId} not found`);
+	}
+	if (isAccountDeleted(account, log)) {
+		return;
+	}
+
+	const [message] = await messageService.get([messageId]);
+
+	// The message row is already gone — some other reconciliation path (body
+	// sync, placement move, a prior flag-push exhaustion) already deleted it.
+	// The marker is orphaned; drop it without touching IMAP.
+	if (!message) {
+		await markerService.delete(messageId, flagName);
+		log.info(
+			{ messageId, flagName, accountId },
+			"Message row no longer exists; flag-push marker dropped without pushing",
+		);
+		return;
+	}
+
+	// The worker has picked up the event and is about to actually attempt the
+	// IMAP STORE — advance the state engine (pending/queued -> processing).
+	// Idempotent to call again on a redelivered event (a prior attempt that
+	// died mid-flight already left it here).
+	await markerService.updateState(messageId, flagName, "processing");
+
+	const mailbox = await mailboxService.get(accountId, message.mailboxId);
+
+	// Cheap frugal skip (epic #1281 invariant 6): a mailbox already known
+	// paused never even borrows a connection. Optimization only — the
+	// guardConnectionCursor wrap below is the structural guarantee (#1272).
+	if (isCursorRebuildNeeded(mailbox.cursorState)) {
+		log.info(
+			{ messageId, flagName, accountId, cursorState: mailbox.cursorState },
+			"Mailbox cursor not normal; pausing outbound flag push this round",
+		);
+		return;
+	}
+
+	await withOAuthLifecycle(
+		buildLifecycleDeps(secrets, accountService),
+		account,
+		log,
+		async (credentials) => {
+			const scope = createConnectionScopeWithCredentials(account, credentials);
+
+			await scope
+				.getConnection()
+				.then(async (rawConnection) => {
+					const connection = guardConnectionCursor(
+						rawConnection,
+						{ mailboxService },
+						accountId,
+						mailbox,
+					);
+
+					await connection.openBox(mailbox.fullPath, false);
+
+					if (marker.operation === "add") {
+						await connection.addFlags([message.uid], [flagName]);
+					} else {
+						await connection.removeFlags([message.uid], [flagName]);
+					}
+
+					// Confirmed IMAP acknowledgement — clears ONLY here, never on
+					// attempt (the defect issue #1273 fixes).
+					await markerService.delete(messageId, flagName);
+
+					log.info(
+						{
+							messageId,
+							flagName,
+							accountId,
+							operation: marker.operation,
+							uid: message.uid,
+							mailboxPath: mailbox.fullPath,
+						},
+						"Flag push confirmed on IMAP; marker cleared",
+					);
+				})
+				.catch(async (error: unknown) => {
+					// Expected pause (epic #1281 invariant 3), not a fault: ack and
+					// skip rather than propagating into queue retry/DLQ. The marker
+					// stays durable; the push resumes once the mailbox returns to
+					// normal.
+					if (error instanceof MailboxCursorPausedError) {
+						log.info(
+							{ messageId, flagName, accountId, cursorState: error.state },
+							"UIDVALIDITY changed; mailbox cursor tripped, pausing outbound flag push",
+						);
+						return;
+					}
+
+					if (receiveCount < FLAG_PUSH_MAX_ATTEMPTS) {
+						// Transient push failure — expected (connections drop). No
+						// alarm; queue redelivery retries from the still-durable marker.
+						throw error;
+					}
+
+					// Redelivery budget exhausted: resolve into exactly one of the
+					// two terminal outcomes (epic invariant 3) instead of
+					// dead-lettering with no diagnosis.
+					const { outcome } = await resolveExhaustedFlagPushFailure(
+						{ markerService, messageService, threadMessageService, log },
+						{
+							accountId,
+							accountConfigId,
+							messageId,
+							flagName,
+							uid: message.uid,
+							mailboxPath: mailbox.fullPath,
+							getConnection: scope.getConnection,
+						},
+					);
+
+					if (outcome === "reconciled") {
+						metrics.addMetric(
+							"flagPushStaleRowReconciled",
+							MetricUnit.Count,
+							1,
+						);
+						return;
+					}
+
+					metrics.addMetric("flagPushFailed", MetricUnit.Count, 1);
+					log.error(
+						{ error: error instanceof Error ? error.message : String(error) },
+						"Flag push retry exhausted; message still exists at its mailbox",
+					);
+					// Terminal — never re-thrown, so the caller acks either way.
+				})
+				.finally(() => scope.disconnect());
+		},
+	);
+};