@remit/imap-worker 0.0.1

package/src/emit.ts

@@ -0,0 +1,142 @@
+import { randomUUID } from "node:crypto";
+import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
+import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
+import { resolveSqsCredentials } from "@remit/sqs-client";
+import { env } from "expect-env";
+import type {
+	ImapEvent,
+	SyncMailboxesEvent,
+	SyncMessagesEvent,
+} from "./events.js";
+
+type EventInput = Omit<ImapEvent, "eventId" | "timestamp">;
+
+const mailboxesQueueUrl = env.SQS_QUEUE_URL_MAILBOXES;
+const messagesQueueUrl = env.SQS_QUEUE_URL_MESSAGES;
+// SYNC_MESSAGE_BODY events route to the single standard body queue (#612). It is
+// a standard (non-FIFO) queue, so isFifoQueue() below skips the FIFO
+// MessageGroupId/dedup parameters automatically.
+const bodyQueueUrl = env.SQS_QUEUE_URL_BODY;
+const flagsQueueUrl = env.SQS_QUEUE_URL_FLAGS;
+const mailboxMgmtQueueUrl = env.SQS_QUEUE_URL_MAILBOX_MGMT;
+const messageMgmtQueueUrl = env.SQS_QUEUE_URL_MESSAGE_MGMT;
+
+const isLocal = mailboxesQueueUrl.startsWith("http://localhost");
+
+const sqs = new SQSClient({
+	endpoint: isLocal ? new URL(mailboxesQueueUrl).origin : undefined,
+	...(isLocal && { protocol: AwsQueryProtocol }),
+	credentials: resolveSqsCredentials(),
+});
+
+const queueUrlMap: Record<ImapEvent["type"], string> = {
+	SYNC_MAILBOXES: mailboxesQueueUrl,
+	SYNC_MESSAGES: messagesQueueUrl,
+	SYNC_MESSAGE_BODY: bodyQueueUrl,
+	MAILBOX_CREATE: mailboxMgmtQueueUrl,
+	MAILBOX_RENAME: mailboxMgmtQueueUrl,
+	MAILBOX_DELETE: mailboxMgmtQueueUrl,
+	MESSAGE_DELETE: messageMgmtQueueUrl,
+	MESSAGE_MOVE: messageMgmtQueueUrl,
+	MESSAGE_COPY: messageMgmtQueueUrl,
+	EMPTY_TRASH: messageMgmtQueueUrl,
+	APPEND_SENT_MESSAGE: messageMgmtQueueUrl,
+	// Rides messageMgmtQueue (issue #1271) rather than a dedicated queue — its
+	// payload carries only our message id (never a UID, unlike the legacy
+	// MESSAGE_MOVE event); per-event-type payload shape, same queue.
+	PLACEMENT_MOVE_PUSH: messageMgmtQueueUrl,
+	// Rides the existing flags queue (issue #1273) — this is imap-worker's OWN
+	// re-arm hint (the periodic per-mailbox sync tick catching up a marker
+	// stuck `pending`), distinct from `FlagPushService`'s user-facing hint
+	// (remit-mailbox-service, sent from the API on its own SQS client onto
+	// SQS_QUEUE_URL). Both land on a queue this worker already consumes;
+	// dispatch is by `type`, not by which queue delivered the message.
+	FLAG_PUSH: flagsQueueUrl,
+};
+
+/**
+ * FIFO queue event types that support deduplication.
+ * Management events (MAILBOX_*, MESSAGE_*, EMPTY_TRASH) are not deduplicated
+ * because each operation is unique.
+ */
+const fifoEventTypes = new Set([
+	"SYNC_MAILBOXES",
+	"SYNC_MESSAGES",
+	// flagsQueue is FIFO and requires a MessageGroupId on every message.
+	// Content-based deduplication (queue-level setting) covers the dedup id —
+	// no per-event case is needed in getDeduplicationId below, same as the
+	// (standard-queue) management events that fall through its default.
+	"FLAG_PUSH",
+]);
+
+/**
+ * Generate a deduplication ID for FIFO queue events.
+ * SQS deduplication window is 5 minutes - duplicate messages within
+ * this window are rejected.
+ */
+export const getDeduplicationId = (event: EventInput): string | undefined => {
+	switch (event.type) {
+		case "SYNC_MAILBOXES": {
+			const e = event as Omit<SyncMailboxesEvent, "eventId" | "timestamp">;
+			return `SYNC_MAILBOXES:${e.accountId}`;
+		}
+
+		case "SYNC_MESSAGES": {
+			const e = event as Omit<SyncMessagesEvent, "eventId" | "timestamp">;
+			// A continuation carries a per-batch cursor so batches 2..N are not
+			// deduped against the initial event (which has none) or each other; the
+			// cursor-less initial id still dedups concurrent fresh syncs of a mailbox.
+			return e.resumeCursor === undefined
+				? `SYNC_MESSAGES:${e.mailboxId}`
+				: `SYNC_MESSAGES:${e.mailboxId}:${e.resumeCursor}`;
+		}
+
+		default:
+			// Management events are not deduplicated
+			return undefined;
+	}
+};
+
+/**
+ * Check if queue URL is a FIFO queue (ends with .fifo)
+ */
+const isFifoQueue = (queueUrl: string): boolean => queueUrl.endsWith(".fifo");
+
+export interface EmitEventOptions {
+	/**
+	 * Delay delivery of the message by this many seconds (0-900).
+	 * Useful for retry backoff to avoid overwhelming IMAP servers.
+	 */
+	delaySeconds?: number;
+}
+
+export const emitEvent = async (
+	event: EventInput,
+	options?: EmitEventOptions,
+) => {
+	const fullEvent: ImapEvent = {
+		...event,
+		eventId: randomUUID(),
+		timestamp: Date.now(),
+	} as ImapEvent;
+
+	const queueUrl = queueUrlMap[event.type];
+	const useFifo = isFifoQueue(queueUrl) && fifoEventTypes.has(event.type);
+
+	// ElasticMQ FIFO queues don't support per-message DelaySeconds
+	const useDelay = options?.delaySeconds && !isLocal;
+
+	await sqs.send(
+		new SendMessageCommand({
+			QueueUrl: queueUrl,
+			MessageBody: JSON.stringify(fullEvent),
+			// Delay delivery for retry backoff (skip for local ElasticMQ)
+			...(useDelay && { DelaySeconds: options.delaySeconds }),
+			// FIFO queue parameters - only set if queue is FIFO
+			...(useFifo && {
+				MessageGroupId: event.accountId,
+				MessageDeduplicationId: getDeduplicationId(event),
+			}),
+		}),
+	);
+};

package/src/events.ts

@@ -0,0 +1,221 @@
+export interface BaseEvent {
+	accountId: string;
+	eventId: string; // Idempotency key
+	timestamp: number; // Unix timestamp
+}
+
+export interface SyncMailboxesEvent extends BaseEvent {
+	type: "SYNC_MAILBOXES";
+}
+
+export interface SyncMessagesEvent extends BaseEvent {
+	type: "SYNC_MESSAGES";
+	mailboxId: string;
+	fullSync?: boolean; // If true, ignore lastSyncUid
+	/**
+	 * Set on a continuation event (the "next batch" a batch emits when it drains
+	 * only part of a mailbox). Carries the batch's remaining-message count so the
+	 * FIFO deduplication id is distinct per batch — otherwise every continuation
+	 * for a mailbox shares one dedup id and the 5-minute window silently drops
+	 * batches 2..N, capping any mailbox over one batch. The value is not used by
+	 * the handler (the resume point comes from the persisted watermark); it only
+	 * makes the dedup id unique.
+	 */
+	resumeCursor?: number;
+}
+
+/**
+ * A message to body-sync, carrying the UID resolved at envelope-sync time so
+ * the consumer can issue one ranged FETCH without a per-message DDB lookup.
+ */
+export interface SyncMessageBodyTarget {
+	messageId: string;
+	uid: number;
+}
+
+export interface SyncMessageBodyEvent extends BaseEvent {
+	type: "SYNC_MESSAGE_BODY";
+	mailboxId: string;
+	/**
+	 * Message ids to sync. Always present for backward compatibility; the
+	 * consumer falls back to this list (looking up each UID) when `messages`
+	 * is absent.
+	 */
+	messageIds: string[];
+	/**
+	 * Preferred shape: messageId+uid pairs. When present, the consumer skips the
+	 * per-message UID lookup and fetches the whole batch in one ranged FETCH.
+	 * Optional so older in-flight events (ids only) still process.
+	 */
+	messages?: SyncMessageBodyTarget[];
+	/**
+	 * Set only by the read-miss re-arm cue (`BodySyncQueueService.requestBodySync`,
+	 * called when a `/content` read finds `bodyStorageKey` set but the storage
+	 * object missing). Tells the consumer to bypass the "already stored" skip
+	 * guard and re-fetch + rewrite the body even though the DB row already
+	 * carries a `bodyStorageKey` — otherwise stale metadata (key set, object
+	 * gone) makes the read-miss retry loop forever, since the bulk skip guard
+	 * would keep treating the message as already synced. Optional so older
+	 * in-flight events (no force) keep the original skip behavior.
+	 */
+	force?: boolean;
+}
+
+export interface MailboxCreateEvent extends BaseEvent {
+	type: "MAILBOX_CREATE";
+	mailboxId: string;
+	path: string;
+	subscribe?: boolean;
+}
+
+export interface MailboxRenameEvent extends BaseEvent {
+	type: "MAILBOX_RENAME";
+	mailboxId: string;
+	oldPath: string;
+	newPath: string;
+}
+
+export interface MailboxDeleteEvent extends BaseEvent {
+	type: "MAILBOX_DELETE";
+	mailboxId: string;
+	path: string;
+}
+
+export type MailboxManagementEvent =
+	| MailboxCreateEvent
+	| MailboxRenameEvent
+	| MailboxDeleteEvent;
+
+/**
+ * Event for deleting a message (move to trash or permanent delete).
+ */
+export interface MessageDeleteEvent extends BaseEvent {
+	type: "MESSAGE_DELETE";
+	messageId: string;
+	mailboxId: string;
+	mailboxPath: string;
+	uid: number;
+	operation: "move_to_trash" | "permanent_delete";
+	destinationMailboxId?: string;
+	destinationMailboxPath?: string;
+}
+
+/**
+ * Event for moving a message to another mailbox.
+ */
+export interface MessageMoveEvent extends BaseEvent {
+	type: "MESSAGE_MOVE";
+	messageId: string;
+	sourceMailboxId: string;
+	sourceMailboxPath: string;
+	destinationMailboxId: string;
+	destinationMailboxPath: string;
+	uid: number;
+}
+
+/**
+ * Event for emptying the Trash mailbox.
+ */
+export interface EmptyTrashEvent extends BaseEvent {
+	type: "EMPTY_TRASH";
+	trashMailboxId: string;
+	trashMailboxPath: string;
+}
+
+/**
+ * Event for copying a message to another mailbox.
+ */
+export interface MessageCopyEvent extends BaseEvent {
+	type: "MESSAGE_COPY";
+	sourceMessageId: string;
+	newMessageId: string;
+	sourceMailboxId: string;
+	sourceMailboxPath: string;
+	destinationMailboxId: string;
+	destinationMailboxPath: string;
+	uid: number;
+}
+
+/**
+ * Event for appending a sent message to the Sent mailbox via IMAP APPEND.
+ */
+export interface AppendSentMessageEvent extends BaseEvent {
+	type: "APPEND_SENT_MESSAGE";
+	outboxMessageId: string;
+}
+
+/**
+ * Drains one pending placement-move marker (issue #1271, epic #1281) to
+ * IMAP. Deliberately carries no UID and no destination — both are resolved
+ * fresh from the Message row and the `MessagePlacementMove` marker at push
+ * time (epic invariant 1), so this event stays valid across any amount of
+ * queue delay or an unrelated UIDVALIDITY rebuild (#1272). If the marker is
+ * gone by the time this is processed (confirmed, superseded, or an external
+ * delete already reconciled it), the handler is a no-op.
+ */
+export interface PlacementMovePushEvent extends BaseEvent {
+	type: "PLACEMENT_MOVE_PUSH";
+	accountConfigId: string;
+	messageId: string;
+}
+
+/**
+ * Wake-up hint for one pending flag-push marker (issue #1273, epic #1281).
+ * Carries only our message id + the flag field — never a UID and never the
+ * desired add/remove transition (both live on the `MessageFlagPush` marker,
+ * resolved fresh at push time, epic invariant 1). Produced two ways: by
+ * `FlagPushService` (remit-mailbox-service) right after a user's flag flip
+ * commits locally, and by the periodic per-mailbox sync tick
+ * (`sync-messages.ts`) re-arming any marker it finds still stuck `pending`
+ * (a prior enqueue that never landed). If the marker is gone or has moved
+ * past `pending` by the time this is processed, the handler is a no-op.
+ */
+export interface FlagPushEvent extends BaseEvent {
+	type: "FLAG_PUSH";
+	accountConfigId: string;
+	messageId: string;
+	flagName: string;
+}
+
+export interface DeleteAccountObjectsEvent {
+	type: "DELETE_ACCOUNT_OBJECTS";
+	accountConfigId: string;
+	continuationToken?: string;
+}
+
+/**
+ * Sent by the account-deletion fanout worker once per accountId under a
+ * deleted AccountConfig. The actual stop semantics already happen via the
+ * account-tombstone fence (`isActive=false` + `deletedAt`) flipped by the
+ * deletion API: any in-flight or future event for the account is dropped
+ * by `isAccountDeleted`. This event exists as an explicit signal in the
+ * cascade contract — when the imap-worker grows real per-account drain or
+ * connection-teardown semantics it hangs off this hook.
+ */
+export interface ImapWorkerStopEvent {
+	type: "IMAP_WORKER_STOP";
+	accountConfigId: string;
+	accountId: string;
+}
+
+/** Union of all event types the worker can process (including non-IMAP ones). */
+export type WorkerEvent =
+	| ImapEvent
+	| DeleteAccountObjectsEvent
+	| ImapWorkerStopEvent;
+
+export type MessageManagementEvent =
+	| MessageDeleteEvent
+	| MessageMoveEvent
+	| EmptyTrashEvent
+	| MessageCopyEvent;
+
+export type ImapEvent =
+	| SyncMailboxesEvent
+	| SyncMessagesEvent
+	| SyncMessageBodyEvent
+	| MailboxManagementEvent
+	| MessageManagementEvent
+	| AppendSentMessageEvent
+	| PlacementMovePushEvent
+	| FlagPushEvent;

package/src/handlers/append-sent-message.ts

@@ -0,0 +1,163 @@
+import { getClient } from "@remit/backend/client";
+import type {
+	IMailboxRepository,
+	IMailboxSpecialUseRepository,
+	OutboxMessageItem,
+} from "@remit/data-ports";
+import { MailboxSpecialUse } from "@remit/domain-enums";
+import type { Logger } from "@remit/logger-lambda";
+import nodemailer from "nodemailer";
+import { isAccountDeleted } from "../account-check.js";
+import { createConnectionScopeWithCredentials } from "../connection-scope.js";
+import type { AppendSentMessageEvent } from "../events.js";
+import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
+import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
+
+const findSentMailbox = async (
+	mailboxSpecialUseService: IMailboxSpecialUseRepository,
+	mailboxService: IMailboxRepository,
+	accountId: string,
+): Promise<{ mailboxId: string; fullPath: string } | null> => {
+	const bySpecialUse = await mailboxSpecialUseService.findBySpecialUse(
+		accountId,
+		MailboxSpecialUse.Sent,
+	);
+	if (bySpecialUse) {
+		return bySpecialUse;
+	}
+
+	const commonSentNames = [
+		"Sent",
+		"Sent Items",
+		"Sent Messages",
+		"[Gmail]/Sent Mail",
+	];
+	const mailboxResult = await mailboxService.listByAccount(accountId);
+
+	for (const name of commonSentNames) {
+		const found = mailboxResult.items.find(
+			(m) => m.fullPath.toLowerCase() === name.toLowerCase(),
+		);
+		if (found) {
+			return { mailboxId: found.mailboxId, fullPath: found.fullPath };
+		}
+	}
+
+	return null;
+};
+
+const buildRawMessage = async (outbox: OutboxMessageItem): Promise<Buffer> => {
+	const from = outbox.fromName
+		? `${outbox.fromName} <${outbox.fromAddress}>`
+		: outbox.fromAddress;
+
+	const transport = nodemailer.createTransport({ streamTransport: true });
+
+	const info = await transport.sendMail({
+		from,
+		to: outbox.toAddresses.join(", "),
+		cc: outbox.ccAddresses?.join(", "),
+		bcc: outbox.bccAddresses?.join(", "),
+		replyTo: outbox.replyToAddress,
+		subject: outbox.subject,
+		text: outbox.textBody,
+		html: outbox.htmlBody,
+		messageId: `<${outbox.messageIdValue}>`,
+		inReplyTo: outbox.inReplyTo ? `<${outbox.inReplyTo}>` : undefined,
+		references: outbox.references?.map((r) => `<${r}>`).join(" "),
+		date: outbox.sentAt ? new Date(outbox.sentAt) : new Date(),
+	});
+
+	const chunks: Buffer[] = [];
+	for await (const chunk of info.message as AsyncIterable<Buffer>) {
+		chunks.push(chunk);
+	}
+	return Buffer.concat(chunks);
+};
+
+export const handleAppendSentMessage = async (
+	event: AppendSentMessageEvent,
+	log: Logger,
+): Promise<void> => {
+	const {
+		account: accountService,
+		outboxMessage: outboxMessageService,
+		mailboxSpecialUse: mailboxSpecialUseService,
+		mailbox: mailboxService,
+		secrets,
+	} = await getClient();
+
+	const { accountId, outboxMessageId } = event;
+
+	log.info({ event: event.type, accountId, outboxMessageId }, "Handling event");
+
+	const account = await accountService.get(accountId);
+	if (isAccountDeleted(account, log)) {
+		return;
+	}
+
+	const outbox = await outboxMessageService.get(
+		account.accountConfigId,
+		outboxMessageId,
+	);
+	if (outbox.status !== "sent") {
+		log.info(
+			{ outboxMessageId, status: outbox.status },
+			"Outbox message not in sent status, skipping APPEND",
+		);
+		return;
+	}
+
+	const sentMailbox = await findSentMailbox(
+		mailboxSpecialUseService,
+		mailboxService,
+		accountId,
+	);
+	if (!sentMailbox) {
+		log.info({ accountId }, "No Sent mailbox found, skipping IMAP APPEND");
+		return;
+	}
+
+	await withOAuthLifecycle(
+		buildLifecycleDeps(secrets, accountService),
+		account,
+		log,
+		async (credentials) => {
+			const scope = createConnectionScopeWithCredentials(account, credentials);
+
+			await scope
+				.getConnection()
+				.then(async (connection) => {
+					const rawMessage = await buildRawMessage(outbox);
+
+					const result = await connection.append(
+						sentMailbox.fullPath,
+						rawMessage,
+						["\\Seen"],
+					);
+
+					log.info(
+						{
+							outboxMessageId,
+							sentMailbox: sentMailbox.fullPath,
+							uid: result.uid,
+							uidValidity: result.uidValidity,
+						},
+						"Appended sent message to Sent mailbox",
+					);
+				})
+				.finally(() => scope.disconnect());
+
+			// The message now lives in the IMAP Sent folder. Drop the outbox row so
+			// the user does not see it twice in the UI (Outbox + Sent). Issue #178.
+			await outboxMessageService.delete(
+				account.accountConfigId,
+				outboxMessageId,
+			);
+			log.info(
+				{ outboxMessageId },
+				"Deleted outbox row after successful APPEND to Sent",
+			);
+		},
+	);
+};

package/src/handlers/delete-account-objects.test.ts

@@ -0,0 +1,81 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type { DeleteAccountObjectsEvent } from "./delete-account-objects.js";
+
+describe("DeleteAccountObjects handler", () => {
+	it("event shape is correct", () => {
+		const event: DeleteAccountObjectsEvent = {
+			type: "DELETE_ACCOUNT_OBJECTS",
+			accountConfigId: "test-account-config-id-12345",
+		};
+
+		assert.equal(event.type, "DELETE_ACCOUNT_OBJECTS");
+		assert.equal(event.accountConfigId, "test-account-config-id-12345");
+		assert.equal(event.continuationToken, undefined);
+	});
+
+	it("event with continuation token is correct", () => {
+		const event: DeleteAccountObjectsEvent = {
+			type: "DELETE_ACCOUNT_OBJECTS",
+			accountConfigId: "test-account-config-id-12345",
+			continuationToken: "abc123",
+		};
+
+		assert.equal(event.continuationToken, "abc123");
+	});
+
+	it("S3 prefix follows expected pattern", () => {
+		const accountConfigId = "test-account-config-id-12345";
+		const prefix = `accounts/${accountConfigId}/`;
+
+		assert.ok(prefix.startsWith("accounts/"));
+		assert.ok(prefix.endsWith("/"));
+		assert.ok(prefix.includes(accountConfigId));
+	});
+
+	it("batch size calculation respects limit", () => {
+		const BATCH_SIZE = 1_000;
+		const keys = Array.from({ length: 2500 }, (_, i) => `key-${i}`);
+
+		// Simulate batching
+		const batches: string[][] = [];
+		for (let i = 0; i < keys.length; i += BATCH_SIZE) {
+			batches.push(keys.slice(i, i + BATCH_SIZE));
+		}
+
+		assert.equal(batches.length, 3);
+		assert.equal(batches[0].length, 1000);
+		assert.equal(batches[1].length, 1000);
+		assert.equal(batches[2].length, 500);
+	});
+
+	it("re-enqueue event preserves continuation token", () => {
+		const accountConfigId = "test-id";
+		const continuationToken = "next-page-token";
+
+		const reenqueueEvent: DeleteAccountObjectsEvent = {
+			type: "DELETE_ACCOUNT_OBJECTS",
+			accountConfigId,
+			continuationToken,
+		};
+
+		const body = JSON.stringify(reenqueueEvent);
+		const parsed = JSON.parse(body) as DeleteAccountObjectsEvent;
+
+		assert.equal(parsed.type, "DELETE_ACCOUNT_OBJECTS");
+		assert.equal(parsed.accountConfigId, accountConfigId);
+		assert.equal(parsed.continuationToken, continuationToken);
+	});
+
+	it("timeout detection triggers re-enqueue", () => {
+		const MIN_REMAINING_MS = 30_000;
+
+		// Simulate near-timeout scenario
+		const getRemainingTimeMs = () => 25_000;
+		assert.ok(getRemainingTimeMs() < MIN_REMAINING_MS);
+
+		// Simulate enough time
+		const getRemainingTimeMsOk = () => 60_000;
+		assert.ok(getRemainingTimeMsOk() >= MIN_REMAINING_MS);
+	});
+});