@remit/imap-worker 0.0.1

package/src/connection-scope.ts

@@ -0,0 +1,335 @@
+/**
+ * Connection scope utility for managing IMAP connections across event processing.
+ *
+ * Creates a lazily-connected, cached connection that can be shared across
+ * multiple operations during a single event's lifetime.
+ *
+ * On top of the per-event scope, this module keeps a MODULE-scoped pool of warm
+ * connections keyed by accountId so a warm Lambda container reuses live IMAP
+ * connections across invocations instead of paying TCP+TLS+LOGIN+SELECT every
+ * time (see #605).
+ */
+
+import {
+	createConnection,
+	createConnectionWithCredentials,
+	type IImapConnection,
+	type ImapConnectionConfig,
+	type MailCredentials,
+} from "@remit/mailbox-service";
+
+export interface ConnectionScope {
+	/**
+	 * Get the connection, connecting lazily if not already connected.
+	 * Returns the same connection instance on subsequent calls.
+	 */
+	getConnection: () => Promise<IImapConnection>;
+
+	/**
+	 * Disconnect the connection if it was ever connected.
+	 * Safe to call multiple times.
+	 */
+	disconnect: () => Promise<void>;
+}
+
+/**
+ * Create a connection scope that manages a single IMAP connection's lifecycle.
+ *
+ * The connection is created lazily on first call to getConnection() and
+ * reused for all subsequent calls. Call disconnect() when done to clean up.
+ *
+ * @example
+ * ```typescript
+ * const scope = createConnectionScope(config);
+ *
+ * await doWork(scope.getConnection)
+ *   .finally(() => scope.disconnect());
+ * ```
+ */
+export const createConnectionScope = (
+	config: ImapConnectionConfig,
+): ConnectionScope => {
+	let connection: IImapConnection | null = null;
+	let connectPromise: Promise<IImapConnection> | null = null;
+
+	const getConnection = async (): Promise<IImapConnection> => {
+		if (connectPromise) {
+			return connectPromise;
+		}
+
+		const conn = createConnection(config);
+		connection = conn;
+		connectPromise = conn.connect().then(() => conn);
+
+		return connectPromise;
+	};
+
+	const disconnect = async (): Promise<void> => {
+		if (connection) {
+			await connection.disconnect();
+			connection = null;
+			connectPromise = null;
+		}
+	};
+
+	return { getConnection, disconnect };
+};
+
+/**
+ * Create a connection scope from account credentials using a password.
+ */
+export const createConnectionScopeFromAccount = (
+	account: {
+		username: string;
+		imapHost: string;
+		imapPort: number;
+		imapTls: boolean;
+	},
+	password: string,
+): ConnectionScope => {
+	return createConnectionScope({
+		user: account.username,
+		credentials: { kind: "password", password },
+		host: account.imapHost,
+		port: account.imapPort,
+		tls: account.imapTls,
+	});
+};
+
+/**
+ * Create a connection scope from account data and a MailCredentials union.
+ * Use this for all handlers that support both password and OAuth accounts.
+ */
+export const createConnectionScopeWithCredentials = (
+	account: {
+		username: string;
+		imapHost: string;
+		imapPort: number;
+		imapTls: boolean;
+	},
+	credentials: MailCredentials,
+): ConnectionScope => {
+	let connection: IImapConnection | null = null;
+	let connectPromise: Promise<IImapConnection> | null = null;
+
+	const getConnection = async (): Promise<IImapConnection> => {
+		if (connectPromise) {
+			return connectPromise;
+		}
+
+		const conn = createConnectionWithCredentials(account, credentials);
+		connection = conn;
+		connectPromise = conn.connect().then(() => conn);
+
+		return connectPromise;
+	};
+
+	const disconnect = async (): Promise<void> => {
+		if (connection) {
+			await connection.disconnect();
+			connection = null;
+			connectPromise = null;
+		}
+	};
+
+	return { getConnection, disconnect };
+};
+
+/**
+ * Number of warm connections to hold per account in a single container.
+ *
+ * Default 2 (Hostnet-conservative). imapflow runs one command at a time, so a
+ * single cached connection cannot be shared by concurrent invocations of the
+ * same account in one container (the worker runs handlers under p-map). The
+ * pool lets concurrent same-account invocations each borrow a live connection
+ * while still reusing them across invocations.
+ *
+ * CAVEAT: warm reuse only helps WITHIN one container. Lambda may run many
+ * containers (each with its own pool) and recycle them, so this cuts login
+ * churn but does NOT by itself bound the connections an account opens against
+ * the provider — the queue concurrency cap (#610) does that.
+ */
+const connectionsPerAccount = (() => {
+	const raw = Number(process.env.CONNECTIONS_PER_ACCOUNT);
+	return Number.isInteger(raw) && raw > 0 ? raw : 2;
+})();
+
+/**
+ * A pooled, lazily-connected connection plus its liveness/borrow bookkeeping.
+ */
+interface PooledConnection {
+	scope: ConnectionScope;
+	/** The established connection once getConnection() has resolved. */
+	connection: IImapConnection | null;
+	/** Borrowed by an in-flight invocation; must not be lent out concurrently. */
+	busy: boolean;
+}
+
+/** accountId -> warm pool. Module scope so it survives across invocations. */
+const warmPools = new Map<string, PooledConnection[]>();
+
+/**
+ * A connection borrowed from the warm pool for the duration of one invocation.
+ */
+export interface BorrowedConnection {
+	/** Get the (possibly cached, liveness-checked) live connection. */
+	getConnection: () => Promise<IImapConnection>;
+	/**
+	 * Return the connection to the pool. Pooled connections stay connected for
+	 * reuse; overflow connections (created when the pool is saturated) are
+	 * disconnected so the pool never grows past connectionsPerAccount.
+	 */
+	release: () => Promise<void>;
+}
+
+/**
+ * A live connection passes liveness when the underlying socket is still
+ * authenticated. imapflow flips `isConnected` to false on its `close`/`error`
+ * events, which fire when the provider drops an idle socket — the dominant
+ * warm-container failure mode. A connection that fails this check is replaced.
+ */
+const isLive = (entry: PooledConnection): boolean =>
+	entry.connection?.isConnected ?? false;
+
+const disconnectQuietly = async (entry: PooledConnection): Promise<void> => {
+	try {
+		await entry.scope.disconnect();
+	} catch {
+		// A dead connection may already be gone; reclaiming it must never throw.
+	}
+};
+
+/**
+ * Borrow a warm connection for one invocation, keyed by accountId.
+ *
+ * Reuses a free, live pooled connection when one exists (no reconnect). A free
+ * connection that has gone dead is disconnected and replaced. When every pooled
+ * connection is busy and the pool is full, an overflow connection is created and
+ * torn down on release so the steady-state pool size stays at
+ * connectionsPerAccount.
+ *
+ * The caller MUST call release() (e.g. in a finally) so the connection returns
+ * to the pool; releasing does NOT disconnect a pooled connection.
+ */
+export const borrowWarmConnection = (
+	accountId: string,
+	createScope: () => ConnectionScope,
+): BorrowedConnection => {
+	const pool = warmPools.get(accountId) ?? [];
+	if (!warmPools.has(accountId)) {
+		warmPools.set(accountId, pool);
+	}
+
+	let entry: PooledConnection | undefined;
+	let isOverflow = false;
+
+	const claimFreeLiveEntry = (): PooledConnection | undefined => {
+		for (const candidate of pool) {
+			if (candidate.busy) {
+				continue;
+			}
+			if (isLive(candidate)) {
+				candidate.busy = true;
+				return candidate;
+			}
+		}
+		return undefined;
+	};
+
+	const claimDeadOrEmptyEntry = (): PooledConnection | undefined => {
+		for (const candidate of pool) {
+			if (!candidate.busy && !isLive(candidate)) {
+				// Claim synchronously BEFORE the awaited disconnect in the recycle
+				// branch, mirroring claimFreeLiveEntry. Without this, a concurrent
+				// same-account borrow could claim the same dead entry during that
+				// await and two invocations would share one imapflow connection.
+				candidate.busy = true;
+				return candidate;
+			}
+		}
+		return undefined;
+	};
+
+	const getConnection = async (): Promise<IImapConnection> => {
+		if (entry) {
+			const conn = await entry.scope.getConnection();
+			entry.connection = conn;
+			return conn;
+		}
+
+		const reused = claimFreeLiveEntry();
+		if (reused) {
+			entry = reused;
+			const conn = await reused.scope.getConnection();
+			reused.connection = conn;
+			return conn;
+		}
+
+		// No free live connection: reuse a dead/never-connected slot if one is
+		// free, otherwise grow the pool, otherwise go overflow.
+		// claimDeadOrEmptyEntry already set busy=true synchronously, so the
+		// recycle slot is fenced off across the awaited disconnect below.
+		const recyclable = claimDeadOrEmptyEntry();
+		if (recyclable) {
+			await disconnectQuietly(recyclable);
+			recyclable.scope = createScope();
+			recyclable.connection = null;
+			entry = recyclable;
+		} else if (pool.length < connectionsPerAccount) {
+			const fresh: PooledConnection = {
+				scope: createScope(),
+				connection: null,
+				busy: true,
+			};
+			pool.push(fresh);
+			entry = fresh;
+		} else {
+			isOverflow = true;
+			entry = { scope: createScope(), connection: null, busy: true };
+		}
+
+		const conn = await entry.scope.getConnection();
+		entry.connection = conn;
+		return conn;
+	};
+
+	const release = async (): Promise<void> => {
+		if (!entry) {
+			return;
+		}
+		if (isOverflow) {
+			await disconnectQuietly(entry);
+			entry = undefined;
+			return;
+		}
+		entry.busy = false;
+		entry = undefined;
+	};
+
+	return { getConnection, release };
+};
+
+/**
+ * Test-only: disconnect and drop an account's warm pool so a test never leaks a
+ * live connection across cases. Not part of the production lifecycle — in
+ * steady state dead entries are recycled lazily on the next borrow.
+ */
+export const __evictWarmConnectionsForTest = async (
+	accountId: string,
+): Promise<void> => {
+	const pool = warmPools.get(accountId);
+	if (!pool) {
+		return;
+	}
+	warmPools.delete(accountId);
+	await Promise.all(pool.map(disconnectQuietly));
+};
+
+/** Test-only: clear all warm pools without disconnecting (fixtures own teardown). */
+export const __resetWarmPoolsForTest = (): void => {
+	warmPools.clear();
+};
+
+/** Test-only: inspect the live pool size for an account. */
+export const __warmPoolSizeForTest = (accountId: string): number =>
+	warmPools.get(accountId)?.length ?? 0;

package/src/e2e-processor-shim.ts

@@ -0,0 +1,248 @@
+#!/usr/bin/env node
+import cluster from "node:cluster";
+import {
+	DeleteMessageCommand,
+	ReceiveMessageCommand,
+	SQSClient,
+} from "@aws-sdk/client-sqs";
+import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
+import { createLogger } from "@remit/logger-lambda";
+import { handler as searchIndexHandler } from "@remit/search-index-worker";
+import { resolveSqsCredentials } from "@remit/sqs-client";
+import type {
+	Context,
+	SQSBatchResponse,
+	SQSEvent,
+	SQSHandler,
+} from "aws-lambda";
+import { env } from "expect-env";
+import { handler } from "./index.js";
+
+/**
+ * E2E-only queue drainer.
+ *
+ * Production binds the Lambda `handler` (`src/index.ts`) to each queue via an
+ * SQS event-source mapping. The e2e/CI stack runs on ElasticMQ, which has no
+ * event-source mapping, so this process supplies that missing piece: it
+ * long-polls each queue, wraps every received batch in an `SQSEvent`, and
+ * invokes the exact production `handler`. It then honours the returned
+ * `batchItemFailures` the way the SQS service would — deleting the messages
+ * that succeeded and leaving the failures un-deleted so their visibility
+ * timeout lapses and SQS redelivers them. No processing or failure logic lives
+ * here; the prod handler owns all of it.
+ *
+ * This is a test harness, not production code. If it crashes the e2e suite
+ * fails loudly, which is the desired signal — there is no crash net.
+ */
+
+// The search-index queue is drained by the production search-index-worker
+// handler instead of the imap handler. It is optional: when its URL is unset
+// (e.g. the e2e stack), the queue is simply not polled.
+const searchIndexQueueUrl = process.env.SQS_QUEUE_URL_SEARCH_INDEX;
+
+// Collect all unique queue URLs to poll. Every required queue URL crashes at
+// init via expect-env instead of silently dropping queues.
+const queueUrls = [
+	...new Set([
+		// FIFO queues for sync operations
+		env.SQS_QUEUE_URL_MAILBOXES,
+		env.SQS_QUEUE_URL_MESSAGES,
+		env.SQS_QUEUE_URL_FLAGS,
+		// Standard body queue (#612) + management queues
+		env.SQS_QUEUE_URL_BODY,
+		env.SQS_QUEUE_URL_MAILBOX_MGMT,
+		env.SQS_QUEUE_URL_MESSAGE_MGMT,
+		// Standard queue for local search indexing (optional)
+		...(searchIndexQueueUrl ? [searchIndexQueueUrl] : []),
+	]),
+];
+
+if (cluster.isPrimary) {
+	// Primary process: fork a worker for each queue
+	const log = createLogger();
+	log.info({ queueUrls, workerCount: queueUrls.length }, "Primary started");
+
+	// Track which queue each worker handles for restart
+	const workerQueues = new Map<number, string>();
+
+	// Fork a worker for each queue
+	for (const queueUrl of queueUrls) {
+		const worker = cluster.fork({ WORKER_QUEUE_URL: queueUrl });
+		workerQueues.set(worker.id, queueUrl);
+		const queueName = new URL(queueUrl).pathname.split("/").pop();
+		log.info({ workerId: worker.id, queueName }, "Forked worker for queue");
+	}
+
+	// Restart workers that crash
+	cluster.on("exit", (worker, code, signal) => {
+		const queueUrl = workerQueues.get(worker.id);
+		const queueName = queueUrl
+			? new URL(queueUrl).pathname.split("/").pop()
+			: "unknown";
+
+		if (signal) {
+			log.info({ workerId: worker.id, queueName, signal }, "Worker killed");
+		} else if (code !== 0) {
+			log.error(
+				{ workerId: worker.id, queueName, code },
+				"Worker crashed, exiting primary",
+			);
+			process.exit(1);
+		} else {
+			log.info({ workerId: worker.id, queueName }, "Worker exited cleanly");
+			workerQueues.delete(worker.id);
+		}
+	});
+
+	// Graceful shutdown: signal all workers
+	const shutdown = () => {
+		log.info("Primary received shutdown signal, stopping workers...");
+		for (const worker of Object.values(cluster.workers ?? {})) {
+			worker?.process.kill("SIGTERM");
+		}
+	};
+
+	process.on("SIGINT", shutdown);
+	process.on("SIGTERM", shutdown);
+} else {
+	// Worker process: poll a single queue
+	const queueUrl = env.WORKER_QUEUE_URL;
+	const queueName = new URL(queueUrl).pathname.split("/").pop();
+	const log = createLogger().child({ queue: queueName });
+
+	// The search-index queue carries DynamoDB-stream-shaped events and is owned
+	// by the search-index-worker handler; every other queue is an imap operation.
+	const activeHandler: SQSHandler =
+		queueUrl === searchIndexQueueUrl ? searchIndexHandler : handler;
+
+	const maxMessages = 10; // SQS API limit
+
+	const isLocal = queueUrl.startsWith("http://localhost");
+	const sqs = new SQSClient({
+		endpoint: isLocal ? new URL(queueUrl).origin : undefined,
+		...(isLocal && { protocol: AwsQueryProtocol }),
+		credentials: resolveSqsCredentials(),
+	});
+
+	let isShuttingDown = false;
+
+	process.on("SIGINT", () => {
+		log.info("Worker received SIGINT, shutting down...");
+		isShuttingDown = true;
+	});
+
+	process.on("SIGTERM", () => {
+		log.info("Worker received SIGTERM, shutting down...");
+		isShuttingDown = true;
+	});
+
+	// Minimal Lambda Context: `withTelemetry` only reads `functionName` and adds
+	// it to the logger; the prod handler never touches the rest.
+	const lambdaContext = {
+		functionName: `e2e-imap-worker-${queueName}`,
+	} as Context;
+
+	const pollQueue = async (): Promise<void> => {
+		log.info({ maxMessages }, "Worker started, polling...");
+
+		let consecutiveEmptyPolls = 0;
+
+		while (!isShuttingDown) {
+			// Use short polling when we just processed messages (likely more waiting)
+			// Use long polling after empty polls to reduce API calls
+			const waitTime = consecutiveEmptyPolls > 0 ? 20 : 0;
+
+			const response = await sqs.send(
+				new ReceiveMessageCommand({
+					QueueUrl: queueUrl,
+					MaxNumberOfMessages: maxMessages,
+					WaitTimeSeconds: waitTime,
+					VisibilityTimeout: 300,
+					MessageSystemAttributeNames: ["ApproximateReceiveCount"],
+				}),
+			);
+
+			if (!response.Messages || response.Messages.length === 0) {
+				consecutiveEmptyPolls++;
+				continue;
+			}
+
+			consecutiveEmptyPolls = 0;
+
+			const messages = response.Messages.flatMap((m) =>
+				m.Body && m.ReceiptHandle && m.MessageId
+					? [
+							{
+								messageId: m.MessageId,
+								receiptHandle: m.ReceiptHandle,
+								body: m.Body,
+								receiveCount: m.Attributes?.ApproximateReceiveCount ?? "1",
+							},
+						]
+					: [],
+			);
+
+			if (messages.length === 0) {
+				continue;
+			}
+
+			log.info({ count: messages.length }, "Invoking handler for batch");
+
+			const event: SQSEvent = {
+				Records: messages.map((m) => ({
+					messageId: m.messageId,
+					receiptHandle: m.receiptHandle,
+					body: m.body,
+					attributes: {
+						ApproximateReceiveCount: m.receiveCount,
+						SentTimestamp: "0",
+						SenderId: "e2e",
+						ApproximateFirstReceiveTimestamp: "0",
+					},
+					messageAttributes: {},
+					md5OfBody: "",
+					eventSource: "aws:sqs",
+					eventSourceARN: queueUrl,
+					awsRegion: "local",
+				})),
+			};
+
+			const result = (await activeHandler(event, lambdaContext, () => {})) as
+				| SQSBatchResponse
+				| undefined;
+
+			const failedIds = new Set(
+				(result?.batchItemFailures ?? []).map((f) => f.itemIdentifier),
+			);
+
+			// Mirror SQS partial-batch-failure semantics: delete the messages the
+			// handler reported as succeeded; leave failures un-deleted so their
+			// visibility timeout lapses and SQS redelivers them (and eventually
+			// dead-letters once maxReceiveCount is hit).
+			const succeeded = messages.filter((m) => !failedIds.has(m.messageId));
+
+			for (const message of succeeded) {
+				await sqs.send(
+					new DeleteMessageCommand({
+						QueueUrl: queueUrl,
+						ReceiptHandle: message.receiptHandle,
+					}),
+				);
+			}
+
+			log.info(
+				{ deleted: succeeded.length, leftForRedelivery: failedIds.size },
+				"Batch processed",
+			);
+		}
+
+		log.info("Worker polling stopped");
+	};
+
+	pollQueue()
+		.then(() => process.exit(0))
+		.catch((error) => {
+			log.error({ error }, "Worker error");
+			process.exit(1);
+		});
+}

package/src/emit.test.ts

@@ -0,0 +1,44 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import { getDeduplicationId } from "./emit.js";
+import type { SyncMessagesEvent } from "./events.js";
+
+type SyncMessagesInput = Omit<SyncMessagesEvent, "eventId" | "timestamp">;
+
+const syncMessages = (over: Partial<SyncMessagesInput>): SyncMessagesInput => ({
+	type: "SYNC_MESSAGES",
+	accountId: "acc-1",
+	mailboxId: "mbx-1",
+	...over,
+});
+
+describe("getDeduplicationId for SYNC_MESSAGES continuation", () => {
+	it("gives an initial (cursor-less) sync a stable id so concurrent fresh syncs dedup", () => {
+		assert.strictEqual(
+			getDeduplicationId(syncMessages({})),
+			"SYNC_MESSAGES:mbx-1",
+		);
+	});
+
+	it("gives each continuation batch a distinct id so FIFO does not drop batches 2..N", () => {
+		// Regression guard: a constant dedup id per mailbox let SQS FIFO reject
+		// every continuation within the 5-minute window, capping a sync at one
+		// batch (~200 messages). Folding the batch's remaining count into the id
+		// keeps sequential batches distinct.
+		const batch2 = getDeduplicationId(syncMessages({ resumeCursor: 800 }));
+		const batch3 = getDeduplicationId(syncMessages({ resumeCursor: 600 }));
+
+		assert.strictEqual(batch2, "SYNC_MESSAGES:mbx-1:800");
+		assert.strictEqual(batch3, "SYNC_MESSAGES:mbx-1:600");
+		assert.notStrictEqual(batch2, batch3);
+	});
+
+	it("separates a continuation from the initial event of the same mailbox", () => {
+		const initial = getDeduplicationId(syncMessages({}));
+		const continuation = getDeduplicationId(
+			syncMessages({ resumeCursor: 800 }),
+		);
+
+		assert.notStrictEqual(initial, continuation);
+	});
+});