@remit/imap-worker 0.0.89 → 0.0.91

package/src/credential-less-sync-fence.test.ts

@@ -1,152 +0,0 @@
-/**
- * Regression test for issue #1120.
- *
- * A sync triggered for an account that stores no credential used to fail the
- * batch item, so the poller left the message undeleted for its whole
- * visibility timeout. `remit-mailboxes.fifo` groups by accountId and a FIFO
- * queue never delivers past an in-flight message in a group, so the account
- * was fenced from every later sync — including the one the user asks for right
- * after saving the password.
- *
- * Runs against the real QueueStore and the real queue definition the stack
- * ships, with the poller's own contract modelled around it: ack deletes the
- * message, a batch item failure leaves it in flight.
- */
-
-import assert from "node:assert/strict";
-import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
-import { join } from "node:path";
-import { after, describe, it } from "node:test";
-import { fileURLToPath } from "node:url";
-import { buildSyncMailboxesMessage } from "@remit/backend/trigger-sync";
-import type { AccountItem } from "@remit/data-ports";
-import { ConnectionState } from "@remit/domain-enums";
-import {
-	bootstrapQueues,
-	loadQueuesConfig,
-	QueueStore,
-} from "@remit/queue-sidecar";
-import { DEFAULT_VISIBILITY_TIMEOUT_SECONDS } from "@remit/sqs-client/poller";
-import {
-	type OAuthLifecycleDeps,
-	withOAuthLifecycle,
-} from "./with-oauth-lifecycle.js";
-
-const QUEUE_NAME = "remit-mailboxes.fifo";
-const ACCOUNT_ID = "acct_imported";
-const T0 = 1_700_000_000_000;
-
-const here = fileURLToPath(new URL(".", import.meta.url));
-const repoRoot = join(here, "..", "..", "..");
-const tmpRoot = join(here, "..", ".tmp", "credential-less-sync-fence");
-
-const silentLogger = {
-	info: () => {},
-	warn: () => {},
-	error: () => {},
-	debug: () => {},
-	trace: () => {},
-	fatal: () => {},
-	child: () => silentLogger,
-} as never;
-
-/** The account the import wizard lands: no password until its owner types one. */
-const importedAccount = {
-	accountId: ACCOUNT_ID,
-	accountConfigId: "acfg_1",
-	username: "alice@example.com",
-	email: "alice@example.com",
-	authType: "password",
-	imapHost: "imap.example.com",
-	imapPort: 993,
-	imapTls: true,
-	imapStartTls: false,
-	isActive: true,
-	connectionState: ConnectionState.CredentialsMissing,
-	createdAt: T0,
-	updatedAt: T0,
-} as unknown as AccountItem;
-
-const enqueueSync = (store: QueueStore, now: number): void => {
-	const message = buildSyncMailboxesMessage({ accountId: ACCOUNT_ID });
-	store.sendMessage({
-		queueName: QUEUE_NAME,
-		body: JSON.stringify(message.body),
-		groupId: message.groupId,
-		deduplicationId: message.deduplicationId,
-		now,
-	});
-};
-
-describe("a credential-less sync trigger (issue #1120)", () => {
-	after(() => {
-		rmSync(tmpRoot, { recursive: true, force: true });
-	});
-
-	it("is acked, leaving the account's queue group deliverable", async () => {
-		mkdirSync(tmpRoot, { recursive: true });
-		const store = new QueueStore(
-			join(mkdtempSync(join(tmpRoot, "store-")), "queue.db"),
-		);
-		bootstrapQueues(
-			store,
-			loadQueuesConfig(join(repoRoot, "deploy/vps/queues.json")),
-		);
-
-		const stateUpdates: string[] = [];
-		const deps: OAuthLifecycleDeps = {
-			secrets: {
-				decrypt: async () => "",
-				encrypt: async () => ({}) as never,
-			},
-			tokenService: { getAccessToken: async () => ({}) as never },
-			persistRotatedToken: async () => {},
-			updateConnectionState: async (_accountId, state) => {
-				stateUpdates.push(state);
-			},
-		};
-
-		// The GET /config poll while the user is still typing the password.
-		enqueueSync(store, T0);
-
-		const [received] = store.receiveMessages({
-			queueName: QUEUE_NAME,
-			maxMessages: 1,
-			visibilityTimeoutSeconds: DEFAULT_VISIBILITY_TIMEOUT_SECONDS,
-			now: T0,
-		});
-		assert.ok(received, "the trigger is on the queue");
-
-		// The poller's contract: an ack deletes the message, a batch item failure
-		// leaves it in flight until the visibility timeout expires.
-		let batchItemFailure: unknown;
-		await withOAuthLifecycle(deps, importedAccount, silentLogger, async () => {
-			throw new Error("IMAP work must not run without a credential");
-		}).then(
-			() => store.deleteMessage(QUEUE_NAME, received.receiptHandle),
-			(err: unknown) => {
-				batchItemFailure = err;
-			},
-		);
-
-		// The refresh the user presses after saving the password.
-		enqueueSync(store, T0 + 2_000);
-		const deliverable = store.receiveMessages({
-			queueName: QUEUE_NAME,
-			maxMessages: 10,
-			now: T0 + 3_000,
-		});
-
-		assert.equal(
-			deliverable.length,
-			1,
-			"the account's group must stay deliverable, not fenced for the visibility timeout",
-		);
-		assert.equal(
-			batchItemFailure,
-			undefined,
-			"the event is acked, not retried",
-		);
-		assert.deepEqual(stateUpdates, [ConnectionState.CredentialsMissing]);
-	});
-});

package/src/e2e-processor-shim.ts

@@ -1,232 +0,0 @@
-#!/usr/bin/env node
-import cluster from "node:cluster";
-import { randomUUID } from "node:crypto";
-import {
-	DeleteMessageCommand,
-	ReceiveMessageCommand,
-} from "@aws-sdk/client-sqs";
-import { createLogger } from "@remit/logger";
-import { installNodeLogger } from "@remit/logger-lambda/node";
-import { createSqsClient } from "@remit/sqs-client/producer";
-import type { Context, SQSBatchResponse, SQSEvent } from "aws-lambda";
-import { env } from "expect-env";
-import { handler } from "./index.js";
-
-installNodeLogger();
-
-/**
- * E2E-only queue drainer.
- *
- * Production binds the Lambda `handler` (`src/index.ts`) to each queue via an
- * SQS event-source mapping. The e2e/CI stack runs on ElasticMQ, which has no
- * event-source mapping, so this process supplies that missing piece: it
- * long-polls each queue, wraps every received batch in an `SQSEvent`, and
- * invokes the exact production `handler`. It then honours the returned
- * `batchItemFailures` the way the SQS service would — deleting the messages
- * that succeeded and leaving the failures un-deleted so their visibility
- * timeout lapses and SQS redelivers them. No processing or failure logic lives
- * here; the prod handler owns all of it.
- *
- * This is a test harness, not production code. If it crashes the e2e suite
- * fails loudly, which is the desired signal — there is no crash net.
- */
-
-// Collect all unique queue URLs to poll. Every required queue URL crashes at
-// init via expect-env instead of silently dropping queues.
-const queueUrls = [
-	...new Set([
-		// FIFO queues for sync operations
-		env.SQS_QUEUE_URL_MAILBOXES,
-		env.SQS_QUEUE_URL_MESSAGES,
-		env.SQS_QUEUE_URL_FLAGS,
-		// Standard body queue (#612) + management queues
-		env.SQS_QUEUE_URL_BODY,
-		env.SQS_QUEUE_URL_MAILBOX_MGMT,
-		env.SQS_QUEUE_URL_MESSAGE_MGMT,
-	]),
-];
-
-if (cluster.isPrimary) {
-	// Primary process: fork a worker for each queue
-	const log = createLogger();
-	log.info({ queueUrls, workerCount: queueUrls.length }, "Primary started");
-
-	// Track which queue each worker handles for restart
-	const workerQueues = new Map<number, string>();
-
-	// Fork a worker for each queue
-	for (const queueUrl of queueUrls) {
-		const worker = cluster.fork({ WORKER_QUEUE_URL: queueUrl });
-		workerQueues.set(worker.id, queueUrl);
-		const queueName = new URL(queueUrl).pathname.split("/").pop();
-		log.info({ workerId: worker.id, queueName }, "Forked worker for queue");
-	}
-
-	// Restart workers that crash
-	cluster.on("exit", (worker, code, signal) => {
-		const queueUrl = workerQueues.get(worker.id);
-		const queueName = queueUrl
-			? new URL(queueUrl).pathname.split("/").pop()
-			: "unknown";
-
-		if (signal) {
-			log.info({ workerId: worker.id, queueName, signal }, "Worker killed");
-		} else if (code !== 0) {
-			log.error(
-				{ workerId: worker.id, queueName, code },
-				"Worker crashed, exiting primary",
-			);
-			process.exit(1);
-		} else {
-			log.info({ workerId: worker.id, queueName }, "Worker exited cleanly");
-			workerQueues.delete(worker.id);
-		}
-	});
-
-	// Graceful shutdown: signal all workers
-	const shutdown = () => {
-		log.info("Primary received shutdown signal, stopping workers...");
-		for (const worker of Object.values(cluster.workers ?? {})) {
-			worker?.process.kill("SIGTERM");
-		}
-	};
-
-	process.on("SIGINT", shutdown);
-	process.on("SIGTERM", shutdown);
-} else {
-	// Worker process: poll a single queue
-	const queueUrl = env.WORKER_QUEUE_URL;
-	const queueName = new URL(queueUrl).pathname.split("/").pop();
-	const log = createLogger().child({ queue: queueName });
-
-	const maxMessages = 10; // SQS API limit
-
-	const sqs = createSqsClient({ queueUrl });
-
-	let isShuttingDown = false;
-
-	process.on("SIGINT", () => {
-		log.info("Worker received SIGINT, shutting down...");
-		isShuttingDown = true;
-	});
-
-	process.on("SIGTERM", () => {
-		log.info("Worker received SIGTERM, shutting down...");
-		isShuttingDown = true;
-	});
-
-	// Minimal Lambda Context: `withTelemetry` reads `functionName` for the line it
-	// logs per invocation and `awsRequestId` for the scope it opens around it, so
-	// every line one batch produces shares a `requestId` (deploy/vps/README.md,
-	// "Logs"). Minted per batch, like the production poller does — a context built
-	// once would put the whole run under one id, which correlates nothing.
-	const nextContext = (): Context =>
-		({
-			functionName: `e2e-imap-worker-${queueName}`,
-			awsRequestId: randomUUID(),
-		}) as Context;
-
-	const pollQueue = async (): Promise<void> => {
-		log.info({ maxMessages }, "Worker started, polling...");
-
-		let consecutiveEmptyPolls = 0;
-
-		while (!isShuttingDown) {
-			// Use short polling when we just processed messages (likely more waiting)
-			// Use long polling after empty polls to reduce API calls
-			const waitTime = consecutiveEmptyPolls > 0 ? 20 : 0;
-
-			const response = await sqs.send(
-				new ReceiveMessageCommand({
-					QueueUrl: queueUrl,
-					MaxNumberOfMessages: maxMessages,
-					WaitTimeSeconds: waitTime,
-					VisibilityTimeout: 300,
-					MessageSystemAttributeNames: ["ApproximateReceiveCount"],
-				}),
-			);
-
-			if (!response.Messages || response.Messages.length === 0) {
-				consecutiveEmptyPolls++;
-				continue;
-			}
-
-			consecutiveEmptyPolls = 0;
-
-			const messages = response.Messages.flatMap((m) =>
-				m.Body && m.ReceiptHandle && m.MessageId
-					? [
-							{
-								messageId: m.MessageId,
-								receiptHandle: m.ReceiptHandle,
-								body: m.Body,
-								receiveCount: m.Attributes?.ApproximateReceiveCount ?? "1",
-							},
-						]
-					: [],
-			);
-
-			if (messages.length === 0) {
-				continue;
-			}
-
-			log.info({ count: messages.length }, "Invoking handler for batch");
-
-			const event: SQSEvent = {
-				Records: messages.map((m) => ({
-					messageId: m.messageId,
-					receiptHandle: m.receiptHandle,
-					body: m.body,
-					attributes: {
-						ApproximateReceiveCount: m.receiveCount,
-						SentTimestamp: "0",
-						SenderId: "e2e",
-						ApproximateFirstReceiveTimestamp: "0",
-					},
-					messageAttributes: {},
-					md5OfBody: "",
-					eventSource: "aws:sqs",
-					eventSourceARN: queueUrl,
-					awsRegion: "local",
-				})),
-			};
-
-			const result = (await handler(event, nextContext(), () => {})) as
-				| SQSBatchResponse
-				| undefined;
-
-			const failedIds = new Set(
-				(result?.batchItemFailures ?? []).map((f) => f.itemIdentifier),
-			);
-
-			// Mirror SQS partial-batch-failure semantics: delete the messages the
-			// handler reported as succeeded; leave failures un-deleted so their
-			// visibility timeout lapses and SQS redelivers them (and eventually
-			// dead-letters once maxReceiveCount is hit).
-			const succeeded = messages.filter((m) => !failedIds.has(m.messageId));
-
-			for (const message of succeeded) {
-				await sqs.send(
-					new DeleteMessageCommand({
-						QueueUrl: queueUrl,
-						ReceiptHandle: message.receiptHandle,
-					}),
-				);
-			}
-
-			log.info(
-				{ deleted: succeeded.length, leftForRedelivery: failedIds.size },
-				"Batch processed",
-			);
-		}
-
-		log.info("Worker polling stopped");
-	};
-
-	pollQueue()
-		.then(() => process.exit(0))
-		.catch((error) => {
-			log.error({ error }, "Worker error");
-			process.exit(1);
-		});
-}

package/src/emit.test.ts

@@ -1,127 +0,0 @@
-import assert from "node:assert/strict";
-import { afterEach, beforeEach, describe, it } from "node:test";
-import {
-	_resetForTest,
-	type RemitClient,
-	setClient,
-} from "@remit/backend/client";
-import {
-	type RecordingQueue,
-	recordingQueue,
-} from "@remit/data-ports/recording-queue";
-import { MailboxLockRepo } from "@remit/drizzle-service";
-import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
-import { emitEvent, SYNC_MESSAGES_PENDING } from "./emit.js";
-import type {
-	FlagPushEvent,
-	SyncMailboxesEvent,
-	SyncMessagesEvent,
-} from "./events.js";
-import { setWorkerQueues } from "./queues.js";
-
-type Emitted<T> = Omit<T, "eventId" | "timestamp">;
-
-let locks: MailboxLockRepo;
-let closeStore: () => void;
-let queue: RecordingQueue;
-
-const sentCommands = () => queue.sent;
-
-beforeEach(() => {
-	queue = recordingQueue();
-	setWorkerQueues({
-		mailboxes: queue,
-		messages: queue,
-		flags: queue,
-		body: queue,
-		mailboxManagement: queue,
-		messageManagement: queue,
-	});
-	const store = createShippedSqliteDb();
-	closeStore = store.close;
-	locks = new MailboxLockRepo(store.db);
-	setClient({ mailboxLock: locks } as unknown as RemitClient);
-});
-
-afterEach(() => {
-	_resetForTest();
-	closeStore();
-});
-
-describe("emitEvent on a FIFO queue", () => {
-	it("groups by account and deduplicates on the event's own id", async () => {
-		const event: Emitted<SyncMessagesEvent> = {
-			type: "SYNC_MESSAGES",
-			accountId: "acc-1",
-			mailboxId: "mbx-1",
-		};
-
-		await emitEvent(event);
-
-		const [sent] = sentCommands();
-		if (!sent) throw new Error("expected a send");
-		assert.equal(sent.groupId, "acc-1");
-		assert.equal(sent.deduplicationId, sent.body.eventId);
-	});
-
-	// Issue #37: every SYNC_MESSAGES for a mailbox carried one dedup id, so
-	// SQS FIFO's 5-minute window discarded the second sync of that mailbox
-	// before any worker saw it — mail appended after a sync could not be
-	// fetched until the window elapsed. Two syncs of one mailbox must both
-	// reach the queue.
-	it("lets a second sync of the same mailbox through", async () => {
-		const event: Emitted<SyncMessagesEvent> = {
-			type: "SYNC_MESSAGES",
-			accountId: "acc-1",
-			mailboxId: "mbx-1",
-		};
-
-		await emitEvent(event);
-		const [first] = sentCommands();
-		if (!first) throw new Error("expected a send");
-		await locks.releaseLock(
-			"acc-1",
-			"mbx-1",
-			SYNC_MESSAGES_PENDING,
-			String(first.body.eventId),
-		);
-		await emitEvent(event);
-
-		const sent = sentCommands();
-		assert.equal(sent.length, 2);
-		assert.notEqual(sent[0]?.deduplicationId, sent[1]?.deduplicationId);
-	});
-
-	it("lets a second sync of the same account's mailbox list through", async () => {
-		const event: Emitted<SyncMailboxesEvent> = {
-			type: "SYNC_MAILBOXES",
-			accountId: "acc-1",
-		};
-
-		await emitEvent(event);
-		await emitEvent(event);
-
-		const sent = sentCommands();
-		assert.equal(sent.length, 2);
-		assert.notEqual(sent[0]?.deduplicationId, sent[1]?.deduplicationId);
-	});
-
-	// A FIFO queue without content-based deduplication rejects a send carrying
-	// no MessageDeduplicationId, so every event routed onto one needs its own —
-	// not only the sync events.
-	it("gives a flag-push re-arm a deduplication id", async () => {
-		const event: Emitted<FlagPushEvent> = {
-			type: "FLAG_PUSH",
-			accountId: "acc-1",
-			accountConfigId: "cfg-1",
-			messageId: "msg-1",
-			flagName: "\\Seen",
-		};
-
-		await emitEvent(event);
-
-		const [sent] = sentCommands();
-		if (!sent) throw new Error("expected a send");
-		assert.equal(sent.deduplicationId, sent.body.eventId);
-	});
-});

package/src/emit.ts

@@ -1,117 +0,0 @@
-import { randomUUID } from "node:crypto";
-import { getClient } from "@remit/backend/client";
-import type { FifoQueueProducer } from "@remit/data-ports";
-import { recordImapFailure } from "@remit/logger-lambda/metrics";
-import type { ImapEvent } from "./events.js";
-import { type WorkerQueues, workerQueues } from "./queues.js";
-
-type EventInput = Omit<ImapEvent, "eventId" | "timestamp">;
-
-export const SYNC_MESSAGES_PENDING = "SYNC_MESSAGES_PENDING";
-
-const SYNC_MESSAGES_PENDING_TTL_SECONDS = 300;
-
-const queueFor = (
-	type: ImapEvent["type"],
-	queues: WorkerQueues,
-): FifoQueueProducer<ImapEvent> => {
-	switch (type) {
-		case "SYNC_MAILBOXES":
-			return queues.mailboxes;
-		case "SYNC_MESSAGES":
-			return queues.messages;
-		// SYNC_MESSAGE_BODY routes to the single standard body queue (#612).
-		case "SYNC_MESSAGE_BODY":
-			return queues.body;
-		case "MAILBOX_CREATE":
-		case "MAILBOX_RENAME":
-		case "MAILBOX_DELETE":
-			return queues.mailboxManagement;
-		case "MESSAGE_DELETE":
-		case "MESSAGE_MOVE":
-		case "MESSAGE_COPY":
-		case "EMPTY_TRASH":
-		case "APPEND_SENT_MESSAGE":
-		// Rides messageMgmtQueue (issue #1271) rather than a dedicated queue — its
-		// payload carries only our message id (never a UID, unlike the legacy
-		// MESSAGE_MOVE event); per-event-type payload shape, same queue.
-		case "PLACEMENT_MOVE_PUSH":
-			return queues.messageManagement;
-		// Rides the existing flags queue (issue #1273) — this is imap-worker's OWN
-		// re-arm hint (the periodic per-mailbox sync tick catching up a marker
-		// stuck `pending`), distinct from `FlagPushService`'s user-facing hint
-		// sent from the API onto the sync queue. Both land on a queue this worker
-		// already consumes; dispatch is by `type`, not by which queue delivered
-		// the message.
-		case "FLAG_PUSH":
-			return queues.flags;
-	}
-};
-
-export interface EmitEventOptions {
-	/**
-	 * Delay delivery of the message by this many seconds (0-900).
-	 * Useful for retry backoff to avoid overwhelming IMAP servers.
-	 */
-	delaySeconds?: number;
-}
-
-export const emitEvent = async (
-	event: EventInput,
-	options?: EmitEventOptions,
-) => {
-	const fullEvent: ImapEvent = {
-		...event,
-		eventId: randomUUID(),
-		timestamp: Date.now(),
-	} as ImapEvent;
-
-	const queue = queueFor(event.type, workerQueues());
-
-	// The deduplication id is the event's own id, so the queue suppresses a
-	// re-send of one event (the retry SQS's own idempotency guard is for) and
-	// nothing else. A shared id per account or per mailbox instead made the
-	// 5-minute window a rate limiter: the second sync of a mailbox within five
-	// minutes was discarded before any worker saw it, so mail that arrived after
-	// a sync could not be fetched until the window elapsed (issue #37).
-	const send = () =>
-		queue.send({
-			body: fullEvent,
-			groupId: event.accountId,
-			deduplicationId: fullEvent.eventId,
-			...(options?.delaySeconds && { delaySeconds: options.delaySeconds }),
-		});
-	if (fullEvent.type !== "SYNC_MESSAGES") {
-		await send();
-		return;
-	}
-
-	const { mailboxLock } = await getClient();
-	const { accountId, mailboxId, eventId } = fullEvent;
-	const previous = await mailboxLock.get(
-		accountId,
-		mailboxId,
-		SYNC_MESSAGES_PENDING,
-	);
-	const acquired = await mailboxLock.tryAcquireLock(
-		mailboxId,
-		SYNC_MESSAGES_PENDING,
-		accountId,
-		eventId,
-		SYNC_MESSAGES_PENDING_TTL_SECONDS,
-	);
-	if (!acquired) return;
-	if (previous && previous.ttl < Math.floor(Date.now() / 1000)) {
-		recordImapFailure("SYNC_PENDING_EXPIRED", "other");
-	}
-
-	await send().catch(async (error: unknown) => {
-		await mailboxLock.releaseLock(
-			accountId,
-			mailboxId,
-			SYNC_MESSAGES_PENDING,
-			eventId,
-		);
-		throw error;
-	});
-};