@remit/imap-worker 0.0.1

package/src/processor.ts

@@ -0,0 +1,66 @@
+import type { Logger } from "@remit/logger-lambda";
+import type { WorkerEvent } from "./events.js";
+import { handleAppendSentMessage } from "./handlers/append-sent-message.js";
+import { handleDeleteAccountObjects } from "./handlers/delete-account-objects.js";
+import { handleEmptyTrash } from "./handlers/empty-trash.js";
+import { handleFlagPush } from "./handlers/flag-push.js";
+import { processMailboxManagement } from "./handlers/mailbox-management.js";
+import { handleMessageCopy } from "./handlers/message-copy.js";
+import { handleMessageDelete } from "./handlers/message-delete.js";
+import { handleMessageMove } from "./handlers/message-move.js";
+import { handlePlacementMovePush } from "./handlers/placement-move-push.js";
+import { syncMailboxes } from "./handlers/sync-mailboxes.js";
+import { syncMessageBody } from "./handlers/sync-message-body.js";
+import { syncMessages } from "./handlers/sync-messages.js";
+
+export const processEvent = async (
+	event: WorkerEvent,
+	log: Logger,
+	/**
+	 * SQS's own delivery count for the record carrying this event (1 on first
+	 * delivery). Only SYNC_MESSAGE_BODY reads it — it's how the handler knows
+	 * this is the last attempt before the queue's own redrive policy would
+	 * DLQ the record, so it can resolve retry exhaustion into a terminal
+	 * outcome (issue #1270) instead of dead-lettering blindly.
+	 */
+	receiveCount = 1,
+): Promise<void> => {
+	switch (event.type) {
+		case "SYNC_MAILBOXES":
+			return syncMailboxes(event, log);
+		case "SYNC_MESSAGES":
+			return syncMessages(event, log);
+		case "SYNC_MESSAGE_BODY":
+			return syncMessageBody(event, log, receiveCount);
+		case "MAILBOX_CREATE":
+		case "MAILBOX_RENAME":
+		case "MAILBOX_DELETE":
+			return processMailboxManagement(event, log);
+		case "MESSAGE_DELETE":
+			return handleMessageDelete(event, log);
+		case "MESSAGE_MOVE":
+			return handleMessageMove(event, log);
+		case "PLACEMENT_MOVE_PUSH":
+			return handlePlacementMovePush(event, log, receiveCount);
+		case "FLAG_PUSH":
+			return handleFlagPush(event, log, receiveCount);
+		case "MESSAGE_COPY":
+			return handleMessageCopy(event, log);
+		case "EMPTY_TRASH":
+			return handleEmptyTrash(event, log);
+		case "APPEND_SENT_MESSAGE":
+			return handleAppendSentMessage(event, log);
+		case "DELETE_ACCOUNT_OBJECTS":
+			return handleDeleteAccountObjects(event, log);
+		case "IMAP_WORKER_STOP":
+			// Tombstone fence on the account row already stops processing;
+			// this event acks the cascade contract and is a no-op today.
+			log.info(
+				{ accountConfigId: event.accountConfigId, accountId: event.accountId },
+				"Imap worker stop signal received",
+			);
+			return;
+		default:
+			throw new Error(`Unknown event type: ${(event as WorkerEvent).type}`);
+	}
+};

package/src/scheduler/config.test.ts

@@ -0,0 +1,40 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
+
+describe("getTickIntervalMs", () => {
+	it("defaults to 1 hour when unset", () => {
+		assert.equal(getTickIntervalMs({}), 60 * 60 * 1000);
+	});
+
+	it("reads MAILBOX_SYNC_TICK_INTERVAL_SECONDS", () => {
+		assert.equal(
+			getTickIntervalMs({ MAILBOX_SYNC_TICK_INTERVAL_SECONDS: "60" }),
+			60_000,
+		);
+	});
+
+	it("falls back to the default on a non-positive or non-numeric value", () => {
+		assert.equal(
+			getTickIntervalMs({ MAILBOX_SYNC_TICK_INTERVAL_SECONDS: "0" }),
+			60 * 60 * 1000,
+		);
+		assert.equal(
+			getTickIntervalMs({ MAILBOX_SYNC_TICK_INTERVAL_SECONDS: "nope" }),
+			60 * 60 * 1000,
+		);
+	});
+});
+
+describe("getOfflineIntervalMs", () => {
+	it("defaults to 12 hours when unset", () => {
+		assert.equal(getOfflineIntervalMs({}), 12 * 60 * 60 * 1000);
+	});
+
+	it("reads MAILBOX_SYNC_OFFLINE_INTERVAL_SECONDS", () => {
+		assert.equal(
+			getOfflineIntervalMs({ MAILBOX_SYNC_OFFLINE_INTERVAL_SECONDS: "3600" }),
+			3_600_000,
+		);
+	});
+});

package/src/scheduler/config.ts

@@ -0,0 +1,52 @@
+/**
+ * Periodic mailbox-sync scheduler (#1247, restructured #1251): the tick rate
+ * and the offline-sync threshold are separate, independently configured
+ * knobs — never hardcode a cadence — following the repo's
+ * env-var-with-a-safe-default convention (see `ACCOUNT_DELETION_GRACE_SECONDS`
+ * in remit-account-worker/src/config.ts).
+ *
+ * `tickIntervalSeconds` drives how often the tick itself runs — the
+ * EventBridge schedule rate in prod, the local-runner loop delay in dev — and
+ * must stay well below `offlineIntervalSeconds` so a tick reliably observes
+ * every account crossing the threshold. CDK and this runtime default must
+ * agree — see infra/lib/config.ts's `mailboxSync` stage config.
+ *
+ * `offlineIntervalSeconds` is the only due-ness threshold: an account is due
+ * once its last successful sync is older than this interval. There is no
+ * "online" tier — client-side polling (useStaleAccountSync) covers an
+ * account while its mail is actively open in the web client.
+ */
+
+const DEFAULT_TICK_INTERVAL_SECONDS = 60 * 60; // 1 hour
+const DEFAULT_OFFLINE_INTERVAL_SECONDS = 12 * 60 * 60; // 12 hours
+
+const parsePositiveIntSeconds = (
+	raw: string | undefined,
+	fallback: number,
+): number => {
+	if (!raw) return fallback;
+	const parsed = Number.parseInt(raw, 10);
+	return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+};
+
+export const getTickIntervalMs = (
+	env: NodeJS.ProcessEnv = process.env,
+): number =>
+	parsePositiveIntSeconds(
+		env.MAILBOX_SYNC_TICK_INTERVAL_SECONDS,
+		DEFAULT_TICK_INTERVAL_SECONDS,
+	) * 1000;
+
+export const getOfflineIntervalMs = (
+	env: NodeJS.ProcessEnv = process.env,
+): number =>
+	parsePositiveIntSeconds(
+		env.MAILBOX_SYNC_OFFLINE_INTERVAL_SECONDS,
+		DEFAULT_OFFLINE_INTERVAL_SECONDS,
+	) * 1000;
+
+// Page size + enqueue concurrency are implementation details of the tick
+// itself, not stage config — unlike the two intervals above, nothing outside
+// this worker needs to agree with them.
+export const SCHEDULER_PAGE_SIZE = 100;
+export const SCHEDULER_ENQUEUE_CONCURRENCY = 10;

package/src/scheduler/decide-due.test.ts

@@ -0,0 +1,44 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { isSyncDue } from "./decide-due.js";
+
+const OFFLINE_INTERVAL_MS = 12 * 60 * 60 * 1000;
+const NOW = 1_700_000_000_000;
+
+describe("isSyncDue", () => {
+	it("is always due when the account has never synced", () => {
+		assert.equal(isSyncDue({ accountId: "a" }, NOW, OFFLINE_INTERVAL_MS), true);
+	});
+
+	it("is due once the sync is older than the offline interval", () => {
+		const account = {
+			accountId: "a",
+			lastSyncAt: NOW - OFFLINE_INTERVAL_MS - 1,
+		};
+		assert.equal(isSyncDue(account, NOW, OFFLINE_INTERVAL_MS), true);
+	});
+
+	it("is not due within the offline interval", () => {
+		const account = {
+			accountId: "a",
+			lastSyncAt: NOW - 60_000,
+		};
+		assert.equal(isSyncDue(account, NOW, OFFLINE_INTERVAL_MS), false);
+	});
+
+	it("is due exactly at the threshold", () => {
+		const account = {
+			accountId: "a",
+			lastSyncAt: NOW - OFFLINE_INTERVAL_MS,
+		};
+		assert.equal(isSyncDue(account, NOW, OFFLINE_INTERVAL_MS), true);
+	});
+
+	it("is not due one millisecond before the threshold", () => {
+		const account = {
+			accountId: "a",
+			lastSyncAt: NOW - OFFLINE_INTERVAL_MS + 1,
+		};
+		assert.equal(isSyncDue(account, NOW, OFFLINE_INTERVAL_MS), false);
+	});
+});

package/src/scheduler/decide-due.ts

@@ -0,0 +1,26 @@
+/**
+ * Pure scheduling decision for the periodic mailbox-sync tick (#1247,
+ * restructured #1251).
+ *
+ * One tier: an account is due once its last successful sync is older than
+ * `offlineIntervalMs`. An account that has never synced is always due. The
+ * tick interval is decoupled from this threshold and runs far more often
+ * than `offlineIntervalMs` (see config.ts), so — unlike the prior two-tier
+ * design — no slack is needed to compensate for sampling lag: a tick that
+ * misses an account by a few seconds simply catches it on the next tick,
+ * which arrives long before the threshold matters again.
+ */
+
+export interface SchedulableAccount {
+	readonly accountId: string;
+	readonly lastSyncAt?: number;
+}
+
+export const isSyncDue = (
+	account: SchedulableAccount,
+	now: number,
+	offlineIntervalMs: number,
+): boolean => {
+	if (!account.lastSyncAt) return true;
+	return now - account.lastSyncAt >= offlineIntervalMs;
+};

package/src/scheduler/handler.ts

@@ -0,0 +1,52 @@
+import { SQSClient } from "@aws-sdk/client-sqs";
+import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
+import { getClient } from "@remit/backend/client";
+import { createLogger, withTelemetry } from "@remit/logger-lambda";
+import { resolveSqsCredentials } from "@remit/sqs-client";
+import type { ScheduledHandler } from "aws-lambda";
+import { env } from "expect-env";
+import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
+import { runSchedulerTick } from "./run-tick.js";
+
+const log = createLogger();
+
+const mailboxesQueueUrl = env.SQS_QUEUE_URL_MAILBOXES;
+const isLocal = mailboxesQueueUrl.startsWith("http://localhost");
+
+const sqsClient = new SQSClient({
+	endpoint: isLocal ? new URL(mailboxesQueueUrl).origin : undefined,
+	...(isLocal && { protocol: AwsQueryProtocol }),
+	credentials: resolveSqsCredentials(),
+});
+
+/**
+ * EventBridge-scheduled entry point for the periodic mailbox-sync tick
+ * (#1247, restructured #1251). Ticks at `MAILBOX_SYNC_TICK_INTERVAL_SECONDS`
+ * (rate schedule, wired in infra/stacks/dev/stacks/remit-worker-stack.ts) and
+ * delegates the actual decision + enqueue to `runSchedulerTick` — the same
+ * function the local dev-stack timer loop calls (see `local-runner.ts`), so
+ * production and local dev run one code path.
+ *
+ * Uses the EventBridge event's own `time` (the scheduled fire time) as the
+ * tick's `now`, rather than `Date.now()` at processing time. This is what
+ * keeps `buildScheduledSyncDedupId`'s time bucket aligned to the schedule
+ * instead of wall-clock/Lambda-cold-start jitter (review #1250): every
+ * distinct scheduled firing gets a `time` exactly `tickIntervalMs` apart
+ * from the last, so consecutive ticks always land in different buckets,
+ * while a genuine retry of the same invocation carries the same `time` and
+ * correctly dedupes.
+ */
+export const handler: ScheduledHandler = withTelemetry(async (event) => {
+	const { account } = await getClient();
+	const now = Date.parse(event.time);
+
+	await runSchedulerTick({
+		accountService: account,
+		sqsClient,
+		queueUrl: mailboxesQueueUrl,
+		log,
+		tickIntervalMs: getTickIntervalMs(),
+		offlineIntervalMs: getOfflineIntervalMs(),
+		now,
+	});
+});

package/src/scheduler/local-runner.ts

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+import { setTimeout as delay } from "node:timers/promises";
+import { SQSClient } from "@aws-sdk/client-sqs";
+import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
+import { getClient } from "@remit/backend/client";
+import { createLogger } from "@remit/logger-lambda";
+import { resolveSqsCredentials } from "@remit/sqs-client";
+import { env } from "expect-env";
+import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
+import { runSchedulerTick } from "./run-tick.js";
+
+/**
+ * Standalone scheduled-sync runner for the local pg-dev docker-compose stack
+ * (#1247, restructured #1251). Production ticks `runSchedulerTick` off an
+ * EventBridge schedule (see handler.ts); ElasticMQ/the pg-dev stack has no
+ * EventBridge, so this process ticks on a plain loop at the same
+ * `MAILBOX_SYNC_TICK_INTERVAL_SECONDS` cadence instead — same function, same
+ * config knobs, so local dev behaves like production rather than needing its
+ * own scheduling logic.
+ *
+ * This is a dev-only harness, not production code: like
+ * `e2e-processor-shim.ts`, a tick failure crashes the process loudly rather
+ * than swallowing it — docker-compose's `restart: unless-stopped` brings it
+ * back for the next tick.
+ */
+
+const log = createLogger();
+
+const mailboxesQueueUrl = env.SQS_QUEUE_URL_MAILBOXES;
+const isLocal = mailboxesQueueUrl.startsWith("http://localhost");
+
+const sqsClient = new SQSClient({
+	endpoint: isLocal ? new URL(mailboxesQueueUrl).origin : undefined,
+	...(isLocal && { protocol: AwsQueryProtocol }),
+	credentials: resolveSqsCredentials(),
+});
+
+const tickIntervalMs = getTickIntervalMs();
+const offlineIntervalMs = getOfflineIntervalMs();
+
+// A persistent failure (e.g. Postgres not up yet at container boot) throws
+// before the loop ever reaches its own `delay`, so `restart: unless-stopped`
+// would otherwise respawn the process immediately — a tight, log-flooding
+// crash loop (review #1250). This fixed pause before exiting is not retry
+// logic (there is nothing to retry here; the container restart IS the
+// retry) — it only paces how fast that restart can happen.
+const CRASH_BACKOFF_MS = 5_000;
+
+log.info(
+	{ tickIntervalMs, offlineIntervalMs },
+	"Local scheduled-sync runner started",
+);
+
+const runLoop = async (): Promise<void> => {
+	const { account } = await getClient();
+	for (;;) {
+		await runSchedulerTick({
+			accountService: account,
+			sqsClient,
+			queueUrl: mailboxesQueueUrl,
+			log,
+			tickIntervalMs,
+			offlineIntervalMs,
+		});
+		await delay(tickIntervalMs);
+	}
+};
+
+runLoop()
+	.catch(async (error) => {
+		log.error({ error }, "Scheduled-sync tick failed");
+		await delay(CRASH_BACKOFF_MS);
+	})
+	.finally(() => {
+		process.exit(1);
+	});

package/src/scheduler/run-tick.test.ts

@@ -0,0 +1,248 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
+import type { AccountItem, AccountSchedulerPage } from "@remit/data-ports";
+import type { Logger } from "@remit/logger-lambda";
+import { runSchedulerTick } from "./run-tick.js";
+
+const TICK_INTERVAL_MS = 60 * 60 * 1000;
+const OFFLINE_INTERVAL_MS = 12 * 60 * 60 * 1000;
+const NOW = 1_700_000_000_000;
+
+const createNoopLogger = (): Logger => {
+	const noop = () => {};
+	const log = {
+		info: noop,
+		warn: noop,
+		error: noop,
+		debug: noop,
+		fatal: noop,
+		trace: noop,
+		child: () => log,
+	} as unknown as Logger;
+	return log;
+};
+
+const createCapturingLogger = (): {
+	log: Logger;
+	calls: { level: string; args: unknown[] }[];
+} => {
+	const calls: { level: string; args: unknown[] }[] = [];
+	const capture =
+		(level: string) =>
+		(...args: unknown[]) =>
+			calls.push({ level, args });
+	const log = {
+		info: capture("info"),
+		warn: capture("warn"),
+		error: capture("error"),
+		debug: capture("debug"),
+		fatal: capture("fatal"),
+		trace: capture("trace"),
+		child: () => log,
+	} as unknown as Logger;
+	return { log, calls };
+};
+
+const baseAccount = (overrides: Partial<AccountItem>): AccountItem =>
+	({
+		accountId: "acct_1",
+		accountConfigId: "acfg_1",
+		username: "user",
+		email: "user@example.com",
+		authType: "password",
+		imapHost: "imap.example.com",
+		imapPort: 993,
+		imapTls: true,
+		imapStartTls: false,
+		smtpEnabled: false,
+		smtpHost: "",
+		smtpPort: 587,
+		smtpTls: false,
+		smtpStartTls: true,
+		smtpUsername: "",
+		isActive: true,
+		connectionState: "authenticated",
+		createdAt: NOW - 1_000_000,
+		updatedAt: NOW - 1_000_000,
+		...overrides,
+	}) as AccountItem;
+
+const fakeAccountService = (pages: AccountSchedulerPage[]) => {
+	let call = 0;
+	return {
+		listAllAccountsPage: async (): Promise<AccountSchedulerPage> => {
+			const page = pages[call];
+			call++;
+			if (!page) throw new Error("no more pages configured");
+			return page;
+		},
+	};
+};
+
+const fakeSqsClient = (): {
+	sqsClient: SQSClient;
+	sent: SendMessageCommand[];
+} => {
+	const sent: SendMessageCommand[] = [];
+	const sqsClient = {
+		send: async (cmd: SendMessageCommand) => {
+			sent.push(cmd);
+			return {};
+		},
+	} as unknown as SQSClient;
+	return { sqsClient, sent };
+};
+
+describe("runSchedulerTick", () => {
+	it("enqueues only accounts that are due, and pages through every account", async () => {
+		const due = baseAccount({
+			accountId: "acct_due",
+			lastSyncAt: NOW - OFFLINE_INTERVAL_MS - 1,
+		});
+		const notDue = baseAccount({
+			accountId: "acct_not_due",
+			lastSyncAt: NOW - 60_000,
+		});
+
+		const accountService = fakeAccountService([
+			{ items: [due], cursor: "page2" },
+			{ items: [notDue], cursor: null },
+		]);
+		const { sqsClient, sent } = fakeSqsClient();
+
+		const result = await runSchedulerTick({
+			accountService,
+			sqsClient,
+			queueUrl:
+				"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
+			log: createNoopLogger(),
+			tickIntervalMs: TICK_INTERVAL_MS,
+			offlineIntervalMs: OFFLINE_INTERVAL_MS,
+			now: NOW,
+		});
+
+		assert.equal(result.scanned, 2);
+		assert.equal(result.enqueued, 1);
+		assert.equal(result.skipped, 1);
+		assert.equal(sent.length, 1);
+		const body = JSON.parse(sent[0]?.input.MessageBody ?? "{}");
+		assert.equal(body.accountId, "acct_due");
+	});
+
+	it("skips deleted, unsyncable-host, and reauth-required accounts", async () => {
+		const deleted = baseAccount({ accountId: "acct_deleted", deletedAt: NOW });
+		const unsyncable = baseAccount({
+			accountId: "acct_unsyncable",
+			imapHost: "mail.invalid",
+		});
+		const reauth = baseAccount({
+			accountId: "acct_reauth",
+			connectionState: "reauth_required",
+		});
+
+		const accountService = fakeAccountService([
+			{ items: [deleted, unsyncable, reauth], cursor: null },
+		]);
+		const { sqsClient, sent } = fakeSqsClient();
+
+		const result = await runSchedulerTick({
+			accountService,
+			sqsClient,
+			queueUrl:
+				"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
+			log: createNoopLogger(),
+			tickIntervalMs: TICK_INTERVAL_MS,
+			offlineIntervalMs: OFFLINE_INTERVAL_MS,
+			now: NOW,
+		});
+
+		assert.equal(result.enqueued, 0);
+		assert.equal(result.skipped, 3);
+		assert.equal(sent.length, 0);
+	});
+
+	it("never logs per-account for ineligible accounts — only the aggregate tick summary (review #1250)", async () => {
+		const deleted = baseAccount({ accountId: "acct_deleted", deletedAt: NOW });
+		const unsyncable = baseAccount({
+			accountId: "acct_unsyncable",
+			imapHost: "mail.invalid",
+		});
+		const reauth = baseAccount({
+			accountId: "acct_reauth",
+			connectionState: "reauth_required",
+		});
+
+		const accountService = fakeAccountService([
+			{ items: [deleted, unsyncable, reauth], cursor: null },
+		]);
+		const { sqsClient } = fakeSqsClient();
+		const { log, calls } = createCapturingLogger();
+
+		await runSchedulerTick({
+			accountService,
+			sqsClient,
+			queueUrl:
+				"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
+			log,
+			tickIntervalMs: TICK_INTERVAL_MS,
+			offlineIntervalMs: OFFLINE_INTERVAL_MS,
+			now: NOW,
+		});
+
+		// Sweeping 3 ineligible accounts must not produce 3 (or more) log
+		// lines through the tick's own logger — only the one aggregate
+		// "tick complete" summary. Per-account noise from the shared
+		// isAccountDeleted/isUnsyncableHost/isAccountReauthRequired helpers
+		// must be swallowed by a silent logger inside the sweep.
+		assert.equal(
+			calls.length,
+			1,
+			`expected exactly one aggregate log line, got ${calls.length}: ${JSON.stringify(calls)}`,
+		);
+	});
+
+	it("gives each enqueued account a scheduler-namespaced, time-bucketed dedup id", async () => {
+		const due = baseAccount({ accountId: "acct_due" });
+		const accountService = fakeAccountService([{ items: [due], cursor: null }]);
+		const { sqsClient, sent } = fakeSqsClient();
+
+		await runSchedulerTick({
+			accountService,
+			sqsClient,
+			queueUrl:
+				"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
+			log: createNoopLogger(),
+			tickIntervalMs: TICK_INTERVAL_MS,
+			offlineIntervalMs: OFFLINE_INTERVAL_MS,
+			now: NOW,
+		});
+
+		const dedupId = sent[0]?.input.MessageDeduplicationId;
+		assert.ok(dedupId?.startsWith("SYNC_MAILBOXES:scheduled:acct_due:"));
+		assert.notEqual(dedupId, "SYNC_MAILBOXES:acct_due");
+	});
+
+	it("terminates pagination and never revisits a page", async () => {
+		const accountService = fakeAccountService([
+			{ items: [], cursor: "page2" },
+			{ items: [], cursor: "page3" },
+			{ items: [], cursor: null },
+		]);
+		const { sqsClient } = fakeSqsClient();
+
+		const result = await runSchedulerTick({
+			accountService,
+			sqsClient,
+			queueUrl:
+				"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
+			log: createNoopLogger(),
+			tickIntervalMs: TICK_INTERVAL_MS,
+			offlineIntervalMs: OFFLINE_INTERVAL_MS,
+			now: NOW,
+		});
+
+		assert.equal(result.scanned, 0);
+		assert.equal(result.enqueued, 0);
+	});
+});