@remit/imap-worker 0.0.1

package/src/scheduler/run-tick.ts

@@ -0,0 +1,141 @@
+import type { SQSClient } from "@aws-sdk/client-sqs";
+import {
+	buildScheduledSyncDedupId,
+	triggerAccountSync,
+} from "@remit/backend/trigger-sync";
+import type { AccountItem, IAccountRepository } from "@remit/data-ports";
+import type { Logger } from "@remit/logger-lambda";
+import pMap from "p-map";
+import {
+	isAccountDeleted,
+	isAccountReauthRequired,
+	isUnsyncableHost,
+} from "../account-check.js";
+import {
+	SCHEDULER_ENQUEUE_CONCURRENCY,
+	SCHEDULER_PAGE_SIZE,
+} from "./config.js";
+import { isSyncDue } from "./decide-due.js";
+
+export interface RunSchedulerTickDeps {
+	accountService: Pick<IAccountRepository, "listAllAccountsPage">;
+	sqsClient: SQSClient;
+	queueUrl: string;
+	log: Logger;
+	offlineIntervalMs: number;
+	/**
+	 * How often this tick itself runs — the dedup-id bucket width for
+	 * `buildScheduledSyncDedupId`, so consecutive ticks each get a fresh id.
+	 */
+	tickIntervalMs: number;
+	/** Injectable for tests; defaults to `Date.now()`. */
+	now?: number;
+}
+
+export interface SchedulerTickResult {
+	scanned: number;
+	enqueued: number;
+	skipped: number;
+}
+
+/**
+ * `isAccountDeleted` / `isUnsyncableHost` / `isAccountReauthRequired` each log
+ * one line per ineligible account — the right volume for their real call site
+ * (once per SYNC_MAILBOXES event). Run across the whole account base every
+ * tick, that becomes one log line per deleted/reauth/placeholder account
+ * every 5 minutes, forever (review #1250). The tick already reports the
+ * aggregate `skipped` count, so eligibility checks here go through a silent
+ * logger — the per-event path (sync-mailboxes.ts) is untouched and keeps
+ * logging normally.
+ */
+const silentLogger: Logger = (() => {
+	const noop = () => {};
+	const stub = {
+		info: noop,
+		warn: noop,
+		error: noop,
+		debug: noop,
+		fatal: noop,
+		trace: noop,
+		child: () => stub,
+	};
+	return stub as unknown as Logger;
+})();
+
+const isEligible = (account: AccountItem): boolean => {
+	if (isAccountDeleted(account, silentLogger)) return false;
+	if (isUnsyncableHost(account, silentLogger)) return false;
+	if (isAccountReauthRequired(account, silentLogger)) return false;
+	return true;
+};
+
+/**
+ * One tick of the periodic mailbox-sync scheduler (#1247, restructured
+ * #1251). Pages through every account (never loading the whole account base
+ * into memory), decides per account via `isSyncDue` against the single
+ * offline threshold, and enqueues SYNC_MAILBOXES for the accounts that are
+ * due — in bounded-concurrency batches, never an unbounded `Promise.all`.
+ *
+ * Every enqueue goes through the same `triggerAccountSync` the manual
+ * POST /sync path uses, with a scheduler-specific, time-bucketed dedup id
+ * (`buildScheduledSyncDedupId`, bucketed by `tickIntervalMs`) so this tick
+ * can never collide with its own previous tick or with a concurrent manual
+ * trigger (see trigger-sync.ts). Concurrent-sync safety for the mailbox
+ * itself is MailboxLockService's job, inside the worker handler — this tick
+ * only decides "is a sync due" and enqueues; it never talks to IMAP.
+ */
+export const runSchedulerTick = async (
+	deps: RunSchedulerTickDeps,
+): Promise<SchedulerTickResult> => {
+	const {
+		accountService,
+		sqsClient,
+		queueUrl,
+		log,
+		offlineIntervalMs,
+		tickIntervalMs,
+	} = deps;
+	const now = deps.now ?? Date.now();
+
+	let cursor: string | undefined;
+	let scanned = 0;
+	let enqueued = 0;
+	let skipped = 0;
+
+	do {
+		const page = await accountService.listAllAccountsPage({
+			limit: SCHEDULER_PAGE_SIZE,
+			cursor,
+		});
+		scanned += page.items.length;
+
+		const due = page.items.filter(
+			(account) =>
+				isEligible(account) && isSyncDue(account, now, offlineIntervalMs),
+		);
+		skipped += page.items.length - due.length;
+
+		await pMap(
+			due,
+			(account) =>
+				triggerAccountSync({
+					sqsClient,
+					queueUrl,
+					accountId: account.accountId,
+					dedupId: buildScheduledSyncDedupId(
+						account.accountId,
+						now,
+						tickIntervalMs,
+					),
+				}),
+			{ concurrency: SCHEDULER_ENQUEUE_CONCURRENCY },
+		);
+		enqueued += due.length;
+
+		cursor = page.cursor ?? undefined;
+	} while (cursor);
+
+	log.info({ scanned, enqueued, skipped }, "Scheduled-sync tick complete");
+
+	return { scanned, enqueued, skipped };
+};

package/src/with-oauth-lifecycle-deps.ts

@@ -0,0 +1,62 @@
+/**
+ * with-oauth-lifecycle-deps.ts
+ *
+ * Shared construction of the deps passed to withOAuthLifecycle by every IMAP
+ * handler. Centralizes the lazy OAuth token service, secrets, refresh-token
+ * rotation persistence, and the connectionState writer so handlers don't each
+ * re-declare the same boilerplate.
+ */
+
+import type { IAccountRepository } from "@remit/data-ports";
+import {
+	createMailOAuthService,
+	microsoftProviderConfig,
+} from "@remit/mail-oauth-service";
+import type { SecretsService } from "@remit/secrets-service";
+import type {
+	ConnectionStateValue,
+	OAuthLifecycleDeps,
+} from "./with-oauth-lifecycle.js";
+
+// Lazy OAuth service: created on first OAuth account.
+// Uses MSOAUTH_* env vars which are only present in deployed Lambdas.
+let _tokenService: ReturnType<typeof createMailOAuthService> | undefined;
+const getTokenService = (): ReturnType<typeof createMailOAuthService> => {
+	if (!_tokenService) {
+		_tokenService = createMailOAuthService(
+			microsoftProviderConfig({
+				clientId: process.env.MSOAUTH_CLIENT_ID ?? "",
+				clientSecret: process.env.MSOAUTH_CLIENT_SECRET ?? "",
+				overrides: process.env.MSOAUTH_TOKEN_ENDPOINT
+					? { tokenEndpoint: process.env.MSOAUTH_TOKEN_ENDPOINT }
+					: undefined,
+			}),
+		);
+	}
+	return _tokenService;
+};
+
+/**
+ * Build the OAuthLifecycleDeps for a handler. `secrets` and `accountService`
+ * are the long-lived singletons already constructed at module scope in each
+ * handler.
+ */
+export const buildLifecycleDeps = (
+	secrets: SecretsService,
+	accountService: IAccountRepository,
+): OAuthLifecycleDeps => ({
+	secrets,
+	tokenService: getTokenService(),
+	persistRotatedToken: async (accountId, encryptedHash, updatedAt) => {
+		await accountService.update(accountId, {
+			oauthRefreshTokenHash: encryptedHash,
+			oauthTokenUpdatedAt: updatedAt,
+		});
+	},
+	updateConnectionState: async (
+		accountId: string,
+		state: ConnectionStateValue,
+	) => {
+		await accountService.update(accountId, { connectionState: state });
+	},
+});

package/src/with-oauth-lifecycle.test.ts

@@ -0,0 +1,227 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type { AccountItem } from "@remit/data-ports";
+import { AccountAuthType, ConnectionState } from "@remit/domain-enums";
+import { RefreshTokenError } from "@remit/mail-oauth-service";
+import {
+	MailConnectionError,
+	type MailCredentials,
+} from "@remit/mailbox-service";
+import {
+	type ConnectionStateValue,
+	type OAuthLifecycleDeps,
+	withOAuthLifecycle,
+} from "./with-oauth-lifecycle.js";
+
+const silentLogger = {
+	info: () => {},
+	warn: () => {},
+	error: () => {},
+	debug: () => {},
+	trace: () => {},
+	fatal: () => {},
+	child: () => silentLogger,
+} as never;
+
+const buildAccount = (overrides: Partial<AccountItem> = {}): AccountItem =>
+	({
+		accountId: "acc-1",
+		accountConfigId: "cfg-1",
+		username: "alice@example.com",
+		email: "alice@example.com",
+		imapHost: "imap.example.com",
+		imapPort: 993,
+		imapTls: true,
+		imapStartTls: false,
+		isActive: true,
+		connectionState: "not_authenticated",
+		createdAt: 0,
+		updatedAt: 0,
+		...overrides,
+	}) as unknown as AccountItem;
+
+interface Recorded {
+	stateUpdates: Array<{ accountId: string; state: ConnectionStateValue }>;
+	resolveCalls: number;
+	workCalls: number;
+}
+
+const passwordCreds: MailCredentials = {
+	kind: "password",
+	password: "secret",
+};
+
+const buildDeps = (
+	options: {
+		resolveCredentials?: OAuthLifecycleDeps["resolveCredentials"];
+	} = {},
+): { deps: OAuthLifecycleDeps; recorded: Recorded } => {
+	const recorded: Recorded = {
+		stateUpdates: [],
+		resolveCalls: 0,
+		workCalls: 0,
+	};
+	const deps: OAuthLifecycleDeps = {
+		secrets: {
+			decrypt: async () => "",
+			encrypt: async () => ({}) as never,
+		},
+		tokenService: { getAccessToken: async () => ({}) as never },
+		persistRotatedToken: async () => {},
+		updateConnectionState: async (accountId, state) => {
+			recorded.stateUpdates.push({ accountId, state });
+		},
+		resolveCredentials:
+			options.resolveCredentials ??
+			(async () => {
+				recorded.resolveCalls += 1;
+				return passwordCreds;
+			}),
+	};
+	// Wrap resolveCredentials to record the call count when a custom one is given.
+	if (options.resolveCredentials) {
+		const inner = options.resolveCredentials;
+		deps.resolveCredentials = async (account, credDeps) => {
+			recorded.resolveCalls += 1;
+			return inner(account, credDeps);
+		};
+	}
+	return { deps, recorded };
+};
+
+describe("withOAuthLifecycle", () => {
+	it("skips work when account is reauth_required", async () => {
+		const { deps, recorded } = buildDeps();
+		const account = buildAccount({ connectionState: "reauth_required" });
+
+		await withOAuthLifecycle(deps, account, silentLogger, async () => {
+			recorded.workCalls += 1;
+		});
+
+		assert.equal(recorded.workCalls, 0, "work must not be called");
+		assert.equal(recorded.resolveCalls, 0, "must not resolve credentials");
+		assert.equal(recorded.stateUpdates.length, 0, "must not update state");
+	});
+
+	it("on RefreshTokenError reauth-required: flips to reauth_required and ACKs (does not rethrow)", async () => {
+		const { deps, recorded } = buildDeps({
+			resolveCredentials: async () => {
+				throw new RefreshTokenError({
+					kind: "reauth-required",
+					code: "invalid_grant",
+				});
+			},
+		});
+		const account = buildAccount();
+
+		await withOAuthLifecycle(deps, account, silentLogger, async () => {
+			recorded.workCalls += 1;
+		});
+
+		assert.equal(
+			recorded.workCalls,
+			0,
+			"work must not run after resolve fails",
+		);
+		assert.equal(recorded.stateUpdates.length, 1);
+		assert.deepEqual(recorded.stateUpdates[0], {
+			accountId: "acc-1",
+			state: ConnectionState.ReauthRequired,
+		});
+	});
+
+	it("on MailConnectionError auth for OAuth account: flips to reauth_required and ACKs", async () => {
+		const { deps, recorded } = buildDeps();
+		const account = buildAccount({ authType: AccountAuthType.OauthMicrosoft });
+
+		await withOAuthLifecycle(deps, account, silentLogger, async () => {
+			throw new MailConnectionError("auth", "auth failed");
+		});
+
+		assert.equal(recorded.stateUpdates.length, 1);
+		assert.deepEqual(recorded.stateUpdates[0], {
+			accountId: "acc-1",
+			state: ConnectionState.ReauthRequired,
+		});
+	});
+
+	it("on MailConnectionError auth for password account: rethrows (batch item failure, no state flip)", async () => {
+		const { deps, recorded } = buildDeps();
+		const account = buildAccount({ authType: AccountAuthType.Password });
+
+		await assert.rejects(
+			() =>
+				withOAuthLifecycle(deps, account, silentLogger, async () => {
+					throw new MailConnectionError("auth", "auth failed");
+				}),
+			/auth failed/,
+		);
+		assert.equal(
+			recorded.stateUpdates.length,
+			0,
+			"must not flip connectionState for password account",
+		);
+	});
+
+	it("on MailConnectionError auth for account with no authType (defaults to password): rethrows", async () => {
+		const { deps, recorded } = buildDeps();
+		// No authType set — defaults to password in resolveConnectionCredentials
+		const account = buildAccount();
+
+		await assert.rejects(
+			() =>
+				withOAuthLifecycle(deps, account, silentLogger, async () => {
+					throw new MailConnectionError("auth", "auth failed");
+				}),
+			/auth failed/,
+		);
+		assert.equal(
+			recorded.stateUpdates.length,
+			0,
+			"must not flip connectionState when authType is unset",
+		);
+	});
+
+	it("on transient error (network): rethrows (batch item failure)", async () => {
+		const { deps, recorded } = buildDeps();
+		const account = buildAccount();
+
+		await assert.rejects(
+			() =>
+				withOAuthLifecycle(deps, account, silentLogger, async () => {
+					throw new MailConnectionError("network", "timeout");
+				}),
+			/timeout/,
+		);
+		assert.equal(recorded.stateUpdates.length, 0, "must not flip state");
+	});
+
+	it("on ordinary Error: rethrows", async () => {
+		const { deps, recorded } = buildDeps();
+		const account = buildAccount();
+
+		await assert.rejects(
+			() =>
+				withOAuthLifecycle(deps, account, silentLogger, async () => {
+					throw new Error("boom");
+				}),
+			/boom/,
+		);
+		assert.equal(recorded.stateUpdates.length, 0);
+	});
+
+	it("on RefreshTokenError transient: rethrows", async () => {
+		const { deps, recorded } = buildDeps({
+			resolveCredentials: async () => {
+				throw new RefreshTokenError({ kind: "transient", code: "503" });
+			},
+		});
+		const account = buildAccount();
+
+		await assert.rejects(
+			() => withOAuthLifecycle(deps, account, silentLogger, async () => {}),
+			/transient/,
+		);
+		assert.equal(recorded.stateUpdates.length, 0, "must not flip state");
+	});
+});

package/src/with-oauth-lifecycle.ts

@@ -0,0 +1,125 @@
+/**
+ * with-oauth-lifecycle.ts
+ *
+ * Centralizes the OAuth reauth/ACK contract shared by every IMAP handler.
+ *
+ * The contract (see issue #472):
+ *  - If the account already requires reauth, skip all IMAP traffic entirely.
+ *  - Credential resolution AND the per-handler work both run inside one
+ *    try/catch so that a revoked OAuth token is caught regardless of where it
+ *    surfaces (token mint vs. first IMAP command).
+ *  - On a terminal auth failure (RefreshTokenError reauth-required, or
+ *    MailConnectionError auth), flip the account to reauth_required and return
+ *    WITHOUT rethrowing — this ACKs the SQS message so it is not retried.
+ *  - On transient / config / network errors, rethrow so SQS retries with
+ *    backoff (let-it-crash).
+ *
+ * Tokens must NEVER appear in logs — only accountId / errorKind / errorCode.
+ */
+
+import type { AccountItem } from "@remit/data-ports";
+import { AccountAuthType, ConnectionState } from "@remit/domain-enums";
+import type { Logger } from "@remit/logger-lambda";
+import { RefreshTokenError } from "@remit/mail-oauth-service";
+import {
+	type AccountCredentialsDeps,
+	MailConnectionError,
+	type MailCredentials,
+	resolveConnectionCredentials,
+} from "@remit/mailbox-service";
+import { isAccountReauthRequired } from "./account-check.js";
+
+/** ConnectionState is a const object (not a TS enum); this is its value type. */
+export type ConnectionStateValue =
+	(typeof ConnectionState)[keyof typeof ConnectionState];
+
+export interface OAuthLifecycleDeps extends AccountCredentialsDeps {
+	/**
+	 * Persist the account's connectionState. Called when a terminal auth
+	 * failure is detected so the account is fenced off until the user re-auths.
+	 */
+	updateConnectionState: (
+		accountId: string,
+		state: ConnectionStateValue,
+	) => Promise<void>;
+	/**
+	 * Resolve credentials for the account. Defaults to
+	 * resolveConnectionCredentials; overridable for testing. Kept as the ONLY
+	 * authType branch in the codebase (see account-credentials.ts).
+	 */
+	resolveCredentials?: (
+		account: AccountItem,
+		deps: AccountCredentialsDeps,
+	) => Promise<MailCredentials>;
+}
+
+/**
+ * Run `work` for an account under the shared OAuth reauth/ACK contract.
+ *
+ * `work` receives the resolved MailCredentials. Both credential resolution and
+ * `work` run inside the same try/catch.
+ */
+export const withOAuthLifecycle = async (
+	deps: OAuthLifecycleDeps,
+	account: AccountItem,
+	log: Logger,
+	work: (credentials: MailCredentials) => Promise<void>,
+): Promise<void> => {
+	// Skip all IMAP traffic for accounts that already require reauth.
+	if (isAccountReauthRequired(account, log)) {
+		return;
+	}
+
+	const resolve = deps.resolveCredentials ?? resolveConnectionCredentials;
+
+	try {
+		const credentials = await resolve(account, deps);
+		await work(credentials);
+	} catch (err) {
+		// Terminal OAuth failure: token revoked / consent withdrawn.
+		if (err instanceof RefreshTokenError) {
+			if (err.error.kind === "reauth-required") {
+				log.warn(
+					{
+						accountId: account.accountId,
+						errorKind: err.error.kind,
+						errorCode: err.error.code,
+					},
+					"OAuth token revoked; marking account reauth_required",
+				);
+				await deps.updateConnectionState(
+					account.accountId,
+					ConnectionState.ReauthRequired,
+				);
+				return; // ACK — do not retry
+			}
+			// transient or config: let-it-crash (SQS retry / DLQ)
+			throw err;
+		}
+
+		// Terminal auth failure at the IMAP layer (bad credentials / expired token).
+		// Only OAuth accounts can recover via the re-auth flow; password accounts
+		// have no such path so we rethrow to preserve pre-PR batch-item-failure
+		// behaviour instead of permanently fencing the account.
+		if (err instanceof MailConnectionError && err.kind === "auth") {
+			if (account.authType !== AccountAuthType.OauthMicrosoft) {
+				throw err;
+			}
+			log.warn(
+				{
+					accountId: account.accountId,
+					errorKind: err.kind,
+				},
+				"IMAP auth rejected; marking account reauth_required",
+			);
+			await deps.updateConnectionState(
+				account.accountId,
+				ConnectionState.ReauthRequired,
+			);
+			return; // ACK — do not retry
+		}
+
+		// Transient / network / unexpected: rethrow so SQS retries with backoff.
+		throw err;
+	}
+};

package/tsconfig.json

@@ -0,0 +1,8 @@
+{
+	"extends": "../../tsconfig.json",
+	"compilerOptions": {
+		"outDir": "dist",
+		"rootDir": "src"
+	},
+	"include": ["src/**/*.ts"]
+}