@remit/imap-worker 0.0.85 → 0.0.87

package/src/scheduler/handler.ts

@@ -1,12 +1,17 @@
 import { getClient } from "@remit/backend/client";
-import { createLogger, withTelemetry } from "@remit/logger-lambda";
+import { createLogger } from "@remit/logger";
+import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
 import { createQueueProducer } from "@remit/sqs-client/producer";
 import type { ScheduledHandler } from "aws-lambda";
 import { env } from "expect-env";
+import { emitEvent } from "../emit.js";
 import { buildAttachmentSweep } from "./attachment-sweep.js";
 import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
 import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
 import { runSchedulerTick } from "./run-tick.js";
+import { redriveStrandedPlacements } from "./stranded-placement-redrive.js";
+
+installNodeLogger();
 
 const log = createLogger();
 
@@ -47,5 +52,11 @@ export const handler: ScheduledHandler = withTelemetry(async (event) => {
 		// already gone and this sweep is the only thing that collects it.
 		sweepAttachments: buildAttachmentSweep(client, log),
 		refreshCalendarSubscriptions: buildCalendarSubscriptionRefresh(client, log),
+		redriveStrandedPlacements: (account, tickNow) =>
+			redriveStrandedPlacements(
+				{ client, emit: emitEvent, log },
+				account,
+				tickNow,
+			),
 	});
 });

package/src/scheduler/loop.test.ts

@@ -11,6 +11,8 @@ const RESULT: SchedulerTickResult = {
 	sweepFailed: 0,
 	subscriptionsRefreshed: 0,
 	subscriptionsFailed: 0,
+	redriven: 0,
+	redriveFailed: 0,
 };
 
 // Only the loop's own two calls matter here; the tick is a stub, so what it is

package/src/scheduler/run-tick.sqlite.test.ts

@@ -0,0 +1,192 @@
+import assert from "node:assert/strict";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
+import type { AccountItem } from "@remit/data-ports";
+import {
+	AccountAuthType,
+	AccountService,
+	ConnectionState,
+} from "@remit/domain-enums";
+import { AccountRepo } from "@remit/drizzle-service";
+import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
+import { capturingLogger, type LogLine } from "../test-helpers/test-world.js";
+import { runSchedulerTick } from "./run-tick.js";
+
+const TICK_INTERVAL_MS = 60 * 60 * 1000;
+const OFFLINE_INTERVAL_MS = 12 * 60 * 60 * 1000;
+const NOW = 1_700_000_000_000;
+
+let accounts: AccountRepo;
+let close: () => void;
+
+beforeEach(() => {
+	const store = createShippedSqliteDb();
+	close = store.close;
+	accounts = new AccountRepo(store.db, store.db);
+});
+
+afterEach(() => close());
+
+const account = async (
+	name: string,
+	overrides: Partial<AccountItem> = {},
+	stored: { credential: boolean } = { credential: true },
+): Promise<AccountItem> => {
+	const created = await accounts.create({
+		accountConfigId: `cfg-${name}`,
+		username: `${name}@example.com`,
+		email: `${name}@example.com`,
+		authType: AccountAuthType.Password,
+		passwordHash: stored.credential ? '{"ciphertext":"x"}' : undefined,
+		imapHost: "imap.example.com",
+		imapPort: 993,
+		imapTls: true,
+		imapStartTls: false,
+		isActive: true,
+		connectionState: ConnectionState.Authenticated,
+	} as Parameters<AccountRepo["create"]>[0]);
+	if (Object.keys(overrides).length === 0) return created;
+	await accounts.update(created.accountId, overrides);
+	return accounts.get(created.accountId);
+};
+
+const tick = async (
+	options: Partial<Parameters<typeof runSchedulerTick>[0]> = {},
+) => {
+	const sent: SendMessageCommand[] = [];
+	const logs: LogLine[] = [];
+	const result = await runSchedulerTick({
+		accountService: accounts,
+		sqsClient: {
+			send: async (command: SendMessageCommand) => {
+				sent.push(command);
+				return {};
+			},
+		} as unknown as SQSClient,
+		queueUrl:
+			"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
+		log: capturingLogger(logs),
+		tickIntervalMs: TICK_INTERVAL_MS,
+		offlineIntervalMs: OFFLINE_INTERVAL_MS,
+		now: NOW,
+		redriveStrandedPlacements: async () => 0,
+		...options,
+	});
+	const enqueued = sent.map(
+		(command) => JSON.parse(command.input.MessageBody ?? "{}").accountId,
+	);
+	return { result, sent, enqueued, logs };
+};
+
+describe("a scheduler tick", () => {
+	it("enqueues the accounts that are due and only those", async () => {
+		const due = await account("due", {
+			lastSyncAt: NOW - OFFLINE_INTERVAL_MS - 1,
+		});
+		const never = await account("never");
+		await account("fresh", { lastSyncAt: NOW - 60_000 });
+
+		const { enqueued } = await tick();
+
+		assert.deepEqual(enqueued.sort(), [due.accountId, never.accountId].sort());
+	});
+
+	it("skips an account it cannot or should not sync, and says so once in aggregate", async () => {
+		const live = await account("live");
+		await account("unsyncable", { imapHost: "mail.invalid" });
+		await account("reauth", {
+			connectionState: ConnectionState.ReauthRequired,
+		});
+		await account("calendar", { syncedServices: [AccountService.Calendar] });
+
+		const { enqueued, result, logs } = await tick();
+
+		assert.deepEqual(enqueued, [live.accountId]);
+		assert.equal(result.skipped, 3);
+		assert.equal(logs.length, 1, "one aggregate line, no per-account noise");
+	});
+
+	// Issue #1120: a sync for an account with no stored credential fenced the
+	// account's FIFO group for its whole visibility timeout.
+	it("skips an account that stores no credential", async () => {
+		const live = await account("live");
+		await account("missing", {}, { credential: false });
+
+		const { enqueued } = await tick();
+
+		assert.deepEqual(enqueued, [live.accountId]);
+	});
+
+	it("gives each enqueued account a scheduler-namespaced, time-bucketed dedup id", async () => {
+		const live = await account("live");
+
+		const { sent } = await tick();
+
+		const dedupId = sent[0]?.input.MessageDeduplicationId;
+		assert.ok(
+			dedupId?.startsWith(`SYNC_MAILBOXES:scheduled:${live.accountId}:`),
+		);
+	});
+
+	it("sweeps every account's attachments, due a sync or not", async () => {
+		const due = await account("due");
+		const fresh = await account("fresh", { lastSyncAt: NOW - 60_000 });
+		const swept: string[] = [];
+
+		const { result } = await tick({
+			sweepAttachments: async (item) => {
+				swept.push(item.accountId);
+			},
+		});
+
+		assert.deepEqual(swept.sort(), [due.accountId, fresh.accountId].sort());
+		assert.equal(result.swept, 2);
+	});
+
+	// Uncontained, one storage error reaches the loop, which exits, and the
+	// container restarts without ever enqueuing an account again.
+	it("keeps enqueuing mail when the attachment sweep throws", async () => {
+		const live = await account("live");
+
+		const { enqueued, result } = await tick({
+			sweepAttachments: async () => {
+				throw new Error("AccessDenied on ListObjectsV2");
+			},
+		});
+
+		assert.deepEqual(enqueued, [live.accountId]);
+		assert.equal(result.sweepFailed, 1);
+	});
+
+	it("keeps the tick alive when the calendar subscription refresh throws", async () => {
+		const live = await account("live");
+
+		const { enqueued, logs } = await tick({
+			refreshCalendarSubscriptions: async () => {
+				throw new Error("database is locked");
+			},
+		});
+
+		assert.deepEqual(enqueued, [live.accountId]);
+		assert.ok(logs.some((line) => line.level === "error"));
+	});
+
+	it("counts what each account re-drove and keeps enqueuing past one that fails", async () => {
+		const live = await account("live");
+		const busy = await account("busy");
+
+		const { enqueued, result, logs } = await tick({
+			redriveStrandedPlacements: async (redriven) => {
+				if (redriven.accountId === busy.accountId) {
+					throw new Error("SQLITE_BUSY");
+				}
+				return 2;
+			},
+		});
+
+		assert.deepEqual(enqueued.sort(), [live.accountId, busy.accountId].sort());
+		assert.equal(result.redriven, 2);
+		assert.equal(result.redriveFailed, 1);
+		assert.ok(logs.some((line) => line.level === "error"));
+	});
+});

package/src/scheduler/run-tick.ts

@@ -4,7 +4,7 @@ import {
 	triggerAccountSync,
 } from "@remit/backend/trigger-sync";
 import type { AccountItem, IAccountRepository } from "@remit/data-ports";
-import type { Logger } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
 import { hasStoredCredential } from "@remit/mailbox-service/account-credentials";
 import pMap from "p-map";
 import {
@@ -53,6 +53,10 @@ export interface RunSchedulerTickDeps {
 		now: number,
 		intervalMs: number,
 	) => Promise<CalendarSubscriptionTally>;
+	redriveStrandedPlacements: (
+		account: AccountItem,
+		now: number,
+	) => Promise<number>;
 }
 
 export interface SchedulerTickResult {
@@ -63,6 +67,8 @@ export interface SchedulerTickResult {
 	sweepFailed: number;
 	subscriptionsRefreshed: number;
 	subscriptionsFailed: number;
+	redriven: number;
+	redriveFailed: number;
 }
 
 /**
@@ -135,6 +141,8 @@ export const runSchedulerTick = async (
 	let skipped = 0;
 	let swept = 0;
 	let sweepFailed = 0;
+	let redriven = 0;
+	let redriveFailed = 0;
 
 	do {
 		const page = await accountService.listAllAccountsPage({
@@ -202,6 +210,32 @@ export const runSchedulerTick = async (
 			sweepFailed += outcomes.filter((ok) => !ok).length;
 		}
 
+		// Contained like the sweep, and for the same reason: a failed re-drive
+		// costs one account this tick, never the scheduler. Eligible accounts
+		// only, since a re-driven event needs a connection to settle anything.
+		const redrives = await pMap(
+			page.items.filter(isEligible),
+			(account) =>
+				deps.redriveStrandedPlacements(account, now).then(
+					(count) => count,
+					(error: unknown) => {
+						log.error(
+							{ accountId: account.accountId, error },
+							"Stranded placement re-drive failed for account",
+						);
+						return null;
+					},
+				),
+			{ concurrency: SCHEDULER_ENQUEUE_CONCURRENCY },
+		);
+		for (const count of redrives) {
+			if (count === null) {
+				redriveFailed += 1;
+				continue;
+			}
+			redriven += count;
+		}
+
 		cursor = page.cursor ?? undefined;
 	} while (cursor);
 
@@ -231,6 +265,8 @@ export const runSchedulerTick = async (
 			sweepFailed,
 			subscriptionsRefreshed,
 			subscriptionsFailed,
+			redriven,
+			redriveFailed,
 		},
 		"Scheduled-sync tick complete",
 	);
@@ -243,5 +279,7 @@ export const runSchedulerTick = async (
 		sweepFailed,
 		subscriptionsRefreshed,
 		subscriptionsFailed,
+		redriven,
+		redriveFailed,
 	};
 };

package/src/scheduler/runner.ts

@@ -1,16 +1,22 @@
 #!/usr/bin/env node
 import { setTimeout as delay } from "node:timers/promises";
+import { finishHalfDeletedAccounts } from "@remit/backend/account-deletion";
 import { getClient } from "@remit/backend/client";
-import { createLogger } from "@remit/logger-lambda";
+import { createLogger } from "@remit/logger";
+import { installNodeLogger } from "@remit/logger-lambda/node";
 import { clearHeartbeats, createHeartbeat } from "@remit/sqs-client/heartbeat";
 import { createQueueProducer } from "@remit/sqs-client/producer";
 
 import { env } from "expect-env";
+import { emitEvent } from "../emit.js";
 import { buildAttachmentSweep } from "./attachment-sweep.js";
 import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
 import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
 import { runSchedulerLoop } from "./loop.js";
 import { runSchedulerTick } from "./run-tick.js";
+import { redriveStrandedPlacements } from "./stranded-placement-redrive.js";
+
+installNodeLogger();
 
 /**
  * The scheduled-sync runner for every deployment that has no EventBridge: the
@@ -36,6 +42,7 @@ const log = createLogger();
 
 const mailboxesQueueUrl = env.SQS_QUEUE_URL_MAILBOXES;
 const sqsClient = createQueueProducer({ queueUrl: mailboxesQueueUrl });
+const accountFanoutQueueUrl = env.SQS_QUEUE_URL_ACCOUNT_FANOUT;
 
 const tickIntervalMs = getTickIntervalMs();
 const offlineIntervalMs = getOfflineIntervalMs();
@@ -71,6 +78,13 @@ const runLoop = async (): Promise<void> => {
 	await clearHeartbeats().catch(onClearError);
 	const client = await getClient();
 	const { account } = client;
+	await finishHalfDeletedAccounts({
+		account,
+		configImport: client.configImport,
+		sqsClient: createQueueProducer({ queueUrl: accountFanoutQueueUrl }),
+		accountFanoutQueueUrl,
+		log,
+	});
 	await runSchedulerLoop({
 		tick: runSchedulerTick,
 		tickDeps: {
@@ -85,6 +99,12 @@ const runLoop = async (): Promise<void> => {
 				client,
 				log,
 			),
+			redriveStrandedPlacements: (stranded, now) =>
+				redriveStrandedPlacements(
+					{ client, emit: emitEvent, log },
+					stranded,
+					now,
+				),
 		},
 		heartbeat: createHeartbeat("tick"),
 		onHeartbeatError: onBeatError,

package/src/scheduler/stranded-placement-redrive.ts

@@ -0,0 +1,265 @@
+import type { RemitClient } from "@remit/backend/client";
+import type { AccountItem, MessageItem } from "@remit/data-ports";
+import { trashMailboxAt } from "@remit/data-ports/folder-role";
+import {
+	deriveCopyMessageId,
+	deriveMessageId,
+	isValidMessageId,
+} from "@remit/data-ports/id";
+import { PATHLESS_MUTATION_EVENT_SCHEMA_VERSION } from "@remit/data-ports/mutation-events";
+import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
+import type { Logger } from "@remit/logger";
+import { isCursorRebuildNeeded } from "@remit/mailbox-service";
+import type {
+	MessageCopyEventV3,
+	MessageDeleteEventV3,
+	MessageMoveEventV3,
+	PlacementMovePushEvent,
+} from "../events.js";
+
+/** Well past three attempts on a 60 s visibility timeout. */
+export const STRANDED_PLACEMENT_AFTER_MILLIS = 60 * 60 * 1000;
+
+type Unstamped<T> = Omit<T, "eventId" | "timestamp">;
+
+export type StrandedPlacementEvent =
+	| Unstamped<PlacementMovePushEvent>
+	| Unstamped<MessageDeleteEventV3>
+	| Unstamped<MessageMoveEventV3>
+	| Unstamped<MessageCopyEventV3>;
+
+export interface StrandedPlacementRedriveDeps {
+	client: Pick<
+		RemitClient,
+		| "mailbox"
+		| "mailboxSpecialUse"
+		| "message"
+		| "placementMove"
+		| "threadMessage"
+	>;
+	emit: (event: StrandedPlacementEvent) => Promise<void>;
+	log: Logger;
+}
+
+const isStrandedCopy = (row: MessageItem): boolean =>
+	row.status === MessageStatus.moving &&
+	row.uid === 0 &&
+	typeof row.originalUid !== "number";
+
+type CopyRedrive =
+	| { kind: "redrive"; event: Unstamped<MessageCopyEventV3> }
+	| { kind: "unresolvable"; reason: string }
+	| { kind: "wait" };
+
+/**
+ * The message a uid-0 copy was made from: its id derives from the Message-ID
+ * the copy's listing row carries, and the copy's own id derives from it.
+ */
+const copyRedriveFor = async (
+	client: StrandedPlacementRedriveDeps["client"],
+	account: AccountItem,
+	copy: MessageItem,
+): Promise<CopyRedrive> => {
+	const listing = await client.threadMessage.findByMessageId(
+		account.accountConfigId,
+		copy.messageId,
+	);
+	if (!listing) return { kind: "unresolvable", reason: "listing row missing" };
+	const header = listing.messageIdHeader;
+	if (!header || !isValidMessageId(header)) {
+		return { kind: "unresolvable", reason: "listing row has no Message-ID" };
+	}
+	const sourceId = deriveMessageId(account.accountId, header);
+	if (deriveCopyMessageId(sourceId, copy.mailboxId) !== copy.messageId) {
+		return {
+			kind: "unresolvable",
+			reason: "copy was not made from the message its Message-ID names",
+		};
+	}
+	const [source] = await client.message.get([sourceId]);
+	if (!source) return { kind: "unresolvable", reason: "source row missing" };
+	if (source.status === MessageStatus.deleted) {
+		return { kind: "unresolvable", reason: "source row deleted" };
+	}
+	if (source.status !== MessageStatus.active) return { kind: "wait" };
+	return {
+		kind: "redrive",
+		event: {
+			type: "MESSAGE_COPY",
+			schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
+			accountId: account.accountId,
+			sourceMessageId: source.messageId,
+			newMessageId: copy.messageId,
+			sourceMailboxId: source.mailboxId,
+			destinationMailboxId: copy.mailboxId,
+			uid: source.uid,
+		},
+	};
+};
+
+/** A copy with no source to copy from has nothing to bind and nothing to hide. */
+const removeUnresolvableCopy = async (
+	deps: StrandedPlacementRedriveDeps,
+	account: AccountItem,
+	row: MessageItem,
+	reason: string,
+	now: number,
+): Promise<boolean> => {
+	const removed = await deps.client.message.deleteIfPlacement(row.messageId, {
+		status: MessageStatus.moving,
+		mailboxId: row.mailboxId,
+		uid: 0,
+	});
+	if (!removed) return false;
+	deps.log.error(
+		{
+			alert: "stranded_copy_unresolvable",
+			accountId: account.accountId,
+			messageId: row.messageId,
+			mailboxId: row.mailboxId,
+			reason,
+			untouchedForMs: now - row.updatedAt,
+		},
+		"A copy stranded at uid 0 has no source to copy from; the copy row was removed",
+	);
+	return true;
+};
+
+const eventFor = async (
+	client: StrandedPlacementRedriveDeps["client"],
+	account: AccountItem,
+	row: MessageItem,
+	trashMailboxId: string | undefined,
+): Promise<StrandedPlacementEvent | undefined> => {
+	const { accountId, accountConfigId } = account;
+	const { messageId } = row;
+
+	if (row.status === MessageStatus.deleting) {
+		return {
+			type: "MESSAGE_DELETE",
+			schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
+			accountId,
+			messageId,
+			mailboxId: row.mailboxId,
+			uid: row.uid,
+			operation: "permanent_delete",
+		};
+	}
+	if (
+		typeof row.originalMailboxId !== "string" ||
+		typeof row.originalUid !== "number"
+	) {
+		return undefined;
+	}
+
+	const marker = await client.placementMove.find(messageId);
+	if (marker && marker.destinationMailboxId === row.mailboxId) {
+		return {
+			type: "PLACEMENT_MOVE_PUSH",
+			accountId,
+			accountConfigId,
+			messageId,
+		};
+	}
+	if (row.mailboxId === trashMailboxId) {
+		return {
+			type: "MESSAGE_DELETE",
+			schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
+			accountId,
+			messageId,
+			mailboxId: row.originalMailboxId,
+			uid: row.originalUid,
+			operation: "move_to_trash",
+			destinationMailboxId: row.mailboxId,
+		};
+	}
+	return {
+		type: "MESSAGE_MOVE",
+		schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
+		accountId,
+		messageId,
+		sourceMailboxId: row.originalMailboxId,
+		destinationMailboxId: row.mailboxId,
+		uid: row.originalUid,
+	};
+};
+
+/** The claim restarts the row's clock; a folder awaiting a cursor rebuild waits until it is normal. */
+export const redriveStrandedPlacements = async (
+	deps: StrandedPlacementRedriveDeps,
+	account: AccountItem,
+	now: number,
+): Promise<number> => {
+	const { client, emit, log } = deps;
+	const { accountId } = account;
+
+	const mailboxes = await client.mailbox.listAllByAccount(accountId);
+	const paused = new Set(
+		mailboxes
+			.filter((mailbox) => isCursorRebuildNeeded(mailbox.cursorState))
+			.map((mailbox) => mailbox.mailboxId),
+	);
+	const stranded = (
+		await client.message.listStrandedPlacements(
+			mailboxes.map((mailbox) => mailbox.mailboxId),
+			now - STRANDED_PLACEMENT_AFTER_MILLIS,
+		)
+	).filter(
+		(row) =>
+			!paused.has(row.mailboxId) &&
+			!(row.originalMailboxId && paused.has(row.originalMailboxId)),
+	);
+	if (stranded.length === 0) return 0;
+
+	const trash = trashMailboxAt(
+		await client.mailboxSpecialUse.resolveTrashRole(accountId),
+		"resolved",
+	);
+	const trashMailboxId = trash.allowed ? trash.mailbox.mailboxId : undefined;
+
+	let redriven = 0;
+	for (const row of stranded) {
+		const copy = isStrandedCopy(row)
+			? await copyRedriveFor(client, account, row)
+			: undefined;
+		if (copy?.kind === "wait") continue;
+		if (copy?.kind === "unresolvable") {
+			if (await removeUnresolvableCopy(deps, account, row, copy.reason, now)) {
+				redriven += 1;
+			}
+			continue;
+		}
+		const event =
+			copy?.kind === "redrive"
+				? copy.event
+				: await eventFor(client, account, row, trashMailboxId);
+		if (!event) continue;
+
+		// A copy's earlier COPY may have landed with its answer lost, so the
+		// re-drive marks it `failed`: the copy handler then asks the destination
+		// before it copies again.
+		const claimed = await client.message.transitionPlacement(
+			row.messageId,
+			{ status: row.status, mailboxId: row.mailboxId, uid: row.uid },
+			{
+				syncStatus: copy ? MessageSyncStatus.failed : MessageSyncStatus.pending,
+			},
+		);
+		if (!claimed) continue;
+
+		await emit(event);
+		redriven += 1;
+		log.warn(
+			{
+				metric: "stranded_placement_redriven",
+				accountId,
+				messageId: row.messageId,
+				event: event.type,
+				syncStatus: row.syncStatus,
+				untouchedForMs: now - row.updatedAt,
+			},
+			"A move or delete had no event left to settle it; re-driven",
+		);
+	}
+	return redriven;
+};