@remit/imap-worker 0.0.86 → 0.0.87

package/src/handlers/sync-mailboxes.test.ts

@@ -56,7 +56,6 @@ describe("mailboxNeedsSync", () => {
 describe("resolveMailboxFreshnessMs", () => {
 	it("defaults to the production window when unset", () => {
 		assert.equal(resolveMailboxFreshnessMs({}), DEFAULT_MAILBOX_FRESHNESS_MS);
-		assert.equal(DEFAULT_MAILBOX_FRESHNESS_MS, 60_000);
 	});
 
 	it("honors a valid override", () => {

package/src/handlers/sync-mailboxes.ts

@@ -13,7 +13,7 @@ import type {
 } from "@remit/data-ports";
 import type { CanonicalMailboxRoleValue } from "@remit/data-ports/folder-role";
 import { SyncPhase } from "@remit/domain-enums";
-import type { Logger } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
 import { RefreshTokenError } from "@remit/mail-oauth-service";
 import {
 	createConnectionWithCredentials,

package/src/handlers/sync-message-body.sqlite.test.ts

@@ -0,0 +1,314 @@
+import assert from "node:assert/strict";
+import { after, afterEach, before, beforeEach, describe, it } from "node:test";
+import { GetParameterCommand, SSMClient } from "@aws-sdk/client-ssm";
+import type { MailboxItem } from "@remit/data-ports";
+import {
+	FilterClauseField,
+	FilterScope,
+	MailboxCursorState,
+	MessageStatus,
+} from "@remit/domain-enums";
+import { isMessageBodySyncBroken } from "@remit/mailbox-service";
+import { mockClient } from "aws-sdk-client-mock";
+import { resetBodySyncGateCache } from "../body-sync-gate.js";
+import type { SyncMessageBodyEvent } from "../events.js";
+import {
+	openStore,
+	type SeededMessage,
+	type World,
+} from "../test-helpers/test-world.js";
+import {
+	BODY_SYNC_MAX_ATTEMPTS,
+	syncMessageBody,
+} from "./sync-message-body.js";
+
+const ssm = mockClient(SSMClient);
+
+let store: Awaited<ReturnType<typeof openStore>>;
+
+before(async () => {
+	store = await openStore();
+});
+
+after(() => store.close());
+
+const bodySync = (enabled: boolean) =>
+	ssm
+		.on(GetParameterCommand)
+		.resolves({ Parameter: { Value: enabled ? "true" : "false" } });
+
+beforeEach(() => {
+	ssm.reset();
+	resetBodySyncGateCache();
+	bodySync(true);
+});
+
+afterEach(() => resetBodySyncGateCache());
+
+interface Arranged {
+	w: World;
+	inbox: MailboxItem;
+	message: SeededMessage;
+	event: SyncMessageBodyEvent;
+}
+
+const arrange = async (): Promise<Arranged> => {
+	const w = await store.world();
+	const inbox = await w.folder("INBOX");
+	const message = await w.message(inbox);
+	return {
+		w,
+		inbox,
+		message,
+		event: {
+			type: "SYNC_MESSAGE_BODY",
+			eventId: `evt-${message.messageId}`,
+			timestamp: Date.now(),
+			accountId: w.account.accountId,
+			mailboxId: inbox.mailboxId,
+			messageIds: [message.messageId],
+			messages: [{ messageId: message.messageId, uid: message.uid }],
+		},
+	};
+};
+
+const run = (a: Arranged, receiveCount = 1) =>
+	syncMessageBody(a.event, a.w.log, receiveCount, a.w.scope);
+
+const bodyFetches = (a: Arranged): string[] => a.w.imap.commands("BODY");
+
+describe("a body sync", () => {
+	it("fetches the body and stores it against the message", async () => {
+		const a = await arrange();
+
+		await run(a);
+
+		assert.equal(bodyFetches(a).length, 1);
+		assert.ok((await a.w.row(a.message.messageId))?.bodyStorageKey);
+	});
+
+	// Issue #1270: swallowing the failure into a fresh enqueue meant the queue's
+	// own redelivery and dead-letter queue never engaged.
+	it("throws within the budget when the server cannot return the body, so the queue redelivers", async () => {
+		const a = await arrange();
+		a.w.imap.failNext.BODY = new Error("Connection reset");
+
+		await assert.rejects(run(a), /Connection reset/);
+
+		assert.equal(
+			(await a.w.row(a.message.messageId))?.bodyStorageKey,
+			undefined,
+		);
+	});
+
+	it("fetches nothing while body sync is paused", async () => {
+		const a = await arrange();
+		ssm.reset();
+		bodySync(false);
+
+		await run(a);
+
+		assert.equal(a.w.imap.connections, 0);
+	});
+
+	it("acks without connecting when the mailbox was deleted", async () => {
+		const a = await arrange();
+		await a.w.client.mailbox.delete(a.w.account.accountId, a.inbox.mailboxId);
+
+		await run(a);
+
+		assert.equal(a.w.imap.connections, 0);
+	});
+
+	it("fetches nothing while the cursor is rebuilding", async () => {
+		const a = await arrange();
+		await a.w.client.mailbox.update(a.w.account.accountId, a.inbox.mailboxId, {
+			cursorState: MailboxCursorState.rebuilding,
+		});
+
+		await run(a);
+
+		assert.equal(a.w.imap.connections, 0);
+	});
+
+	it("fetches nothing when the mailbox's UIDVALIDITY changed under it", async () => {
+		const a = await arrange();
+		a.w.imap.setUidValidity("INBOX", 2);
+
+		await run(a);
+
+		assert.deepEqual(bodyFetches(a), []);
+		assert.equal(
+			(await a.w.row(a.message.messageId))?.bodyStorageKey,
+			undefined,
+		);
+	});
+});
+
+const batchOf = (
+	w: World,
+	inbox: MailboxItem,
+	messages: SeededMessage[],
+): SyncMessageBodyEvent => ({
+	type: "SYNC_MESSAGE_BODY",
+	eventId: `evt-${messages.map((m) => m.messageId).join("-")}`,
+	timestamp: Date.now(),
+	accountId: w.account.accountId,
+	mailboxId: inbox.mailboxId,
+	messageIds: messages.map((m) => m.messageId),
+	messages: messages.map((m) => ({ messageId: m.messageId, uid: m.uid })),
+});
+
+const runBatch = (w: World, event: SyncMessageBodyEvent, receiveCount = 1) =>
+	syncMessageBody(event, w.log, receiveCount, w.scope);
+
+const arrangeFiling = async () => {
+	const w = await store.world();
+	const inbox = await w.folder("INBOX");
+	const travel = await w.folder("Travel");
+	await w.client.filter.create({
+		accountConfigId: w.account.accountConfigId,
+		name: "From a friend",
+		scope: FilterScope.Standing,
+		literalClauses: [
+			{ field: FilterClauseField.From, value: "friend@example.com" },
+		],
+		actionMailboxId: travel.mailboxId,
+	});
+	const first = await w.message(inbox);
+	const second = await w.message(inbox);
+	return {
+		w,
+		travel,
+		first,
+		second,
+		event: batchOf(w, inbox, [first, second]),
+	};
+};
+
+describe("a body batch whose filter files mail while the queue is refusing", () => {
+	it("fails at the first filing it cannot enqueue and leaves the rest untouched", async () => {
+		const { w, first, second, event } = await arrangeFiling();
+		w.refuseSends(new Error("connect ECONNREFUSED 127.0.0.1:9324"));
+
+		await assert.rejects(runBatch(w, event), /ECONNREFUSED/);
+
+		const untouched = await w.row(second.messageId);
+		assert.equal(untouched?.status, MessageStatus.active);
+		assert.equal(untouched?.bodyStorageKey, undefined);
+		assert.equal(await w.client.placementMove.find(second.messageId), null);
+		assert.equal(
+			(await w.client.placementMove.find(first.messageId))?.state,
+			"pending",
+		);
+	});
+
+	it("files and stores the whole batch on the redelivery once the queue accepts", async () => {
+		const { w, travel, first, second, event } = await arrangeFiling();
+		w.refuseSends(new Error("connect ECONNREFUSED 127.0.0.1:9324"));
+		await assert.rejects(runBatch(w, event), /ECONNREFUSED/);
+
+		w.refuseSends(undefined);
+		await runBatch(w, event, 2);
+
+		for (const message of [first, second]) {
+			const row = await w.row(message.messageId);
+			assert.equal(row?.mailboxId, travel.mailboxId);
+			assert.ok(row?.bodyStorageKey, `${message.messageId} body stored`);
+			assert.equal(row?.movedByRemit, true);
+			assert.equal(row?.filterMove?.sourceMailboxId, event.mailboxId);
+			assert.equal(
+				(await w.client.placementMove.find(message.messageId))?.state,
+				"queued",
+			);
+		}
+	});
+});
+
+describe("a body batch on its last attempt while the queue is refusing", () => {
+	it("throws, so the batch dead-letters instead of its messages being marked broken", async () => {
+		const { w, first: message, event } = await arrangeFiling();
+		w.refuseSends(new Error("connect ECONNREFUSED 127.0.0.1:9324"));
+
+		await assert.rejects(
+			runBatch(w, event, BODY_SYNC_MAX_ATTEMPTS),
+			/ECONNREFUSED/,
+		);
+
+		assert.equal(
+			await isMessageBodySyncBroken(
+				w.client.storage,
+				w.account.accountConfigId,
+				w.account.accountId,
+				message.messageId,
+			),
+			false,
+		);
+	});
+});
+
+const startMove = (
+	w: World,
+	message: SeededMessage,
+	from: MailboxItem,
+	to: MailboxItem,
+) =>
+	w.client.message.transitionPlacement(
+		message.messageId,
+		{
+			status: MessageStatus.active,
+			mailboxId: from.mailboxId,
+			uid: message.uid,
+		},
+		{
+			mailboxId: to.mailboxId,
+			status: MessageStatus.moving,
+			originalMailboxId: from.mailboxId,
+			originalUid: message.uid,
+		},
+	);
+
+describe("a body batch that has gone stale", () => {
+	it("does not fetch a message from a folder its confirmed move took it out of", async () => {
+		const w = await store.world();
+		const inbox = await w.folder("INBOX");
+		const travel = await w.folder("Travel");
+		const moved = await w.message(inbox);
+		const neighbour = await w.message(inbox);
+		await startMove(w, moved, inbox, travel);
+		await w.client.message.updateUid(
+			moved.messageId,
+			neighbour.uid,
+			travel.mailboxId,
+		);
+
+		await runBatch(w, batchOf(w, inbox, [moved]));
+
+		assert.deepEqual(w.imap.commands("BODY"), []);
+		assert.equal((await w.row(moved.messageId))?.bodyStorageKey, undefined);
+	});
+
+	it("fetches a message whose move has not confirmed from the folder its uid is in", async () => {
+		const w = await store.world();
+		const inbox = await w.folder("INBOX");
+		const travel = await w.folder("Travel");
+		const filing = await w.message(inbox);
+		await startMove(w, filing, inbox, travel);
+
+		await runBatch(w, batchOf(w, inbox, [filing]));
+
+		assert.equal(w.imap.commands("BODY").length, 1);
+		assert.ok((await w.row(filing.messageId))?.bodyStorageKey);
+	});
+
+	it("acks a batch whose message row has been deleted since", async () => {
+		const w = await store.world();
+		const inbox = await w.folder("INBOX");
+		const gone = await w.message(inbox);
+		await w.client.message.delete(gone.messageId);
+
+		await runBatch(w, batchOf(w, inbox, [gone]));
+
+		assert.equal(w.imap.connections, 0);
+	});
+});

package/src/handlers/sync-message-body.test.ts

@@ -1,37 +1,12 @@
 import assert from "node:assert";
-import { afterEach, before, describe, mock, test } from "node:test";
-import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
-import { GetParameterCommand, SSMClient } from "@aws-sdk/client-ssm";
-import { getClient, type RemitClient, setClient } from "@remit/backend/client";
-import type { AccountItem, MailboxItem } from "@remit/data-ports";
-import type { Logger } from "@remit/logger-lambda";
-import { BodySyncService, type SyncedMessage } from "@remit/mailbox-service";
-import { mockClient } from "aws-sdk-client-mock";
-import { resetBodySyncGateCache } from "../body-sync-gate.js";
-import { __warmPoolSizeForTest } from "../connection-scope.js";
+import { describe, test } from "node:test";
+import type { SyncedMessage } from "@remit/mailbox-service";
 import type { SyncMessageBodyEvent } from "../events.js";
 import {
-	BODY_SYNC_MAX_ATTEMPTS,
 	buildRetryableFailureError,
-	getBodySyncMaxAttempts,
 	resolveBatch,
-	syncMessageBody,
 } from "./sync-message-body.js";
-import { BODY_BATCH_SIZE, batchSyncedMessages } from "./sync-messages.js";
-
-const silentLogger = (() => {
-	const noop = () => {};
-	const log = {
-		info: noop,
-		warn: noop,
-		error: noop,
-		debug: noop,
-		fatal: noop,
-		trace: noop,
-		child: () => log,
-	} as unknown as Logger;
-	return log;
-})();
+import { batchSyncedMessages } from "./sync-messages.js";
 
 const baseEvent = {
 	type: "SYNC_MESSAGE_BODY" as const,
@@ -135,273 +110,6 @@ describe("buildRetryableFailureError — the DLQ-propagation signal", () => {
 		assert.match(error.message, /msg-2/);
 		assert.match(error.message, /attempt 1\/3/);
 	});
-
-	test("is a real Error instance so it propagates like any other failure", () => {
-		const error = buildRetryableFailureError(["msg-1"], 2);
-		assert.ok(error instanceof Error);
-	});
-
-	test("BODY_SYNC_MAX_ATTEMPTS matches the body queue's maxReceiveCount (3)", () => {
-		// See MAX_RECEIVE_COUNT in infra/stacks/dev/stacks/remit-queue-stack.ts.
-		assert.equal(BODY_SYNC_MAX_ATTEMPTS, 3);
-	});
-});
-
-describe("getBodySyncMaxAttempts — env-derived, CDK-injected threshold (#1270)", () => {
-	test("parses the CDK-injected env var (derived from the queue's MAX_RECEIVE_COUNT)", () => {
-		assert.equal(getBodySyncMaxAttempts({ BODY_SYNC_MAX_ATTEMPTS: "3" }), 3);
-		assert.equal(getBodySyncMaxAttempts({ BODY_SYNC_MAX_ATTEMPTS: "5" }), 5);
-	});
-
-	test("defaults to 3 when unset (local dev, unit tests) — matches the queue's own default", () => {
-		assert.equal(getBodySyncMaxAttempts({}), 3);
-	});
-
-	test("defaults to 3 on a non-numeric or non-positive value", () => {
-		assert.equal(getBodySyncMaxAttempts({ BODY_SYNC_MAX_ATTEMPTS: "nope" }), 3);
-		assert.equal(getBodySyncMaxAttempts({ BODY_SYNC_MAX_ATTEMPTS: "0" }), 3);
-		assert.equal(getBodySyncMaxAttempts({ BODY_SYNC_MAX_ATTEMPTS: "-1" }), 3);
-	});
-});
-
-describe("syncMessageBody — pause gate runs before connection reuse", () => {
-	afterEach(() => {
-		mockClient(SSMClient).reset();
-		resetBodySyncGateCache();
-	});
-
-	test("paused: acks-and-skips before borrowing a warm connection", async () => {
-		const accountId = "paused-account-zzz";
-		mockClient(SSMClient)
-			.on(GetParameterCommand)
-			.resolves({ Parameter: { Value: "false" } });
-
-		const event: SyncMessageBodyEvent = {
-			...baseEvent,
-			accountId,
-			messageIds: ["msg-1"],
-			messages: [{ messageId: "msg-1", uid: 101 }],
-		};
-
-		await syncMessageBody(event, silentLogger);
-
-		// Returning at the gate must not touch the warm pool (no account lookup,
-		// no IMAP connection) — proves the gate is first.
-		assert.strictEqual(
-			__warmPoolSizeForTest(accountId),
-			0,
-			"paused handler must not create a warm connection",
-		);
-	});
-});
-
-describe("syncMessageBody — DLQ propagation (integrated, #1270)", () => {
-	const accountId = "cap-account-zzz";
-	const mailboxId = "cap-mailbox-zzz";
-
-	const cappedAccount = (): AccountItem =>
-		({
-			accountId,
-			accountConfigId: "cap-acfg-zzz",
-			connectionState: "authenticated",
-			username: "cap@imap.example.com",
-			imapHost: "imap.example.com",
-			imapPort: 993,
-			imapTls: true,
-			// deserializeEncryptedPayload just needs base64-decodable strings; the
-			// mocked secrets.decrypt below never inspects the actual bytes.
-			passwordHash: JSON.stringify({
-				encryptedDek: "",
-				encryptedData: "",
-				iv: "",
-				authTag: "",
-			}),
-		}) as unknown as AccountItem;
-
-	const cappedMailbox = (): MailboxItem =>
-		({ fullPath: "INBOX" }) as unknown as MailboxItem;
-
-	// The client is supplied by injection (`setClient`), so these tests register
-	// the repositories they then mock rather than reaching a composition.
-	before(() => {
-		setClient({
-			account: { get: async () => cappedAccount() },
-			mailbox: { get: async () => cappedMailbox() },
-			mailboxSpecialUse: {
-				findJunkMailbox: async () => null,
-				findInboxMailbox: async () => null,
-			},
-			message: {},
-			threadMessage: {},
-			address: {},
-			envelope: {},
-			placementMove: {},
-			filter: {},
-			filterAnchor: {},
-			messageLabel: {},
-			storage: {},
-			secrets: { decrypt: async () => "fake-password" },
-		} as unknown as RemitClient);
-	});
-
-	afterEach(() => {
-		mock.restoreAll();
-		mockClient(SSMClient).reset();
-		mockClient(SQSClient).reset();
-		resetBodySyncGateCache();
-	});
-
-	test("below BODY_SYNC_MAX_ATTEMPTS: throws instead of re-enqueueing, so SQS redelivers the record (issue #1270)", async () => {
-		// This is the mechanism that lets a genuine processing failure ever reach
-		// the body-dlq: the handler used to swallow every failure into a fresh
-		// SQS SendMessage and always return successfully, so the queue's own
-		// maxReceiveCount/DLQ never engaged. Throwing here is what index.ts's SQS
-		// handler turns into a batchItemFailure, which SQS then redelivers.
-		mockClient(SSMClient)
-			.on(GetParameterCommand)
-			.resolves({ Parameter: { Value: "true" } });
-		const sqsMock = mockClient(SQSClient);
-
-		mock.method((await getClient()).account, "get", async () =>
-			cappedAccount(),
-		);
-		mock.method((await getClient()).mailbox, "get", async () =>
-			cappedMailbox(),
-		);
-		mock.method(
-			(await getClient()).secrets,
-			"decrypt",
-			async () => "fake-password",
-		);
-		mock.method(BodySyncService.prototype, "syncBodies", async () => ({
-			syncedCount: 0,
-			syncedMessageIds: [],
-			skippedCount: 0,
-			failedCount: 1,
-			failedMessageIds: ["msg-1"],
-		}));
-
-		const event: SyncMessageBodyEvent = {
-			...baseEvent,
-			accountId,
-			mailboxId,
-			messageIds: ["msg-1"],
-			messages: [{ messageId: "msg-1", uid: 101 }],
-		};
-
-		await assert.rejects(
-			() => syncMessageBody(event, silentLogger, 1),
-			(err: unknown) =>
-				err instanceof Error &&
-				/Body sync failed for 1 message/.test(err.message),
-		);
-
-		assert.equal(
-			sqsMock.commandCalls(SendMessageCommand).length,
-			0,
-			"no manual re-enqueue — SQS's own redelivery owns the retry now",
-		);
-	});
-
-	test("deleted mailbox: acks-and-skips without ever calling BodySyncService.syncBodies (#287/#289)", async () => {
-		// A SYNC_MESSAGE_BODY trigger outlived the folder it targeted. The mailbox
-		// lookup throws NotFoundError, which would poison the account's per-group
-		// body FIFO forever; the handler must resolve it terminally instead.
-		mockClient(SSMClient)
-			.on(GetParameterCommand)
-			.resolves({ Parameter: { Value: "true" } });
-
-		mock.method((await getClient()).account, "get", async () =>
-			cappedAccount(),
-		);
-		mock.method((await getClient()).mailbox, "get", async () => {
-			throw Object.assign(new Error(`Mailbox not found: ${mailboxId}`), {
-				name: "NotFoundError",
-			});
-		});
-		mock.method(
-			(await getClient()).secrets,
-			"decrypt",
-			async () => "fake-password",
-		);
-		const syncBodies = mock.method(
-			BodySyncService.prototype,
-			"syncBodies",
-			async () => {
-				throw new Error("must not be called for a deleted mailbox");
-			},
-		);
-
-		const event: SyncMessageBodyEvent = {
-			...baseEvent,
-			accountId,
-			mailboxId,
-			messageIds: ["msg-1"],
-			messages: [{ messageId: "msg-1", uid: 101 }],
-		};
-
-		// Must resolve, not reject — a deleted folder is an expected terminal
-		// outcome, not a fault to retry/DLQ.
-		await syncMessageBody(event, silentLogger, 1);
-
-		assert.equal(
-			syncBodies.mock.calls.length,
-			0,
-			"a deleted mailbox must never reach the body fetch",
-		);
-	});
-
-	test("cursor_invalid: acks-and-skips without ever calling BodySyncService.syncBodies (#1272)", async () => {
-		// The cheap pre-check (isCursorRebuildNeeded, run before borrowing any
-		// connection — frugal, epic #1281 invariant 6) catches an already-paused
-		// mailbox before BodySyncService is ever invoked. The structural backstop
-		// for a cursor that trips *during* this call — guardConnectionCursor's
-		// openBox override — is covered directly in mailbox-cursor.test.ts.
-		mockClient(SSMClient)
-			.on(GetParameterCommand)
-			.resolves({ Parameter: { Value: "true" } });
-
-		mock.method((await getClient()).account, "get", async () =>
-			cappedAccount(),
-		);
-		mock.method((await getClient()).mailbox, "get", async () => ({
-			...cappedMailbox(),
-			mailboxId,
-			cursorState: "cursor_invalid",
-		}));
-		mock.method(
-			(await getClient()).secrets,
-			"decrypt",
-			async () => "fake-password",
-		);
-		const syncBodies = mock.method(
-			BodySyncService.prototype,
-			"syncBodies",
-			async () => {
-				throw new Error(
-					"must not be called while the mailbox cursor is paused",
-				);
-			},
-		);
-
-		const event: SyncMessageBodyEvent = {
-			...baseEvent,
-			accountId,
-			mailboxId,
-			messageIds: ["msg-1"],
-			messages: [{ messageId: "msg-1", uid: 101 }],
-		};
-
-		// Must resolve, not reject — a paused cursor is a routine, expected pause
-		// (epic #1281 invariant 3), not a fault to retry/DLQ.
-		await syncMessageBody(event, silentLogger, 1);
-
-		assert.equal(
-			syncBodies.mock.calls.length,
-			0,
-			"outbound body fetch must not run while the mailbox cursor is paused",
-		);
-	});
 });
 
 describe("batchSyncedMessages — one batch == one ranged fetch", () => {
@@ -411,10 +119,6 @@ describe("batchSyncedMessages — one batch == one ranged fetch", () => {
 			uid: i + 1,
 		}));
 
-	test("batch size is raised to 200", () => {
-		assert.equal(BODY_BATCH_SIZE, 200);
-	});
-
 	test("packs up to 200 messages into a single batch", () => {
 		const batches = batchSyncedMessages(makeSynced(200));