@remit/imap-worker 0.0.90 → 0.0.91

package/src/handlers/sync-message-body.sqlite.test.ts

@@ -1,314 +0,0 @@
-import assert from "node:assert/strict";
-import { after, afterEach, before, beforeEach, describe, it } from "node:test";
-import { GetParameterCommand, SSMClient } from "@aws-sdk/client-ssm";
-import type { MailboxItem } from "@remit/data-ports";
-import {
-	FilterClauseField,
-	FilterScope,
-	MailboxCursorState,
-	MessageStatus,
-} from "@remit/domain-enums";
-import { isMessageBodySyncBroken } from "@remit/mailbox-service";
-import { mockClient } from "aws-sdk-client-mock";
-import { resetBodySyncGateCache } from "../body-sync-gate.js";
-import type { SyncMessageBodyEvent } from "../events.js";
-import {
-	openStore,
-	type SeededMessage,
-	type World,
-} from "../test-helpers/test-world.js";
-import {
-	BODY_SYNC_MAX_ATTEMPTS,
-	syncMessageBody,
-} from "./sync-message-body.js";
-
-const ssm = mockClient(SSMClient);
-
-let store: Awaited<ReturnType<typeof openStore>>;
-
-before(async () => {
-	store = await openStore();
-});
-
-after(() => store.close());
-
-const bodySync = (enabled: boolean) =>
-	ssm
-		.on(GetParameterCommand)
-		.resolves({ Parameter: { Value: enabled ? "true" : "false" } });
-
-beforeEach(() => {
-	ssm.reset();
-	resetBodySyncGateCache();
-	bodySync(true);
-});
-
-afterEach(() => resetBodySyncGateCache());
-
-interface Arranged {
-	w: World;
-	inbox: MailboxItem;
-	message: SeededMessage;
-	event: SyncMessageBodyEvent;
-}
-
-const arrange = async (): Promise<Arranged> => {
-	const w = await store.world();
-	const inbox = await w.folder("INBOX");
-	const message = await w.message(inbox);
-	return {
-		w,
-		inbox,
-		message,
-		event: {
-			type: "SYNC_MESSAGE_BODY",
-			eventId: `evt-${message.messageId}`,
-			timestamp: Date.now(),
-			accountId: w.account.accountId,
-			mailboxId: inbox.mailboxId,
-			messageIds: [message.messageId],
-			messages: [{ messageId: message.messageId, uid: message.uid }],
-		},
-	};
-};
-
-const run = (a: Arranged, receiveCount = 1) =>
-	syncMessageBody(a.event, a.w.log, a.w.queues, receiveCount, a.w.scope);
-
-const bodyFetches = (a: Arranged): string[] => a.w.imap.commands("BODY");
-
-describe("a body sync", () => {
-	it("fetches the body and stores it against the message", async () => {
-		const a = await arrange();
-
-		await run(a);
-
-		assert.equal(bodyFetches(a).length, 1);
-		assert.ok((await a.w.row(a.message.messageId))?.bodyStorageKey);
-	});
-
-	// Issue #1270: swallowing the failure into a fresh enqueue meant the queue's
-	// own redelivery and dead-letter queue never engaged.
-	it("throws within the budget when the server cannot return the body, so the queue redelivers", async () => {
-		const a = await arrange();
-		a.w.imap.failNext.BODY = new Error("Connection reset");
-
-		await assert.rejects(run(a), /Connection reset/);
-
-		assert.equal(
-			(await a.w.row(a.message.messageId))?.bodyStorageKey,
-			undefined,
-		);
-	});
-
-	it("fetches nothing while body sync is paused", async () => {
-		const a = await arrange();
-		ssm.reset();
-		bodySync(false);
-
-		await run(a);
-
-		assert.equal(a.w.imap.connections, 0);
-	});
-
-	it("acks without connecting when the mailbox was deleted", async () => {
-		const a = await arrange();
-		await a.w.client.mailbox.delete(a.w.account.accountId, a.inbox.mailboxId);
-
-		await run(a);
-
-		assert.equal(a.w.imap.connections, 0);
-	});
-
-	it("fetches nothing while the cursor is rebuilding", async () => {
-		const a = await arrange();
-		await a.w.client.mailbox.update(a.w.account.accountId, a.inbox.mailboxId, {
-			cursorState: MailboxCursorState.rebuilding,
-		});
-
-		await run(a);
-
-		assert.equal(a.w.imap.connections, 0);
-	});
-
-	it("fetches nothing when the mailbox's UIDVALIDITY changed under it", async () => {
-		const a = await arrange();
-		a.w.imap.setUidValidity("INBOX", 2);
-
-		await run(a);
-
-		assert.deepEqual(bodyFetches(a), []);
-		assert.equal(
-			(await a.w.row(a.message.messageId))?.bodyStorageKey,
-			undefined,
-		);
-	});
-});
-
-const batchOf = (
-	w: World,
-	inbox: MailboxItem,
-	messages: SeededMessage[],
-): SyncMessageBodyEvent => ({
-	type: "SYNC_MESSAGE_BODY",
-	eventId: `evt-${messages.map((m) => m.messageId).join("-")}`,
-	timestamp: Date.now(),
-	accountId: w.account.accountId,
-	mailboxId: inbox.mailboxId,
-	messageIds: messages.map((m) => m.messageId),
-	messages: messages.map((m) => ({ messageId: m.messageId, uid: m.uid })),
-});
-
-const runBatch = (w: World, event: SyncMessageBodyEvent, receiveCount = 1) =>
-	syncMessageBody(event, w.log, w.queues, receiveCount, w.scope);
-
-const arrangeFiling = async () => {
-	const w = await store.world();
-	const inbox = await w.folder("INBOX");
-	const travel = await w.folder("Travel");
-	await w.client.filter.create({
-		accountConfigId: w.account.accountConfigId,
-		name: "From a friend",
-		scope: FilterScope.Standing,
-		literalClauses: [
-			{ field: FilterClauseField.From, value: "friend@example.com" },
-		],
-		actionMailboxId: travel.mailboxId,
-	});
-	const first = await w.message(inbox);
-	const second = await w.message(inbox);
-	return {
-		w,
-		travel,
-		first,
-		second,
-		event: batchOf(w, inbox, [first, second]),
-	};
-};
-
-describe("a body batch whose filter files mail while the queue is refusing", () => {
-	it("fails at the first filing it cannot enqueue and leaves the rest untouched", async () => {
-		const { w, first, second, event } = await arrangeFiling();
-		w.refuseSends(new Error("connect ECONNREFUSED 127.0.0.1:9324"));
-
-		await assert.rejects(runBatch(w, event), /ECONNREFUSED/);
-
-		const untouched = await w.row(second.messageId);
-		assert.equal(untouched?.status, MessageStatus.active);
-		assert.equal(untouched?.bodyStorageKey, undefined);
-		assert.equal(await w.client.placementMove.find(second.messageId), null);
-		assert.equal(
-			(await w.client.placementMove.find(first.messageId))?.state,
-			"pending",
-		);
-	});
-
-	it("files and stores the whole batch on the redelivery once the queue accepts", async () => {
-		const { w, travel, first, second, event } = await arrangeFiling();
-		w.refuseSends(new Error("connect ECONNREFUSED 127.0.0.1:9324"));
-		await assert.rejects(runBatch(w, event), /ECONNREFUSED/);
-
-		w.refuseSends(undefined);
-		await runBatch(w, event, 2);
-
-		for (const message of [first, second]) {
-			const row = await w.row(message.messageId);
-			assert.equal(row?.mailboxId, travel.mailboxId);
-			assert.ok(row?.bodyStorageKey, `${message.messageId} body stored`);
-			assert.equal(row?.movedByRemit, true);
-			assert.equal(row?.filterMove?.sourceMailboxId, event.mailboxId);
-			assert.equal(
-				(await w.client.placementMove.find(message.messageId))?.state,
-				"queued",
-			);
-		}
-	});
-});
-
-describe("a body batch on its last attempt while the queue is refusing", () => {
-	it("throws, so the batch dead-letters instead of its messages being marked broken", async () => {
-		const { w, first: message, event } = await arrangeFiling();
-		w.refuseSends(new Error("connect ECONNREFUSED 127.0.0.1:9324"));
-
-		await assert.rejects(
-			runBatch(w, event, BODY_SYNC_MAX_ATTEMPTS),
-			/ECONNREFUSED/,
-		);
-
-		assert.equal(
-			await isMessageBodySyncBroken(
-				w.client.storage,
-				w.account.accountConfigId,
-				w.account.accountId,
-				message.messageId,
-			),
-			false,
-		);
-	});
-});
-
-const startMove = (
-	w: World,
-	message: SeededMessage,
-	from: MailboxItem,
-	to: MailboxItem,
-) =>
-	w.client.message.transitionPlacement(
-		message.messageId,
-		{
-			status: MessageStatus.active,
-			mailboxId: from.mailboxId,
-			uid: message.uid,
-		},
-		{
-			mailboxId: to.mailboxId,
-			status: MessageStatus.moving,
-			originalMailboxId: from.mailboxId,
-			originalUid: message.uid,
-		},
-	);
-
-describe("a body batch that has gone stale", () => {
-	it("does not fetch a message from a folder its confirmed move took it out of", async () => {
-		const w = await store.world();
-		const inbox = await w.folder("INBOX");
-		const travel = await w.folder("Travel");
-		const moved = await w.message(inbox);
-		const neighbour = await w.message(inbox);
-		await startMove(w, moved, inbox, travel);
-		await w.client.message.updateUid(
-			moved.messageId,
-			neighbour.uid,
-			travel.mailboxId,
-		);
-
-		await runBatch(w, batchOf(w, inbox, [moved]));
-
-		assert.deepEqual(w.imap.commands("BODY"), []);
-		assert.equal((await w.row(moved.messageId))?.bodyStorageKey, undefined);
-	});
-
-	it("fetches a message whose move has not confirmed from the folder its uid is in", async () => {
-		const w = await store.world();
-		const inbox = await w.folder("INBOX");
-		const travel = await w.folder("Travel");
-		const filing = await w.message(inbox);
-		await startMove(w, filing, inbox, travel);
-
-		await runBatch(w, batchOf(w, inbox, [filing]));
-
-		assert.equal(w.imap.commands("BODY").length, 1);
-		assert.ok((await w.row(filing.messageId))?.bodyStorageKey);
-	});
-
-	it("acks a batch whose message row has been deleted since", async () => {
-		const w = await store.world();
-		const inbox = await w.folder("INBOX");
-		const gone = await w.message(inbox);
-		await w.client.message.delete(gone.messageId);
-
-		await runBatch(w, batchOf(w, inbox, [gone]));
-
-		assert.equal(w.imap.connections, 0);
-	});
-});

package/src/handlers/sync-message-body.test.ts

@@ -1,150 +0,0 @@
-import assert from "node:assert";
-import { describe, test } from "node:test";
-import type { SyncedMessage } from "@remit/mailbox-service";
-import type { SyncMessageBodyEvent } from "../events.js";
-import {
-	buildRetryableFailureError,
-	resolveBatch,
-} from "./sync-message-body.js";
-import { batchSyncedMessages } from "./sync-messages.js";
-
-const baseEvent = {
-	type: "SYNC_MESSAGE_BODY" as const,
-	accountId: "test-account-123",
-	mailboxId: "test-mailbox-456",
-	eventId: "event-789",
-	timestamp: 1700000000000,
-};
-
-describe("resolveBatch — event-shape preference", () => {
-	test("prefers the new messages[] shape and exposes the uid map", () => {
-		const event: SyncMessageBodyEvent = {
-			...baseEvent,
-			messageIds: ["msg-1", "msg-2"],
-			messages: [
-				{ messageId: "msg-1", uid: 101 },
-				{ messageId: "msg-2", uid: 102 },
-			],
-		};
-
-		const { messageIds, uidByMessageId } = resolveBatch(event);
-
-		assert.deepEqual(messageIds, ["msg-1", "msg-2"]);
-		assert.ok(uidByMessageId);
-		assert.equal(uidByMessageId.get("msg-1"), 101);
-		assert.equal(uidByMessageId.get("msg-2"), 102);
-	});
-
-	test("derives messageIds from messages[] when both disagree", () => {
-		// messages[] is authoritative; a stale messageIds list must not leak in.
-		const event: SyncMessageBodyEvent = {
-			...baseEvent,
-			messageIds: ["stale"],
-			messages: [{ messageId: "msg-1", uid: 101 }],
-		};
-
-		const { messageIds } = resolveBatch(event);
-
-		assert.deepEqual(messageIds, ["msg-1"]);
-	});
-
-	test("falls back to legacy messageIds[] with no uid map", () => {
-		const event: SyncMessageBodyEvent = {
-			...baseEvent,
-			messageIds: ["msg-1", "msg-2", "msg-3"],
-		};
-
-		const { messageIds, uidByMessageId } = resolveBatch(event);
-
-		assert.deepEqual(messageIds, ["msg-1", "msg-2", "msg-3"]);
-		assert.equal(uidByMessageId, undefined);
-	});
-
-	test("force defaults to false when the event omits it (legacy/bulk events)", () => {
-		const event: SyncMessageBodyEvent = {
-			...baseEvent,
-			messageIds: ["msg-1"],
-		};
-
-		assert.equal(resolveBatch(event).force, false);
-	});
-
-	test("force is carried from the event's read-miss re-arm cue", () => {
-		const event: SyncMessageBodyEvent = {
-			...baseEvent,
-			messageIds: ["msg-1"],
-			messages: [{ messageId: "msg-1", uid: 101 }],
-			force: true,
-		};
-
-		assert.equal(resolveBatch(event).force, true);
-	});
-
-	test("empty messages[] resolves to an empty batch, not the legacy list", () => {
-		const event: SyncMessageBodyEvent = {
-			...baseEvent,
-			messageIds: ["should-be-ignored"],
-			messages: [],
-		};
-
-		const { messageIds, uidByMessageId } = resolveBatch(event);
-
-		assert.deepEqual(messageIds, []);
-		assert.ok(uidByMessageId);
-		assert.equal(uidByMessageId.size, 0);
-	});
-});
-
-describe("buildRetryableFailureError — the DLQ-propagation signal", () => {
-	// Genuine processing failures must propagate (issue #1270): syncMessageBody
-	// throws this while SQS redelivery budget remains, instead of swallowing the
-	// failure into a fresh re-enqueue. index.ts's SQS handler catches it and
-	// reports the record as a batch item failure, so SQS redelivers it — and
-	// once the queue's own maxReceiveCount is hit, the record dead-letters into
-	// the body-dlq (alarmed in infra/stacks/dev/stacks/remit-worker-monitoring-stack.ts).
-
-	test("names every failed message id and the current attempt", () => {
-		const error = buildRetryableFailureError(["msg-1", "msg-2"], 1);
-
-		assert.match(error.message, /msg-1/);
-		assert.match(error.message, /msg-2/);
-		assert.match(error.message, /attempt 1\/3/);
-	});
-});
-
-describe("batchSyncedMessages — one batch == one ranged fetch", () => {
-	const makeSynced = (count: number): SyncedMessage[] =>
-		Array.from({ length: count }, (_, i) => ({
-			messageId: `msg-${i}`,
-			uid: i + 1,
-		}));
-
-	test("packs up to 200 messages into a single batch", () => {
-		const batches = batchSyncedMessages(makeSynced(200));
-
-		assert.equal(batches.length, 1);
-		assert.equal(batches[0].length, 200);
-	});
-
-	test("splits 201 messages into 200 + 1", () => {
-		const batches = batchSyncedMessages(makeSynced(201));
-
-		assert.equal(batches.length, 2);
-		assert.equal(batches[0].length, 200);
-		assert.equal(batches[1].length, 1);
-	});
-
-	test("keeps messageId+uid pairs intact per batch", () => {
-		const batches = batchSyncedMessages(makeSynced(3));
-
-		assert.deepEqual(batches[0], [
-			{ messageId: "msg-0", uid: 1 },
-			{ messageId: "msg-1", uid: 2 },
-			{ messageId: "msg-2", uid: 3 },
-		]);
-	});
-
-	test("empty input yields no batches", () => {
-		assert.deepEqual(batchSyncedMessages([]), []);
-	});
-});