@remit/mailbox-service 0.0.110 → 0.0.112

package/src/body-sync-terminal.test.ts

@@ -1,310 +1,148 @@
 import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import type {
-	IMessageRepository,
-	IThreadMessageRepository,
-} from "@remit/data-ports";
-import type {
-	StorageService,
-	StoreBodyPartParams,
-} from "@remit/storage-service";
-import type { BodySyncLogger } from "./body-sync.js";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import type { AccountItem, MailboxItem } from "@remit/data-ports";
+import type { StorageService } from "@remit/storage-service";
 import {
 	isMessageBodySyncBroken,
-	type ResolveExhaustedBodySyncDeps,
 	resolveExhaustedBodySyncFailures,
 } from "./body-sync-terminal.js";
-import type { IImapConnection } from "./types.js";
-
-interface LogEntry {
-	obj: Record<string, unknown>;
-	msg: string;
-}
-
-const buildLogger = (): {
-	log: BodySyncLogger;
-	infos: LogEntry[];
-	errors: LogEntry[];
-} => {
-	const infos: LogEntry[] = [];
-	const errors: LogEntry[] = [];
-	return {
-		log: {
-			info: (obj, msg) => infos.push({ obj, msg: msg ?? "" }),
-			error: (obj, msg) => errors.push({ obj, msg: msg ?? "" }),
-		},
-		infos,
-		errors,
-	};
-};
-
-/** Message rows keyed by messageId; `uid` is the only field the resolver reads. */
-const buildMessageService = (
-	uidByMessageId: Record<string, number>,
-	deleted: string[],
-): Pick<IMessageRepository, "get" | "delete"> =>
-	({
-		get: async (messageId: string) => {
-			const uid = uidByMessageId[messageId];
-			if (uid === undefined) {
-				throw new Error(`no fixture uid for ${messageId}`);
-			}
-			return { messageId, uid };
-		},
-		delete: async (messageId: string) => {
-			deleted.push(messageId);
-		},
-	}) as unknown as Pick<IMessageRepository, "get" | "delete">;
+import { eml, FakeImapServer } from "./test-helpers/fake-imap.js";
+import {
+	messageAtUid,
+	openStore,
+	type Store,
+	seedAccount,
+	seedMailbox,
+	silentLog,
+	syncFolder,
+} from "./test-helpers/test-store.js";
+
+let store: Store;
+let storage: StorageService;
+let imap: FakeImapServer;
+let account: AccountItem;
+let inbox: MailboxItem;
+
+beforeEach(async () => {
+	store = await openStore();
+	storage = store.storage();
+	imap = new FakeImapServer();
+	account = await seedAccount(store);
+	inbox = await seedMailbox(store, account, "INBOX");
+	imap.addFolder("INBOX");
+});
 
-const buildThreadMessageService = (
-	deletedKeys: Array<{ accountConfigId: string; threadMessageId: string }>,
-): Pick<IThreadMessageRepository, "findAllByMessageId" | "deleteMany"> =>
-	({
-		findAllByMessageId: async (accountConfigId: string, messageId: string) => [
-			{
-				accountConfigId,
-				threadMessageId: `tm-${messageId}`,
-				messageId,
-			},
-		],
-		deleteMany: async (
-			keys: Array<{ accountConfigId: string; threadMessageId: string }>,
-		) => {
-			deletedKeys.push(...keys);
-		},
-	}) as unknown as Pick<
-		IThreadMessageRepository,
-		"findAllByMessageId" | "deleteMany"
-	>;
+afterEach(() => store.close());
 
-const buildStorageService = (
-	brokenAlready: Set<string> = new Set(),
-): {
-	storageService: Pick<StorageService, "storeBodyPart" | "bodyPartExists">;
-	stored: StoreBodyPartParams[];
-} => {
-	const stored: StoreBodyPartParams[] = [];
-	return {
-		stored,
-		storageService: {
-			storeBodyPart: async (params: StoreBodyPartParams) => {
-				stored.push(params);
-				brokenAlready.add(params.messageId);
-				return {} as never;
-			},
-			bodyPartExists: async (
-				_accountConfigId: string,
-				_accountId: string,
-				messageId: string,
-				partPath: string,
-			) => partPath === ".sync-failed" && brokenAlready.has(messageId),
-		},
-	};
+const arrive = async (subject: string) => {
+	const uid = imap.deliver("INBOX", eml({ Subject: subject }));
+	await syncFolder(store, account, inbox, imap);
+	return { uid, messageId: (await messageAtUid(store, inbox, uid)).messageId };
 };
 
-/**
- * A connection whose `fetchMessages` returns a hit for every uid in `present`,
- * except those in `fetchDrops` — messages the server still has and SEARCH
- * still lists, whose FETCH row imapflow drops (#408).
- */
-const buildConnection = (
-	present: Set<number>,
-	fetchDrops: Set<number> = new Set(),
-): IImapConnection =>
-	({
-		openBox: async () => ({}) as never,
-		fetchMessages: async (uids: number[]) =>
-			uids
-				.filter((uid) => present.has(uid) && !fetchDrops.has(uid))
-				.map((uid) => ({ uid }) as unknown as never),
-		search: async (criteria: unknown[]) => {
-			const [, value] = (criteria as Array<[string, string]>)[0];
-			const uid = Number(value);
-			return present.has(uid) ? [uid] : [];
+const resolve = (failedMessageIds: string[], connection = imap.connection()) =>
+	resolveExhaustedBodySyncFailures(
+		{
+			messageService: store.message,
+			storageService: storage,
+			log: silentLog,
 		},
-	}) as unknown as IImapConnection;
-
-describe("resolveExhaustedBodySyncFailures — the two terminal outcomes", () => {
-	it("outcome 1 (EXPECTED): a uid missing from IMAP reconciles the stale row, no alarm", async () => {
-		const deletedMessages: string[] = [];
-		const deletedThreadMessages: Array<{
-			accountConfigId: string;
-			threadMessageId: string;
-		}> = [];
-		const { storageService, stored } = buildStorageService();
-		const { log, infos, errors } = buildLogger();
-
-		const deps: ResolveExhaustedBodySyncDeps = {
-			messageService: buildMessageService({ "msg-gone": 101 }, deletedMessages),
-			threadMessageService: buildThreadMessageService(deletedThreadMessages),
-			storageService,
-			log,
-		};
-
-		const result = await resolveExhaustedBodySyncFailures(deps, {
-			accountId: "acc-1",
-			accountConfigId: "cfg-1",
-			mailboxId: "mbx-1",
+		{
+			accountId: account.accountId,
+			accountConfigId: account.accountConfigId,
+			mailboxId: inbox.mailboxId,
 			mailboxPath: "INBOX",
-			failedMessageIds: ["msg-gone"],
-			getConnection: async () => buildConnection(new Set()),
-		});
-
-		assert.deepEqual(result.reconciledMessageIds, ["msg-gone"]);
-		assert.deepEqual(result.brokenMessageIds, []);
-		assert.deepEqual(deletedMessages, ["msg-gone"]);
-		assert.deepEqual(deletedThreadMessages, [
-			{ accountConfigId: "cfg-1", threadMessageId: "tm-msg-gone" },
-		]);
-		// No sentinel written, no alert-shaped log for the expected case.
-		assert.equal(stored.length, 0);
-		assert.equal(errors.length, 0);
-		assert.ok(
-			infos.some((e) => e.obj.metric === "body_sync_stale_row_reconciled"),
-			"expects an info-level metric log, not an alert",
+			failedMessageIds,
+			getConnection: async () => connection,
+		},
+	);
+
+const isBroken = (messageId: string) =>
+	isMessageBodySyncBroken(
+		storage,
+		account.accountConfigId,
+		account.accountId,
+		messageId,
+	);
+
+describe("a body sync that exhausted its retries", () => {
+	it("removes the row and every thread copy of a message the server expunged", async () => {
+		const { uid, messageId } = await arrive("gone");
+		const [row] = await store.threadMessage.findAllByMessageId(
+			account.accountConfigId,
+			messageId,
 		);
-	});
-
-	it("outcome 2 (BROKEN): a uid IMAP still has persists the failed-forever sentinel and alerts", async () => {
-		const deletedMessages: string[] = [];
-		const { storageService, stored } = buildStorageService();
-		const { log, errors } = buildLogger();
+		await store.threadMessage.create({ ...row, threadId: "drifted-root" });
+		imap.expunge("INBOX", uid);
 
-		const deps: ResolveExhaustedBodySyncDeps = {
-			messageService: buildMessageService(
-				{ "msg-broken": 202 },
-				deletedMessages,
-			),
-			threadMessageService: buildThreadMessageService([]),
-			storageService,
-			log,
-		};
+		const result = await resolve([messageId]);
 
-		const result = await resolveExhaustedBodySyncFailures(deps, {
-			accountId: "acc-1",
-			accountConfigId: "cfg-1",
-			mailboxId: "mbx-1",
-			mailboxPath: "INBOX",
-			failedMessageIds: ["msg-broken"],
-			getConnection: async () => buildConnection(new Set([202])),
+		assert.deepEqual(result, {
+			reconciledMessageIds: [messageId],
+			brokenMessageIds: [],
 		});
-
-		assert.deepEqual(result.brokenMessageIds, ["msg-broken"]);
-		assert.deepEqual(result.reconciledMessageIds, []);
-		// The row is never deleted for a broken (still-real) message.
-		assert.deepEqual(deletedMessages, []);
-		assert.equal(stored.length, 1);
-		assert.equal(stored[0]?.partPath, ".sync-failed");
-		assert.ok(
-			errors.some((e) => e.obj.alert === "body_sync_message_broken"),
-			"expects an alert-shaped error log for the broken case",
+		await assert.rejects(store.message.get(messageId));
+		assert.deepEqual(
+			await store.threadMessage.findAllByMessageId(
+				account.accountConfigId,
+				messageId,
+			),
+			[],
 		);
+		assert.equal(await isBroken(messageId), false);
 	});
 
-	it("a dropped FETCH row mid-batch is not absence: the live message keeps its rows", async () => {
-		const deletedMessages: string[] = [];
-		const deletedThreadMessages: Array<{
-			accountConfigId: string;
-			threadMessageId: string;
-		}> = [];
-		const { storageService } = buildStorageService();
-		const { log } = buildLogger();
-
-		const deps: ResolveExhaustedBodySyncDeps = {
-			messageService: buildMessageService({ m1: 1, m2: 2 }, deletedMessages),
-			threadMessageService: buildThreadMessageService(deletedThreadMessages),
-			storageService,
-			log,
-		};
+	it("marks a message the server still holds as broken and keeps its row", async () => {
+		const { messageId } = await arrive("broken");
 
-		// Both messages are live. The FETCH for m1 returns its row; the
-		// immediately following FETCH for m2 drops its row (#408).
-		const result = await resolveExhaustedBodySyncFailures(deps, {
-			accountId: "acc-1",
-			accountConfigId: "cfg-1",
-			mailboxId: "mbx-1",
-			mailboxPath: "INBOX",
-			failedMessageIds: ["m1", "m2"],
-			getConnection: async () => buildConnection(new Set([1, 2]), new Set([2])),
-		});
+		const result = await resolve([messageId]);
 
-		assert.deepEqual(result.reconciledMessageIds, []);
-		assert.deepEqual(result.brokenMessageIds, ["m1", "m2"]);
-		assert.deepEqual(deletedMessages, []);
-		assert.deepEqual(deletedThreadMessages, []);
+		assert.deepEqual(result.brokenMessageIds, [messageId]);
+		assert.ok(await store.message.get(messageId));
+		assert.equal(await isBroken(messageId), true);
 	});
 
-	it("resolves a mixed batch into both outcomes independently", async () => {
-		const deletedMessages: string[] = [];
-		const { storageService } = buildStorageService();
-		const { log } = buildLogger();
-
-		const deps: ResolveExhaustedBodySyncDeps = {
-			messageService: buildMessageService(
-				{ "msg-gone": 1, "msg-broken": 2 },
-				deletedMessages,
-			),
-			threadMessageService: buildThreadMessageService([]),
-			storageService,
-			log,
-		};
-
-		const result = await resolveExhaustedBodySyncFailures(deps, {
-			accountId: "acc-1",
-			accountConfigId: "cfg-1",
-			mailboxId: "mbx-1",
-			mailboxPath: "INBOX",
-			failedMessageIds: ["msg-gone", "msg-broken"],
-			getConnection: async () => buildConnection(new Set([2])),
-		});
+	it("does not read a dropped FETCH row as absence", async () => {
+		const first = await arrive("first");
+		const second = await arrive("second");
+		const connection = imap.connection();
+		const fetch = connection.fetchMessages;
+		connection.fetchMessages = async (uids) =>
+			uids.includes(second.uid) ? [] : fetch(uids);
+
+		const result = await resolve(
+			[first.messageId, second.messageId],
+			connection,
+		);
 
-		assert.deepEqual(result.reconciledMessageIds, ["msg-gone"]);
-		assert.deepEqual(result.brokenMessageIds, ["msg-broken"]);
+		assert.deepEqual(result.reconciledMessageIds, []);
+		assert.deepEqual(result.brokenMessageIds, [
+			first.messageId,
+			second.messageId,
+		]);
+		assert.ok(await store.message.get(second.messageId));
 	});
 
-	it("no failed ids is a no-op — never opens a connection", async () => {
-		const { storageService } = buildStorageService();
-		const { log } = buildLogger();
-		let connectionRequested = false;
-
-		const deps: ResolveExhaustedBodySyncDeps = {
-			messageService: buildMessageService({}, []),
-			threadMessageService: buildThreadMessageService([]),
-			storageService,
-			log,
-		};
-
-		const result = await resolveExhaustedBodySyncFailures(deps, {
-			accountId: "acc-1",
-			accountConfigId: "cfg-1",
-			mailboxId: "mbx-1",
-			mailboxPath: "INBOX",
-			failedMessageIds: [],
-			getConnection: async () => {
-				connectionRequested = true;
-				return buildConnection(new Set());
+	it("never opens a connection when nothing failed", async () => {
+		const result = await resolveExhaustedBodySyncFailures(
+			{
+				messageService: store.message,
+				storageService: storage,
+				log: silentLog,
 			},
-		});
+			{
+				accountId: account.accountId,
+				accountConfigId: account.accountConfigId,
+				mailboxId: inbox.mailboxId,
+				mailboxPath: "INBOX",
+				failedMessageIds: [],
+				getConnection: async () => assert.fail("no connection is needed"),
+			},
+		);
 
 		assert.deepEqual(result, {
 			reconciledMessageIds: [],
 			brokenMessageIds: [],
 		});
-		assert.equal(connectionRequested, false);
-	});
-});
-
-describe("isMessageBodySyncBroken", () => {
-	it("delegates to bodyPartExists with the sentinel path", async () => {
-		const { storageService } = buildStorageService(new Set(["msg-1"]));
-
-		assert.equal(
-			await isMessageBodySyncBroken(storageService, "cfg-1", "acc-1", "msg-1"),
-			true,
-		);
-		assert.equal(
-			await isMessageBodySyncBroken(storageService, "cfg-1", "acc-1", "msg-2"),
-			false,
-		);
 	});
 });

package/src/body-sync-terminal.ts

@@ -2,10 +2,6 @@ import type { IMessageRepository } from "@remit/data-ports";
 import type { StorageService } from "@remit/storage-service";
 import type { BodySyncLogger } from "./body-sync.js";
 import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
-import {
-	reconcileStaleMessage,
-	type StaleMessageReconcileDeps,
-} from "./stale-message-reconcile.js";
 import type { IImapConnection } from "./types.js";
 
 /**
@@ -17,9 +13,8 @@ import type { IImapConnection } from "./types.js";
  */
 const BODY_SYNC_FAILED_MARKER_PATH = ".sync-failed";
 
-export interface ResolveExhaustedBodySyncDeps
-	extends StaleMessageReconcileDeps {
-	messageService: Pick<IMessageRepository, "get" | "delete">;
+export interface ResolveExhaustedBodySyncDeps {
+	messageService: Pick<IMessageRepository, "get" | "deleteIfPlacement">;
 	storageService: Pick<StorageService, "storeBodyPart" | "bodyPartExists">;
 	log: BodySyncLogger;
 }
@@ -90,7 +85,7 @@ const markMessageBodySyncFailed = async (
  *    the four sites that used to read an empty FETCH as absence: it issues
  *    back-to-back single-UID FETCHes on one connection, which is the
  *    condition imapflow #408 drops rows under. The stale row is deleted via
- *    {@link reconcileStaleMessage} so the existing missing-row 404 path
+ *    `deleteIfPlacement` so the existing missing-row 404 path
  *    takes over. Callers should emit a metric only — this is routine, not an
  *    incident.
  * 2. BROKEN — the message still exists on IMAP but its body could not be
@@ -130,10 +125,13 @@ export const resolveExhaustedBodySyncFailures = async (
 		const message = await deps.messageService.get(messageId);
 
 		if (await isMessageGoneFromOpenMailbox(connection, message.uid)) {
-			const { threadMessagesDeleted } = await reconcileStaleMessage(
-				deps,
-				accountConfigId,
+			const reconciled = await deps.messageService.deleteIfPlacement(
 				messageId,
+				{
+					status: message.status,
+					mailboxId: message.mailboxId,
+					uid: message.uid,
+				},
 			);
 			deps.log.info(
 				{
@@ -143,7 +141,7 @@ export const resolveExhaustedBodySyncFailures = async (
 					mailboxId,
 					messageId,
 					uid: message.uid,
-					threadMessagesDeleted,
+					reconciled,
 				},
 				"Message no longer exists on IMAP after retry exhaustion; stale row reconciled",
 			);