@remit/mailbox-service 0.0.111 → 0.0.113

package/src/body-sync-terminal.test.ts

@@ -1,310 +0,0 @@
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import type {
-	IMessageRepository,
-	IThreadMessageRepository,
-} from "@remit/data-ports";
-import type {
-	StorageService,
-	StoreBodyPartParams,
-} from "@remit/storage-service";
-import type { BodySyncLogger } from "./body-sync.js";
-import {
-	isMessageBodySyncBroken,
-	type ResolveExhaustedBodySyncDeps,
-	resolveExhaustedBodySyncFailures,
-} from "./body-sync-terminal.js";
-import type { IImapConnection } from "./types.js";
-
-interface LogEntry {
-	obj: Record<string, unknown>;
-	msg: string;
-}
-
-const buildLogger = (): {
-	log: BodySyncLogger;
-	infos: LogEntry[];
-	errors: LogEntry[];
-} => {
-	const infos: LogEntry[] = [];
-	const errors: LogEntry[] = [];
-	return {
-		log: {
-			info: (obj, msg) => infos.push({ obj, msg: msg ?? "" }),
-			error: (obj, msg) => errors.push({ obj, msg: msg ?? "" }),
-		},
-		infos,
-		errors,
-	};
-};
-
-/** Message rows keyed by messageId; `uid` is the only field the resolver reads. */
-const buildMessageService = (
-	uidByMessageId: Record<string, number>,
-	deleted: string[],
-): Pick<IMessageRepository, "get" | "delete"> =>
-	({
-		get: async (messageId: string) => {
-			const uid = uidByMessageId[messageId];
-			if (uid === undefined) {
-				throw new Error(`no fixture uid for ${messageId}`);
-			}
-			return { messageId, uid };
-		},
-		delete: async (messageId: string) => {
-			deleted.push(messageId);
-		},
-	}) as unknown as Pick<IMessageRepository, "get" | "delete">;
-
-const buildThreadMessageService = (
-	deletedKeys: Array<{ accountConfigId: string; threadMessageId: string }>,
-): Pick<IThreadMessageRepository, "findAllByMessageId" | "deleteMany"> =>
-	({
-		findAllByMessageId: async (accountConfigId: string, messageId: string) => [
-			{
-				accountConfigId,
-				threadMessageId: `tm-${messageId}`,
-				messageId,
-			},
-		],
-		deleteMany: async (
-			keys: Array<{ accountConfigId: string; threadMessageId: string }>,
-		) => {
-			deletedKeys.push(...keys);
-		},
-	}) as unknown as Pick<
-		IThreadMessageRepository,
-		"findAllByMessageId" | "deleteMany"
-	>;
-
-const buildStorageService = (
-	brokenAlready: Set<string> = new Set(),
-): {
-	storageService: Pick<StorageService, "storeBodyPart" | "bodyPartExists">;
-	stored: StoreBodyPartParams[];
-} => {
-	const stored: StoreBodyPartParams[] = [];
-	return {
-		stored,
-		storageService: {
-			storeBodyPart: async (params: StoreBodyPartParams) => {
-				stored.push(params);
-				brokenAlready.add(params.messageId);
-				return {} as never;
-			},
-			bodyPartExists: async (
-				_accountConfigId: string,
-				_accountId: string,
-				messageId: string,
-				partPath: string,
-			) => partPath === ".sync-failed" && brokenAlready.has(messageId),
-		},
-	};
-};
-
-/**
- * A connection whose `fetchMessages` returns a hit for every uid in `present`,
- * except those in `fetchDrops` — messages the server still has and SEARCH
- * still lists, whose FETCH row imapflow drops (#408).
- */
-const buildConnection = (
-	present: Set<number>,
-	fetchDrops: Set<number> = new Set(),
-): IImapConnection =>
-	({
-		openBox: async () => ({}) as never,
-		fetchMessages: async (uids: number[]) =>
-			uids
-				.filter((uid) => present.has(uid) && !fetchDrops.has(uid))
-				.map((uid) => ({ uid }) as unknown as never),
-		search: async (criteria: unknown[]) => {
-			const [, value] = (criteria as Array<[string, string]>)[0];
-			const uid = Number(value);
-			return present.has(uid) ? [uid] : [];
-		},
-	}) as unknown as IImapConnection;
-
-describe("resolveExhaustedBodySyncFailures — the two terminal outcomes", () => {
-	it("outcome 1 (EXPECTED): a uid missing from IMAP reconciles the stale row, no alarm", async () => {
-		const deletedMessages: string[] = [];
-		const deletedThreadMessages: Array<{
-			accountConfigId: string;
-			threadMessageId: string;
-		}> = [];
-		const { storageService, stored } = buildStorageService();
-		const { log, infos, errors } = buildLogger();
-
-		const deps: ResolveExhaustedBodySyncDeps = {
-			messageService: buildMessageService({ "msg-gone": 101 }, deletedMessages),
-			threadMessageService: buildThreadMessageService(deletedThreadMessages),
-			storageService,
-			log,
-		};
-
-		const result = await resolveExhaustedBodySyncFailures(deps, {
-			accountId: "acc-1",
-			accountConfigId: "cfg-1",
-			mailboxId: "mbx-1",
-			mailboxPath: "INBOX",
-			failedMessageIds: ["msg-gone"],
-			getConnection: async () => buildConnection(new Set()),
-		});
-
-		assert.deepEqual(result.reconciledMessageIds, ["msg-gone"]);
-		assert.deepEqual(result.brokenMessageIds, []);
-		assert.deepEqual(deletedMessages, ["msg-gone"]);
-		assert.deepEqual(deletedThreadMessages, [
-			{ accountConfigId: "cfg-1", threadMessageId: "tm-msg-gone" },
-		]);
-		// No sentinel written, no alert-shaped log for the expected case.
-		assert.equal(stored.length, 0);
-		assert.equal(errors.length, 0);
-		assert.ok(
-			infos.some((e) => e.obj.metric === "body_sync_stale_row_reconciled"),
-			"expects an info-level metric log, not an alert",
-		);
-	});
-
-	it("outcome 2 (BROKEN): a uid IMAP still has persists the failed-forever sentinel and alerts", async () => {
-		const deletedMessages: string[] = [];
-		const { storageService, stored } = buildStorageService();
-		const { log, errors } = buildLogger();
-
-		const deps: ResolveExhaustedBodySyncDeps = {
-			messageService: buildMessageService(
-				{ "msg-broken": 202 },
-				deletedMessages,
-			),
-			threadMessageService: buildThreadMessageService([]),
-			storageService,
-			log,
-		};
-
-		const result = await resolveExhaustedBodySyncFailures(deps, {
-			accountId: "acc-1",
-			accountConfigId: "cfg-1",
-			mailboxId: "mbx-1",
-			mailboxPath: "INBOX",
-			failedMessageIds: ["msg-broken"],
-			getConnection: async () => buildConnection(new Set([202])),
-		});
-
-		assert.deepEqual(result.brokenMessageIds, ["msg-broken"]);
-		assert.deepEqual(result.reconciledMessageIds, []);
-		// The row is never deleted for a broken (still-real) message.
-		assert.deepEqual(deletedMessages, []);
-		assert.equal(stored.length, 1);
-		assert.equal(stored[0]?.partPath, ".sync-failed");
-		assert.ok(
-			errors.some((e) => e.obj.alert === "body_sync_message_broken"),
-			"expects an alert-shaped error log for the broken case",
-		);
-	});
-
-	it("a dropped FETCH row mid-batch is not absence: the live message keeps its rows", async () => {
-		const deletedMessages: string[] = [];
-		const deletedThreadMessages: Array<{
-			accountConfigId: string;
-			threadMessageId: string;
-		}> = [];
-		const { storageService } = buildStorageService();
-		const { log } = buildLogger();
-
-		const deps: ResolveExhaustedBodySyncDeps = {
-			messageService: buildMessageService({ m1: 1, m2: 2 }, deletedMessages),
-			threadMessageService: buildThreadMessageService(deletedThreadMessages),
-			storageService,
-			log,
-		};
-
-		// Both messages are live. The FETCH for m1 returns its row; the
-		// immediately following FETCH for m2 drops its row (#408).
-		const result = await resolveExhaustedBodySyncFailures(deps, {
-			accountId: "acc-1",
-			accountConfigId: "cfg-1",
-			mailboxId: "mbx-1",
-			mailboxPath: "INBOX",
-			failedMessageIds: ["m1", "m2"],
-			getConnection: async () => buildConnection(new Set([1, 2]), new Set([2])),
-		});
-
-		assert.deepEqual(result.reconciledMessageIds, []);
-		assert.deepEqual(result.brokenMessageIds, ["m1", "m2"]);
-		assert.deepEqual(deletedMessages, []);
-		assert.deepEqual(deletedThreadMessages, []);
-	});
-
-	it("resolves a mixed batch into both outcomes independently", async () => {
-		const deletedMessages: string[] = [];
-		const { storageService } = buildStorageService();
-		const { log } = buildLogger();
-
-		const deps: ResolveExhaustedBodySyncDeps = {
-			messageService: buildMessageService(
-				{ "msg-gone": 1, "msg-broken": 2 },
-				deletedMessages,
-			),
-			threadMessageService: buildThreadMessageService([]),
-			storageService,
-			log,
-		};
-
-		const result = await resolveExhaustedBodySyncFailures(deps, {
-			accountId: "acc-1",
-			accountConfigId: "cfg-1",
-			mailboxId: "mbx-1",
-			mailboxPath: "INBOX",
-			failedMessageIds: ["msg-gone", "msg-broken"],
-			getConnection: async () => buildConnection(new Set([2])),
-		});
-
-		assert.deepEqual(result.reconciledMessageIds, ["msg-gone"]);
-		assert.deepEqual(result.brokenMessageIds, ["msg-broken"]);
-	});
-
-	it("no failed ids is a no-op — never opens a connection", async () => {
-		const { storageService } = buildStorageService();
-		const { log } = buildLogger();
-		let connectionRequested = false;
-
-		const deps: ResolveExhaustedBodySyncDeps = {
-			messageService: buildMessageService({}, []),
-			threadMessageService: buildThreadMessageService([]),
-			storageService,
-			log,
-		};
-
-		const result = await resolveExhaustedBodySyncFailures(deps, {
-			accountId: "acc-1",
-			accountConfigId: "cfg-1",
-			mailboxId: "mbx-1",
-			mailboxPath: "INBOX",
-			failedMessageIds: [],
-			getConnection: async () => {
-				connectionRequested = true;
-				return buildConnection(new Set());
-			},
-		});
-
-		assert.deepEqual(result, {
-			reconciledMessageIds: [],
-			brokenMessageIds: [],
-		});
-		assert.equal(connectionRequested, false);
-	});
-});
-
-describe("isMessageBodySyncBroken", () => {
-	it("delegates to bodyPartExists with the sentinel path", async () => {
-		const { storageService } = buildStorageService(new Set(["msg-1"]));
-
-		assert.equal(
-			await isMessageBodySyncBroken(storageService, "cfg-1", "acc-1", "msg-1"),
-			true,
-		);
-		assert.equal(
-			await isMessageBodySyncBroken(storageService, "cfg-1", "acc-1", "msg-2"),
-			false,
-		);
-	});
-});

package/src/body-sync-terminal.ts

@@ -1,176 +0,0 @@
-import type { IMessageRepository } from "@remit/data-ports";
-import type { StorageService } from "@remit/storage-service";
-import type { BodySyncLogger } from "./body-sync.js";
-import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
-import {
-	reconcileStaleMessage,
-	type StaleMessageReconcileDeps,
-} from "./stale-message-reconcile.js";
-import type { IImapConnection } from "./types.js";
-
-/**
- * Body-part sentinel path a permanently-broken message's failure context is
- * written under, mirroring the `.materialized` deferred-parts sentinel in
- * `body-sync.ts`. Using the existing per-message body-part storage (instead
- * of a new Message-row column) means no schema change: the read path checks
- * for it with the same `bodyPartExists` HEAD it already issues.
- */
-const BODY_SYNC_FAILED_MARKER_PATH = ".sync-failed";
-
-export interface ResolveExhaustedBodySyncDeps
-	extends StaleMessageReconcileDeps {
-	messageService: Pick<IMessageRepository, "get" | "delete">;
-	storageService: Pick<StorageService, "storeBodyPart" | "bodyPartExists">;
-	log: BodySyncLogger;
-}
-
-export interface ResolveExhaustedBodySyncInput {
-	accountId: string;
-	accountConfigId: string;
-	mailboxId: string;
-	mailboxPath: string;
-	failedMessageIds: string[];
-	getConnection: () => Promise<IImapConnection>;
-}
-
-export interface ResolveExhaustedBodySyncResult {
-	/** Outcome 1 (EXPECTED): message gone upstream, row deleted. */
-	reconciledMessageIds: string[];
-	/** Outcome 2 (BROKEN): message exists but is unfetchable/unparseable. */
-	brokenMessageIds: string[];
-}
-
-/**
- * Whether a message's body-sync has already been marked permanently failed
- * (outcome 2 of {@link resolveExhaustedBodySyncFailures}). The read path
- * calls this before attempting or re-arming any fetch so a client that opens
- * a broken message gets the explicit unrecoverable error immediately instead
- * of looping through another 202 or another IMAP round-trip.
- */
-export const isMessageBodySyncBroken = (
-	storageService: Pick<StorageService, "bodyPartExists">,
-	accountConfigId: string,
-	accountId: string,
-	messageId: string,
-): Promise<boolean> =>
-	storageService.bodyPartExists(
-		accountConfigId,
-		accountId,
-		messageId,
-		BODY_SYNC_FAILED_MARKER_PATH,
-	);
-
-const markMessageBodySyncFailed = async (
-	storageService: Pick<StorageService, "storeBodyPart">,
-	accountConfigId: string,
-	accountId: string,
-	messageId: string,
-	context: Record<string, unknown>,
-): Promise<void> => {
-	await storageService.storeBodyPart({
-		accountConfigId,
-		accountId,
-		messageId,
-		partPath: BODY_SYNC_FAILED_MARKER_PATH,
-		content: Buffer.from(JSON.stringify({ ...context, failedAt: Date.now() })),
-		contentType: "application/json",
-	});
-};
-
-/**
- * Resolve every SYNC_MESSAGE_BODY failure that has exhausted the body
- * queue's redelivery budget (see `BODY_SYNC_MAX_ATTEMPTS` in
- * `sync-message-body.ts`) into exactly one of the two terminal outcomes
- * issue #1270 / epic #1281 invariant 3 require. There is no third, softer
- * outcome — every failed id lands in one of the two result lists.
- *
- * 1. EXPECTED — the message no longer exists on IMAP (expunged, or a
- *    UIDVALIDITY change moved it, #1272), confirmed by
- *    {@link isMessageGoneFromOpenMailbox}. This loop is the most exposed of
- *    the four sites that used to read an empty FETCH as absence: it issues
- *    back-to-back single-UID FETCHes on one connection, which is the
- *    condition imapflow #408 drops rows under. The stale row is deleted via
- *    {@link reconcileStaleMessage} so the existing missing-row 404 path
- *    takes over. Callers should emit a metric only — this is routine, not an
- *    incident.
- * 2. BROKEN — the message still exists on IMAP but its body could not be
- *    fetched or parsed. The failure is persisted as a body-part sentinel
- *    (survives across invocations and warm/cold starts) and logged with an
- *    `alert`-shaped entry so an operator alarm can key off it. Callers
- *    should treat this as loud: the message is unrecoverable until an
- *    operator investigates.
- *
- * Neither outcome re-enqueues: both are terminal, so the caller acks the SQS
- * message either way — retrying a stale or broken message can never succeed.
- */
-export const resolveExhaustedBodySyncFailures = async (
-	deps: ResolveExhaustedBodySyncDeps,
-	input: ResolveExhaustedBodySyncInput,
-): Promise<ResolveExhaustedBodySyncResult> => {
-	const {
-		accountId,
-		accountConfigId,
-		mailboxId,
-		mailboxPath,
-		failedMessageIds,
-		getConnection,
-	} = input;
-
-	const reconciledMessageIds: string[] = [];
-	const brokenMessageIds: string[] = [];
-
-	if (failedMessageIds.length === 0) {
-		return { reconciledMessageIds, brokenMessageIds };
-	}
-
-	const connection = await getConnection();
-	await connection.openBox(mailboxPath);
-
-	for (const messageId of failedMessageIds) {
-		const message = await deps.messageService.get(messageId);
-
-		if (await isMessageGoneFromOpenMailbox(connection, message.uid)) {
-			const { threadMessagesDeleted } = await reconcileStaleMessage(
-				deps,
-				accountConfigId,
-				messageId,
-			);
-			deps.log.info(
-				{
-					metric: "body_sync_stale_row_reconciled",
-					accountId,
-					accountConfigId,
-					mailboxId,
-					messageId,
-					uid: message.uid,
-					threadMessagesDeleted,
-				},
-				"Message no longer exists on IMAP after retry exhaustion; stale row reconciled",
-			);
-			reconciledMessageIds.push(messageId);
-			continue;
-		}
-
-		await markMessageBodySyncFailed(
-			deps.storageService,
-			accountConfigId,
-			accountId,
-			messageId,
-			{ accountId, accountConfigId, mailboxId, messageId, uid: message.uid },
-		);
-		deps.log.error?.(
-			{
-				alert: "body_sync_message_broken",
-				accountId,
-				accountConfigId,
-				mailboxId,
-				messageId,
-				uid: message.uid,
-			},
-			"Message body could not be fetched/parsed after retry exhaustion; message exists on IMAP but is unrecoverable",
-		);
-		brokenMessageIds.push(messageId);
-	}
-
-	return { reconciledMessageIds, brokenMessageIds };
-};