@remit/mailbox-service 0.0.16 → 0.0.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.16",
+  "version": "0.0.17",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/body-sync-terminal.test.ts

@@ -102,14 +102,26 @@ const buildStorageService = (
 	};
 };
 
-/** A connection whose `fetchMessages` returns a hit for every uid in `present`. */
-const buildConnection = (present: Set<number>): IImapConnection =>
+/**
+ * A connection whose `fetchMessages` returns a hit for every uid in `present`,
+ * except those in `fetchDrops` — messages the server still has and SEARCH
+ * still lists, whose FETCH row imapflow drops (#408).
+ */
+const buildConnection = (
+	present: Set<number>,
+	fetchDrops: Set<number> = new Set(),
+): IImapConnection =>
 	({
 		openBox: async () => ({}) as never,
 		fetchMessages: async (uids: number[]) =>
 			uids
-				.filter((uid) => present.has(uid))
+				.filter((uid) => present.has(uid) && !fetchDrops.has(uid))
 				.map((uid) => ({ uid }) as unknown as never),
+		search: async (criteria: unknown[]) => {
+			const [, value] = (criteria as Array<[string, string]>)[0];
+			const uid = Number(value);
+			return present.has(uid) ? [uid] : [];
+		},
 	}) as unknown as IImapConnection;
 
 describe("resolveExhaustedBodySyncFailures — the two terminal outcomes", () => {
@@ -189,6 +201,39 @@ describe("resolveExhaustedBodySyncFailures — the two terminal outcomes", () =>
 		);
 	});
 
+	it("a dropped FETCH row mid-batch is not absence: the live message keeps its rows", async () => {
+		const deletedMessages: string[] = [];
+		const deletedThreadMessages: Array<{
+			accountConfigId: string;
+			threadMessageId: string;
+		}> = [];
+		const { storageService } = buildStorageService();
+		const { log } = buildLogger();
+
+		const deps: ResolveExhaustedBodySyncDeps = {
+			messageService: buildMessageService({ m1: 1, m2: 2 }, deletedMessages),
+			threadMessageService: buildThreadMessageService(deletedThreadMessages),
+			storageService,
+			log,
+		};
+
+		// Both messages are live. The FETCH for m1 returns its row; the
+		// immediately following FETCH for m2 drops its row (#408).
+		const result = await resolveExhaustedBodySyncFailures(deps, {
+			accountId: "acc-1",
+			accountConfigId: "cfg-1",
+			mailboxId: "mbx-1",
+			mailboxPath: "INBOX",
+			failedMessageIds: ["m1", "m2"],
+			getConnection: async () => buildConnection(new Set([1, 2]), new Set([2])),
+		});
+
+		assert.deepEqual(result.reconciledMessageIds, []);
+		assert.deepEqual(result.brokenMessageIds, ["m1", "m2"]);
+		assert.deepEqual(deletedMessages, []);
+		assert.deepEqual(deletedThreadMessages, []);
+	});
+
 	it("resolves a mixed batch into both outcomes independently", async () => {
 		const deletedMessages: string[] = [];
 		const { storageService } = buildStorageService();

package/src/body-sync-terminal.ts

@@ -1,6 +1,7 @@
 import type { IMessageRepository } from "@remit/data-ports";
 import type { StorageService } from "@remit/storage-service";
 import type { BodySyncLogger } from "./body-sync.js";
+import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
 import {
 	reconcileStaleMessage,
 	type StaleMessageReconcileDeps,
@@ -84,7 +85,11 @@ const markMessageBodySyncFailed = async (
  * outcome — every failed id lands in one of the two result lists.
  *
  * 1. EXPECTED — the message no longer exists on IMAP (expunged, or a
- *    UIDVALIDITY change moved it, #1272). The stale row is deleted via
+ *    UIDVALIDITY change moved it, #1272), confirmed by
+ *    {@link isMessageGoneFromOpenMailbox}. This loop is the most exposed of
+ *    the four sites that used to read an empty FETCH as absence: it issues
+ *    back-to-back single-UID FETCHes on one connection, which is the
+ *    condition imapflow #408 drops rows under. The stale row is deleted via
  *    {@link reconcileStaleMessage} so the existing missing-row 404 path
  *    takes over. Callers should emit a metric only — this is routine, not an
  *    incident.
@@ -123,9 +128,8 @@ export const resolveExhaustedBodySyncFailures = async (
 
 	for (const messageId of failedMessageIds) {
 		const message = await deps.messageService.get(messageId);
-		const found = await connection.fetchMessages([message.uid]);
 
-		if (found.length === 0) {
+		if (await isMessageGoneFromOpenMailbox(connection, message.uid)) {
 			const { threadMessagesDeleted } = await reconcileStaleMessage(
 				deps,
 				accountConfigId,

package/src/flag-push-terminal.test.ts

@@ -33,13 +33,21 @@ const buildLogger = (): {
 	};
 };
 
-const buildConnection = (present: Set<number>): IImapConnection =>
+const buildConnection = (
+	present: Set<number>,
+	fetchDrops: Set<number> = new Set(),
+): IImapConnection =>
 	({
 		openBox: async () => ({}) as never,
 		fetchMessages: async (uids: number[]) =>
 			uids
-				.filter((uid) => present.has(uid))
+				.filter((uid) => present.has(uid) && !fetchDrops.has(uid))
 				.map((uid) => ({ uid }) as unknown as never),
+		search: async (criteria: unknown[]) => {
+			const [, value] = (criteria as Array<[string, string]>)[0];
+			const uid = Number(value);
+			return present.has(uid) ? [uid] : [];
+		},
 	}) as unknown as IImapConnection;
 
 describe("resolveExhaustedFlagPushFailure — the two terminal outcomes (mirrors #1289/#1270 for flag pushes)", () => {
@@ -104,6 +112,51 @@ describe("resolveExhaustedFlagPushFailure — the two terminal outcomes (mirrors
 		assert.ok(metricLog, "expected a routine reconciliation metric log");
 	});
 
+	it("a dropped FETCH row is not absence: the message is still in the mailbox — marker and local rows survive", async () => {
+		const markerDeletes: Array<{ messageId: string; flagName: string }> = [];
+		const { log, errors } = buildLogger();
+
+		const deps: ResolveExhaustedFlagPushDeps = {
+			markerService: {
+				delete: async (messageId: string, flagName: string) => {
+					markerDeletes.push({ messageId, flagName });
+				},
+			},
+			messageService: {
+				delete: async () => {
+					throw new Error("must not be called — the message still exists");
+				},
+			} as unknown as Pick<IMessageRepository, "delete">,
+			threadMessageService: {
+				findAllByMessageId: async () => {
+					throw new Error("must not be called — the message still exists");
+				},
+				deleteMany: async () => {
+					throw new Error("must not be called — the message still exists");
+				},
+			} as unknown as Pick<
+				IThreadMessageRepository,
+				"findAllByMessageId" | "deleteMany"
+			>,
+			log,
+		};
+
+		const result = await resolveExhaustedFlagPushFailure(deps, {
+			accountId: "acc-1",
+			accountConfigId: "cfg-1",
+			messageId: "msg-live",
+			flagName: "\\Seen",
+			uid: 303,
+			mailboxPath: "INBOX",
+			getConnection: async () =>
+				buildConnection(new Set([303]), new Set([303])),
+		});
+
+		assert.equal(result.outcome, "broken");
+		assert.deepEqual(markerDeletes, []);
+		assert.ok(errors.some((e) => e.obj.alert === "flag_push_failed"));
+	});
+
 	it("BROKEN (should never happen): the message still exists — marker left in place, alert logged", async () => {
 		const markerDeletes: Array<{ messageId: string; flagName: string }> = [];
 		const { log, errors } = buildLogger();

package/src/flag-push-terminal.ts

@@ -1,4 +1,5 @@
 import type { FlagPushLogger } from "./flag-push.js";
+import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
 import {
 	reconcileStaleMessage,
 	type StaleMessageReconcileDeps,
@@ -35,10 +36,11 @@ export interface ResolveExhaustedFlagPushResult {
  * (epic #1281 invariant 3) — no third, softer outcome.
  *
  * 1. RECONCILED (expected) — the message no longer exists at its mailbox on
- *    IMAP. Per invariant 2, an external delete supersedes the marker
- *    entirely: the marker is dropped and the stale Message/ThreadMessage rows
- *    are deleted via {@link reconcileStaleMessage}. Metric only, no alarm —
- *    routine.
+ *    IMAP, confirmed by {@link isMessageGoneFromOpenMailbox} rather than by a
+ *    FETCH coming back empty. Per invariant 2, an external delete supersedes
+ *    the marker entirely: the marker is dropped and the stale
+ *    Message/ThreadMessage rows are deleted via {@link reconcileStaleMessage}.
+ *    Metric only, no alarm — routine.
  * 2. BROKEN — the message still exists, but the flag push keeps failing.
  *    Broken code or a broken account, not a transient blip. The marker is
  *    left in place (not cleared) — while pending, resync never reverts the
@@ -48,6 +50,15 @@ export interface ResolveExhaustedFlagPushResult {
  *    entry for an operator alarm; never re-thrown (terminal — the caller acks
  *    either way, since retrying a stale or permanently-broken push can never
  *    succeed).
+ *
+ * An operator reading `flag_push_failed` should know one case where the
+ * message is not actually there: a message another client expunged
+ * mid-session can answer an empty FETCH while the server still lists its UID
+ * in SEARCH, until it is allowed to send the untagged EXPUNGE. That message
+ * lands in BROKEN, and BROKEN is terminal — the marker stays pending and the
+ * alert stands until someone clears it. The reverse mistake deletes live
+ * mail, so the cost is paid deliberately: a stale alert is recoverable, a
+ * deleted message is not.
  */
 export const resolveExhaustedFlagPushFailure = async (
 	deps: ResolveExhaustedFlagPushDeps,
@@ -65,9 +76,8 @@ export const resolveExhaustedFlagPushFailure = async (
 
 	const connection = await getConnection();
 	await connection.openBox(mailboxPath, true);
-	const found = await connection.fetchMessages([uid]);
 
-	if (found.length === 0) {
+	if (await isMessageGoneFromOpenMailbox(connection, uid)) {
 		await deps.markerService.delete(messageId, flagName);
 		const { threadMessagesDeleted } = await reconcileStaleMessage(
 			deps,

package/src/imapflow-connection.test.ts

@@ -112,6 +112,52 @@ const fakeMailbox = (path: string, exists: number) => ({
 	readOnly: true,
 });
 
+describe("ImapFlowConnection.search — the probe every stale-row reconcile rests on (#102)", () => {
+	const buildSearchConnection = (
+		result: unknown,
+		calls: Array<{ query: unknown; options: unknown }> = [],
+	): ImapFlowConnection => {
+		const connection = buildConnectionWithClient({
+			mailboxOpen: async (path: string) => fakeMailbox(path, 1),
+			search: async (query: unknown, options: unknown) => {
+				calls.push({ query, options });
+				return result;
+			},
+		});
+		return connection;
+	};
+
+	it("translates a UID criterion into a UID SEARCH answering UIDs, not sequence numbers", async () => {
+		const calls: Array<{ query: unknown; options: unknown }> = [];
+		const connection = buildSearchConnection([42], calls);
+		await connection.openBox("INBOX", true);
+
+		const result = await connection.search([["UID", "42"]]);
+
+		assert.deepStrictEqual(result, [42]);
+		assert.deepStrictEqual(calls, [
+			{ query: { uid: "42" }, options: { uid: true } },
+		]);
+	});
+
+	it("answers an empty array when the server matched nothing", async () => {
+		const connection = buildSearchConnection([]);
+		await connection.openBox("INBOX", true);
+
+		assert.deepStrictEqual(await connection.search([["UID", "42"]]), []);
+	});
+
+	it("throws on a failed SEARCH instead of reporting it as no matches", async () => {
+		const connection = buildSearchConnection(false);
+		await connection.openBox("INBOX", true);
+
+		await assert.rejects(
+			() => connection.search([["UID", "42"]]),
+			/SEARCH failed/,
+		);
+	});
+});
+
 describe("ImapFlowConnection.getMailboxStatus — deletedCount (#1042)", () => {
 	it("returns the SEARCH \\Deleted count alongside the STATUS counts", async () => {
 		const searchQueries: Array<Record<string, unknown>> = [];
@@ -152,7 +198,7 @@ describe("ImapFlowConnection.getMailboxStatus — deletedCount (#1042)", () => {
 				highestModseq: 0n,
 			}),
 			mailboxOpen: async (path: string) => fakeMailbox(path, 4),
-			search: async () => false,
+			search: async () => [],
 		});
 
 		const status = await connection.getMailboxStatus("INBOX");
@@ -173,7 +219,7 @@ describe("ImapFlowConnection.getMailboxStatus — deletedCount (#1042)", () => {
 				highestModseq: modseq,
 			}),
 			mailboxOpen: async (path: string) => fakeMailbox(path, 1),
-			search: async () => false,
+			search: async () => [],
 		});
 
 		const status = await connection.getMailboxStatus("INBOX");

package/src/imapflow-connection.ts

@@ -397,7 +397,15 @@ export class ImapFlowConnection {
 	};
 
 	/**
-	 * Search for messages
+	 * Search for messages, answering UIDs.
+	 *
+	 * imapflow answers a successful SEARCH with an array — empty when the
+	 * server matched nothing — and `false` when the command failed or no
+	 * mailbox was selected. Collapsing the two into `[]` makes a failed SEARCH
+	 * indistinguishable from an empty mailbox, and every caller here acts on
+	 * emptiness by deleting something: the cursor rebuild reads it as "every
+	 * local row is stale", and {@link isMessageGoneFromOpenMailbox} reads it as
+	 * "this message is gone". A failure therefore throws.
 	 */
 	search = async (criteria: unknown[]): Promise<number[]> => {
 		this.ensureConnected();
@@ -410,9 +418,8 @@ export class ImapFlowConnection {
 		const searchQuery = this.convertSearchCriteria(criteria);
 
 		const result = await this.client?.search(searchQuery, { uid: true });
-		// search can return false if no messages match, or undefined if client is null
-		if (!result) {
-			return [];
+		if (!Array.isArray(result)) {
+			throw new Error(`IMAP SEARCH failed in mailbox ${this.currentMailbox}`);
 		}
 		return result;
 	};

package/src/index.ts

@@ -167,6 +167,7 @@ export {
 	type ParsedMessageContent,
 	parseMessageContent,
 } from "./message-parser.js";
+export { isMessageGoneFromOpenMailbox } from "./message-presence.js";
 export {
 	type ImapConnectionFactory,
 	MessageSyncService,

package/src/message-presence.test.ts

@@ -0,0 +1,50 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
+import type { IImapConnection } from "./types.js";
+
+type Probe = Pick<IImapConnection, "fetchMessages" | "search">;
+
+const buildProbe = (
+	fetched: number[],
+	searched: number[],
+	searchCalls: unknown[][] = [],
+): Probe => ({
+	fetchMessages: async () => fetched.map((uid) => ({ uid }) as never),
+	search: async (criteria: unknown[]) => {
+		searchCalls.push(criteria);
+		return searched;
+	},
+});
+
+describe("isMessageGoneFromOpenMailbox", () => {
+	it("a FETCH row is proof of presence — no SEARCH needed", async () => {
+		const searchCalls: unknown[][] = [];
+		const gone = await isMessageGoneFromOpenMailbox(
+			buildProbe([7], [], searchCalls),
+			7,
+		);
+
+		assert.equal(gone, false);
+		assert.equal(searchCalls.length, 0);
+	});
+
+	it("an empty FETCH the SEARCH contradicts is a dropped row, not an absence", async () => {
+		const gone = await isMessageGoneFromOpenMailbox(buildProbe([], [7]), 7);
+
+		assert.equal(gone, false);
+	});
+
+	it("only a SEARCH that does not list the uid confirms it is gone", async () => {
+		const gone = await isMessageGoneFromOpenMailbox(buildProbe([], []), 7);
+
+		assert.equal(gone, true);
+	});
+
+	it("asks the server for the uid it is about to reconcile", async () => {
+		const searchCalls: unknown[][] = [];
+		await isMessageGoneFromOpenMailbox(buildProbe([], [], searchCalls), 42);
+
+		assert.deepEqual(searchCalls, [[["UID", "42"]]]);
+	});
+});

package/src/message-presence.ts

@@ -0,0 +1,35 @@
+import type { IImapConnection } from "./types.js";
+
+/**
+ * Whether a UID is confirmed gone from the currently open mailbox.
+ *
+ * A FETCH that yields no row is not proof of absence. imapflow drops rows on
+ * back-to-back FETCHes (#408) — the same client glitch #100 had to stop
+ * reading as authoritative on the sync path — so a transient blip returns an
+ * empty result for a message that is still on the server. Anything that
+ * deletes local rows on that reading destroys live mail.
+ *
+ * Absence is therefore confirmed with a UID SEARCH, which the server answers
+ * with a plain UID set rather than a stream of message rows, and which
+ * `placement-move-push.ts` already uses as its verification probe. Only a
+ * SEARCH that does not list the UID counts as gone; an empty FETCH the SEARCH
+ * contradicts leaves the message present, and the caller treats it as such.
+ *
+ * The absence verdict rests on an empty SEARCH meaning the server matched
+ * nothing, so `IImapConnection.search` must throw when a SEARCH fails rather
+ * than answer `[]` — an implementation that reports failure as an empty array
+ * hands this function a deletion authority it cannot tell apart from a genuine
+ * miss.
+ */
+export const isMessageGoneFromOpenMailbox = async (
+	connection: Pick<IImapConnection, "fetchMessages" | "search">,
+	uid: number,
+): Promise<boolean> => {
+	const found = await connection.fetchMessages([uid]);
+	if (found.length > 0) {
+		return false;
+	}
+
+	const matched = await connection.search([["UID", String(uid)]]);
+	return !matched.includes(uid);
+};

package/src/placement-move-terminal.test.ts

@@ -33,13 +33,21 @@ const buildLogger = (): {
 	};
 };
 
-const buildConnection = (present: Set<number>): IImapConnection =>
+const buildConnection = (
+	present: Set<number>,
+	fetchDrops: Set<number> = new Set(),
+): IImapConnection =>
 	({
 		openBox: async () => ({}) as never,
 		fetchMessages: async (uids: number[]) =>
 			uids
-				.filter((uid) => present.has(uid))
+				.filter((uid) => present.has(uid) && !fetchDrops.has(uid))
 				.map((uid) => ({ uid }) as unknown as never),
+		search: async (criteria: unknown[]) => {
+			const [, value] = (criteria as Array<[string, string]>)[0];
+			const uid = Number(value);
+			return present.has(uid) ? [uid] : [];
+		},
 	}) as unknown as IImapConnection;
 
 describe("resolveExhaustedPlacementMoveFailure — the two terminal outcomes (mirrors #1270 for placement moves)", () => {
@@ -148,6 +156,54 @@ describe("resolveExhaustedPlacementMoveFailure — the two terminal outcomes (mi
 		);
 	});
 
+	it("a dropped FETCH row is not absence: the message is still at the source — marker and local rows survive", async () => {
+		const deletedMessages: string[] = [];
+		const deletedThreadMessages: unknown[] = [];
+		const markerDeletes: string[] = [];
+		const { log, errors } = buildLogger();
+
+		const deps: ResolveExhaustedPlacementMoveDeps = {
+			markerService: {
+				delete: async (id: string) => {
+					markerDeletes.push(id);
+				},
+			},
+			messageService: {
+				delete: async (id: string) => {
+					deletedMessages.push(id);
+				},
+			} as unknown as Pick<IMessageRepository, "delete">,
+			threadMessageService: {
+				findAllByMessageId: async () => [
+					{ accountConfigId: "cfg-1", threadMessageId: "tm-msg-live" },
+				],
+				deleteMany: async (keys: unknown[]) => {
+					deletedThreadMessages.push(...keys);
+				},
+			} as unknown as Pick<
+				IThreadMessageRepository,
+				"findAllByMessageId" | "deleteMany"
+			>,
+			log,
+		};
+
+		const result = await resolveExhaustedPlacementMoveFailure(deps, {
+			accountId: "acc-1",
+			accountConfigId: "cfg-1",
+			messageId: "msg-live",
+			uid: 303,
+			sourceMailboxPath: "INBOX",
+			getConnection: async () =>
+				buildConnection(new Set([303]), new Set([303])),
+		});
+
+		assert.equal(result.outcome, "broken");
+		assert.deepEqual(markerDeletes, []);
+		assert.deepEqual(deletedMessages, []);
+		assert.deepEqual(deletedThreadMessages, []);
+		assert.ok(errors.some((e) => e.obj.alert === "placement_move_failed"));
+	});
+
 	it("never throws — both outcomes are terminal, the caller always acks", async () => {
 		const { log } = buildLogger();
 		const deps: ResolveExhaustedPlacementMoveDeps = {

package/src/placement-move-terminal.ts

@@ -1,3 +1,4 @@
+import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
 import type { PlacementMoveLogger } from "./placement-move.js";
 import {
 	reconcileStaleMessage,
@@ -33,9 +34,11 @@ export interface ResolveExhaustedPlacementMoveResult {
  * taxonomy (epic #1281 invariant 3) — no third, softer outcome.
  *
  * 1. RECONCILED (expected) — the message no longer exists at its pending-move
- *    source on IMAP. Per invariant 2, an external delete supersedes the
- *    marker entirely: the marker is dropped and the stale Message/ThreadMessage
- *    rows are deleted via {@link reconcileStaleMessage}. This is also the
+ *    source on IMAP, confirmed by {@link isMessageGoneFromOpenMailbox} rather
+ *    than by a FETCH coming back empty. Per invariant 2, an external delete
+ *    supersedes the marker entirely: the marker is dropped and the stale
+ *    Message/ThreadMessage rows are deleted via
+ *    {@link reconcileStaleMessage}. This is also the
  *    outcome for the (rarer, functionally indistinguishable from here) case
  *    where a foreign client moved the message elsewhere — either way, our
  *    prediction no longer holds, and the marker cannot be honoured. Metric
@@ -49,6 +52,15 @@ export interface ResolveExhaustedPlacementMoveResult {
  *    `alert`-shaped entry for an operator alarm; never re-thrown (terminal —
  *    the caller acks either way, since retrying a stale or permanently-broken
  *    move can never succeed).
+ *
+ * An operator reading `placement_move_failed` should know one case where the
+ * message is not actually at the source: a message another client expunged
+ * mid-session can answer an empty FETCH while the server still lists its UID
+ * in SEARCH, until it is allowed to send the untagged EXPUNGE. That message
+ * lands in BROKEN, and BROKEN is terminal — the marker stays pending and the
+ * alert stands until someone clears it. The reverse mistake deletes live
+ * mail, so the cost is paid deliberately: a stale alert is recoverable, a
+ * deleted message is not.
  */
 export const resolveExhaustedPlacementMoveFailure = async (
 	deps: ResolveExhaustedPlacementMoveDeps,
@@ -65,9 +77,8 @@ export const resolveExhaustedPlacementMoveFailure = async (
 
 	const connection = await getConnection();
 	await connection.openBox(sourceMailboxPath);
-	const found = await connection.fetchMessages([uid]);
 
-	if (found.length === 0) {
+	if (await isMessageGoneFromOpenMailbox(connection, uid)) {
 		await deps.markerService.delete(messageId);
 		const { threadMessagesDeleted } = await reconcileStaleMessage(
 			deps,