@remit/mailbox-service 0.0.15 → 0.0.17

package/src/flag-push-terminal.test.ts

@@ -33,13 +33,21 @@ const buildLogger = (): {
 	};
 };
 
-const buildConnection = (present: Set<number>): IImapConnection =>
+const buildConnection = (
+	present: Set<number>,
+	fetchDrops: Set<number> = new Set(),
+): IImapConnection =>
 	({
 		openBox: async () => ({}) as never,
 		fetchMessages: async (uids: number[]) =>
 			uids
-				.filter((uid) => present.has(uid))
+				.filter((uid) => present.has(uid) && !fetchDrops.has(uid))
 				.map((uid) => ({ uid }) as unknown as never),
+		search: async (criteria: unknown[]) => {
+			const [, value] = (criteria as Array<[string, string]>)[0];
+			const uid = Number(value);
+			return present.has(uid) ? [uid] : [];
+		},
 	}) as unknown as IImapConnection;
 
 describe("resolveExhaustedFlagPushFailure — the two terminal outcomes (mirrors #1289/#1270 for flag pushes)", () => {
@@ -104,6 +112,51 @@ describe("resolveExhaustedFlagPushFailure — the two terminal outcomes (mirrors
 		assert.ok(metricLog, "expected a routine reconciliation metric log");
 	});
 
+	it("a dropped FETCH row is not absence: the message is still in the mailbox — marker and local rows survive", async () => {
+		const markerDeletes: Array<{ messageId: string; flagName: string }> = [];
+		const { log, errors } = buildLogger();
+
+		const deps: ResolveExhaustedFlagPushDeps = {
+			markerService: {
+				delete: async (messageId: string, flagName: string) => {
+					markerDeletes.push({ messageId, flagName });
+				},
+			},
+			messageService: {
+				delete: async () => {
+					throw new Error("must not be called — the message still exists");
+				},
+			} as unknown as Pick<IMessageRepository, "delete">,
+			threadMessageService: {
+				findAllByMessageId: async () => {
+					throw new Error("must not be called — the message still exists");
+				},
+				deleteMany: async () => {
+					throw new Error("must not be called — the message still exists");
+				},
+			} as unknown as Pick<
+				IThreadMessageRepository,
+				"findAllByMessageId" | "deleteMany"
+			>,
+			log,
+		};
+
+		const result = await resolveExhaustedFlagPushFailure(deps, {
+			accountId: "acc-1",
+			accountConfigId: "cfg-1",
+			messageId: "msg-live",
+			flagName: "\\Seen",
+			uid: 303,
+			mailboxPath: "INBOX",
+			getConnection: async () =>
+				buildConnection(new Set([303]), new Set([303])),
+		});
+
+		assert.equal(result.outcome, "broken");
+		assert.deepEqual(markerDeletes, []);
+		assert.ok(errors.some((e) => e.obj.alert === "flag_push_failed"));
+	});
+
 	it("BROKEN (should never happen): the message still exists — marker left in place, alert logged", async () => {
 		const markerDeletes: Array<{ messageId: string; flagName: string }> = [];
 		const { log, errors } = buildLogger();

package/src/flag-push-terminal.ts

@@ -1,4 +1,5 @@
 import type { FlagPushLogger } from "./flag-push.js";
+import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
 import {
 	reconcileStaleMessage,
 	type StaleMessageReconcileDeps,
@@ -35,10 +36,11 @@ export interface ResolveExhaustedFlagPushResult {
  * (epic #1281 invariant 3) — no third, softer outcome.
  *
  * 1. RECONCILED (expected) — the message no longer exists at its mailbox on
- *    IMAP. Per invariant 2, an external delete supersedes the marker
- *    entirely: the marker is dropped and the stale Message/ThreadMessage rows
- *    are deleted via {@link reconcileStaleMessage}. Metric only, no alarm —
- *    routine.
+ *    IMAP, confirmed by {@link isMessageGoneFromOpenMailbox} rather than by a
+ *    FETCH coming back empty. Per invariant 2, an external delete supersedes
+ *    the marker entirely: the marker is dropped and the stale
+ *    Message/ThreadMessage rows are deleted via {@link reconcileStaleMessage}.
+ *    Metric only, no alarm — routine.
  * 2. BROKEN — the message still exists, but the flag push keeps failing.
  *    Broken code or a broken account, not a transient blip. The marker is
  *    left in place (not cleared) — while pending, resync never reverts the
@@ -48,6 +50,15 @@ export interface ResolveExhaustedFlagPushResult {
  *    entry for an operator alarm; never re-thrown (terminal — the caller acks
  *    either way, since retrying a stale or permanently-broken push can never
  *    succeed).
+ *
+ * An operator reading `flag_push_failed` should know one case where the
+ * message is not actually there: a message another client expunged
+ * mid-session can answer an empty FETCH while the server still lists its UID
+ * in SEARCH, until it is allowed to send the untagged EXPUNGE. That message
+ * lands in BROKEN, and BROKEN is terminal — the marker stays pending and the
+ * alert stands until someone clears it. The reverse mistake deletes live
+ * mail, so the cost is paid deliberately: a stale alert is recoverable, a
+ * deleted message is not.
  */
 export const resolveExhaustedFlagPushFailure = async (
 	deps: ResolveExhaustedFlagPushDeps,
@@ -65,9 +76,8 @@ export const resolveExhaustedFlagPushFailure = async (
 
 	const connection = await getConnection();
 	await connection.openBox(mailboxPath, true);
-	const found = await connection.fetchMessages([uid]);
 
-	if (found.length === 0) {
+	if (await isMessageGoneFromOpenMailbox(connection, uid)) {
 		await deps.markerService.delete(messageId, flagName);
 		const { threadMessagesDeleted } = await reconcileStaleMessage(
 			deps,

package/src/imapflow-connection.test.ts

@@ -5,6 +5,7 @@ import {
 	toInternalDate,
 	toIsoDateString,
 } from "./imapflow-connection.js";
+import type { ImapMessage } from "./types.js";
 
 describe("toIsoDateString", () => {
 	it("converts a Date to an ISO string", () => {
@@ -111,6 +112,52 @@ const fakeMailbox = (path: string, exists: number) => ({
 	readOnly: true,
 });
 
+describe("ImapFlowConnection.search — the probe every stale-row reconcile rests on (#102)", () => {
+	const buildSearchConnection = (
+		result: unknown,
+		calls: Array<{ query: unknown; options: unknown }> = [],
+	): ImapFlowConnection => {
+		const connection = buildConnectionWithClient({
+			mailboxOpen: async (path: string) => fakeMailbox(path, 1),
+			search: async (query: unknown, options: unknown) => {
+				calls.push({ query, options });
+				return result;
+			},
+		});
+		return connection;
+	};
+
+	it("translates a UID criterion into a UID SEARCH answering UIDs, not sequence numbers", async () => {
+		const calls: Array<{ query: unknown; options: unknown }> = [];
+		const connection = buildSearchConnection([42], calls);
+		await connection.openBox("INBOX", true);
+
+		const result = await connection.search([["UID", "42"]]);
+
+		assert.deepStrictEqual(result, [42]);
+		assert.deepStrictEqual(calls, [
+			{ query: { uid: "42" }, options: { uid: true } },
+		]);
+	});
+
+	it("answers an empty array when the server matched nothing", async () => {
+		const connection = buildSearchConnection([]);
+		await connection.openBox("INBOX", true);
+
+		assert.deepStrictEqual(await connection.search([["UID", "42"]]), []);
+	});
+
+	it("throws on a failed SEARCH instead of reporting it as no matches", async () => {
+		const connection = buildSearchConnection(false);
+		await connection.openBox("INBOX", true);
+
+		await assert.rejects(
+			() => connection.search([["UID", "42"]]),
+			/SEARCH failed/,
+		);
+	});
+});
+
 describe("ImapFlowConnection.getMailboxStatus — deletedCount (#1042)", () => {
 	it("returns the SEARCH \\Deleted count alongside the STATUS counts", async () => {
 		const searchQueries: Array<Record<string, unknown>> = [];
@@ -151,7 +198,7 @@ describe("ImapFlowConnection.getMailboxStatus — deletedCount (#1042)", () => {
 				highestModseq: 0n,
 			}),
 			mailboxOpen: async (path: string) => fakeMailbox(path, 4),
-			search: async () => false,
+			search: async () => [],
 		});
 
 		const status = await connection.getMailboxStatus("INBOX");
@@ -172,7 +219,7 @@ describe("ImapFlowConnection.getMailboxStatus — deletedCount (#1042)", () => {
 				highestModseq: modseq,
 			}),
 			mailboxOpen: async (path: string) => fakeMailbox(path, 1),
-			search: async () => false,
+			search: async () => [],
 		});
 
 		const status = await connection.getMailboxStatus("INBOX");
@@ -266,3 +313,51 @@ describe("ImapFlowConnection CONDSTORE (reader#20)", () => {
 		assert.deepStrictEqual(messages[0].flags, ["\\Seen"]);
 	});
 });
+
+describe("ImapFlowConnection message envelopes (issue #72)", () => {
+	const fetchRows = async (
+		rows: Array<Record<string, unknown>>,
+	): Promise<ImapMessage[]> => {
+		const connection = buildConnectionWithClient({
+			enabled: new Set(),
+			mailbox: fakeMailbox("INBOX", 1),
+			mailboxOpen: async (path: string) => fakeMailbox(path, 1),
+			fetch: () =>
+				(async function* () {
+					for (const row of rows) yield row;
+				})(),
+		});
+		Object.assign(connection as unknown as Record<string, unknown>, {
+			currentMailbox: "INBOX",
+		});
+		return connection.fetchMessages([1]);
+	};
+
+	it("leaves an absent ENVELOPE absent instead of synthesising an empty one", async () => {
+		// A synthesised envelope made every `if (!msg.envelope)` guard downstream
+		// unreachable, so the row was saved as a message with no sender, subject
+		// or date under a `generated:` key — indistinguishable from real mail.
+		const [message] = await fetchRows([
+			{ uid: 1, seq: 1, internalDate: new Date(0), size: 10 },
+		]);
+		assert.equal(message?.envelope, undefined);
+	});
+
+	it("still converts an envelope the server did send", async () => {
+		const [message] = await fetchRows([
+			{
+				uid: 1,
+				seq: 1,
+				internalDate: new Date(0),
+				size: 10,
+				envelope: {
+					subject: "Hello",
+					messageId: "<a@example.com>",
+					from: [{ address: "sender@example.com" }],
+				},
+			},
+		]);
+		assert.equal(message?.envelope?.subject, "Hello");
+		assert.equal(message?.envelope?.from[0]?.mailbox, "sender");
+	});
+});

package/src/imapflow-connection.ts

@@ -397,7 +397,15 @@ export class ImapFlowConnection {
 	};
 
 	/**
-	 * Search for messages
+	 * Search for messages, answering UIDs.
+	 *
+	 * imapflow answers a successful SEARCH with an array — empty when the
+	 * server matched nothing — and `false` when the command failed or no
+	 * mailbox was selected. Collapsing the two into `[]` makes a failed SEARCH
+	 * indistinguishable from an empty mailbox, and every caller here acts on
+	 * emptiness by deleting something: the cursor rebuild reads it as "every
+	 * local row is stale", and {@link isMessageGoneFromOpenMailbox} reads it as
+	 * "this message is gone". A failure therefore throws.
 	 */
 	search = async (criteria: unknown[]): Promise<number[]> => {
 		this.ensureConnected();
@@ -410,9 +418,8 @@ export class ImapFlowConnection {
 		const searchQuery = this.convertSearchCriteria(criteria);
 
 		const result = await this.client?.search(searchQuery, { uid: true });
-		// search can return false if no messages match, or undefined if client is null
-		if (!result) {
-			return [];
+		if (!Array.isArray(result)) {
+			throw new Error(`IMAP SEARCH failed in mailbox ${this.currentMailbox}`);
 		}
 		return result;
 	};
@@ -591,20 +598,23 @@ export class ImapFlowConnection {
 		for await (const msg of fetchIterator) {
 			// imapflow occasionally yields a row with undefined uid or internalDate
 			// on back-to-back FETCH calls (e.g. after a body-fetch on the same UID).
-			// Skipping the row is safe: the caller asked for a specific UID set and
-			// will simply not see that entry rather than the whole call crashing.
-			// See #408 for the investigation.
+			// See #408 for the investigation. This is the client library glitching,
+			// not the message being malformed, so the row is dropped rather than
+			// quarantined — recording it as a defective message would advance a
+			// cursor past mail that is fine (issue #72).
+			//
+			// Dropping it is only safe because the caller treats a requested UID
+			// with no row as unconsumed and holds its watermark below it. It used
+			// to advance regardless, which is how a transient client glitch turned
+			// into a message never fetched again.
 			if (msg.uid == null || msg.internalDate == null) {
 				continue;
 			}
 
 			// Coerce INTERNALDATE without ever throwing: a malformed value must not
-			// abort the whole fetch batch. `null` only for an absent value (already
-			// handled by the guard above); a bad value falls back to now.
-			const internalDate = toInternalDate(msg.internalDate);
-			if (internalDate === null) {
-				continue;
-			}
+			// abort the whole fetch batch. A bad value falls back to now; an absent
+			// one cannot reach here, having been dropped above.
+			const internalDate = toInternalDate(msg.internalDate) ?? new Date();
 
 			// Parse References header if present
 			const references = await this.parseReferencesHeader(msg.headers);
@@ -840,20 +850,13 @@ export class ImapFlowConnection {
 			  }
 			| undefined,
 	): ImapMessage["envelope"] => {
-		if (!envelope) {
-			return {
-				date: "",
-				subject: "",
-				from: [],
-				sender: [],
-				replyTo: [],
-				to: [],
-				cc: [],
-				bcc: [],
-				inReplyTo: "",
-				messageId: "",
-			};
-		}
+		// An absent ENVELOPE stays absent. Synthesising an empty one here made
+		// every `if (!msg.envelope)` guard downstream unreachable, so a FETCH row
+		// that carried no envelope was saved as a row with no sender, no subject
+		// and no date, keyed by the `generated:` fallback — indistinguishable
+		// from a real message. The field is optional on `ImapMessage` precisely
+		// so callers can see the difference (issue #72).
+		if (!envelope) return undefined;
 
 		const convertAddresses = (
 			addrs?: Array<{ name?: string; address?: string }>,

package/src/index.ts

@@ -15,6 +15,10 @@ export {
 	type ParsedAttributes,
 	parseImapAttributes,
 } from "./attribute-mapper.js";
+export {
+	BodyParseError,
+	parseMessageBody,
+} from "./body-parse.js";
 export {
 	type BodySyncLogger,
 	BodySyncService,
@@ -22,6 +26,7 @@ export {
 	extractPrimaryFromEmail,
 	type FetchBodyResult,
 	type PlacementConfig,
+	type QuarantineConfig,
 	type SyncBodiesResult,
 	toParsedBody,
 } from "./body-sync.js";
@@ -162,6 +167,7 @@ export {
 	type ParsedMessageContent,
 	parseMessageContent,
 } from "./message-parser.js";
+export { isMessageGoneFromOpenMailbox } from "./message-presence.js";
 export {
 	type ImapConnectionFactory,
 	MessageSyncService,
@@ -188,6 +194,16 @@ export {
 	type ResolveExhaustedPlacementMoveResult,
 	resolveExhaustedPlacementMoveFailure,
 } from "./placement-move-terminal.js";
+export {
+	type QuarantineContext,
+	QuarantinedUids,
+	type QuarantineFailure,
+	type QuarantineLogger,
+	type QuarantineMessageShape,
+	QuarantineService,
+	resolveMailboxRole,
+	shapeFromMessageData,
+} from "./quarantine.js";
 export {
 	extractSnippetFromEmail,
 	generateSnippet,

package/src/message-presence.test.ts

@@ -0,0 +1,50 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
+import type { IImapConnection } from "./types.js";
+
+type Probe = Pick<IImapConnection, "fetchMessages" | "search">;
+
+const buildProbe = (
+	fetched: number[],
+	searched: number[],
+	searchCalls: unknown[][] = [],
+): Probe => ({
+	fetchMessages: async () => fetched.map((uid) => ({ uid }) as never),
+	search: async (criteria: unknown[]) => {
+		searchCalls.push(criteria);
+		return searched;
+	},
+});
+
+describe("isMessageGoneFromOpenMailbox", () => {
+	it("a FETCH row is proof of presence — no SEARCH needed", async () => {
+		const searchCalls: unknown[][] = [];
+		const gone = await isMessageGoneFromOpenMailbox(
+			buildProbe([7], [], searchCalls),
+			7,
+		);
+
+		assert.equal(gone, false);
+		assert.equal(searchCalls.length, 0);
+	});
+
+	it("an empty FETCH the SEARCH contradicts is a dropped row, not an absence", async () => {
+		const gone = await isMessageGoneFromOpenMailbox(buildProbe([], [7]), 7);
+
+		assert.equal(gone, false);
+	});
+
+	it("only a SEARCH that does not list the uid confirms it is gone", async () => {
+		const gone = await isMessageGoneFromOpenMailbox(buildProbe([], []), 7);
+
+		assert.equal(gone, true);
+	});
+
+	it("asks the server for the uid it is about to reconcile", async () => {
+		const searchCalls: unknown[][] = [];
+		await isMessageGoneFromOpenMailbox(buildProbe([], [], searchCalls), 42);
+
+		assert.deepEqual(searchCalls, [[["UID", "42"]]]);
+	});
+});

package/src/message-presence.ts

@@ -0,0 +1,35 @@
+import type { IImapConnection } from "./types.js";
+
+/**
+ * Whether a UID is confirmed gone from the currently open mailbox.
+ *
+ * A FETCH that yields no row is not proof of absence. imapflow drops rows on
+ * back-to-back FETCHes (#408) — the same client glitch #100 had to stop
+ * reading as authoritative on the sync path — so a transient blip returns an
+ * empty result for a message that is still on the server. Anything that
+ * deletes local rows on that reading destroys live mail.
+ *
+ * Absence is therefore confirmed with a UID SEARCH, which the server answers
+ * with a plain UID set rather than a stream of message rows, and which
+ * `placement-move-push.ts` already uses as its verification probe. Only a
+ * SEARCH that does not list the UID counts as gone; an empty FETCH the SEARCH
+ * contradicts leaves the message present, and the caller treats it as such.
+ *
+ * The absence verdict rests on an empty SEARCH meaning the server matched
+ * nothing, so `IImapConnection.search` must throw when a SEARCH fails rather
+ * than answer `[]` — an implementation that reports failure as an empty array
+ * hands this function a deletion authority it cannot tell apart from a genuine
+ * miss.
+ */
+export const isMessageGoneFromOpenMailbox = async (
+	connection: Pick<IImapConnection, "fetchMessages" | "search">,
+	uid: number,
+): Promise<boolean> => {
+	const found = await connection.fetchMessages([uid]);
+	if (found.length > 0) {
+		return false;
+	}
+
+	const matched = await connection.search([["UID", String(uid)]]);
+	return !matched.includes(uid);
+};