@remit/mailbox-service 0.0.109 → 0.0.111

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.109",
+  "version": "0.0.111",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/adversarial-mime.e2e.test.ts

@@ -223,7 +223,7 @@ describe("Adversarial MIME corpus (Dovecot) — #402", {
 				// (multipart walking, base64/qp decoding, charset folding).
 				const rfc822 = await connection.fetchMessageBody(appendResult.uid);
 				assert.ok(
-					rfc822.length > 0,
+					rfc822 && rfc822.length > 0,
 					`fetchMessageBody should return bytes for ${name}`,
 				);
 

package/src/body-sync-message-gone.test.ts

@@ -0,0 +1,139 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	IAddressRepository,
+	IEnvelopeRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+	MessageItem,
+	PlacementPredicate,
+} from "@remit/data-ports";
+import {
+	MessageGoneFromServerError,
+	MessagePlacementUnsettledError,
+} from "@remit/data-ports/errors";
+import type { StorageService } from "@remit/storage-service";
+import { BodySyncService } from "./body-sync.js";
+import type { IImapConnection } from "./types.js";
+
+const MESSAGE_ID = "m-gone";
+
+interface Scenario {
+	status: MessageItem["status"];
+	rowStillMatches?: boolean;
+	serverStillLists?: boolean;
+}
+
+const readUnstoredMessage = async ({
+	status,
+	rowStillMatches = true,
+	serverStillLists = false,
+}: Scenario) => {
+	const predicates: PlacementPredicate[] = [];
+	const deletedMessages: string[] = [];
+	const deletedThreadMessages: string[] = [];
+
+	const messageService = {
+		get: async () =>
+			({
+				messageId: MESSAGE_ID,
+				mailboxId: "mb-inbox",
+				uid: 42,
+				status,
+			}) as unknown as MessageItem,
+		deleteIfPlacement: async (
+			messageId: string,
+			expected: PlacementPredicate,
+		) => {
+			predicates.push(expected);
+			if (!rowStillMatches) return false;
+			deletedMessages.push(messageId);
+			return true;
+		},
+	} as unknown as IMessageRepository;
+
+	const threadMessageService = {
+		findAllByMessageId: async () => [
+			{ accountConfigId: "cfg-1", threadMessageId: "tm-1" },
+		],
+		deleteMany: async (keys: { threadMessageId: string }[]) => {
+			deletedThreadMessages.push(...keys.map((key) => key.threadMessageId));
+		},
+	} as unknown as IThreadMessageRepository;
+
+	const service = new BodySyncService(
+		messageService,
+		{} as unknown as StorageService,
+		threadMessageService,
+		{} as unknown as IAddressRepository,
+		{} as unknown as IEnvelopeRepository,
+		{ info: () => {}, error: () => {} },
+	);
+
+	const connection = {
+		openBox: async () => {},
+		fetchMessageBody: async () => null,
+		fetchMessages: async () => [],
+		search: async () => (serverStillLists ? [42] : []),
+	} as unknown as IImapConnection;
+
+	const outcome = await service
+		.fetchAndGetBody(
+			MESSAGE_ID,
+			"acc-1",
+			"cfg-1",
+			{ fullPath: "INBOX", placement: "inbox" },
+			async () => connection,
+		)
+		.then(
+			() => assert.fail("expected the read to be refused"),
+			(error: unknown) => error,
+		);
+
+	return { outcome, predicates, deletedMessages, deletedThreadMessages };
+};
+
+describe("reading a message the mail server no longer holds", () => {
+	it("answers 404 and removes the settled row with its thread entries", async () => {
+		const { outcome, predicates, deletedMessages, deletedThreadMessages } =
+			await readUnstoredMessage({ status: "active" });
+
+		assert.ok(outcome instanceof MessageGoneFromServerError);
+		assert.equal(outcome.statusCode, 404);
+		assert.deepEqual(predicates, [
+			{ status: "active", mailboxId: "mb-inbox", uid: 42 },
+		]);
+		assert.deepEqual(deletedMessages, [MESSAGE_ID]);
+		assert.deepEqual(deletedThreadMessages, ["tm-1"]);
+	});
+
+	it("answers 409 and deletes nothing while our own move is in flight", async () => {
+		const { outcome, predicates, deletedThreadMessages } =
+			await readUnstoredMessage({ status: "moving" });
+
+		assert.ok(outcome instanceof MessagePlacementUnsettledError);
+		assert.equal(outcome.statusCode, 409);
+		assert.deepEqual(predicates, []);
+		assert.deepEqual(deletedThreadMessages, []);
+	});
+
+	it("answers 409 and keeps the thread entries when the row changed after it was read", async () => {
+		const { outcome, deletedMessages, deletedThreadMessages } =
+			await readUnstoredMessage({ status: "active", rowStillMatches: false });
+
+		assert.ok(outcome instanceof MessagePlacementUnsettledError);
+		assert.deepEqual(deletedMessages, []);
+		assert.deepEqual(deletedThreadMessages, []);
+	});
+
+	it("fails loudly and deletes nothing when the server still lists the UID", async () => {
+		const { outcome, predicates } = await readUnstoredMessage({
+			status: "active",
+			serverStillLists: true,
+		});
+
+		assert.ok(outcome instanceof Error);
+		assert.ok(!(outcome instanceof MessageGoneFromServerError));
+		assert.deepEqual(predicates, []);
+	});
+});

package/src/body-sync.ts

@@ -13,12 +13,17 @@ import {
 	type IMailboxSpecialUseRepository,
 	type IMessageRepository,
 	type IThreadMessageRepository,
+	type IUnitOfWork,
 	isSenderMuted,
 	type MessageItem,
 	type ThreadMessageItem,
 	type UpdateMessageInput,
 } from "@remit/data-ports";
-import { NotFoundError } from "@remit/data-ports/errors";
+import {
+	MessageGoneFromServerError,
+	MessagePlacementUnsettledError,
+	NotFoundError,
+} from "@remit/data-ports/errors";
 import { deriveAddressId, deriveBodyPartId } from "@remit/data-ports/id";
 import { isBulkSender } from "@remit/data-ports/wellknown";
 import {
@@ -62,7 +67,10 @@ import {
 	denormalizeMessageCategory,
 	hasDecidedCategory,
 } from "./message-category.js";
+import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
+import { PassThroughUnitOfWork } from "./pass-through-unit-of-work.js";
 import type { PlacementMoveService } from "./placement-move.js";
+import { isPlacementUnsettled } from "./placement-settled.js";
 import { type QuarantineService, shapeFromMessageData } from "./quarantine.js";
 import { extractSnippetFromEmail } from "./snippet.js";
 import { type IImapConnection, MailConnectionError } from "./types.js";
@@ -308,6 +316,7 @@ export interface CalendarSuggestionConfig {
 export class BodySyncService {
 	private log: BodySyncLogger;
 	private readonly filterPipeline?: FilterPipeline;
+	private readonly unitOfWork: IUnitOfWork;
 
 	constructor(
 		private messageService: IMessageRepository,
@@ -321,8 +330,17 @@ export class BodySyncService {
 		private readonly quarantineConfig?: QuarantineConfig,
 		private readonly unsubscribeConfig?: UnsubscribeConfig,
 		private readonly calendarConfig?: CalendarSuggestionConfig,
+		unitOfWork?: IUnitOfWork,
 	) {
 		this.log = logger ?? noopLogger;
+		this.unitOfWork =
+			unitOfWork ??
+			new PassThroughUnitOfWork({
+				message: messageService,
+				envelope: envelopeService,
+				address: addressService,
+				threadMessage: threadMessageService,
+			});
 		this.filterPipeline = filterConfig
 			? new FilterPipeline(filterConfig, this.log)
 			: undefined;
@@ -1047,7 +1065,9 @@ export class BodySyncService {
 				);
 				needsStore = true;
 				body = await this.fetchFromImap(
-					message.uid,
+					message,
+					accountId,
+					accountConfigId,
 					folder.fullPath,
 					getConnection,
 				);
@@ -1056,7 +1076,9 @@ export class BodySyncService {
 			// Fetch from IMAP and store
 			needsStore = true;
 			body = await this.fetchFromImap(
-				message.uid,
+				message,
+				accountId,
+				accountConfigId,
 				folder.fullPath,
 				getConnection,
 			);
@@ -1099,14 +1121,97 @@ export class BodySyncService {
 	}
 
 	private async fetchFromImap(
-		uid: number,
+		message: MessageItem,
+		accountId: string,
+		accountConfigId: string,
 		mailboxPath: string,
 		getConnection: ConnectionGetter,
 	): Promise<Buffer> {
+		const { uid } = message;
 		this.log.debug?.({ uid }, "Fetching body from IMAP");
 		const connection = await getConnection();
 		await connection.openBox(mailboxPath);
-		return connection.fetchMessageBody(uid);
+		const body = await connection.fetchMessageBody(uid);
+		if (body) return body;
+
+		if (!(await isMessageGoneFromOpenMailbox(connection, uid))) {
+			throw new Error(
+				`IMAP FETCH returned no source for UID ${uid} in ${mailboxPath}, which the server still lists`,
+			);
+		}
+
+		return this.reconcileGoneMessage(
+			message,
+			accountId,
+			accountConfigId,
+			mailboxPath,
+		);
+	}
+
+	/**
+	 * The server no longer holds the message at the row's UID. A row with our
+	 * own move or delete in flight belongs to that mutation (409). A settled row
+	 * is removed with its thread entries, but only while its placement still
+	 * matches what was read (imap-mutations R3); losing that predicate is a 409
+	 * like any other lost placement transition.
+	 */
+	private async reconcileGoneMessage(
+		message: MessageItem,
+		accountId: string,
+		accountConfigId: string,
+		mailboxPath: string,
+	): Promise<never> {
+		const { messageId, uid } = message;
+		if (isPlacementUnsettled(message)) {
+			throw new MessagePlacementUnsettledError(
+				`Message ${messageId} is not at its UID while a move or delete is in flight`,
+				accountId,
+				messageId,
+				"in_flight",
+			);
+		}
+
+		const reconciled = await this.unitOfWork.transaction(async (repos) => {
+			const deleted = await repos.message.deleteIfPlacement(messageId, {
+				status: message.status,
+				mailboxId: message.mailboxId,
+				uid,
+			});
+			if (!deleted) return false;
+			const threadMessages = await repos.threadMessage.findAllByMessageId(
+				accountConfigId,
+				messageId,
+			);
+			if (threadMessages.length > 0) {
+				await repos.threadMessage.deleteMany(
+					threadMessages.map((row) => ({
+						accountConfigId: row.accountConfigId,
+						threadMessageId: row.threadMessageId,
+					})),
+				);
+			}
+			return true;
+		});
+
+		if (!reconciled) {
+			throw new MessagePlacementUnsettledError(
+				`Message ${messageId} changed placement while its absence from ${mailboxPath} was being reconciled`,
+				accountId,
+				messageId,
+				"in_flight",
+			);
+		}
+
+		this.log.info(
+			{
+				metric: "message_gone_stale_row_reconciled",
+				messageId,
+				uid,
+				mailboxPath,
+			},
+			"Message no longer on the mail server (expunged or moved by another client); stale row reconciled",
+		);
+		throw new MessageGoneFromServerError(messageId);
 	}
 
 	/**

package/src/filters/match.test.ts

@@ -65,6 +65,42 @@ describe("clauseMatches", () => {
 		);
 	});
 
+	it("matches HasWords on the sender display name alone, as search does", () => {
+		const msg = message({
+			from: "booking@mail.example",
+			fromName: "Austrian Airlines Booking",
+			subject: "Your trip to Vienna",
+			text: "Check in opens 24 hours before departure.",
+		});
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.HasWords, "Austrian airlines"),
+				msg,
+			),
+			true,
+		);
+	});
+
+	it("matches HasWords regardless of word order and case", () => {
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.HasWords, "ALICE invoice"),
+				message(),
+			),
+			true,
+		);
+	});
+
+	it("does not match HasWords when one word is missing", () => {
+		assert.equal(
+			clauseMatches(
+				clause(FilterClauseField.HasWords, "invoice receipt"),
+				message(),
+			),
+			false,
+		);
+	});
+
 	it("never matches an empty clause value", () => {
 		assert.equal(
 			clauseMatches(clause(FilterClauseField.HasWords, "   "), message()),

package/src/filters/match.ts

@@ -49,6 +49,20 @@ export interface FilterMessage {
 const includesFold = (haystack: string, needle: string): boolean =>
 	haystack.toLowerCase().includes(needle.toLowerCase());
 
+/**
+ * Whether every whitespace-separated word of `value` appears somewhere in the
+ * sender address, sender name, subject or body — the shape free-text search
+ * matches with (`buildSearchConditions`), so a filter made from a search keeps
+ * the mail that search showed.
+ */
+const hasEveryWord = (value: string, msg: FilterMessage): boolean => {
+	const fields = [msg.from, msg.fromName, msg.subject, msg.text];
+	return value
+		.split(/\s+/)
+		.filter(Boolean)
+		.every((word) => fields.some((field) => includesFold(field, word)));
+};
+
 const hostOf = (address: string): string => {
 	const at = address.lastIndexOf("@");
 	return at >= 0 ? address.slice(at + 1) : address;
@@ -65,8 +79,8 @@ const registrableDomain = (addressOrHost: string): string | null =>
 
 /**
  * Whether one literal clause matches the message. From matches against the
- * sender address and display name; Subject against the subject; HasWords against
- * subject or body; ListId against the exact normalized `List-Id`; FromDomain
+ * sender address and display name; Subject against the subject; HasWords
+ * requires every word somewhere in sender, subject or body; ListId against the exact normalized `List-Id`; FromDomain
  * against the sender's registrable domain. An empty clause value never matches.
  */
 export const clauseMatches = (
@@ -81,7 +95,7 @@ export const clauseMatches = (
 		case FilterClauseField.Subject:
 			return includesFold(msg.subject, value);
 		case FilterClauseField.HasWords:
-			return includesFold(msg.subject, value) || includesFold(msg.text, value);
+			return hasEveryWord(value, msg);
 		case FilterClauseField.ListId: {
 			const target = normalizeListId(value);
 			return target !== "" && normalizeListId(msg.listId) === target;

package/src/imapflow-connection.e2e.test.ts

@@ -113,7 +113,7 @@ describe("Message bodies (Dovecot)", {
 			assert.ok(uids.length > 0, "Should have messages to fetch body for");
 
 			const body = await connection.fetchMessageBody(uids[0]);
-			assert.ok(body.length > 0, "Body should not be empty");
+			assert.ok(body && body.length > 0, "Body should not be empty");
 
 			const bodyText = body.toString("utf-8");
 			assert.ok(bodyText.length > 0, "Body text should not be empty");

package/src/imapflow-connection.ts

@@ -721,9 +721,9 @@ export class ImapFlowConnection {
 	 * Fetch the full message body (RFC822 source) for a single message by UID.
 	 *
 	 * @param uid - The UID of the message to fetch
-	 * @returns The raw message body as a Buffer
+	 * @returns The raw message body, or null when the UID FETCH matched no message
 	 */
-	fetchMessageBody = async (uid: number): Promise<Buffer> => {
+	fetchMessageBody = async (uid: number): Promise<Buffer | null> => {
 		this.ensureConnected();
 		const { client } = this;
 
@@ -739,19 +739,22 @@ export class ImapFlowConnection {
 			uid: true,
 		});
 
-		// Connection may have been lost during download
-		if (!result || !result.content) {
-			throw new Error(
-				`IMAP connection lost while downloading message UID ${uid}`,
-			);
+		if (result?.content) {
+			const chunks: Buffer[] = [];
+			for await (const chunk of result.content) {
+				chunks.push(chunk);
+			}
+			return Buffer.concat(chunks);
 		}
 
-		const chunks: Buffer[] = [];
-		for await (const chunk of result.content) {
-			chunks.push(chunk);
+		if (!client.usable || !client.mailbox) {
+			throw new MailConnectionError(
+				"network",
+				`IMAP connection lost while downloading message UID ${uid}`,
+			);
 		}
 
-		return Buffer.concat(chunks);
+		return null;
 	};
 
 	/**

package/src/types.ts

@@ -253,7 +253,7 @@ export interface IImapConnection {
 	 * no body. Requires a mailbox to be open.
 	 */
 	fetchEnvelopeSnapshots(uids: number[]): Promise<ImapEnvelopeSnapshot[]>;
-	fetchMessageBody(uid: number): Promise<Buffer>;
+	fetchMessageBody(uid: number): Promise<Buffer | null>;
 	/**
 	 * Fetch full message bodies for many UIDs in ONE pipelined ranged UID FETCH.
 	 * Yields `{ uid, source }` per message; `source` is a readable stream the