@remit/mailbox-service 0.0.14 → 0.0.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.14",
+  "version": "0.0.16",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/body-parse.test.ts

@@ -0,0 +1,55 @@
+/**
+ * The boundary the whole quarantine feature rests on: a message defect is a
+ * `BodyParseError`, everything else is not. If this type ever wraps something
+ * that is not the parser's refusal, an outage becomes a pile of records saying
+ * the user's mail is unreadable — and the cursor moves past it.
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { BodyParseError, parseMessageBody } from "./body-parse.js";
+
+const WELL_FORMED = Buffer.from(
+	[
+		"From: someone@example.com",
+		"Subject: hello",
+		"Content-Type: text/plain",
+		"",
+		"body",
+	].join("\r\n"),
+);
+
+describe("parseMessageBody", () => {
+	it("returns the parsed message when the body is readable", async () => {
+		const parsed = await parseMessageBody(WELL_FORMED);
+		assert.equal(parsed.subject, "hello");
+	});
+
+	it("wraps a parser refusal so the catch site can attribute it", async () => {
+		const error = await parseMessageBody(
+			// A source the parser has to reject rather than tolerate.
+			null as unknown as Buffer,
+		).then(
+			() => null,
+			(err: unknown) => err,
+		);
+
+		assert.ok(error instanceof BodyParseError);
+		assert.equal(error.name, "BodyParseError");
+	});
+
+	it("names an unknown charset, because that defect is identifiable", () => {
+		const error = new BodyParseError(new Error("Unknown charset: x-nonesuch"));
+		assert.equal(error.failureCode, "UnknownCharset");
+	});
+
+	it("refuses to guess a code from parser prose it cannot read", () => {
+		const error = new BodyParseError(new Error("something went sideways"));
+		assert.equal(error.failureCode, "UnreadableBody");
+	});
+
+	it("keeps the parser's own words, which stay on screen and off a report", () => {
+		const error = new BodyParseError(new Error("boundary never closed"));
+		assert.equal(error.message, "boundary never closed");
+	});
+});

package/src/body-parse.ts

@@ -0,0 +1,61 @@
+import type { QuarantineItem } from "@remit/data-ports";
+import { QuarantineFailureCode } from "@remit/domain-enums";
+import { type ParsedMail, simpleParser } from "mailparser";
+
+type FailureCode = QuarantineItem["failureCode"];
+
+/**
+ * The message body could not be parsed.
+ *
+ * This type is the whole reason the sync path can quarantine anything. Before
+ * it, every catch site on the body path saw one undifferentiated `unknown`
+ * covering mailparser, S3, DynamoDB and SQS alike, so "the message is built in
+ * a way Remit could not read" was indistinguishable from "S3 returned a 503".
+ * Recording the second as the first advances the cursor past mail that is
+ * perfectly fine and never fetches it again.
+ *
+ * Only {@link parseMessageBody} constructs one, and its try block contains the
+ * parse call and nothing else — so an instance of this type is proof that the
+ * message, not the infrastructure, is what failed. Everything else propagates.
+ */
+export class BodyParseError extends Error {
+	readonly failureCode: FailureCode;
+
+	constructor(cause: unknown) {
+		const message = cause instanceof Error ? cause.message : String(cause);
+		super(message, { cause });
+		this.name = "BodyParseError";
+		this.failureCode = classifyBodyParseFailure(message);
+	}
+}
+
+/**
+ * mailparser reports its refusals as free text with no stable type, code or
+ * class, so the closed failure vocabulary can only name a defect it can
+ * recognise without reading parser prose. The charset decoder is the one that
+ * says the same thing every time, because the sentence comes from iconv rather
+ * than the parser. Anything else is `UnreadableBody`, and the parser's own
+ * words go to `failureMessage`, which is shown on screen and never published.
+ *
+ * Guessing more finely than this would put a wrong, unfalsifiable code in the
+ * title of an issue filed under the user's own account.
+ */
+const UNKNOWN_CHARSET = /unknown charset|unsupported charset|invalid encoding/i;
+
+const classifyBodyParseFailure = (message: string): FailureCode =>
+	UNKNOWN_CHARSET.test(message)
+		? QuarantineFailureCode.UnknownCharset
+		: QuarantineFailureCode.UnreadableBody;
+
+/**
+ * Parse a raw RFC822 body, distinguishing a message defect from everything
+ * else. The try block is the parse call alone; nothing that touches storage,
+ * the queue or the database may be moved inside it.
+ */
+export const parseMessageBody = async (body: Buffer): Promise<ParsedMail> => {
+	try {
+		return await simpleParser(body);
+	} catch (error) {
+		throw new BodyParseError(error);
+	}
+};

package/src/body-sync-quarantine.test.ts

@@ -0,0 +1,219 @@
+/**
+ * The boundary this feature lives or dies on (issue #72).
+ *
+ * A message the parser refuses is set aside and stops being requeued. Storage
+ * and database failures — which the same per-message frame catches — keep
+ * propagating to the requeue path, because recording one as a quarantine would
+ * tell the user that mail Remit could not reach was mail Remit could not read,
+ * and let go of it.
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	IAddressRepository,
+	IEnvelopeRepository,
+	IMailboxSpecialUseRepository,
+	IMessageRepository,
+	IQuarantineRepository,
+	IThreadMessageRepository,
+	MessageItem,
+	QuarantineItem,
+	QuarantineUpsertInput,
+	UpdateMessageInput,
+} from "@remit/data-ports";
+import { MessageCategory } from "@remit/domain-enums";
+import type { StorageService } from "@remit/storage-service";
+import { BodySyncService } from "./body-sync.js";
+import { QuarantineService } from "./quarantine.js";
+
+const UNPARSEABLE = Symbol("unparseable");
+
+const buildHarness = (options: {
+	retrieve?: (key: string) => Promise<Buffer>;
+	existing?: QuarantineItem[];
+	upsertFails?: boolean;
+}) => {
+	const writes: QuarantineUpsertInput[] = [];
+	const messageUpdates: string[] = [];
+
+	const message: MessageItem = {
+		messageId: "m-1",
+		mailboxId: "mbx-1",
+		uid: 40217,
+		rfc822Size: 2048,
+		messageIdHeader: "<abc@example.com>",
+		bodyStorageKey: "s3://bodies/m-1",
+		category: MessageCategory.uncategorized,
+	} as MessageItem;
+
+	const messageService = {
+		get: async () => message,
+		update: async (messageId: string, _input: UpdateMessageInput) => {
+			messageUpdates.push(messageId);
+		},
+	} as unknown as IMessageRepository;
+
+	const storageService = {
+		retrieve:
+			options.retrieve ??
+			(async () => {
+				throw new Error("no body configured");
+			}),
+	} as unknown as StorageService;
+
+	const envelopeService = {
+		getMessageData: async () => ({ bodyPart: [], bodyPartParameter: [] }),
+	} as unknown as IEnvelopeRepository;
+
+	const repository = {
+		listByAccountConfigId: async () => options.existing ?? [],
+		upsert: async (input: QuarantineUpsertInput) => {
+			if (options.upsertFails) throw new Error("database unavailable");
+			writes.push(input);
+		},
+	} satisfies IQuarantineRepository;
+
+	const service = new BodySyncService(
+		messageService,
+		storageService,
+		{
+			getByMessageId: async () => ({
+				threadMessageId: "tm-1",
+				sentDate: 1,
+				mailboxId: "mbx-1",
+				isRead: false,
+				isDeleted: false,
+				hasStars: false,
+				hasAttachment: false,
+			}),
+			update: async () => {},
+		} as unknown as IThreadMessageRepository,
+		{} as unknown as IAddressRepository,
+		envelopeService,
+		{ info: () => {}, error: () => {}, debug: () => {} },
+		undefined,
+		undefined,
+		{
+			quarantineService: new QuarantineService(
+				repository,
+				{
+					listByMailboxId: async () => [],
+				} as unknown as IMailboxSpecialUseRepository,
+				"sha-abc",
+				{ info: () => {}, warn: () => {} },
+			),
+			mailboxId: "mbx-1",
+			uidValidity: 1_712_000_000,
+			attempts: 2,
+		},
+	);
+
+	return { service, writes, messageUpdates };
+};
+
+const sync = (service: BodySyncService) =>
+	service.syncBodies(["m-1"], "acc-1", "cfg-1", "INBOX", async () => {
+		throw new Error("this test must not open IMAP");
+	});
+
+describe("body sync quarantines a message the parser refuses", () => {
+	const retrieveUnparseable = async () => UNPARSEABLE as unknown as Buffer;
+
+	it("records the failure instead of requeueing the message forever", async () => {
+		const harness = buildHarness({ retrieve: retrieveUnparseable });
+
+		const result = await sync(harness.service);
+
+		assert.equal(harness.writes.length, 1);
+		assert.deepEqual(result.failedMessageIds, []);
+	});
+
+	it("names the message by uid and UIDVALIDITY, so the record is idempotent", async () => {
+		const harness = buildHarness({ retrieve: retrieveUnparseable });
+
+		await sync(harness.service);
+
+		assert.equal(harness.writes[0]?.uid, 40217);
+		assert.equal(harness.writes[0]?.uidValidity, 1_712_000_000);
+		assert.equal(harness.writes[0]?.failureStage, "BodyParse");
+	});
+
+	it("does not mark the message synced — it was set aside, not applied", async () => {
+		const harness = buildHarness({ retrieve: retrieveUnparseable });
+
+		const result = await sync(harness.service);
+
+		assert.deepEqual(result.syncedMessageIds, []);
+		assert.deepEqual(harness.messageUpdates, []);
+	});
+});
+
+describe("body sync leaves infrastructure failures alone", () => {
+	it("requeues a storage failure rather than calling the message unreadable", async () => {
+		const harness = buildHarness({
+			retrieve: async () => {
+				throw new Error("S3 503 SlowDown");
+			},
+		});
+
+		const result = await sync(harness.service);
+
+		assert.deepEqual(harness.writes, []);
+		assert.deepEqual(result.failedMessageIds, ["m-1"]);
+	});
+
+	it("requeues a database failure the same way", async () => {
+		const harness = buildHarness({
+			retrieve: async () => {
+				const error = new Error("ProvisionedThroughputExceededException");
+				error.name = "ProvisionedThroughputExceededException";
+				throw error;
+			},
+		});
+
+		const result = await sync(harness.service);
+
+		assert.deepEqual(harness.writes, []);
+		assert.deepEqual(result.failedMessageIds, ["m-1"]);
+	});
+});
+
+describe("body sync contains a failure to write the record", () => {
+	it("requeues the message instead of aborting the batch", async () => {
+		const harness = buildHarness({
+			retrieve: async () => UNPARSEABLE as unknown as Buffer,
+			upsertFails: true,
+		});
+
+		// Writing the record is database work, so its failure is infrastructure:
+		// the message keeps its place in the queue rather than being let go of
+		// with nothing written down, and the rest of the batch is unaffected.
+		const result = await sync(harness.service);
+
+		assert.deepEqual(harness.writes, []);
+		assert.deepEqual(result.failedMessageIds, ["m-1"]);
+	});
+});
+
+describe("body sync skips what is already quarantined", () => {
+	it("does not fetch or re-parse a uid already set aside", async () => {
+		const harness = buildHarness({
+			existing: [
+				{
+					mailboxId: "mbx-1",
+					uidValidity: 1_712_000_000,
+					uid: 40217,
+				} as QuarantineItem,
+			],
+			retrieve: async () => {
+				throw new Error("a quarantined message must not be read again");
+			},
+		});
+
+		const result = await sync(harness.service);
+
+		assert.equal(result.skippedCount, 1);
+		assert.deepEqual(harness.writes, []);
+	});
+});

package/src/body-sync.ts

@@ -18,6 +18,7 @@ import {
 	MessageCategory,
 	PlacementAction,
 	PlacementConfidence,
+	QuarantineFailureStage,
 	SenderTrust,
 } from "@remit/domain-enums";
 import {
@@ -27,6 +28,7 @@ import {
 } from "@remit/storage-service";
 import { type ParsedMail, simpleParser } from "mailparser";
 import pMap from "p-map";
+import { BodyParseError, parseMessageBody } from "./body-parse.js";
 import { mapBodyPartsToContent } from "./body-part-mapper.js";
 import type { FilterMessage } from "./filters/match.js";
 import {
@@ -46,6 +48,7 @@ import {
 	type FolderPlacement,
 } from "./heuristics/classifyPlacement.js";
 import type { PlacementMoveService } from "./placement-move.js";
+import { type QuarantineService, shapeFromMessageData } from "./quarantine.js";
 import { extractSnippetFromEmail } from "./snippet.js";
 import { type IImapConnection, MailConnectionError } from "./types.js";
 
@@ -186,6 +189,24 @@ export interface PlacementConfig {
 	placementMoveService: PlacementMoveService;
 }
 
+/**
+ * What body sync needs to set a message aside (issue #72). The mailbox facts
+ * are supplied by the caller because they are properties of the round, not of
+ * any one message.
+ */
+export interface QuarantineConfig {
+	quarantineService: QuarantineService;
+	mailboxId: string;
+	/**
+	 * The mailbox's stored UIDVALIDITY. Taken from the mailbox row rather than
+	 * from the open box because a round whose two disagree is paused by the
+	 * cursor guard before it reaches any UID.
+	 */
+	uidValidity: number;
+	/** Redeliveries of this batch so far, so the record says how hard we tried. */
+	attempts: number;
+}
+
 export class BodySyncService {
 	private log: BodySyncLogger;
 	private readonly filterPipeline?: FilterPipeline;
@@ -199,6 +220,7 @@ export class BodySyncService {
 		logger?: BodySyncLogger,
 		private readonly placementConfig?: PlacementConfig,
 		private readonly filterConfig?: FilterConfig,
+		private readonly quarantineConfig?: QuarantineConfig,
 	) {
 		this.log = logger ?? noopLogger;
 		this.filterPipeline = filterConfig
@@ -235,6 +257,7 @@ export class BodySyncService {
 	): Promise<SyncBodiesResult> {
 		const syncedMessageIds: string[] = [];
 		let skippedCount = 0;
+		const location = { accountId, accountConfigId, mailboxPath };
 
 		// Resolve every message up front so we can issue ONE ranged FETCH for the
 		// whole batch (the desktop-client pattern) instead of a SELECT + download
@@ -249,8 +272,34 @@ export class BodySyncService {
 		// classification failed. They are NOT in `pending` — nothing about them
 		// needs fetching — so they are merged into failedMessageIds separately.
 		const backfillFailedMessageIds: string[] = [];
+
+		// One read per round, not per message (issue #72). The list is small by
+		// design and almost always empty, so a lookup per message would put a
+		// query on the hot path for a state that is nearly never set.
+		const quarantined =
+			await this.quarantineConfig?.quarantineService.load(accountConfigId);
+
 		for (const messageId of messageIds) {
 			const message = await this.messageService.get(messageId);
+
+			// A quarantined message has no stored body, so nothing else here would
+			// stop it being fetched and re-parsed on every redelivery, forever.
+			if (
+				this.quarantineConfig &&
+				quarantined?.has(
+					this.quarantineConfig.mailboxId,
+					this.quarantineConfig.uidValidity,
+					message.uid,
+				)
+			) {
+				this.log.debug?.(
+					{ messageId, uid: message.uid },
+					"Message is quarantined, skipping",
+				);
+				skippedCount++;
+				continue;
+			}
+
 			if (message.bodyStorageKey && !force) {
 				this.log.debug?.({ messageId }, "Body already stored, skipping");
 				// The skip guard keys on the body, but classification is a separate
@@ -270,6 +319,21 @@ export class BodySyncService {
 					() => null,
 					(error: unknown) => error,
 				);
+				// The stored bytes will not parse, and they will not start parsing on
+				// a later attempt. Requeueing forever is the stall; set the message
+				// aside instead. Everything else this call can throw is storage or
+				// database work and stays on the requeue path below.
+				if (
+					backfillError instanceof BodyParseError &&
+					(await this.quarantineBodyParse(
+						message.messageId,
+						location,
+						backfillError,
+					))
+				) {
+					skippedCount++;
+					continue;
+				}
 				if (backfillError !== null) {
 					this.log.error?.(
 						{
@@ -330,6 +394,22 @@ export class BodySyncService {
 					// batch requeues just this message. A dropped connection is handled
 					// by the outer catch (fail-fast on the whole remaining batch).
 					if (isConnectionDrop(error)) throw error;
+
+					// The one error in this frame that is the message's own fault. It
+					// reaches here from the parse call and nowhere else, so the S3, DDB
+					// and SQS work this same frame wraps cannot be mistaken for it and
+					// keeps requeueing below. The row is durable before the UID leaves
+					// `pending`, so the message is never let go of unrecorded.
+					if (
+						error instanceof BodyParseError &&
+						(await this.quarantineBodyParse(messageId, location, error))
+					) {
+						pending.delete(uid);
+						skippedCount++;
+						source.resume();
+						continue;
+					}
+
 					this.log.error?.(
 						{
 							messageId,
@@ -381,6 +461,86 @@ export class BodySyncService {
 		return this.buildResult(syncedMessageIds, skippedCount, failedMessageIds);
 	}
 
+	/**
+	 * Set a message aside because its body will not parse (issue #72).
+	 *
+	 * Returns false when no record was written — either because this service was
+	 * built without a quarantine writer, or because writing it failed. The
+	 * caller then falls back to requeueing rather than dropping a message no
+	 * record survives. The failure the row carries is the parser's, and only the
+	 * parser's — see {@link BodyParseError}.
+	 *
+	 * Reading and writing the record is storage and database work, so its own
+	 * failure is contained per message the same way the save it replaces is: it
+	 * must not take down a batch that has other messages to fetch, and in the
+	 * stream loop a throw would also skip `source.resume()` and leave the
+	 * literal undrained on an otherwise usable connection.
+	 *
+	 * The fingerprint comes from the BodyPart rows metadata sync already wrote:
+	 * the body path streams raw bytes and has no FETCH result to read, and
+	 * re-reading headers the parser has just refused would be reading the very
+	 * thing that is broken.
+	 */
+	private async quarantineBodyParse(
+		messageId: string,
+		location: {
+			accountId: string;
+			accountConfigId: string;
+			mailboxPath: string;
+		},
+		error: BodyParseError,
+	): Promise<boolean> {
+		const config = this.quarantineConfig;
+		if (!config) return false;
+
+		return this.recordQuarantine(config, messageId, location, error).then(
+			() => true,
+			(writeError: unknown) => {
+				this.log.error?.(
+					{
+						messageId,
+						errorName: (writeError as { name?: string }).name,
+						error: inspect(writeError),
+					},
+					"Could not record quarantine; leaving the message for requeue",
+				);
+				return false;
+			},
+		);
+	}
+
+	private async recordQuarantine(
+		config: QuarantineConfig,
+		messageId: string,
+		location: {
+			accountId: string;
+			accountConfigId: string;
+			mailboxPath: string;
+		},
+		error: BodyParseError,
+	): Promise<void> {
+		const message = await this.messageService.get(messageId);
+		const messageData = await this.envelopeService.getMessageData(messageId);
+
+		await config.quarantineService.record(
+			{
+				accountId: location.accountId,
+				accountConfigId: location.accountConfigId,
+				mailboxId: config.mailboxId,
+				mailboxPath: location.mailboxPath,
+				uidValidity: config.uidValidity,
+				attempts: config.attempts,
+			},
+			message.uid,
+			{
+				stage: QuarantineFailureStage.BodyParse,
+				code: error.failureCode,
+				message: error.message,
+			},
+			shapeFromMessageData(message, messageData),
+		);
+	}
+
 	private buildResult(
 		syncedMessageIds: string[],
 		skippedCount: number,
@@ -749,7 +909,7 @@ export class BodySyncService {
 		}
 
 		const body = await this.storageService.retrieve(message.bodyStorageKey);
-		const parsed = await simpleParser(body);
+		const parsed = await parseMessageBody(body);
 		const classification = this.classifyMessage(parsed);
 
 		await this.messageService.update(message.messageId, classification);
@@ -1247,8 +1407,12 @@ export class BodySyncService {
 		accountConfigId: string,
 		body: Buffer,
 	): Promise<ParsedMail> {
-		// Parse the email body
-		const parsed = await simpleParser(body);
+		// The one operation on this path that can fail because of how the message
+		// is built. Its own try block lives in `parseMessageBody`, so the caller's
+		// frame — which also wraps the S3 body write, the parsed-body cache, the
+		// placement move, the label writes and the counter update — can tell the
+		// two apart and quarantine only this one (issue #72).
+		const parsed = await parseMessageBody(body);
 
 		// Extract snippet from text or HTML content
 		const snippet = extractSnippetFromEmail(

package/src/imapflow-connection.test.ts

@@ -5,6 +5,7 @@ import {
 	toInternalDate,
 	toIsoDateString,
 } from "./imapflow-connection.js";
+import type { ImapMessage } from "./types.js";
 
 describe("toIsoDateString", () => {
 	it("converts a Date to an ISO string", () => {
@@ -266,3 +267,51 @@ describe("ImapFlowConnection CONDSTORE (reader#20)", () => {
 		assert.deepStrictEqual(messages[0].flags, ["\\Seen"]);
 	});
 });
+
+describe("ImapFlowConnection message envelopes (issue #72)", () => {
+	const fetchRows = async (
+		rows: Array<Record<string, unknown>>,
+	): Promise<ImapMessage[]> => {
+		const connection = buildConnectionWithClient({
+			enabled: new Set(),
+			mailbox: fakeMailbox("INBOX", 1),
+			mailboxOpen: async (path: string) => fakeMailbox(path, 1),
+			fetch: () =>
+				(async function* () {
+					for (const row of rows) yield row;
+				})(),
+		});
+		Object.assign(connection as unknown as Record<string, unknown>, {
+			currentMailbox: "INBOX",
+		});
+		return connection.fetchMessages([1]);
+	};
+
+	it("leaves an absent ENVELOPE absent instead of synthesising an empty one", async () => {
+		// A synthesised envelope made every `if (!msg.envelope)` guard downstream
+		// unreachable, so the row was saved as a message with no sender, subject
+		// or date under a `generated:` key — indistinguishable from real mail.
+		const [message] = await fetchRows([
+			{ uid: 1, seq: 1, internalDate: new Date(0), size: 10 },
+		]);
+		assert.equal(message?.envelope, undefined);
+	});
+
+	it("still converts an envelope the server did send", async () => {
+		const [message] = await fetchRows([
+			{
+				uid: 1,
+				seq: 1,
+				internalDate: new Date(0),
+				size: 10,
+				envelope: {
+					subject: "Hello",
+					messageId: "<a@example.com>",
+					from: [{ address: "sender@example.com" }],
+				},
+			},
+		]);
+		assert.equal(message?.envelope?.subject, "Hello");
+		assert.equal(message?.envelope?.from[0]?.mailbox, "sender");
+	});
+});