@remit/mailbox-service 0.0.15 → 0.0.17

package/src/placement-move-terminal.test.ts

@@ -33,13 +33,21 @@ const buildLogger = (): {
 	};
 };
 
-const buildConnection = (present: Set<number>): IImapConnection =>
+const buildConnection = (
+	present: Set<number>,
+	fetchDrops: Set<number> = new Set(),
+): IImapConnection =>
 	({
 		openBox: async () => ({}) as never,
 		fetchMessages: async (uids: number[]) =>
 			uids
-				.filter((uid) => present.has(uid))
+				.filter((uid) => present.has(uid) && !fetchDrops.has(uid))
 				.map((uid) => ({ uid }) as unknown as never),
+		search: async (criteria: unknown[]) => {
+			const [, value] = (criteria as Array<[string, string]>)[0];
+			const uid = Number(value);
+			return present.has(uid) ? [uid] : [];
+		},
 	}) as unknown as IImapConnection;
 
 describe("resolveExhaustedPlacementMoveFailure — the two terminal outcomes (mirrors #1270 for placement moves)", () => {
@@ -148,6 +156,54 @@ describe("resolveExhaustedPlacementMoveFailure — the two terminal outcomes (mi
 		);
 	});
 
+	it("a dropped FETCH row is not absence: the message is still at the source — marker and local rows survive", async () => {
+		const deletedMessages: string[] = [];
+		const deletedThreadMessages: unknown[] = [];
+		const markerDeletes: string[] = [];
+		const { log, errors } = buildLogger();
+
+		const deps: ResolveExhaustedPlacementMoveDeps = {
+			markerService: {
+				delete: async (id: string) => {
+					markerDeletes.push(id);
+				},
+			},
+			messageService: {
+				delete: async (id: string) => {
+					deletedMessages.push(id);
+				},
+			} as unknown as Pick<IMessageRepository, "delete">,
+			threadMessageService: {
+				findAllByMessageId: async () => [
+					{ accountConfigId: "cfg-1", threadMessageId: "tm-msg-live" },
+				],
+				deleteMany: async (keys: unknown[]) => {
+					deletedThreadMessages.push(...keys);
+				},
+			} as unknown as Pick<
+				IThreadMessageRepository,
+				"findAllByMessageId" | "deleteMany"
+			>,
+			log,
+		};
+
+		const result = await resolveExhaustedPlacementMoveFailure(deps, {
+			accountId: "acc-1",
+			accountConfigId: "cfg-1",
+			messageId: "msg-live",
+			uid: 303,
+			sourceMailboxPath: "INBOX",
+			getConnection: async () =>
+				buildConnection(new Set([303]), new Set([303])),
+		});
+
+		assert.equal(result.outcome, "broken");
+		assert.deepEqual(markerDeletes, []);
+		assert.deepEqual(deletedMessages, []);
+		assert.deepEqual(deletedThreadMessages, []);
+		assert.ok(errors.some((e) => e.obj.alert === "placement_move_failed"));
+	});
+
 	it("never throws — both outcomes are terminal, the caller always acks", async () => {
 		const { log } = buildLogger();
 		const deps: ResolveExhaustedPlacementMoveDeps = {

package/src/placement-move-terminal.ts

@@ -1,3 +1,4 @@
+import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
 import type { PlacementMoveLogger } from "./placement-move.js";
 import {
 	reconcileStaleMessage,
@@ -33,9 +34,11 @@ export interface ResolveExhaustedPlacementMoveResult {
  * taxonomy (epic #1281 invariant 3) — no third, softer outcome.
  *
  * 1. RECONCILED (expected) — the message no longer exists at its pending-move
- *    source on IMAP. Per invariant 2, an external delete supersedes the
- *    marker entirely: the marker is dropped and the stale Message/ThreadMessage
- *    rows are deleted via {@link reconcileStaleMessage}. This is also the
+ *    source on IMAP, confirmed by {@link isMessageGoneFromOpenMailbox} rather
+ *    than by a FETCH coming back empty. Per invariant 2, an external delete
+ *    supersedes the marker entirely: the marker is dropped and the stale
+ *    Message/ThreadMessage rows are deleted via
+ *    {@link reconcileStaleMessage}. This is also the
  *    outcome for the (rarer, functionally indistinguishable from here) case
  *    where a foreign client moved the message elsewhere — either way, our
  *    prediction no longer holds, and the marker cannot be honoured. Metric
@@ -49,6 +52,15 @@ export interface ResolveExhaustedPlacementMoveResult {
  *    `alert`-shaped entry for an operator alarm; never re-thrown (terminal —
  *    the caller acks either way, since retrying a stale or permanently-broken
  *    move can never succeed).
+ *
+ * An operator reading `placement_move_failed` should know one case where the
+ * message is not actually at the source: a message another client expunged
+ * mid-session can answer an empty FETCH while the server still lists its UID
+ * in SEARCH, until it is allowed to send the untagged EXPUNGE. That message
+ * lands in BROKEN, and BROKEN is terminal — the marker stays pending and the
+ * alert stands until someone clears it. The reverse mistake deletes live
+ * mail, so the cost is paid deliberately: a stale alert is recoverable, a
+ * deleted message is not.
  */
 export const resolveExhaustedPlacementMoveFailure = async (
 	deps: ResolveExhaustedPlacementMoveDeps,
@@ -65,9 +77,8 @@ export const resolveExhaustedPlacementMoveFailure = async (
 
 	const connection = await getConnection();
 	await connection.openBox(sourceMailboxPath);
-	const found = await connection.fetchMessages([uid]);
 
-	if (found.length === 0) {
+	if (await isMessageGoneFromOpenMailbox(connection, uid)) {
 		await deps.markerService.delete(messageId);
 		const { threadMessagesDeleted } = await reconcileStaleMessage(
 			deps,

package/src/quarantine.test.ts

@@ -0,0 +1,191 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	IMailboxSpecialUseRepository,
+	IQuarantineRepository,
+	MessageData,
+	MessageItem,
+	QuarantineItem,
+	QuarantineUpsertInput,
+} from "@remit/data-ports";
+import {
+	QuarantinedUids,
+	QuarantineService,
+	resolveMailboxRole,
+	shapeFromMessageData,
+} from "./quarantine.js";
+
+const noopLog = { info: () => {}, warn: () => {} };
+
+const buildService = (
+	specialUse: string[] = [],
+): {
+	service: QuarantineService;
+	writes: QuarantineUpsertInput[];
+	listed: string[];
+} => {
+	const writes: QuarantineUpsertInput[] = [];
+	const listed: string[] = [];
+	const repository = {
+		listByAccountConfigId: async (accountConfigId: string) => {
+			listed.push(accountConfigId);
+			return [] as QuarantineItem[];
+		},
+		upsert: async (input: QuarantineUpsertInput) => {
+			writes.push(input);
+		},
+	} satisfies IQuarantineRepository;
+
+	const mailboxSpecialUseService = {
+		listByMailboxId: async () => specialUse.map((use) => ({ specialUse: use })),
+	} as unknown as IMailboxSpecialUseRepository;
+
+	return {
+		service: new QuarantineService(
+			repository,
+			mailboxSpecialUseService,
+			"sha-abc",
+			noopLog,
+		),
+		writes,
+		listed,
+	};
+};
+
+const context = {
+	accountId: "acct-1",
+	accountConfigId: "cfg-1",
+	mailboxId: "mbx-1",
+	mailboxPath: "INBOX",
+	uidValidity: 1_712_000_000,
+	attempts: 2,
+};
+
+const failure = {
+	stage: "BodyParse" as const,
+	code: "UnreadableBody" as const,
+	message: "the parser said no",
+};
+
+describe("QuarantineService.record", () => {
+	it("stamps the worker build, not the client's", async () => {
+		const { service, writes } = buildService();
+		await service.record(context, 40217, failure);
+		assert.equal(writes[0]?.workerVersion, "sha-abc");
+	});
+
+	it("names the message by mailbox, uidValidity and uid", async () => {
+		const { service, writes } = buildService();
+		await service.record(context, 40217, failure);
+		assert.deepEqual(
+			{
+				mailboxId: writes[0]?.mailboxId,
+				uidValidity: writes[0]?.uidValidity,
+				uid: writes[0]?.uid,
+			},
+			{ mailboxId: "mbx-1", uidValidity: 1_712_000_000, uid: 40217 },
+		);
+	});
+
+	it("supplies no quarantineId, so the random-id default is unreachable", async () => {
+		const { service, writes } = buildService();
+		await service.record(context, 40217, failure);
+		assert.ok(!("quarantineId" in (writes[0] as object)));
+	});
+
+	it("omits an absent diagnostic rather than writing a null through", async () => {
+		const { service, writes } = buildService();
+		await service.record(
+			{ ...context, mailboxPath: "Clients/Acme" },
+			1,
+			failure,
+		);
+		assert.ok(!("failurePartPath" in (writes[0] as object)));
+		assert.ok(!("mailboxRole" in (writes[0] as object)));
+	});
+
+	it("reads the folder's role off the server's SPECIAL-USE", async () => {
+		const { service, writes } = buildService(["Junk"]);
+		await service.record({ ...context, mailboxPath: "Spam" }, 1, failure);
+		assert.equal(writes[0]?.mailboxRole, "Junk");
+	});
+});
+
+describe("resolveMailboxRole", () => {
+	it("gives INBOX its role without a SPECIAL-USE flag, which it never has", () => {
+		assert.equal(resolveMailboxRole("INBOX", []), "Inbox");
+	});
+
+	it("leaves a plain folder roleless instead of inventing one", () => {
+		assert.equal(resolveMailboxRole("Clients/Acme", []), undefined);
+	});
+
+	it("drops a SPECIAL-USE that is not a canonical role", () => {
+		assert.equal(resolveMailboxRole("Priority", ["Important"]), undefined);
+	});
+});
+
+describe("QuarantinedUids", () => {
+	const entries = [
+		{ mailboxId: "mbx-1", uidValidity: 10, uid: 5 },
+	] as QuarantineItem[];
+
+	it("matches a uid on the same mailbox and UIDVALIDITY", () => {
+		assert.equal(new QuarantinedUids(entries).has("mbx-1", 10, 5), true);
+	});
+
+	it("does not match the same uid on a new UIDVALIDITY, which is a different message", () => {
+		assert.equal(new QuarantinedUids(entries).has("mbx-1", 11, 5), false);
+	});
+
+	it("does not match the same uid in another mailbox", () => {
+		assert.equal(new QuarantinedUids(entries).has("mbx-2", 10, 5), false);
+	});
+});
+
+describe("shapeFromMessageData", () => {
+	const message = {
+		rfc822Size: 2048,
+		messageIdHeader: "<xyz@example.com>",
+	} as MessageItem;
+
+	const data = {
+		bodyPart: [
+			{
+				bodyPartId: "bp-0",
+				partPath: "0",
+				mediaType: "multipart",
+				mediaSubtype: "alternative",
+				transferEncoding: "7bit",
+			},
+			{
+				bodyPartId: "bp-1",
+				partPath: "1",
+				mediaType: "text",
+				mediaSubtype: "plain",
+				transferEncoding: "quoted-printable",
+			},
+		],
+		bodyPartParameter: [
+			{
+				bodyPartId: "bp-0",
+				parameterName: "charset",
+				parameterValue: "iso-8859-1",
+			},
+		],
+	} as unknown as MessageData;
+
+	it("rebuilds the tree from the rows metadata sync already wrote", () => {
+		assert.deepEqual(shapeFromMessageData(message, data).structure, [
+			{ depth: 0, contentType: "multipart/alternative" },
+			{ depth: 1, contentType: "text/plain" },
+		]);
+	});
+
+	it("takes the root part's declared charset and encoding", () => {
+		const shape = shapeFromMessageData(message, data);
+		assert.equal(shape.charset, "iso-8859-1");
+		assert.equal(shape.transferEncoding, "7bit");
+		assert.equal(shape.contentType, "multipart/alternative");
+	});
+});

package/src/quarantine.ts

@@ -0,0 +1,228 @@
+import type {
+	IMailboxSpecialUseRepository,
+	IQuarantineRepository,
+	MessageData,
+	MessageItem,
+	QuarantineItem,
+	QuarantineMimeNodeItem,
+} from "@remit/data-ports";
+import { quarantineMessageIdHash, ROOT_PART_PATH } from "@remit/data-ports/id";
+import { CanonicalMailboxRole } from "@remit/domain-enums";
+
+type CanonicalRole = NonNullable<QuarantineItem["mailboxRole"]>;
+type FailureStage = QuarantineItem["failureStage"];
+type FailureCode = QuarantineItem["failureCode"];
+
+const CANONICAL_ROLES = new Set<string>(Object.values(CanonicalMailboxRole));
+
+/**
+ * The canonical role of the folder a quarantined message arrived in, read from
+ * the server's own RFC 6154 SPECIAL-USE declaration — the same source the
+ * appointment flow seeds from. A folder the server declares nothing about has
+ * no role, which is the normal state for a plain folder and is why the field
+ * is optional. `Important` is a SPECIAL-USE with no canonical role and drops
+ * out here rather than being invented into one.
+ */
+export const resolveMailboxRole = (
+	mailboxPath: string,
+	specialUse: readonly string[],
+): CanonicalRole | undefined => {
+	if (mailboxPath.toUpperCase() === "INBOX") return CanonicalMailboxRole.Inbox;
+	const matched = specialUse.find((use) => CANONICAL_ROLES.has(use));
+	return matched as CanonicalRole | undefined;
+};
+
+/** Everything about the quarantine that is the same for every uid in a round. */
+export interface QuarantineContext {
+	accountId: string;
+	accountConfigId: string;
+	mailboxId: string;
+	mailboxPath: string;
+	uidValidity: number;
+	/** Rounds tried before the message was set aside. */
+	attempts: number;
+}
+
+/**
+ * The repro fingerprint: what the message declared itself to be. Every field
+ * is optional-by-absence because a message can fail before its shape was read,
+ * and a required column would force the writer to invent a value or drop the
+ * record.
+ */
+export interface QuarantineMessageShape {
+	contentType?: string;
+	transferEncoding?: string;
+	charset?: string;
+	sizeBytes?: number;
+	structure: QuarantineMimeNodeItem[];
+	messageIdHash?: string;
+}
+
+export interface QuarantineFailure {
+	stage: FailureStage;
+	code: FailureCode;
+	/** Parser error text. Stored, shown on screen, never published. */
+	message: string;
+	partPath?: string;
+}
+
+const EMPTY_SHAPE: QuarantineMessageShape = { structure: [] };
+
+/**
+ * Message shape off the rows metadata sync already wrote. The body path has no
+ * FETCH result to read — it streams raw bytes — but the MIME tree it needs was
+ * walked into BodyPart rows when the message's metadata was synced, so the
+ * fingerprint comes from there rather than from re-reading headers the parser
+ * has just refused.
+ */
+export const shapeFromMessageData = (
+	message: MessageItem,
+	data: MessageData,
+): QuarantineMessageShape => {
+	const parts = [...data.bodyPart].sort((a, b) =>
+		a.partPath.localeCompare(b.partPath, "en", { numeric: true }),
+	);
+	const root = parts.find((part) => part.partPath === ROOT_PART_PATH);
+	const rootCharset = root
+		? data.bodyPartParameter.find(
+				(param) =>
+					param.bodyPartId === root.bodyPartId &&
+					param.parameterName.toLowerCase() === "charset",
+			)?.parameterValue
+		: undefined;
+
+	return {
+		...(root ? { contentType: `${root.mediaType}/${root.mediaSubtype}` } : {}),
+		...(root ? { transferEncoding: root.transferEncoding } : {}),
+		...(rootCharset ? { charset: rootCharset } : {}),
+		...(message.rfc822Size ? { sizeBytes: message.rfc822Size } : {}),
+		structure: parts.map((part) => ({
+			depth: partPathDepth(part.partPath),
+			contentType: `${part.mediaType}/${part.mediaSubtype}`,
+		})),
+		...hashOf(message.messageIdHeader),
+	};
+};
+
+const partPathDepth = (partPath: string): number =>
+	partPath === ROOT_PART_PATH ? 0 : partPath.split(".").length;
+
+const hashOf = (
+	messageIdHeader: string | undefined,
+): { messageIdHash?: string } => {
+	const messageIdHash = quarantineMessageIdHash(messageIdHeader);
+	return messageIdHash ? { messageIdHash } : {};
+};
+
+/**
+ * The set of messages already set aside, held for the duration of one sync
+ * round. A round loads it once and filters against it in memory: the list is
+ * small by design — a growing one is a bug being reported, not a page to
+ * paginate — and a lookup per message would put a query on the hot path for a
+ * state that is almost always empty.
+ */
+export class QuarantinedUids {
+	private readonly keys: ReadonlySet<string>;
+
+	constructor(entries: readonly QuarantineItem[]) {
+		this.keys = new Set(
+			entries.map((entry) =>
+				uidKey(entry.mailboxId, entry.uidValidity, entry.uid),
+			),
+		);
+	}
+
+	get size(): number {
+		return this.keys.size;
+	}
+
+	has(mailboxId: string, uidValidity: number, uid: number): boolean {
+		return this.keys.has(uidKey(mailboxId, uidValidity, uid));
+	}
+}
+
+const uidKey = (mailboxId: string, uidValidity: number, uid: number): string =>
+	`${mailboxId}:${uidValidity}:${uid}`;
+
+export interface QuarantineLogger {
+	info(obj: Record<string, unknown>, msg: string): void;
+	warn(obj: Record<string, unknown>, msg: string): void;
+}
+
+/**
+ * Writes the record a message becomes when the sync path cannot apply it
+ * (issue #72), and loads the set a round filters against.
+ *
+ * Only a message defect reaches this class. Deciding that is the caller's job
+ * and it is made at exactly one kind of catch site — one narrow enough that
+ * the error can only have come from the message itself. An S3, queue or
+ * database failure propagates; writing one here would advance a cursor past
+ * mail that is fine and tell the user it was unreadable.
+ */
+export class QuarantineService {
+	constructor(
+		private readonly repository: IQuarantineRepository,
+		private readonly mailboxSpecialUseService: IMailboxSpecialUseRepository,
+		private readonly workerVersion: string,
+		private readonly log: QuarantineLogger,
+	) {}
+
+	async load(accountConfigId: string): Promise<QuarantinedUids> {
+		return new QuarantinedUids(
+			await this.repository.listByAccountConfigId(accountConfigId),
+		);
+	}
+
+	/**
+	 * Set a message aside. Resolves only once the row is durable, because the
+	 * caller's next act is to let a cursor move past this uid — and a cursor
+	 * that moves past work no record survives is the silent loss this whole
+	 * feature exists to end.
+	 */
+	async record(
+		context: QuarantineContext,
+		uid: number,
+		failure: QuarantineFailure,
+		shape: QuarantineMessageShape = EMPTY_SHAPE,
+	): Promise<void> {
+		// Resolved here rather than per round: a round almost never writes a row,
+		// so the lookup belongs on the write and not on the hot path.
+		const specialUse = await this.mailboxSpecialUseService.listByMailboxId(
+			context.mailboxId,
+		);
+		const mailboxRole = resolveMailboxRole(
+			context.mailboxPath,
+			specialUse.map((entry) => entry.specialUse),
+		);
+
+		await this.repository.upsert({
+			accountConfigId: context.accountConfigId,
+			accountId: context.accountId,
+			mailboxId: context.mailboxId,
+			uidValidity: context.uidValidity,
+			uid,
+			...(mailboxRole ? { mailboxRole } : {}),
+			mailboxPath: context.mailboxPath,
+			quarantinedAt: Date.now(),
+			attempts: context.attempts,
+			failureStage: failure.stage,
+			failureCode: failure.code,
+			failureMessage: failure.message,
+			...(failure.partPath ? { failurePartPath: failure.partPath } : {}),
+			workerVersion: this.workerVersion,
+			...shape,
+		});
+
+		this.log.warn(
+			{
+				mailboxId: context.mailboxId,
+				mailboxPath: context.mailboxPath,
+				uid,
+				uidValidity: context.uidValidity,
+				failureStage: failure.stage,
+				failureCode: failure.code,
+			},
+			"Message quarantined; cursor may advance past it",
+		);
+	}
+}