@remit/mailbox-service 0.0.111 → 0.0.113

package/src/body-part-mapper.ts

@@ -1,360 +0,0 @@
-/**
- * Pair every persisted `BodyPart` row with the decoded bytes mailparser
- * produced for the same MIME node. Total function: returns one
- * `BodyPartContentPair` per non-multipart leaf, never undefined, never
- * throws on well-formed input.
- *
- * Source of truth (per issue #395):
- *   - `partPath`, `contentType`, `contentId`, `dispositionFilename`,
- *     `sizeOctets` — owned by BODYSTRUCTURE (the `BodyPart` row).
- *   - Decoded bytes — owned by mailparser (`parsed.text`, `parsed.html`,
- *     `parsed.attachments[i].content`).
- *   - `parsed.attachments[i].partId` is also dot-numbered (set at runtime
- *     in `mail-parser.js:860`; missing from `@types/mailparser`; surfaced
- *     here via `./types/mailparser-augment.d.ts`). Structural pairing on
- *     `partId === partPath` is the dominant strategy for non-text leaves.
- *
- * Pipeline (per leaf in declaration order; each step consumes one
- * attachment when it pairs, subsequent steps work on the residual set):
- *   1. text/plain → first leaf gets `parsed.text` (empty Buffer if
- *      `sizeOctets === 0`); subsequent text/plain leaves get empty.
- *   2. text/html → first leaf gets `parsed.html` (empty Buffer if
- *      `sizeOctets === 0`); subsequent text/html leaves get empty.
- *   3. Non-text leaves with `sizeOctets === 0` short-circuit to an empty
- *      Buffer without consuming an attachment — symmetric with the text
- *      path, so the positional fallback can't hand a zero-sized leaf real
- *      bytes that belong to another leaf.
- *   4. Remaining non-text leaves try, in order:
- *      a. `attachment.partId === bodyPart.partPath`
- *      b. contentId match (case + angle-insensitive)
- *      c. dispositionFilename match (case-insensitive)
- *      d. content-type compatibility class: exact → binary-family
- *         (any non-text/*) → text-family
- *      e. positional fallback: next unconsumed attachment in declaration
- *         order. Logs at `debug` so operators can see "we guessed"
- *         pairings without spamming higher levels.
- *   5. Anything still unpaired → zero-byte Buffer + structured `warn` log.
- *
- * The mapper assumes well-formed input — malformed MIME is rejected
- * upstream in `mime-walker.ts`. If a leaf truly has no source bytes
- * available (e.g. mailparser tolerated a malformed message that
- * BODYSTRUCTURE disagrees with on leaf count), it gets the empty-Buffer
- * fallback rather than a throw, so the call site stays simple.
- */
-
-/// <reference path="./types/mailparser-augment.d.ts" />
-
-import { Buffer } from "node:buffer";
-import type { BodyPartItem } from "@remit/data-ports";
-import type { Attachment, ParsedMail } from "mailparser";
-
-export interface BodyPartContentPair {
-	partPath: string;
-	contentType: string;
-	content: Buffer;
-}
-
-export type MapperInput = Pick<
-	BodyPartItem,
-	| "partPath"
-	| "isMultipart"
-	| "mediaType"
-	| "mediaSubtype"
-	| "contentId"
-	| "dispositionFilename"
-	| "disposition"
-> & {
-	/**
-	 * Bytes the BODYSTRUCTURE leaf declares. Optional so hand-built test
-	 * inputs don't have to fabricate it — in production every BodyPart row
-	 * carries this field. `sizeOctets === 0` is the explicit empty-leaf
-	 * signal that text-leaf routing uses to produce a zero-byte Buffer
-	 * (see fixture 15-empty-text-part).
-	 */
-	sizeOctets?: number;
-};
-
-export interface MapperLogger {
-	warn(obj: Record<string, unknown>, msg: string): void;
-	debug?(obj: Record<string, unknown>, msg: string): void;
-}
-
-export interface MapperContext {
-	messageId?: string;
-	logger?: MapperLogger;
-}
-
-const EMPTY_BUFFER = Buffer.alloc(0);
-
-const stripAngles = (value: string): string => value.replace(/^<+|>+$/g, "");
-
-const buildContentType = (
-	part: Pick<BodyPartItem, "mediaType" | "mediaSubtype">,
-): string => `${part.mediaType.toLowerCase()}/${part.mediaSubtype}`;
-
-const textToBuffer = (value: string | false | undefined): Buffer => {
-	if (typeof value !== "string" || value.length === 0) return EMPTY_BUFFER;
-	return Buffer.from(value, "utf8");
-};
-
-const isTextStar = (contentType: string): boolean =>
-	contentType.toLowerCase().startsWith("text/");
-
-/** Match `partId === partPath` against an unconsumed attachment. */
-const findByPartId = (
-	attachments: readonly Attachment[],
-	consumed: ReadonlySet<number>,
-	partPath: string,
-): number | null => {
-	for (let i = 0; i < attachments.length; i++) {
-		if (consumed.has(i)) continue;
-		const att = attachments[i];
-		if (typeof att.partId !== "string") continue;
-		if (att.partId === partPath) return i;
-	}
-	return null;
-};
-
-/** Match by contentId (case + angle-insensitive). */
-const findByContentId = (
-	attachments: readonly Attachment[],
-	consumed: ReadonlySet<number>,
-	contentId: string,
-): number | null => {
-	const target = stripAngles(contentId).toLowerCase();
-	for (let i = 0; i < attachments.length; i++) {
-		if (consumed.has(i)) continue;
-		const att = attachments[i];
-		const candidates = [att.cid, att.contentId];
-		for (const cand of candidates) {
-			if (typeof cand !== "string") continue;
-			if (stripAngles(cand).toLowerCase() === target) return i;
-		}
-	}
-	return null;
-};
-
-/** Match by dispositionFilename (case-insensitive). */
-const findByFilename = (
-	attachments: readonly Attachment[],
-	consumed: ReadonlySet<number>,
-	filename: string,
-): number | null => {
-	const target = filename.toLowerCase();
-	for (let i = 0; i < attachments.length; i++) {
-		if (consumed.has(i)) continue;
-		const att = attachments[i];
-		if (typeof att.filename !== "string") continue;
-		if (att.filename.toLowerCase() === target) return i;
-	}
-	return null;
-};
-
-/**
- * Content-type compatibility class. The BodyPart row's content-type is the
- * sender's verbatim label; mailparser's `att.contentType` is sniffed from
- * filename extension. They often disagree (the Odido bug — postmortem
- * #394 — is `application/octet-stream` vs `application/pdf`).
- *
- * Order:
- *   1. Exact match (`application/pdf` ↔ `application/pdf`).
- *   2. Binary family — leaf is non-`text/*`, attachment is non-`text/*`.
- *      Covers octet-stream ↔ pdf, octet-stream ↔ image, etc.
- *   3. Text family — leaf is `text/*`, attachment is `text/*`. Catches
- *      `text/calendar` / `text/csv` after the inline `text/plain` and
- *      `text/html` slots are taken.
- */
-const findByContentTypeClass = (
-	attachments: readonly Attachment[],
-	consumed: ReadonlySet<number>,
-	contentType: string,
-): number | null => {
-	const target = contentType.toLowerCase();
-	const leafIsText = isTextStar(target);
-
-	for (let i = 0; i < attachments.length; i++) {
-		if (consumed.has(i)) continue;
-		const att = attachments[i];
-		if (att.contentType?.toLowerCase() === target) return i;
-	}
-
-	if (!leafIsText) {
-		for (let i = 0; i < attachments.length; i++) {
-			if (consumed.has(i)) continue;
-			const att = attachments[i];
-			const attType = att.contentType?.toLowerCase();
-			if (!attType) continue;
-			if (!isTextStar(attType)) return i;
-		}
-	} else {
-		for (let i = 0; i < attachments.length; i++) {
-			if (consumed.has(i)) continue;
-			const att = attachments[i];
-			const attType = att.contentType?.toLowerCase();
-			if (!attType) continue;
-			if (isTextStar(attType)) return i;
-		}
-	}
-
-	return null;
-};
-
-/** Positional fallback: next unconsumed attachment in declaration order. */
-const findPositional = (
-	attachments: readonly Attachment[],
-	consumed: ReadonlySet<number>,
-): number | null => {
-	for (let i = 0; i < attachments.length; i++) {
-		if (!consumed.has(i)) return i;
-	}
-	return null;
-};
-
-const pairNonTextLeaf = (
-	part: MapperInput,
-	attachments: readonly Attachment[],
-	consumed: Set<number>,
-	context: MapperContext,
-): { content: Buffer; paired: true } | { paired: false } => {
-	const contentType = buildContentType(part);
-
-	// Symmetry with the text-leaf path: a leaf whose BODYSTRUCTURE-declared
-	// size is zero has no bytes to find, so do not consume a residual
-	// attachment for it (the positional fallback would otherwise hand it
-	// real bytes belonging to another leaf — see PR D, review note b).
-	if (part.sizeOctets === 0) {
-		return { content: EMPTY_BUFFER, paired: true };
-	}
-
-	const byPartId = findByPartId(attachments, consumed, part.partPath);
-	if (byPartId !== null) {
-		consumed.add(byPartId);
-		return { content: attachments[byPartId].content, paired: true };
-	}
-
-	if (part.contentId) {
-		const byCid = findByContentId(attachments, consumed, part.contentId);
-		if (byCid !== null) {
-			consumed.add(byCid);
-			return { content: attachments[byCid].content, paired: true };
-		}
-	}
-
-	if (part.dispositionFilename) {
-		const byName = findByFilename(
-			attachments,
-			consumed,
-			part.dispositionFilename,
-		);
-		if (byName !== null) {
-			consumed.add(byName);
-			return { content: attachments[byName].content, paired: true };
-		}
-	}
-
-	const byType = findByContentTypeClass(attachments, consumed, contentType);
-	if (byType !== null) {
-		consumed.add(byType);
-		return { content: attachments[byType].content, paired: true };
-	}
-
-	const positional = findPositional(attachments, consumed);
-	if (positional !== null) {
-		consumed.add(positional);
-		const chosen = attachments[positional];
-		// Audit log so operators can see "we guessed" pairings when a
-		// well-formed message takes the structural-fallback path. Helps
-		// triage cases where mailparser and BODYSTRUCTURE disagree more
-		// than the earlier matchers can handle.
-		context.logger?.debug?.(
-			{
-				messageId: context.messageId,
-				partPath: part.partPath,
-				leafContentType: contentType,
-				chosenAttachmentPartId: chosen.partId ?? null,
-				chosenAttachmentContentType: chosen.contentType ?? null,
-				chosenAttachmentFilename: chosen.filename ?? null,
-			},
-			"body-part-mapper: positional fallback paired leaf with next residual attachment",
-		);
-		return { content: chosen.content, paired: true };
-	}
-
-	return { paired: false };
-};
-
-/**
- * Pair body-part rows with mailparser bytes. Returns one pair per
- * non-multipart leaf, in the order the leaves appear in `bodyParts`.
- *
- * Never throws on well-formed input. Leaves with no source bytes get an
- * empty Buffer; the call site does not need a try/catch.
- */
-export const mapBodyPartsToContent = (
-	bodyParts: readonly MapperInput[],
-	parsed: ParsedMail,
-	context: MapperContext = {},
-): BodyPartContentPair[] => {
-	const pairs: BodyPartContentPair[] = [];
-	const attachments = parsed.attachments ?? [];
-	const consumed = new Set<number>();
-	let textRouted = false;
-	let htmlRouted = false;
-
-	for (const part of bodyParts) {
-		if (part.isMultipart) continue;
-
-		const contentType = buildContentType(part);
-		const isPlain = part.mediaType === "TEXT" && part.mediaSubtype === "plain";
-		const isHtml = part.mediaType === "TEXT" && part.mediaSubtype === "html";
-
-		// `sizeOctets === 0` is the explicit empty-leaf signal; an undefined
-		// sizeOctets (hand-built test input) is treated as non-empty so we
-		// fall through to the text-routing path.
-		const isEmptyLeaf = part.sizeOctets === 0;
-
-		if (isPlain) {
-			const content =
-				!textRouted && !isEmptyLeaf ? textToBuffer(parsed.text) : EMPTY_BUFFER;
-			textRouted = true;
-			pairs.push({ partPath: part.partPath, contentType, content });
-			continue;
-		}
-
-		if (isHtml) {
-			const content =
-				!htmlRouted && !isEmptyLeaf
-					? textToBuffer(
-							typeof parsed.html === "string" ? parsed.html : undefined,
-						)
-					: EMPTY_BUFFER;
-			htmlRouted = true;
-			pairs.push({ partPath: part.partPath, contentType, content });
-			continue;
-		}
-
-		const result = pairNonTextLeaf(part, attachments, consumed, context);
-		if (result.paired) {
-			pairs.push({
-				partPath: part.partPath,
-				contentType,
-				content: result.content,
-			});
-			continue;
-		}
-
-		context.logger?.warn(
-			{
-				messageId: context.messageId,
-				partPath: part.partPath,
-				contentType,
-				contentId: part.contentId,
-				filename: part.dispositionFilename,
-			},
-			"body-part-mapper: no source bytes for leaf; pairing with zero-byte buffer",
-		);
-		pairs.push({ partPath: part.partPath, contentType, content: EMPTY_BUFFER });
-	}
-
-	return pairs;
-};

package/src/body-sync-auto-read-writeonce.test.ts

@@ -1,219 +0,0 @@
-/**
- * Issue #499: `Address.flags.unsubscribed` auto-mark-read is a first-
- * classification decision, like `Message.category` (#355) and the placement
- * verdict (#383). Re-deriving it on the two shipped re-entrant paths —
- * `fetchAndGetBody`'s `NoSuchKey` fallback and `syncBodies(..., force: true)`
- * — undid a user who had deliberately marked such a message unread.
- *
- * The re-entrant fixtures below keep `flags.unsubscribed` set, so a regression
- * that drops the guard shows up as a real `FlagQueueService.markAsRead`
- * round-trip against a message the user owns, not as a fixture that happens to
- * disagree with the flag.
- */
-
-import assert from "node:assert/strict";
-import { Readable } from "node:stream";
-import { describe, it } from "node:test";
-import type {
-	AddressItem,
-	IAddressRepository,
-	IEnvelopeRepository,
-	IMessageRepository,
-	IThreadMessageRepository,
-	MessageItem,
-	UpdateMessageInput,
-} from "@remit/data-ports";
-import type { StorageService } from "@remit/storage-service";
-import { BodySyncService } from "./body-sync.js";
-import type { FlagQueueService } from "./flag-queue.js";
-import type { IImapConnection } from "./types.js";
-
-const UNSUBSCRIBED_SENDER = "newsletter@example.com";
-
-const PLAIN_EML = Buffer.from(
-	[
-		`From: Newsletter <${UNSUBSCRIBED_SENDER}>`,
-		"To: me@example.com",
-		"Subject: This week",
-		"List-Unsubscribe: <mailto:stop@example.com>",
-		"Content-Type: text/plain",
-		"",
-		"body",
-	].join("\r\n"),
-);
-
-interface MarkReadCall {
-	accountConfigId: string;
-	messageId: string;
-	accountId: string;
-}
-
-interface Harness {
-	service: BodySyncService;
-	markReadCalls: MarkReadCall[];
-}
-
-const buildHarness = (
-	message: Partial<MessageItem> & Pick<MessageItem, "messageId">,
-	retrieve: () => Promise<Buffer>,
-): Harness => {
-	const markReadCalls: MarkReadCall[] = [];
-
-	const messageRow = {
-		uid: 1,
-		mailboxId: "mb-inbox",
-		...message,
-	} as unknown as MessageItem;
-
-	const messageService = {
-		get: async () => messageRow,
-		update: async (_messageId: string, input: UpdateMessageInput) => {
-			Object.assign(messageRow, input);
-		},
-	} as unknown as IMessageRepository;
-
-	const threadMessageService = {
-		findAllByMessageId: async () => [
-			{
-				threadMessageId: "tm-1",
-				messageId: message.messageId,
-				mailboxId: messageRow.mailboxId,
-				sentDate: 1,
-				isRead: false,
-				isDeleted: false,
-				hasStars: false,
-				hasAttachment: false,
-			},
-		],
-		update: async () => {},
-	} as unknown as IThreadMessageRepository;
-
-	const storageService = {
-		retrieve,
-		storeMessageBody: async () => ({ uri: `s3://bodies/${message.messageId}` }),
-		storeMessageBodyStream: async () => ({
-			uri: `s3://bodies/${message.messageId}`,
-		}),
-		storeParsedBody: async () => {},
-		listBodyParts: async () => [],
-	} as unknown as StorageService;
-
-	const addressService = {
-		getAddress: async () =>
-			({
-				flags: { unsubscribed: { value: true, setAt: 1 } },
-			}) as unknown as AddressItem,
-		incrementInboundCount: async () => {},
-	} as unknown as IAddressRepository;
-
-	const envelopeService = {
-		listBodyParts: async () => [],
-	} as unknown as IEnvelopeRepository;
-
-	const flagQueueService = {
-		markAsRead: async (
-			accountConfigId: string,
-			messageId: string,
-			accountId: string,
-		) => {
-			markReadCalls.push({ accountConfigId, messageId, accountId });
-		},
-	} as unknown as FlagQueueService;
-
-	const service = new BodySyncService(
-		messageService,
-		storageService,
-		threadMessageService,
-		addressService,
-		envelopeService,
-		{ info: () => {}, error: () => {} },
-		undefined,
-		undefined,
-		undefined,
-		{ flagQueueService },
-	);
-
-	return { service, markReadCalls };
-};
-
-const noSuchKeyError = () =>
-	Object.assign(new Error("missing"), { name: "NoSuchKey" });
-
-const bodyConnection = () =>
-	({
-		openBox: async () => {},
-		fetchMessageBody: async () => PLAIN_EML,
-		async *fetchMessageBodies(uids: number[]) {
-			for (const uid of uids) {
-				yield { uid, source: Readable.from([PLAIN_EML]) };
-			}
-		},
-	}) as unknown as IImapConnection;
-
-describe("unsubscribed auto-mark-read is decided once (issue #499)", () => {
-	it("marks a message read on the pass that first classifies it", async () => {
-		const harness = buildHarness({ messageId: "m-1" }, async () => {
-			throw new Error("no body stored yet; must not retrieve");
-		});
-
-		await harness.service.fetchAndGetBody(
-			"m-1",
-			"acc-1",
-			"cfg-1",
-			{ fullPath: "INBOX", placement: "inbox" },
-			async () => bodyConnection(),
-		);
-
-		assert.deepEqual(harness.markReadCalls, [
-			{ accountConfigId: "cfg-1", messageId: "m-1", accountId: "acc-1" },
-		]);
-	});
-
-	it("leaves a manually-unread message alone through the NoSuchKey IMAP re-fetch", async () => {
-		const harness = buildHarness(
-			{ messageId: "m-1", bodyStorageKey: "s3://bodies/m-1" },
-			async () => {
-				throw noSuchKeyError();
-			},
-		);
-
-		await harness.service.fetchAndGetBody(
-			"m-1",
-			"acc-1",
-			"cfg-1",
-			{ fullPath: "INBOX", placement: "inbox" },
-			async () => bodyConnection(),
-		);
-
-		assert.deepEqual(
-			harness.markReadCalls,
-			[],
-			"a re-entrant pass must not re-apply auto-read over the user's unread",
-		);
-	});
-
-	it("leaves a manually-unread message alone when syncBodies re-fetches with force", async () => {
-		const harness = buildHarness(
-			{ messageId: "m-1", bodyStorageKey: "s3://bodies/m-1" },
-			async () => {
-				throw new Error("force path must not retrieve from storage");
-			},
-		);
-
-		const result = await harness.service.syncBodies(
-			["m-1"],
-			"acc-1",
-			"cfg-1",
-			{ fullPath: "INBOX", placement: "inbox" },
-			async () => bodyConnection(),
-			true,
-		);
-
-		assert.deepEqual(result.syncedMessageIds, ["m-1"]);
-		assert.deepEqual(
-			harness.markReadCalls,
-			[],
-			"a forced re-sync must not re-apply auto-read over the user's unread",
-		);
-	});
-});