@remit/mailbox-service 0.0.1

package/src/mime-walker.test.ts

@@ -0,0 +1,311 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { MediaType, TransferEncoding } from "@remit/domain-enums";
+import {
+	type MimeNode,
+	ROOT_PART_PATH,
+	walkMimeStructure,
+} from "./mime-walker.js";
+
+describe("walkMimeStructure", () => {
+	it("flattens a simple text/plain message into a single record", () => {
+		const root: MimeNode = {
+			type: "text/plain",
+			parameters: { charset: "utf-8" },
+			encoding: "7bit",
+			size: 42,
+			lineCount: 3,
+		};
+
+		const records = walkMimeStructure(root);
+
+		assert.equal(records.length, 1);
+		const [only] = records;
+		assert.equal(only.partPath, ROOT_PART_PATH);
+		assert.equal(only.parentPartPath, null);
+		assert.equal(only.mediaType, "TEXT");
+		assert.equal(only.mediaSubtype, "plain");
+		assert.equal(only.transferEncoding, "7BIT");
+		assert.equal(only.sizeOctets, 42);
+		assert.equal(only.lineCount, 3);
+		assert.equal(only.isMultipart, false);
+		assert.equal(only.disposition, undefined);
+		assert.deepEqual(only.parameters, [
+			{ parameterName: "charset", parameterValue: "utf-8" },
+		]);
+	});
+
+	it("walks multipart/alternative + multipart/mixed + attachment + inline image (alice/bob fixture)", () => {
+		// multipart/mixed
+		//   ├── multipart/alternative   (1)
+		//   │     ├── text/plain        (1.1)
+		//   │     └── text/html         (1.2)
+		//   ├── application/pdf  (attachment)  (2)
+		//   └── multipart/related        (3)
+		//         ├── text/html          (3.1)
+		//         └── image/png  (inline) (3.2)
+		const root: MimeNode = {
+			type: "multipart/mixed",
+			parameters: { boundary: "outer" },
+			childNodes: [
+				{
+					part: "1",
+					type: "multipart/alternative",
+					parameters: { boundary: "alt" },
+					childNodes: [
+						{
+							part: "1.1",
+							type: "text/plain",
+							parameters: { charset: "utf-8" },
+							encoding: "7bit",
+							size: 120,
+							lineCount: 4,
+						},
+						{
+							part: "1.2",
+							type: "text/html",
+							parameters: { charset: "utf-8" },
+							encoding: "quoted-printable",
+							size: 350,
+							lineCount: 6,
+						},
+					],
+				},
+				{
+					part: "2",
+					type: "application/pdf",
+					parameters: { name: "alice-resume.pdf" },
+					encoding: "base64",
+					size: 9001,
+					disposition: "attachment",
+					dispositionParameters: { filename: "alice-resume.pdf" },
+					md5: "deadbeef",
+				},
+				{
+					part: "3",
+					type: "multipart/related",
+					parameters: { boundary: "rel" },
+					childNodes: [
+						{
+							part: "3.1",
+							type: "text/html",
+							parameters: { charset: "utf-8" },
+							encoding: "quoted-printable",
+							size: 200,
+							lineCount: 5,
+						},
+						{
+							part: "3.2",
+							type: "image/png",
+							parameters: { name: "bob-avatar.png" },
+							id: "<bob-avatar@example.com>",
+							encoding: "base64",
+							size: 4096,
+							disposition: "inline",
+							dispositionParameters: { filename: "bob-avatar.png" },
+						},
+					],
+				},
+			],
+		};
+
+		const records = walkMimeStructure(root);
+
+		const byPath = new Map(records.map((r) => [r.partPath, r]));
+
+		assert.equal(records.length, 8);
+
+		const rootRec = byPath.get(ROOT_PART_PATH);
+		assert.ok(rootRec, "root present");
+		assert.equal(rootRec.parentPartPath, null);
+		assert.equal(rootRec.isMultipart, true);
+		assert.equal(rootRec.mediaType, "MULTIPART");
+		assert.equal(rootRec.mediaSubtype, "mixed");
+		assert.equal(rootRec.multipartSubtype, "mixed");
+
+		const alt = byPath.get("1");
+		assert.ok(alt);
+		assert.equal(alt.parentPartPath, ROOT_PART_PATH);
+		assert.equal(alt.isMultipart, true);
+		assert.equal(alt.multipartSubtype, "alternative");
+
+		const text = byPath.get("1.1");
+		assert.ok(text);
+		assert.equal(text.parentPartPath, "1");
+		assert.equal(text.mediaType, "TEXT");
+		assert.equal(text.mediaSubtype, "plain");
+		assert.equal(text.transferEncoding, "7BIT");
+
+		const html = byPath.get("1.2");
+		assert.ok(html);
+		assert.equal(html.parentPartPath, "1");
+		assert.equal(html.transferEncoding, "QUOTED-PRINTABLE");
+
+		const pdf = byPath.get("2");
+		assert.ok(pdf);
+		assert.equal(pdf.parentPartPath, ROOT_PART_PATH);
+		assert.equal(pdf.mediaType, "APPLICATION");
+		assert.equal(pdf.disposition, "attachment");
+		assert.equal(pdf.dispositionFilename, "alice-resume.pdf");
+		assert.equal(pdf.md5Hash, "deadbeef");
+		assert.equal(pdf.sizeOctets, 9001);
+
+		const related = byPath.get("3");
+		assert.ok(related);
+		assert.equal(related.multipartSubtype, "related");
+
+		const inlineImg = byPath.get("3.2");
+		assert.ok(inlineImg);
+		assert.equal(inlineImg.parentPartPath, "3");
+		assert.equal(inlineImg.disposition, "inline");
+		assert.equal(inlineImg.contentId, "bob-avatar@example.com");
+		assert.equal(inlineImg.dispositionFilename, "bob-avatar.png");
+	});
+
+	it("falls back to Content-Type 'name' parameter when no Content-Disposition filename is set", () => {
+		const root: MimeNode = {
+			type: "application/octet-stream",
+			parameters: { name: "alice-data.bin" },
+			encoding: "base64",
+			size: 100,
+			disposition: "attachment",
+		};
+
+		const [rec] = walkMimeStructure(root);
+		assert.equal(rec.dispositionFilename, "alice-data.bin");
+	});
+
+	it("emits stable, sorted parameters (no input-order dependence)", () => {
+		const root: MimeNode = {
+			type: "text/plain",
+			parameters: { charset: "utf-8", format: "flowed", name: "alice.txt" },
+			encoding: "7bit",
+			size: 1,
+		};
+
+		const [rec] = walkMimeStructure(root);
+		assert.deepEqual(
+			rec.parameters.map((p) => p.parameterName),
+			["charset", "format", "name"],
+		);
+	});
+
+	it("falls back to application/octet-stream on an unknown MIME top-level type", () => {
+		const [rec] = walkMimeStructure({
+			type: "carrierpigeon/scroll",
+			encoding: "7bit",
+			size: 1,
+		});
+		assert.equal(rec.mediaType, MediaType.Application);
+		assert.equal(rec.mediaSubtype, "scroll");
+	});
+
+	it("falls back to application/octet-stream on a Content-Type with no '/' separator", () => {
+		const [rec] = walkMimeStructure({
+			type: "text",
+			encoding: "7bit",
+			size: 1,
+		});
+		assert.equal(rec.mediaType, MediaType.Application);
+		assert.equal(rec.mediaSubtype, "octet-stream");
+	});
+
+	it("falls back to octet-stream subtype on a Content-Type missing the subtype", () => {
+		const [rec] = walkMimeStructure({
+			type: "text/",
+			encoding: "7bit",
+			size: 1,
+		});
+		assert.equal(rec.mediaType, MediaType.Text);
+		assert.equal(rec.mediaSubtype, "octet-stream");
+	});
+
+	it("falls back to BINARY on an unknown transfer encoding", () => {
+		const [rec] = walkMimeStructure({
+			type: "text/plain",
+			encoding: "rot13",
+			size: 1,
+		});
+		assert.equal(rec.transferEncoding, TransferEncoding.Binary);
+	});
+
+	it("drops an unknown Content-Disposition rather than throwing", () => {
+		const [rec] = walkMimeStructure({
+			type: "application/pdf",
+			encoding: "base64",
+			size: 1,
+			disposition: "secretly-mine",
+		});
+		assert.equal(rec.disposition, undefined);
+	});
+
+	it("keeps an empty Content-Type total (application/octet-stream)", () => {
+		const [rec] = walkMimeStructure({ type: "", encoding: "7bit", size: 1 });
+		assert.equal(rec.mediaType, MediaType.Application);
+		assert.equal(rec.mediaSubtype, "octet-stream");
+	});
+
+	it("defaults transferEncoding to 7BIT and sizeOctets to 0 when absent", () => {
+		const [rec] = walkMimeStructure({ type: "text/plain" });
+		assert.equal(rec.transferEncoding, "7BIT");
+		assert.equal(rec.sizeOctets, 0);
+	});
+
+	// Regression test for the DynamoDB "multiple operations on one item" flake.
+	// When imapflow returns a message/rfc822 attachment, it may attach the inner
+	// message body as a childNode with part="" (no path). The old code gave every
+	// such node ROOT_PART_PATH ("0"), colliding with the outer root. The fix
+	// assigns a synthetic path "<parentPath>.<siblingIndex>" to keep keys unique.
+	it("assigns unique partPaths to message/rfc822 inner-body nodes that have no part field", () => {
+		const rfc822: MimeNode = {
+			type: "multipart/mixed",
+			// root has no part
+			childNodes: [
+				{
+					type: "text/plain",
+					part: "1",
+					encoding: "7bit",
+					size: 20,
+				},
+				{
+					type: "message/rfc822",
+					part: "2",
+					size: 300,
+					// imapflow surfaces the inner body as a child with no part
+					childNodes: [
+						{
+							type: "text/plain",
+							// no part — this was the bug
+							encoding: "7bit",
+							size: 50,
+						},
+					],
+				},
+			],
+		};
+
+		const records = walkMimeStructure(rfc822);
+		const paths = records.map((r) => r.partPath);
+
+		// All four paths must be unique.
+		assert.equal(
+			new Set(paths).size,
+			records.length,
+			`Duplicate partPaths detected: ${paths.join(", ")}`,
+		);
+
+		// Root gets the canonical ROOT_PART_PATH.
+		assert.equal(paths[0], ROOT_PART_PATH);
+
+		// The inner body without a part field gets a synthetic path, not "0".
+		const innerBody = records.find(
+			(r) => r.mediaSubtype === "plain" && r.partPath !== "1",
+		);
+		assert.ok(innerBody, "inner text/plain should be in the records");
+		assert.notEqual(
+			innerBody.partPath,
+			ROOT_PART_PATH,
+			`inner body should not reuse ROOT_PART_PATH but got ${innerBody.partPath}`,
+		);
+	});
+});

package/src/mime-walker.ts

@@ -0,0 +1,305 @@
+/**
+ * MIME structure walker.
+ *
+ * Flattens an IMAP BODYSTRUCTURE tree (as produced by ImapFlow's
+ * `MessageStructureObject`) into a list of `BodyPartRecord`s ready to be
+ * persisted to the BodyPart / BodyPartParameter tables.
+ *
+ * Inputs are intentionally typed against a structural subset rather than
+ * pulling the imapflow types directly so the walker can be unit-tested
+ * without spinning up an IMAP connection.
+ */
+
+import { ROOT_PART_PATH } from "@remit/data-ports/id";
+import {
+	ContentDisposition,
+	MediaType,
+	MultipartSubtype,
+	TransferEncoding,
+} from "@remit/domain-enums";
+
+export { ROOT_PART_PATH };
+
+export type MediaTypeValue = (typeof MediaType)[keyof typeof MediaType];
+export type TransferEncodingValue =
+	(typeof TransferEncoding)[keyof typeof TransferEncoding];
+export type ContentDispositionValue =
+	(typeof ContentDisposition)[keyof typeof ContentDisposition];
+export type MultipartSubtypeValue =
+	(typeof MultipartSubtype)[keyof typeof MultipartSubtype];
+
+/**
+ * Structural shape of an IMAP BODYSTRUCTURE node. Mirrors imapflow's
+ * `MessageStructureObject` minus the fields we don't read so this module
+ * stays free of an imapflow dependency.
+ */
+export interface MimeNode {
+	/**
+	 * Dot-numbered MIME path (e.g., "1", "1.1", "2"). Empty/undefined for
+	 * the root node — the walker assigns "0" to it for storage stability.
+	 */
+	part?: string;
+	/** Full Content-Type (e.g., "text/plain", "multipart/mixed"). */
+	type: string;
+	/** Content-Type parameters (charset, boundary, name, ...). */
+	parameters?: Record<string, string>;
+	/** Content-ID header value (without angle brackets). */
+	id?: string;
+	/** Content-Description header value. */
+	description?: string;
+	/** Transfer encoding (7bit, 8bit, base64, quoted-printable, binary). */
+	encoding?: string;
+	/** Encoded byte size of this part. */
+	size?: number;
+	/** Line count for text/* parts. */
+	lineCount?: number;
+	/** MD5 hash. */
+	md5?: string;
+	/** Content-Disposition (inline | attachment). */
+	disposition?: string;
+	/** Content-Disposition parameters (filename, ...). */
+	dispositionParameters?: Record<string, string>;
+	/** Content-Language values. */
+	language?: string[];
+	/** Content-Location URI. */
+	location?: string;
+	/** Children for multipart nodes. */
+	childNodes?: MimeNode[];
+}
+
+export interface BodyPartParameterRecord {
+	parameterName: string;
+	parameterValue: string;
+}
+
+/**
+ * Flattened MIME node ready to upsert. `parentPartPath` is the parent
+ * node's `partPath` (or null for the root) so the caller can map paths to
+ * the deterministic `bodyPartId`s without re-walking the tree.
+ */
+export interface BodyPartRecord {
+	partPath: string;
+	parentPartPath: string | null;
+	mediaType: MediaTypeValue;
+	mediaSubtype: string;
+	contentId?: string;
+	contentDescription?: string;
+	transferEncoding: TransferEncodingValue;
+	sizeOctets: number;
+	lineCount?: number;
+	md5Hash?: string;
+	disposition?: ContentDispositionValue;
+	dispositionFilename?: string;
+	language?: string;
+	location?: string;
+	isMultipart: boolean;
+	multipartSubtype?: MultipartSubtypeValue;
+	parameters: BodyPartParameterRecord[];
+}
+
+const MEDIA_TYPE_BY_TOP_LEVEL: Record<string, MediaTypeValue> = {
+	text: MediaType.Text,
+	image: MediaType.Image,
+	audio: MediaType.Audio,
+	video: MediaType.Video,
+	application: MediaType.Application,
+	multipart: MediaType.Multipart,
+	message: MediaType.Message,
+};
+
+const MULTIPART_SUBTYPE_BY_VALUE: Record<string, MultipartSubtypeValue> = {
+	mixed: MultipartSubtype.Mixed,
+	alternative: MultipartSubtype.Alternative,
+	digest: MultipartSubtype.Digest,
+	parallel: MultipartSubtype.Parallel,
+	related: MultipartSubtype.Related,
+	signed: MultipartSubtype.Signed,
+	encrypted: MultipartSubtype.Encrypted,
+	"form-data": MultipartSubtype.FormData,
+	report: MultipartSubtype.Report,
+};
+
+const TRANSFER_ENCODING_BY_VALUE: Record<string, TransferEncodingValue> = {
+	"7bit": TransferEncoding.SevenBit,
+	"8bit": TransferEncoding.EightBit,
+	binary: TransferEncoding.Binary,
+	base64: TransferEncoding.Base64,
+	"quoted-printable": TransferEncoding.QuotedPrintable,
+};
+
+const DISPOSITION_BY_VALUE: Record<string, ContentDispositionValue> = {
+	inline: ContentDisposition.Inline,
+	attachment: ContentDisposition.Attachment,
+};
+
+const FALLBACK_MEDIA_SUBTYPE = "octet-stream";
+
+/**
+ * Split a Content-Type into a known media type and its subtype, tolerating
+ * malformed values. The Content-Type is copied from an untrusted header, so a
+ * missing "/", an unknown top-level type, or an empty subtype must not throw
+ * (that would drop the whole message). RFC 2046 §4.5.3 makes
+ * `application/octet-stream` the defined default for an unrecognized type, so
+ * that is the fallback; the raw subtype is preserved where present.
+ */
+const splitContentType = (
+	type: string,
+): { mediaType: MediaTypeValue; mediaSubtype: string } => {
+	const slashIndex = type.indexOf("/");
+	if (slashIndex < 0) {
+		return {
+			mediaType: MediaType.Application,
+			mediaSubtype: FALLBACK_MEDIA_SUBTYPE,
+		};
+	}
+	const topRaw = type.slice(0, slashIndex);
+	const subRaw = type.slice(slashIndex + 1);
+	const top = topRaw.toLowerCase();
+	const mediaType = MEDIA_TYPE_BY_TOP_LEVEL[top] ?? MediaType.Application;
+	const subLower = subRaw.toLowerCase();
+	const mediaSubtype = subLower.length > 0 ? subLower : FALLBACK_MEDIA_SUBTYPE;
+	return { mediaType, mediaSubtype };
+};
+
+/**
+ * An absent Content-Transfer-Encoding means 7bit (RFC 2045 §6.1); an
+ * unrecognized token is treated as opaque bytes rather than throwing, so a
+ * message carrying an exotic encoding still syncs.
+ */
+const mapTransferEncoding = (
+	encoding: string | undefined,
+): TransferEncodingValue => {
+	if (!encoding) return TransferEncoding.SevenBit;
+	return (
+		TRANSFER_ENCODING_BY_VALUE[encoding.toLowerCase()] ??
+		TransferEncoding.Binary
+	);
+};
+
+/**
+ * An unknown Content-Disposition is treated as absent (undefined) rather than
+ * throwing, so a message with a malformed disposition still syncs.
+ */
+const mapDisposition = (
+	disposition: string | undefined,
+): ContentDispositionValue | undefined => {
+	if (!disposition) return undefined;
+	return DISPOSITION_BY_VALUE[disposition.toLowerCase()];
+};
+
+const mapMultipartSubtype = (
+	subtype: string,
+): MultipartSubtypeValue | undefined => {
+	const mapped = MULTIPART_SUBTYPE_BY_VALUE[subtype.toLowerCase()];
+	if (!mapped) return undefined;
+	return mapped;
+};
+
+const stripAngles = (value: string): string => value.replace(/^<+|>+$/g, "");
+
+const flattenParameters = (
+	params: Record<string, string> | undefined,
+): BodyPartParameterRecord[] => {
+	if (!params) return [];
+	const entries = Object.entries(params).filter(([name, value]) => {
+		if (typeof name !== "string" || name.length === 0) return false;
+		if (typeof value !== "string") return false;
+		return true;
+	});
+	entries.sort((a, b) => a[0].localeCompare(b[0]));
+	return entries.map(([parameterName, parameterValue]) => ({
+		parameterName,
+		parameterValue,
+	}));
+};
+
+const toRecord = (
+	node: MimeNode,
+	partPath: string,
+	parentPartPath: string | null,
+): BodyPartRecord => {
+	const { mediaType, mediaSubtype } = splitContentType(node.type);
+	const isMultipart = mediaType === MediaType.Multipart;
+	const multipartSubtype = isMultipart
+		? mapMultipartSubtype(mediaSubtype)
+		: undefined;
+
+	const dispositionFilename =
+		node.dispositionParameters?.filename ?? node.parameters?.name ?? undefined;
+
+	const language = node.language?.[0];
+
+	const record: BodyPartRecord = {
+		partPath,
+		parentPartPath,
+		mediaType,
+		mediaSubtype,
+		transferEncoding: mapTransferEncoding(node.encoding),
+		sizeOctets: typeof node.size === "number" ? node.size : 0,
+		isMultipart,
+		parameters: flattenParameters(node.parameters),
+	};
+
+	if (node.id) record.contentId = stripAngles(node.id);
+	if (node.description) record.contentDescription = node.description;
+	if (typeof node.lineCount === "number") record.lineCount = node.lineCount;
+	if (node.md5) record.md5Hash = node.md5;
+	const mappedDisposition = mapDisposition(node.disposition);
+	if (mappedDisposition) record.disposition = mappedDisposition;
+	if (dispositionFilename) record.dispositionFilename = dispositionFilename;
+	if (language) record.language = language;
+	if (node.location) record.location = node.location;
+	if (multipartSubtype) record.multipartSubtype = multipartSubtype;
+
+	return record;
+};
+
+/**
+ * Walk an IMAP BODYSTRUCTURE tree depth-first and return a flat list of
+ * `BodyPartRecord`s. The root node uses `ROOT_PART_PATH` ("0"); all other
+ * nodes use the dot-numbered IMAP path that ImapFlow assigns (e.g., "1",
+ * "1.2", "2.1.3"). Every field comes from an untrusted BODYSTRUCTURE, so an
+ * unrecognized MIME type, transfer encoding, or Content-Disposition maps to a
+ * safe default rather than throwing — one malformed part must never drop the
+ * whole message.
+ *
+ * **Part-path uniqueness**: some IMAP servers (and the `message/rfc822`
+ * inner-body convention) return child nodes with an empty `part` field.
+ * Assigning ROOT_PART_PATH to every such node would produce duplicate keys
+ * and cause a DynamoDB "multiple operations on one item" error in
+ * `upsertBodyParts`. Non-root nodes without a `part` therefore receive a
+ * synthetic path `<parentPath>.<siblingIndex>` that is stable across
+ * repeated syncs of the same message.
+ */
+export const walkMimeStructure = (root: MimeNode): BodyPartRecord[] => {
+	const out: BodyPartRecord[] = [];
+
+	const visit = (
+		node: MimeNode,
+		parentPath: string | null,
+		siblingIndex: number,
+	) => {
+		let partPath: string;
+		if (node.part && node.part.length > 0) {
+			partPath = node.part;
+		} else if (parentPath === null) {
+			// True root of the BODYSTRUCTURE tree.
+			partPath = ROOT_PART_PATH;
+		} else {
+			// Non-root node without an IMAP part path — synthesise one so
+			// the DynamoDB keys remain unique. This happens most commonly
+			// for the body of a message/rfc822 attachment, whose inner
+			// structure imapflow attaches as a childNode with part="".
+			partPath = `${parentPath}.${siblingIndex}`;
+		}
+		out.push(toRecord(node, partPath, parentPath));
+		const children = node.childNodes ?? [];
+		for (let i = 0; i < children.length; i++) {
+			// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
+			visit(children[i]!, partPath, i + 1);
+		}
+	};
+
+	visit(root, null, 0);
+	return out;
+};