@remit/mailbox-service 0.0.110 → 0.0.112

package/src/adversarial-mime.e2e.test.ts

@@ -1,307 +0,0 @@
-/**
- * E2E adversarial MIME corpus (#402).
- *
- * Each .eml fixture under `test/fixtures/mime-adversarial-e2e/` is APPENDed to
- * the mailfuzz Dovecot INBOX, fetched back via IMAP, then parsed end-to-end
- * with mailparser. The suite asserts the two behaviours called out in #402:
- *
- *   1. body renders            — `parsed.text` or `parsed.html` is non-empty
- *   2. attachments are listable — when the fixture declares attachments, every
- *                                 expected filename appears in `parsed.attachments`
- *                                 with non-zero content length
- *
- * The shapes here are content-driven adversarial cases that real-world senders
- * actually emit (PDF as application/octet-stream, TNEF, nested multipart/related,
- * forwarded message/rfc822, calendar invites, quoted-printable soft breaks,
- * 8-bit UTF-8 bodies, empty text/plain alternatives) — the same class of shape
- * that caused the body-sync regression in #394.
- *
- * Mapper unit-level defence lives in `body-part-mapper.property.test.ts`; this
- * suite is the integration-layer belt-and-braces against the same class.
- */
-
-import assert from "node:assert/strict";
-import { readdirSync, readFileSync } from "node:fs";
-import { dirname, join, resolve } from "node:path";
-import { after, before, describe, test } from "node:test";
-import { fileURLToPath } from "node:url";
-import { type ParsedMail, simpleParser } from "mailparser";
-import { uniqueMailboxName } from "./test-helpers/isolated-mailbox.js";
-import {
-	createMailfuzzConnection,
-	withMailfuzzConnection,
-} from "./test-helpers/mailfuzz-connection.js";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-const FIXTURES_DIR = resolve(
-	__dirname,
-	"../test/fixtures/mime-adversarial-e2e",
-);
-
-interface AttachmentExpectation {
-	filename: string;
-	contentType?: string;
-}
-
-interface FixtureExpectation {
-	/** Subject must round-trip — proves the message we fetched is the one we appended. */
-	subjectIncludes: string;
-	/** At least one of `text` or `html` must be non-empty after parsing. */
-	bodyRenders: true;
-	/** Attachments expected after parsing. Empty array means "no attachments". */
-	attachments: AttachmentExpectation[];
-	/** Optional substring that must appear in parsed.text or parsed.html. */
-	bodyContains?: string;
-	/**
-	 * Skip the BODYSTRUCTURE re-fetch for this fixture.
-	 * Set when the fixture shape is synthetic and doesn't produce a meaningful
-	 * BODYSTRUCTURE response (e.g. bare TNEF blob — see fixture 05 and #408).
-	 */
-	skipBodyStructure?: true;
-}
-
-/**
- * Expectations indexed by .eml basename. Each fixture must have one entry.
- *
- * Filename is the contract — match it exactly to the file on disk.
- */
-const EXPECTATIONS: Record<string, FixtureExpectation> = {
-	"01-octet-stream-pdf.eml": {
-		subjectIncludes: "Adversarial fixture 01",
-		bodyRenders: true,
-		attachments: [{ filename: "contract.pdf" }],
-		bodyContains: "contract PDF",
-	},
-	"02-nested-related-cid-image.eml": {
-		subjectIncludes: "Adversarial fixture 02",
-		bodyRenders: true,
-		attachments: [
-			{ filename: "logo.png", contentType: "image/png" },
-			{ filename: "invoice.pdf" },
-		],
-		bodyContains: "Inline logo",
-	},
-	"03-calendar-invite.eml": {
-		subjectIncludes: "Adversarial fixture 03",
-		bodyRenders: true,
-		attachments: [{ filename: "invite.ics" }],
-		bodyContains: "Meeting invite",
-	},
-	"04-forwarded-rfc822-with-attachments.eml": {
-		subjectIncludes: "Adversarial fixture 04",
-		bodyRenders: true,
-		// mailparser surfaces the forwarded rfc822 as an attachment whose name
-		// derives from the Content-Disposition filename. The inner pdf is not
-		// surfaced as a top-level attachment because it lives inside the nested
-		// rfc822 — listing the rfc822 itself is the integration-layer contract.
-		attachments: [{ filename: "forwarded.eml" }],
-		bodyContains: "Forwarding",
-	},
-	"05-tnef-winmail.eml": {
-		subjectIncludes: "Adversarial fixture 05",
-		bodyRenders: true,
-		attachments: [{ filename: "winmail.dat" }],
-		bodyContains: "TNEF",
-		// skipBodyStructure: The TNEF fixture uses a synthetic magic-byte stub
-		// rather than a real-world winmail.dat capture, so the BODYSTRUCTURE
-		// returned by Dovecot for this fixture is not meaningful. The re-fetch
-		// is skipped here to avoid asserting on a shape that will change once
-		// the fixture is replaced with a real capture. See #408.
-		skipBodyStructure: true,
-	},
-	"06-8bit-non-ascii.eml": {
-		subjectIncludes: "Café façade",
-		bodyRenders: true,
-		attachments: [],
-		bodyContains: "café",
-	},
-	"07-quoted-printable-soft-breaks.eml": {
-		subjectIncludes: "Adversarial fixture 07",
-		bodyRenders: true,
-		attachments: [],
-		// "single" is split across a soft line break in the encoded source;
-		// only an actually-decoded quoted-printable reassembles it.
-		bodyContains: "single logical line",
-	},
-	"08-alternative-empty-text.eml": {
-		subjectIncludes: "Adversarial fixture 08",
-		bodyRenders: true,
-		attachments: [],
-		// text/plain is empty; rendering must fall back to html.
-		bodyContains: "intentionally empty",
-	},
-};
-
-const listEmlFixtures = (): string[] =>
-	readdirSync(FIXTURES_DIR)
-		.filter((name) => name.endsWith(".eml"))
-		.sort();
-
-const loadFixtureRaw = (name: string): string => {
-	const path = join(FIXTURES_DIR, name);
-	// Read as utf8 — fixture 06 includes raw 8-bit UTF-8 bytes which IMAP
-	// servers accept on APPEND as long as they're valid UTF-8 octets.
-	return readFileSync(path, "utf8");
-};
-
-const bodyText = (parsed: ParsedMail): string => {
-	const text = parsed.text ?? "";
-	const html = typeof parsed.html === "string" ? parsed.html : "";
-	return `${text}\n${html}`;
-};
-
-describe("Adversarial MIME corpus (Dovecot) — #402", {
-	skip: !process.env.RUN_E2E_TESTS,
-}, () => {
-	// Own mailbox instead of the shared INBOX so appending fixtures never
-	// collides with another e2e file's count/flag assertions (#508).
-	const inbox = uniqueMailboxName("E2E_adversarial");
-
-	before(async () => {
-		await withMailfuzzConnection(async (connection) => {
-			await connection.createMailbox(inbox);
-		});
-	});
-
-	after(async () => {
-		const connection = createMailfuzzConnection();
-		await connection.connect();
-		await connection.deleteMailbox(inbox).catch(() => {});
-		if (connection.isConnected) {
-			await connection.disconnect();
-		}
-	});
-
-	// Sanity: every .eml on disk has an expectation entry, and every
-	// expectation entry points at an existing file. Catches a fixture being
-	// added without an assertion (or vice versa).
-	test("every fixture has an expectation entry", () => {
-		const onDisk = new Set(listEmlFixtures());
-		const expected = new Set(Object.keys(EXPECTATIONS));
-		for (const name of onDisk) {
-			assert.ok(
-				expected.has(name),
-				`Fixture ${name} has no expectation entry in EXPECTATIONS`,
-			);
-		}
-		for (const name of expected) {
-			assert.ok(
-				onDisk.has(name),
-				`Expectation ${name} has no .eml file on disk`,
-			);
-		}
-		// Pin the count: #402 calls for 8 adversarial shapes.
-		assert.equal(
-			onDisk.size,
-			8,
-			"Expected exactly 8 adversarial MIME fixtures",
-		);
-	});
-
-	for (const name of Object.keys(EXPECTATIONS).sort()) {
-		const expectation = EXPECTATIONS[name];
-		if (!expectation) continue;
-
-		test(`fixture ${name} round-trips through IMAP and parses`, async () => {
-			const raw = loadFixtureRaw(name);
-
-			await withMailfuzzConnection(async (connection) => {
-				// APPEND the fixture into this file's own mailbox. Mailfuzz uses
-				// Dovecot which returns a proper UID on APPEND (mokapi does not —
-				// see outbox-roundtrip.e2e.test.ts and the RFC 025 notes).
-				const appendResult = await connection.append(inbox, raw);
-				assert.ok(
-					appendResult.uid > 0,
-					`APPEND should return a UID for ${name}`,
-				);
-
-				await connection.openBox(inbox, true);
-
-				// Fetch the message we just appended back. fetchMessageBody
-				// returns the raw RFC822 source; mailparser handles the rest
-				// (multipart walking, base64/qp decoding, charset folding).
-				const rfc822 = await connection.fetchMessageBody(appendResult.uid);
-				assert.ok(
-					rfc822.length > 0,
-					`fetchMessageBody should return bytes for ${name}`,
-				);
-
-				// BODYSTRUCTURE re-fetch: exercises the IMAP mapper path at the
-				// integration boundary (the exact regression class from #394).
-				// Skipped for fixtures with synthetic shapes that don't produce
-				// meaningful BODYSTRUCTURE (see skipBodyStructure flag + #408).
-				// The guard in fetchMessages skips rows with undefined uid/internalDate
-				// so back-to-back FETCH is safe on Dovecot (see imapflow-connection.ts).
-				if (!expectation.skipBodyStructure) {
-					const refetched = await connection.fetchMessages([appendResult.uid]);
-					assert.ok(
-						refetched.length > 0,
-						`BODYSTRUCTURE re-fetch should return a message for ${name}`,
-					);
-					assert.ok(
-						refetched[0]?.bodyStructure != null,
-						`BODYSTRUCTURE should be present for ${name}`,
-					);
-				}
-
-				const parsed = await simpleParser(rfc822);
-
-				// Subject round-trips (proves we fetched the right message and
-				// any encoded-word headers decoded correctly).
-				assert.ok(
-					parsed.subject,
-					`Parsed subject should be present for ${name}`,
-				);
-				assert.ok(
-					parsed.subject?.includes(expectation.subjectIncludes),
-					`Subject "${parsed.subject}" should include "${expectation.subjectIncludes}" for ${name}`,
-				);
-
-				// Body renders — at least one of text/html is non-empty. This
-				// is the headline "body renders" assertion from #402.
-				const text = parsed.text ?? "";
-				const html = typeof parsed.html === "string" ? parsed.html : "";
-				assert.ok(
-					text.trim().length > 0 || html.trim().length > 0,
-					`Body should render (text or html non-empty) for ${name}`,
-				);
-
-				if (expectation.bodyContains) {
-					const haystack = bodyText(parsed).toLowerCase();
-					const needle = expectation.bodyContains.toLowerCase();
-					assert.ok(
-						haystack.includes(needle),
-						`Body should contain "${expectation.bodyContains}" for ${name}`,
-					);
-				}
-
-				// Attachments are listable — every expected attachment appears
-				// in parsed.attachments with a matching filename and non-zero
-				// content length. This is the "attachments listable" half of
-				// the #402 contract.
-				const attachments = parsed.attachments ?? [];
-				for (const expectedAtt of expectation.attachments) {
-					const found = attachments.find(
-						(a) => a.filename === expectedAtt.filename,
-					);
-					assert.ok(
-						found,
-						`Attachment "${expectedAtt.filename}" should be listable for ${name}; got [${attachments.map((a) => a.filename).join(", ")}]`,
-					);
-					assert.ok(
-						(found.size ?? 0) > 0,
-						`Attachment "${expectedAtt.filename}" should have non-zero size for ${name}`,
-					);
-					if (expectedAtt.contentType) {
-						assert.equal(
-							found.contentType,
-							expectedAtt.contentType,
-							`Attachment "${expectedAtt.filename}" should have contentType "${expectedAtt.contentType}" for ${name}`,
-						);
-					}
-				}
-			});
-		});
-	}
-});

package/src/body-sync-auto-read-writeonce.test.ts

@@ -1,219 +0,0 @@
-/**
- * Issue #499: `Address.flags.unsubscribed` auto-mark-read is a first-
- * classification decision, like `Message.category` (#355) and the placement
- * verdict (#383). Re-deriving it on the two shipped re-entrant paths —
- * `fetchAndGetBody`'s `NoSuchKey` fallback and `syncBodies(..., force: true)`
- * — undid a user who had deliberately marked such a message unread.
- *
- * The re-entrant fixtures below keep `flags.unsubscribed` set, so a regression
- * that drops the guard shows up as a real `FlagQueueService.markAsRead`
- * round-trip against a message the user owns, not as a fixture that happens to
- * disagree with the flag.
- */
-
-import assert from "node:assert/strict";
-import { Readable } from "node:stream";
-import { describe, it } from "node:test";
-import type {
-	AddressItem,
-	IAddressRepository,
-	IEnvelopeRepository,
-	IMessageRepository,
-	IThreadMessageRepository,
-	MessageItem,
-	UpdateMessageInput,
-} from "@remit/data-ports";
-import type { StorageService } from "@remit/storage-service";
-import { BodySyncService } from "./body-sync.js";
-import type { FlagQueueService } from "./flag-queue.js";
-import type { IImapConnection } from "./types.js";
-
-const UNSUBSCRIBED_SENDER = "newsletter@example.com";
-
-const PLAIN_EML = Buffer.from(
-	[
-		`From: Newsletter <${UNSUBSCRIBED_SENDER}>`,
-		"To: me@example.com",
-		"Subject: This week",
-		"List-Unsubscribe: <mailto:stop@example.com>",
-		"Content-Type: text/plain",
-		"",
-		"body",
-	].join("\r\n"),
-);
-
-interface MarkReadCall {
-	accountConfigId: string;
-	messageId: string;
-	accountId: string;
-}
-
-interface Harness {
-	service: BodySyncService;
-	markReadCalls: MarkReadCall[];
-}
-
-const buildHarness = (
-	message: Partial<MessageItem> & Pick<MessageItem, "messageId">,
-	retrieve: () => Promise<Buffer>,
-): Harness => {
-	const markReadCalls: MarkReadCall[] = [];
-
-	const messageRow = {
-		uid: 1,
-		mailboxId: "mb-inbox",
-		...message,
-	} as unknown as MessageItem;
-
-	const messageService = {
-		get: async () => messageRow,
-		update: async (_messageId: string, input: UpdateMessageInput) => {
-			Object.assign(messageRow, input);
-		},
-	} as unknown as IMessageRepository;
-
-	const threadMessageService = {
-		findAllByMessageId: async () => [
-			{
-				threadMessageId: "tm-1",
-				messageId: message.messageId,
-				mailboxId: messageRow.mailboxId,
-				sentDate: 1,
-				isRead: false,
-				isDeleted: false,
-				hasStars: false,
-				hasAttachment: false,
-			},
-		],
-		update: async () => {},
-	} as unknown as IThreadMessageRepository;
-
-	const storageService = {
-		retrieve,
-		storeMessageBody: async () => ({ uri: `s3://bodies/${message.messageId}` }),
-		storeMessageBodyStream: async () => ({
-			uri: `s3://bodies/${message.messageId}`,
-		}),
-		storeParsedBody: async () => {},
-		listBodyParts: async () => [],
-	} as unknown as StorageService;
-
-	const addressService = {
-		getAddress: async () =>
-			({
-				flags: { unsubscribed: { value: true, setAt: 1 } },
-			}) as unknown as AddressItem,
-		incrementInboundCount: async () => {},
-	} as unknown as IAddressRepository;
-
-	const envelopeService = {
-		listBodyParts: async () => [],
-	} as unknown as IEnvelopeRepository;
-
-	const flagQueueService = {
-		markAsRead: async (
-			accountConfigId: string,
-			messageId: string,
-			accountId: string,
-		) => {
-			markReadCalls.push({ accountConfigId, messageId, accountId });
-		},
-	} as unknown as FlagQueueService;
-
-	const service = new BodySyncService(
-		messageService,
-		storageService,
-		threadMessageService,
-		addressService,
-		envelopeService,
-		{ info: () => {}, error: () => {} },
-		undefined,
-		undefined,
-		undefined,
-		{ flagQueueService },
-	);
-
-	return { service, markReadCalls };
-};
-
-const noSuchKeyError = () =>
-	Object.assign(new Error("missing"), { name: "NoSuchKey" });
-
-const bodyConnection = () =>
-	({
-		openBox: async () => {},
-		fetchMessageBody: async () => PLAIN_EML,
-		async *fetchMessageBodies(uids: number[]) {
-			for (const uid of uids) {
-				yield { uid, source: Readable.from([PLAIN_EML]) };
-			}
-		},
-	}) as unknown as IImapConnection;
-
-describe("unsubscribed auto-mark-read is decided once (issue #499)", () => {
-	it("marks a message read on the pass that first classifies it", async () => {
-		const harness = buildHarness({ messageId: "m-1" }, async () => {
-			throw new Error("no body stored yet; must not retrieve");
-		});
-
-		await harness.service.fetchAndGetBody(
-			"m-1",
-			"acc-1",
-			"cfg-1",
-			{ fullPath: "INBOX", placement: "inbox" },
-			async () => bodyConnection(),
-		);
-
-		assert.deepEqual(harness.markReadCalls, [
-			{ accountConfigId: "cfg-1", messageId: "m-1", accountId: "acc-1" },
-		]);
-	});
-
-	it("leaves a manually-unread message alone through the NoSuchKey IMAP re-fetch", async () => {
-		const harness = buildHarness(
-			{ messageId: "m-1", bodyStorageKey: "s3://bodies/m-1" },
-			async () => {
-				throw noSuchKeyError();
-			},
-		);
-
-		await harness.service.fetchAndGetBody(
-			"m-1",
-			"acc-1",
-			"cfg-1",
-			{ fullPath: "INBOX", placement: "inbox" },
-			async () => bodyConnection(),
-		);
-
-		assert.deepEqual(
-			harness.markReadCalls,
-			[],
-			"a re-entrant pass must not re-apply auto-read over the user's unread",
-		);
-	});
-
-	it("leaves a manually-unread message alone when syncBodies re-fetches with force", async () => {
-		const harness = buildHarness(
-			{ messageId: "m-1", bodyStorageKey: "s3://bodies/m-1" },
-			async () => {
-				throw new Error("force path must not retrieve from storage");
-			},
-		);
-
-		const result = await harness.service.syncBodies(
-			["m-1"],
-			"acc-1",
-			"cfg-1",
-			{ fullPath: "INBOX", placement: "inbox" },
-			async () => bodyConnection(),
-			true,
-		);
-
-		assert.deepEqual(result.syncedMessageIds, ["m-1"]);
-		assert.deepEqual(
-			harness.markReadCalls,
-			[],
-			"a forced re-sync must not re-apply auto-read over the user's unread",
-		);
-	});
-});