@remit/mailbox-service 0.0.85 → 0.0.86

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.85",
+  "version": "0.0.86",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/body-sync-category-denormalize.test.ts

@@ -3,21 +3,22 @@
  * on, and its write-path defects only appear under conditions a happy-path test
  * never creates (issue #320).
  *
- * 1. The retro classification path wrote the Message before the ThreadMessage
- *    while its skip guard keyed off `message.category`. A failure between the
- *    two left the guard satisfied and the denormalized row stranded at
- *    `uncategorized` forever, because the requeued retry returned early.
+ * 1. The classifying pass wrote the Message before the ThreadMessage while the
+ *    skip guard keyed off the Message. A failure between the two left the guard
+ *    satisfied and the denormalized row stranded at `uncategorized` forever,
+ *    because the requeued retry returned early.
  * 2. `denormalizeCategory` resolved one arbitrary row per message through an
  *    unordered `.limit(1)`. More than one row per messageId is schema-legal —
  *    reachable through thread-root drift, not through a second mailbox, whose ids
  *    collapse to one row — and every row a message has must end up correct.
- * 3. It also wrote only `category` on the retro path, so a copied message, which
- *    reaches no other denormalizing path, never got a `listId` at all.
+ * 3. It also wrote only `category` on one of its paths, so the snippet and the
+ *    `listId` derived from the same bytes were dropped on the floor.
  *
  * None of these show up on a single row in a single successful pass.
  */
 
 import assert from "node:assert/strict";
+import { Readable } from "node:stream";
 import { describe, it } from "node:test";
 import type {
 	IAddressRepository,
@@ -104,7 +105,6 @@ const buildHarness = (
 	rows: Row[],
 	options: {
 		failThreadUpdates?: number;
-		bodyStored?: boolean;
 	} = {},
 ): Harness => {
 	const messageUpdates: Array<{
@@ -120,9 +120,6 @@ const buildHarness = (
 		mailboxId: "mb-inbox",
 		uid: 1,
 		category: MessageCategory.uncategorized,
-		...(options.bodyStored === false
-			? {}
-			: { bodyStorageKey: "s3://bodies/m-1" }),
 	} as unknown as MessageItem;
 
 	const messageService = {
@@ -164,6 +161,7 @@ const buildHarness = (
 	const storageService = {
 		retrieve: async () => LINKEDIN_EML,
 		storeMessageBody: async () => ({ uri: "s3://bodies/m-1" }),
+		storeMessageBodyStream: async () => ({ uri: "s3://bodies/m-1" }),
 		storeParsedBody: async () => {},
 		listBodyParts: async () => [],
 	} as unknown as StorageService;
@@ -192,10 +190,22 @@ const buildHarness = (
 	};
 };
 
-const backfill = (harness: Harness) =>
-	harness.service.syncBodies(["m-1"], "acc-1", "cfg-1", "INBOX", async () => {
-		throw new Error("the backfill must not open IMAP");
-	});
+const syncBatch = (harness: Harness) =>
+	harness.service.syncBodies(
+		["m-1"],
+		"acc-1",
+		"cfg-1",
+		"INBOX",
+		async () =>
+			({
+				openBox: async () => {},
+				async *fetchMessageBodies(uids: number[]) {
+					for (const uid of uids) {
+						yield { uid, source: Readable.from([LINKEDIN_EML]) };
+					}
+				},
+			}) as unknown as IImapConnection,
+	);
 
 const syncFromImap = (harness: Harness) =>
 	harness.service.fetchAndGetBody(
@@ -210,19 +220,19 @@ const syncFromImap = (harness: Harness) =>
 			}) as unknown as IImapConnection,
 	);
 
-describe("the classification backfill survives an interrupted write", () => {
+describe("the classifying pass survives an interrupted write", () => {
 	it("leaves the Message unclassified when the ThreadMessage write fails, so the retry runs again", async () => {
 		const harness = buildHarness([buildRow({ threadMessageId: "tm-1" })], {
 			failThreadUpdates: 1,
 		});
 
-		const failed = await backfill(harness);
+		const failed = await syncBatch(harness);
 
 		assert.deepEqual(failed.failedMessageIds, ["m-1"]);
 		assert.equal(harness.message.category, MessageCategory.uncategorized);
 		assert.deepEqual(harness.messageUpdates, []);
 
-		const retried = await backfill(harness);
+		const retried = await syncBatch(harness);
 
 		assert.deepEqual(retried.failedMessageIds, []);
 		assert.equal(harness.message.category, MessageCategory.social);
@@ -232,14 +242,14 @@ describe("the classification backfill survives an interrupted write", () => {
 	it("writes the Message only after the denormalized row is durable", async () => {
 		const harness = buildHarness([buildRow({ threadMessageId: "tm-1" })]);
 
-		await backfill(harness);
+		await syncBatch(harness);
 
 		assert.deepEqual(harness.writeOrder, ["thread", "message"]);
 	});
 });
 
 describe("denormalization reaches every row a message has", () => {
-	it("carries the backfilled category onto all of a message's rows", async () => {
+	it("carries the classified category onto all of a message's rows", async () => {
 		const harness = buildHarness([
 			buildRow({ threadMessageId: "tm-inbox", mailboxId: "mb-inbox" }),
 			buildRow({
@@ -249,7 +259,7 @@ describe("denormalization reaches every row a message has", () => {
 			}),
 		]);
 
-		await backfill(harness);
+		await syncBatch(harness);
 
 		assert.deepEqual(
 			harness.rows.map((r) => r.category),
@@ -267,7 +277,7 @@ describe("denormalization reaches every row a message has", () => {
 			}),
 		]);
 
-		await backfill(harness);
+		await syncBatch(harness);
 
 		assert.deepEqual(
 			harness.threadUpdates.map((u) => ({
@@ -286,16 +296,13 @@ describe("denormalization reaches every row a message has", () => {
 		);
 	});
 
-	it("fans the snippet and the list id out to every row on the sync path", async () => {
+	it("fans the snippet and the list id out to every row on the read path", async () => {
 		// A snippet and a List-Id belong to the message, not to the mailbox a row
 		// happens to name, so every row carries them.
-		const harness = buildHarness(
-			[
-				buildRow({ threadMessageId: "tm-inbox", mailboxId: "mb-inbox" }),
-				buildRow({ threadMessageId: "tm-archive", mailboxId: "mb-archive" }),
-			],
-			{ bodyStored: false },
-		);
+		const harness = buildHarness([
+			buildRow({ threadMessageId: "tm-inbox", mailboxId: "mb-inbox" }),
+			buildRow({ threadMessageId: "tm-archive", mailboxId: "mb-archive" }),
+		]);
 
 		await syncFromImap(harness);
 
@@ -318,7 +325,7 @@ describe("denormalization reaches every row a message has", () => {
 			buildRow({ threadMessageId: "tm-archive", mailboxId: "mb-archive" }),
 		]);
 
-		await backfill(harness);
+		await syncBatch(harness);
 
 		assert.deepEqual(
 			harness.threadUpdates.map((u) => u.threadMessageId),
@@ -327,15 +334,13 @@ describe("denormalization reaches every row a message has", () => {
 	});
 });
 
-describe("the classification backfill writes the whole denormalized set", () => {
-	// A copied message inherits its source's stored body AND its decided
-	// category, so the full body-store path skips it and this path's guard
-	// returns early — and nothing but these two paths ever writes `listId`. The
-	// backfill passing only `category` left every copy's `list_id` NULL for good.
+describe("the classifying pass writes the whole denormalized set", () => {
+	// Nothing but this pass ever derives `listId` from a message's bytes, so a
+	// write that carried only `category` left the row's `list_id` NULL for good.
 	it("carries the list id and the snippet, not just the category", async () => {
 		const harness = buildHarness([buildRow({ threadMessageId: "tm-1" })]);
 
-		await backfill(harness);
+		await syncBatch(harness);
 
 		assert.deepEqual(harness.threadUpdates.length, 1);
 		assert.deepEqual(harness.threadUpdates[0].input, {

package/src/body-sync-category-override.test.ts

@@ -15,6 +15,14 @@
  * `NoSuchKey` IMAP re-fetch and `syncBodies(..., force: true)` — so a
  * regression that let the override bypass the guard shows up here, not just
  * in an isolated unit check.
+ *
+ * The override reaches a message on the pass that examines it, and only then
+ * (issue #331). Mail already stored and examined is not revisited, whatever the
+ * user later says about the sender: an ordinary sync pass does not re-read a
+ * stored body, and the last test here pins that. Applying a reclassification
+ * backwards over mail already on disk is a deliberate operation over
+ * `Message.classificationState`, tracked in issue #415, and nothing in this
+ * file claims it happens today.
  */
 
 import assert from "node:assert/strict";
@@ -278,15 +286,50 @@ describe("Address.flags.category overrides classification at sync time (issue #2
 		);
 	});
 
-	it("backfillClassification picks up the same override for a body stored before classification existed", async () => {
+	it("reaches a message still awaiting examination on the batch sync path too", async () => {
+		const harness = buildHarness(
+			{ messageId: "m-1", category: MessageCategory.uncategorized },
+			overrideFlags(MessageCategory.transactional),
+		);
+
+		const result = await harness.service.syncBodies(
+			["m-1"],
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			async () =>
+				({
+					openBox: async () => {},
+					async *fetchMessageBodies(uids: number[]) {
+						for (const uid of uids) {
+							yield { uid, source: Readable.from([LINKEDIN_EML]) };
+						}
+					},
+				}) as unknown as IImapConnection,
+		);
+
+		assert.deepEqual(result.syncedMessageIds, ["m-1"]);
+		assert.equal(harness.messageUpdates.length, 1);
+		assert.equal(
+			harness.messageUpdates[0].input.category,
+			MessageCategory.transactional,
+		);
+		assert.equal(harness.rows[0].category, MessageCategory.transactional);
+		assert.equal(
+			harness.messageUpdates[0].input.classificationState,
+			"Examined",
+		);
+	});
+
+	it("does not reach back to mail it already examined when the sender is reclassified later", async () => {
 		const harness = buildHarness(
 			{
 				messageId: "m-1",
 				bodyStorageKey: "s3://bodies/m-1",
 				category: MessageCategory.uncategorized,
-			},
+				classificationState: "Examined",
+			} as Partial<MessageItem> & Pick<MessageItem, "messageId">,
 			overrideFlags(MessageCategory.transactional),
-			async () => LINKEDIN_EML,
 		);
 
 		const result = await harness.service.syncBodies(
@@ -295,17 +338,18 @@ describe("Address.flags.category overrides classification at sync time (issue #2
 			"cfg-1",
 			"INBOX",
 			async () => {
-				throw new Error("backfill must not open IMAP");
+				throw new Error("an examined message must not be fetched again");
 			},
 		);
 
+		// The harness's default `retrieve` throws, so a storage read would fail
+		// this outright. Issue #415 is where applying the override to this message
+		// deliberately belongs; a sync pass is not it.
 		assert.equal(result.skippedCount, 1);
-		assert.equal(harness.messageUpdates.length, 1);
-		assert.equal(
-			harness.messageUpdates[0].input.category,
-			MessageCategory.transactional,
-		);
-		assert.equal(harness.rows[0].category, MessageCategory.transactional);
+		assert.deepEqual(result.failedMessageIds, []);
+		assert.deepEqual(harness.messageUpdates, []);
+		assert.deepEqual(harness.threadUpdates, []);
+		assert.equal(harness.message.category, MessageCategory.uncategorized);
 	});
 
 	describe("survives a re-entrant pass with a differing override already set", () => {

package/src/body-sync-quarantine.test.ts

@@ -5,10 +5,19 @@
  * and database failures — which the same per-message frame catches — keep
  * propagating to the requeue path, because recording one as a quarantine would
  * tell the user that mail Remit could not reach was mail Remit could not read,
- * and let go of it.
+ * and let go of it. That discrimination is the `error instanceof BodyParseError`
+ * branch in `syncBodies`' stream loop, and these tests drive it there.
+ *
+ * The parse cases inject the `BodyParseError` at the storage write rather than
+ * feeding mailparser bytes it refuses: mailparser accepts every byte sequence a
+ * test can construct, and the branch under test asks only what type of error
+ * reached it. That only `parseMessageBody` ever constructs one — so an instance
+ * of it really is proof the message and not the infrastructure failed — is
+ * pinned in `body-parse.test.ts`.
  */
 
 import assert from "node:assert/strict";
+import { Readable } from "node:stream";
 import { describe, it } from "node:test";
 import type {
 	IAddressRepository,
@@ -22,16 +31,29 @@ import type {
 	QuarantineUpsertInput,
 	UpdateMessageInput,
 } from "@remit/data-ports";
+import { NotFoundError } from "@remit/data-ports/errors";
 import { MessageCategory } from "@remit/domain-enums";
 import type { StorageService } from "@remit/storage-service";
+import { BodyParseError } from "./body-parse.js";
 import { BodySyncService } from "./body-sync.js";
 import { QuarantineService } from "./quarantine.js";
-
-const UNPARSEABLE = Symbol("unparseable");
+import type { IImapConnection } from "./types.js";
+
+const WELL_FORMED = Buffer.from(
+	[
+		"From: someone@example.com",
+		"To: me@example.com",
+		"Subject: hello",
+		"Content-Type: text/plain",
+		"",
+		"body",
+	].join("\r\n"),
+);
 
 const buildHarness = (options: {
-	retrieve?: (key: string) => Promise<Buffer>;
 	existing?: QuarantineItem[];
+	storeBodyFails?: unknown;
+	storeParsedFails?: unknown;
 	upsertFails?: boolean;
 }) => {
 	const writes: QuarantineUpsertInput[] = [];
@@ -43,8 +65,8 @@ const buildHarness = (options: {
 		uid: 40217,
 		rfc822Size: 2048,
 		messageIdHeader: "<abc@example.com>",
-		bodyStorageKey: "s3://bodies/m-1",
 		category: MessageCategory.uncategorized,
+		classificationState: "NotExamined",
 	} as MessageItem;
 
 	const messageService = {
@@ -55,17 +77,32 @@ const buildHarness = (options: {
 	} as unknown as IMessageRepository;
 
 	const storageService = {
-		retrieve:
-			options.retrieve ??
-			(async () => {
-				throw new Error("no body configured");
-			}),
+		retrieve: async () => {
+			throw new Error("the sync path never reads a body back");
+		},
+		storeMessageBodyStream: async () => {
+			if (options.storeBodyFails !== undefined) throw options.storeBodyFails;
+			return { uri: "s3://bodies/m-1" };
+		},
+		storeParsedBody: async () => {
+			if (options.storeParsedFails !== undefined)
+				throw options.storeParsedFails;
+		},
+		listBodyParts: async () => [],
 	} as unknown as StorageService;
 
 	const envelopeService = {
 		getMessageData: async () => ({ bodyPart: [], bodyPartParameter: [] }),
+		listBodyParts: async () => [],
 	} as unknown as IEnvelopeRepository;
 
+	const addressService = {
+		getAddress: async () => {
+			throw new NotFoundError("Address not found");
+		},
+		incrementInboundCount: async () => {},
+	} as unknown as IAddressRepository;
+
 	const repository = {
 		listByAccountConfigId: async () => options.existing ?? [],
 		upsert: async (input: QuarantineUpsertInput) => {
@@ -91,7 +128,7 @@ const buildHarness = (options: {
 			],
 			update: async () => {},
 		} as unknown as IThreadMessageRepository,
-		{} as unknown as IAddressRepository,
+		addressService,
 		envelopeService,
 		{ info: () => {}, error: () => {}, debug: () => {} },
 		undefined,
@@ -114,16 +151,27 @@ const buildHarness = (options: {
 	return { service, writes, messageUpdates };
 };
 
+const connection = () =>
+	({
+		openBox: async () => {},
+		async *fetchMessageBodies(uids: number[]) {
+			for (const uid of uids) {
+				yield { uid, source: Readable.from([WELL_FORMED]) };
+			}
+		},
+	}) as unknown as IImapConnection;
+
 const sync = (service: BodySyncService) =>
-	service.syncBodies(["m-1"], "acc-1", "cfg-1", "INBOX", async () => {
-		throw new Error("this test must not open IMAP");
-	});
+	service.syncBodies(["m-1"], "acc-1", "cfg-1", "INBOX", async () =>
+		connection(),
+	);
 
-describe("body sync quarantines a message the parser refuses", () => {
-	const retrieveUnparseable = async () => UNPARSEABLE as unknown as Buffer;
+const parseRefusal = () =>
+	new BodyParseError(new Error("boundary never closed"));
 
+describe("body sync quarantines a message the parser refuses", () => {
 	it("records the failure instead of requeueing the message forever", async () => {
-		const harness = buildHarness({ retrieve: retrieveUnparseable });
+		const harness = buildHarness({ storeBodyFails: parseRefusal() });
 
 		const result = await sync(harness.service);
 
@@ -132,7 +180,7 @@ describe("body sync quarantines a message the parser refuses", () => {
 	});
 
 	it("names the message by uid and UIDVALIDITY, so the record is idempotent", async () => {
-		const harness = buildHarness({ retrieve: retrieveUnparseable });
+		const harness = buildHarness({ storeBodyFails: parseRefusal() });
 
 		await sync(harness.service);
 
@@ -142,7 +190,7 @@ describe("body sync quarantines a message the parser refuses", () => {
 	});
 
 	it("does not mark the message synced — it was set aside, not applied", async () => {
-		const harness = buildHarness({ retrieve: retrieveUnparseable });
+		const harness = buildHarness({ storeBodyFails: parseRefusal() });
 
 		const result = await sync(harness.service);
 
@@ -154,9 +202,7 @@ describe("body sync quarantines a message the parser refuses", () => {
 describe("body sync leaves infrastructure failures alone", () => {
 	it("requeues a storage failure rather than calling the message unreadable", async () => {
 		const harness = buildHarness({
-			retrieve: async () => {
-				throw new Error("S3 503 SlowDown");
-			},
+			storeBodyFails: new Error("S3 503 SlowDown"),
 		});
 
 		const result = await sync(harness.service);
@@ -165,15 +211,13 @@ describe("body sync leaves infrastructure failures alone", () => {
 		assert.deepEqual(result.failedMessageIds, ["m-1"]);
 	});
 
-	it("requeues a database failure the same way", async () => {
-		const harness = buildHarness({
-			retrieve: async () => {
-				const error = new Error("ProvisionedThroughputExceededException");
-				error.name = "ProvisionedThroughputExceededException";
-				throw error;
-			},
-		});
+	it("requeues a database failure the same way, parse already past", async () => {
+		const throughput = new Error("ProvisionedThroughputExceededException");
+		throughput.name = "ProvisionedThroughputExceededException";
+		const harness = buildHarness({ storeParsedFails: throughput });
 
+		// The body parsed cleanly and the failure came after it, so nothing here
+		// may say the message is unreadable.
 		const result = await sync(harness.service);
 
 		assert.deepEqual(harness.writes, []);
@@ -184,7 +228,7 @@ describe("body sync leaves infrastructure failures alone", () => {
 describe("body sync contains a failure to write the record", () => {
 	it("requeues the message instead of aborting the batch", async () => {
 		const harness = buildHarness({
-			retrieve: async () => UNPARSEABLE as unknown as Buffer,
+			storeBodyFails: parseRefusal(),
 			upsertFails: true,
 		});
 
@@ -208,14 +252,21 @@ describe("body sync skips what is already quarantined", () => {
 					uid: 40217,
 				} as QuarantineItem,
 			],
-			retrieve: async () => {
-				throw new Error("a quarantined message must not be read again");
-			},
 		});
 
-		const result = await sync(harness.service);
+		const result = await harness.service.syncBodies(
+			["m-1"],
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			async () => {
+				throw new Error("a quarantined message must not be fetched again");
+			},
+		);
 
 		assert.equal(result.skippedCount, 1);
+		assert.deepEqual(result.failedMessageIds, []);
 		assert.deepEqual(harness.writes, []);
+		assert.deepEqual(harness.messageUpdates, []);
 	});
 });