@remit/mailbox-service 0.0.10 → 0.0.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.10",
+  "version": "0.0.12",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/body-sync-backfill.test.ts

@@ -0,0 +1,285 @@
+/**
+ * Body-sync's skip guard keys on `bodyStorageKey`, but classification is a
+ * separate derived field written by the same pass. A message that got its body
+ * before it got a classifier — or whose classifying pass failed after the body
+ * landed — is skipped forever and stays `uncategorized` (issue #45).
+ *
+ * These tests pin the backfill that closes that gap: it reads the stored body
+ * rather than IMAP, writes only the classification, and never silently absorbs
+ * a storage failure.
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	IAddressRepository,
+	IEnvelopeRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+	MessageItem,
+	UpdateMessageInput,
+	UpdateThreadMessageInput,
+} from "@remit/data-ports";
+import { MessageCategory } from "@remit/domain-enums";
+import type { StorageService } from "@remit/storage-service";
+import { BodySyncService } from "./body-sync.js";
+
+const LINKEDIN_EML = Buffer.from(
+	[
+		"From: LinkedIn <messages-noreply@linkedin.com>",
+		"To: me@example.com",
+		"Subject: You have a new invitation",
+		"List-Unsubscribe: <https://www.linkedin.com/e/unsub>",
+		"Content-Type: text/plain",
+		"",
+		"invitation",
+	].join("\r\n"),
+);
+
+interface Harness {
+	service: BodySyncService;
+	messageUpdates: Array<{ messageId: string; input: UpdateMessageInput }>;
+	threadUpdates: UpdateThreadMessageInput[];
+	retrieved: string[];
+	loggedErrors: Array<Record<string, unknown>>;
+}
+
+type MessageFixture = Partial<MessageItem> & Pick<MessageItem, "messageId">;
+
+const buildHarness = (
+	messages: MessageFixture[],
+	retrieve: (key: string) => Promise<Buffer> = async () => LINKEDIN_EML,
+): Harness => {
+	const messageUpdates: Array<{
+		messageId: string;
+		input: UpdateMessageInput;
+	}> = [];
+	const threadUpdates: UpdateThreadMessageInput[] = [];
+	const retrieved: string[] = [];
+	const loggedErrors: Array<Record<string, unknown>> = [];
+
+	const byId = new Map(messages.map((m) => [m.messageId, m]));
+
+	const messageService = {
+		get: async (messageId: string) => {
+			const message = byId.get(messageId);
+			if (!message) throw new Error(`no fixture for ${messageId}`);
+			return { uid: 1, category: MessageCategory.uncategorized, ...message };
+		},
+		update: async (messageId: string, input: UpdateMessageInput) => {
+			messageUpdates.push({ messageId, input });
+		},
+	} as unknown as IMessageRepository;
+
+	const threadMessageService = {
+		getByMessageId: async () => ({
+			threadMessageId: "tm-1",
+			sentDate: 1,
+			mailboxId: "mb-1",
+			isRead: false,
+			isDeleted: false,
+			hasStars: false,
+			hasAttachment: false,
+		}),
+		update: async (
+			_accountConfigId: string,
+			_threadMessageId: string,
+			input: UpdateThreadMessageInput,
+		) => {
+			threadUpdates.push(input);
+		},
+	} as unknown as IThreadMessageRepository;
+
+	const storageService = {
+		retrieve: async (key: string) => {
+			retrieved.push(key);
+			return retrieve(key);
+		},
+	} as unknown as StorageService;
+
+	const service = new BodySyncService(
+		messageService,
+		storageService,
+		threadMessageService,
+		{} as unknown as IAddressRepository,
+		{} as unknown as IEnvelopeRepository,
+		{
+			info: () => {},
+			error: (obj: Record<string, unknown>) => {
+				loggedErrors.push(obj);
+			},
+		},
+	);
+
+	return { service, messageUpdates, threadUpdates, retrieved, loggedErrors };
+};
+
+const failingConnection = async () => {
+	throw new Error("body-sync must not open IMAP to backfill a classification");
+};
+
+const stored = (messageId: string) => ({
+	messageId,
+	bodyStorageKey: `s3://bodies/${messageId}`,
+	category: MessageCategory.uncategorized,
+});
+
+describe("body-sync classification backfill", () => {
+	it("classifies a skipped message whose body is stored but category is uncategorized", async () => {
+		const harness = buildHarness([stored("m-1")]);
+
+		const result = await harness.service.syncBodies(
+			["m-1"],
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			failingConnection,
+		);
+
+		assert.equal(result.skippedCount, 1);
+		assert.equal(harness.messageUpdates.length, 1);
+		assert.equal(
+			harness.messageUpdates[0].input.category,
+			MessageCategory.social,
+		);
+	});
+
+	it("denormalizes the backfilled category onto the ThreadMessage", async () => {
+		const harness = buildHarness([stored("m-1")]);
+
+		await harness.service.syncBodies(
+			["m-1"],
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			failingConnection,
+		);
+
+		assert.equal(harness.threadUpdates.length, 1);
+		assert.equal(harness.threadUpdates[0].category, MessageCategory.social);
+	});
+
+	it("reads the stored body instead of reconnecting to IMAP", async () => {
+		const harness = buildHarness([stored("m-1")]);
+
+		await harness.service.syncBodies(
+			["m-1"],
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			failingConnection,
+		);
+
+		assert.deepEqual(harness.retrieved, ["s3://bodies/m-1"]);
+	});
+
+	it("classifies a message whose category field is absent entirely", async () => {
+		// Rows written before the column existed carry no value at all. Reading
+		// that as "already classified" would strand the oldest mail — exactly
+		// what this backfill exists to reach.
+		const harness = buildHarness([
+			{
+				messageId: "m-1",
+				bodyStorageKey: "s3://bodies/m-1",
+				category: undefined,
+			},
+		]);
+
+		await harness.service.syncBodies(
+			["m-1"],
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			failingConnection,
+		);
+
+		assert.equal(harness.messageUpdates.length, 1);
+		assert.equal(
+			harness.messageUpdates[0].input.category,
+			MessageCategory.social,
+		);
+	});
+
+	it("leaves an already-classified message untouched", async () => {
+		const harness = buildHarness([
+			{ ...stored("m-1"), category: MessageCategory.marketing },
+		]);
+
+		const result = await harness.service.syncBodies(
+			["m-1"],
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			failingConnection,
+		);
+
+		assert.equal(result.skippedCount, 1);
+		assert.deepEqual(harness.messageUpdates, []);
+		assert.deepEqual(harness.retrieved, []);
+	});
+
+	it("requeues the message when its stored body cannot be read", async () => {
+		// An unreadable body object is an infra fault. Absorbing it would leave
+		// the message permanently unclassified with nothing to show for it.
+		const harness = buildHarness([stored("m-1")], async () => {
+			throw new Error("AccessDenied");
+		});
+
+		const result = await harness.service.syncBodies(
+			["m-1"],
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			failingConnection,
+		);
+
+		assert.deepEqual(result.failedMessageIds, ["m-1"]);
+		assert.equal(result.skippedCount, 0);
+	});
+
+	it("logs a backfill failure rather than swallowing it", async () => {
+		const harness = buildHarness([stored("m-1")], async () => {
+			throw new Error("AccessDenied");
+		});
+
+		await harness.service.syncBodies(
+			["m-1"],
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			failingConnection,
+		);
+
+		assert.equal(harness.loggedErrors.length, 1);
+		assert.equal(harness.loggedErrors[0].messageId, "m-1");
+	});
+
+	it("does not abort the batch when one message's backfill fails", async () => {
+		// The failure happens in the message-resolution loop, before any body is
+		// fetched. Letting it escape would strand every other message in the
+		// batch — including ones that need a genuine IMAP fetch — behind one
+		// unreadable S3 object.
+		const harness = buildHarness(
+			[stored("m-bad"), stored("m-good")],
+			async (key) => {
+				if (key.endsWith("m-bad")) throw new Error("AccessDenied");
+				return LINKEDIN_EML;
+			},
+		);
+
+		const result = await harness.service.syncBodies(
+			["m-bad", "m-good"],
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			failingConnection,
+		);
+
+		assert.deepEqual(result.failedMessageIds, ["m-bad"]);
+		assert.equal(result.skippedCount, 1);
+		assert.deepEqual(
+			harness.messageUpdates.map((u) => u.messageId),
+			["m-good"],
+		);
+	});
+});

package/src/body-sync.ts

@@ -6,6 +6,8 @@ import type {
 	IMailboxSpecialUseRepository,
 	IMessageRepository,
 	IThreadMessageRepository,
+	MessageItem,
+	ThreadMessageItem,
 	UpdateMessageInput,
 } from "@remit/data-ports";
 import { NotFoundError } from "@remit/data-ports/errors";
@@ -13,6 +15,7 @@ import { deriveAddressId } from "@remit/data-ports/id";
 import { isBulkSender } from "@remit/data-ports/wellknown";
 import {
 	MailboxSpecialUse,
+	MessageCategory,
 	PlacementAction,
 	PlacementConfidence,
 	SenderTrust,
@@ -52,6 +55,8 @@ type MessagePlacementVerdict = NonNullable<
 	UpdateMessageInput["placementVerdict"]
 >;
 
+type ThreadMessageCategory = ThreadMessageItem["category"];
+
 /**
  * Outcome of {@link BodySyncService.resolvePlacement}: the audit `verdict` to
  * persist on the Message (present whenever Remit decided to act, i.e. action
@@ -240,10 +245,45 @@ export class BodySyncService {
 		// messageId so we can match FETCH rows back and re-enqueue any UID the
 		// server never returns.
 		const pending = new Map<number, string>();
+		// Messages that were skipped (body already stored) but whose backfill
+		// classification failed. They are NOT in `pending` — nothing about them
+		// needs fetching — so they are merged into failedMessageIds separately.
+		const backfillFailedMessageIds: string[] = [];
 		for (const messageId of messageIds) {
 			const message = await this.messageService.get(messageId);
 			if (message.bodyStorageKey && !force) {
 				this.log.debug?.({ messageId }, "Body already stored, skipping");
+				// The skip guard keys on the body, but classification is a separate
+				// derived field written by the same pass. A message that got its body
+				// before it got a classifier — or whose classifying pass failed after
+				// the body landed — is skipped here forever and stays `uncategorized`
+				// (issue #45). Classify it from the stored bytes: no IMAP, no
+				// placement/filter side effects, and it skips cleanly once done.
+				//
+				// Contained per-message: one unreadable body object must not abort a
+				// batch that has not fetched anything yet. The failure is loud and
+				// the id is requeued, but the other messages still get their bodies.
+				const backfillError = await this.backfillClassification(
+					message,
+					accountConfigId,
+				).then(
+					() => null,
+					(error: unknown) => error,
+				);
+				if (backfillError !== null) {
+					this.log.error?.(
+						{
+							messageId,
+							storageKey: message.bodyStorageKey,
+							errorName: (backfillError as { name?: string }).name,
+							errorCode: (backfillError as { Code?: string }).Code,
+							error: inspect(backfillError),
+						},
+						"Classification backfill failed for an already-stored body; leaving for requeue",
+					);
+					backfillFailedMessageIds.push(messageId);
+					continue;
+				}
 				skippedCount++;
 				continue;
 			}
@@ -251,7 +291,11 @@ export class BodySyncService {
 		}
 
 		if (pending.size === 0) {
-			return this.buildResult(syncedMessageIds, skippedCount, []);
+			return this.buildResult(
+				syncedMessageIds,
+				skippedCount,
+				backfillFailedMessageIds,
+			);
 		}
 
 		const connection = await getConnection();
@@ -321,7 +365,7 @@ export class BodySyncService {
 
 		// Anything still pending was never yielded (mid-stream drop or a UID the
 		// server silently omitted) — re-enqueue it.
-		const failedMessageIds = [...pending.values()];
+		const failedMessageIds = [...pending.values(), ...backfillFailedMessageIds];
 
 		this.log.info(
 			{
@@ -671,13 +715,65 @@ export class BodySyncService {
 		return connection.fetchMessageBody(uid);
 	}
 
+	/**
+	 * Classify a message whose body is already stored but which carries no
+	 * decided category, reading the body from storage instead of IMAP.
+	 *
+	 * "No decided category" is `uncategorized` OR the field being absent: rows
+	 * written before the column existed have no value at all, and treating that
+	 * as already-classified would strand exactly the oldest mail this backfill
+	 * exists to reach.
+	 *
+	 * Deliberately narrower than {@link applyPostStoreSteps}: it writes the
+	 * derived classification fields and the denormalized ThreadMessage category,
+	 * and nothing else. Placement moves and filter actions are index-time
+	 * decisions that already ran (or were declined) when the body first landed;
+	 * re-running them here would move mail the user has since filed by hand.
+	 *
+	 * A storage or write failure propagates to the caller, which contains it per
+	 * message: the id lands in `failedMessageIds` and SQS requeues it, while the
+	 * rest of the batch still gets its bodies. An unreadable body object is an
+	 * infra fault, never absorbed — but it is also not a reason to abort a batch
+	 * that has fetched nothing yet.
+	 */
+	private async backfillClassification(
+		message: MessageItem,
+		accountConfigId: string,
+	): Promise<void> {
+		if (!message.bodyStorageKey) return;
+		if (
+			message.category !== undefined &&
+			message.category !== MessageCategory.uncategorized
+		) {
+			return;
+		}
+
+		const body = await this.storageService.retrieve(message.bodyStorageKey);
+		const parsed = await simpleParser(body);
+		const classification = this.classifyMessage(parsed);
+
+		await this.messageService.update(message.messageId, classification);
+		await this.denormalizeCategory(
+			accountConfigId,
+			message.messageId,
+			classification.category,
+		);
+
+		this.log.info(
+			{ messageId: message.messageId, category: classification.category },
+			"Backfilled classification for an already-stored body",
+		);
+	}
+
 	/**
 	 * Pure header classification. Returns the subset of the Message update that
 	 * carries the derived fields; the caller folds it into a single UpdateItem
 	 * alongside `bodyStorageKey`. Optional signals are omitted when absent so we
 	 * never overwrite an existing value with `undefined`.
 	 */
-	private classifyMessage(parsed: ParsedMail): UpdateMessageInput {
+	private classifyMessage(
+		parsed: ParsedMail,
+	): UpdateMessageInput & { category: ThreadMessageCategory } {
 		const category = classifyByHeaders(parsed);
 		const authenticity = extractAuthenticity(parsed);
 		const authResult = extractAuthResult(parsed);
@@ -1163,19 +1259,43 @@ export class BodySyncService {
 
 		const category = classifyByHeaders(parsed);
 
-		// Get the ThreadMessage by messageId (efficient GSI lookup). The write is
-		// keyed on messageId, so it does not depend on the RFC822 Message-ID
-		// header — a headerless message still gets its category/snippet
-		// denormalized, matching the unconditional Message.category write.
+		await this.denormalizeCategory(
+			accountConfigId,
+			messageId,
+			category,
+			snippet,
+		);
+
+		this.log.debug?.(
+			{ messageId, category, snippetLength: snippet?.length ?? 0 },
+			"ThreadMessage snippet + category updated",
+		);
+
+		return parsed;
+	}
+
+	/**
+	 * Write the denormalized `category` (and optionally the snippet) onto the
+	 * message's ThreadMessage row — the copy the list/search read path serves
+	 * without a per-row Message fetch.
+	 *
+	 * The ThreadMessage is looked up by messageId (GSI), so it does not depend
+	 * on the RFC822 Message-ID header — a headerless message still gets
+	 * denormalized, matching the unconditional Message.category write. The full
+	 * composite set is passed so that a future key-attribute addition touching
+	 * the lsi3/lsi4/lsi5/gsi2 sort keys keeps the index rows consistent.
+	 */
+	private async denormalizeCategory(
+		accountConfigId: string,
+		messageId: string,
+		category: ThreadMessageCategory,
+		snippet?: string,
+	): Promise<void> {
 		const threadMessage = await this.threadMessageService.getByMessageId(
 			accountConfigId,
 			messageId,
 		);
 
-		// Update ThreadMessage snippet + denormalized category.
-		// Pass the full composite set so that if a future key-attribute addition
-		// touches lsi3/lsi4/lsi5/gsi2 sort keys, the index rows remain consistent.
-		// The threadMessage was fetched just above, so the values are already in scope.
 		await this.threadMessageService.update(
 			accountConfigId,
 			threadMessage.threadMessageId,
@@ -1191,12 +1311,5 @@ export class BodySyncService {
 				},
 			},
 		);
-
-		this.log.debug?.(
-			{ messageId, category, snippetLength: snippet?.length ?? 0 },
-			"ThreadMessage snippet + category updated",
-		);
-
-		return parsed;
 	}
 }

package/src/heuristics/classifyByHeaders.test.ts

@@ -256,7 +256,7 @@ describe("classifyByHeaders", () => {
 		assert.equal(classifyByHeaders(parsed), MessageCategory.transactional);
 	});
 
-	it("Auto-Submitted wins over List-Unsubscribe", async () => {
+	it("List-Unsubscribe wins over Auto-Submitted", async () => {
 		const parsed = await parse([
 			"From: alice@example.com",
 			"To: bob@example.com",
@@ -266,7 +266,7 @@ describe("classifyByHeaders", () => {
 			"",
 			"body",
 		]);
-		assert.equal(classifyByHeaders(parsed), MessageCategory.automated);
+		assert.equal(classifyByHeaders(parsed), MessageCategory.marketing);
 	});
 });
 

package/src/heuristics/classifyByHeaders.ts

@@ -6,6 +6,7 @@ import type {
 	ParsedMail,
 	StructuredHeader,
 } from "mailparser";
+import { hasMachineHeader, isMachineLocalPart } from "./machineSenders.js";
 import { SOCIAL_DOMAINS } from "./socialDomains.js";
 import { TRANSACTIONAL_DOMAINS } from "./transactionalDomains.js";
 
@@ -46,24 +47,32 @@ export interface MessageAuthenticity {
  * Header-only classification. Pure function. First match wins. Falls through
  * to `personal` so misclassification stays in the safest bucket.
  *
- * Rule order matches the EDD heuristic table:
+ * Rules are ordered most-specific-signal first. Two properties drive the
+ * order, and both were violated by the original table (issue #45):
  *
- * 1. `Auto-Submitted: auto-generated|auto-replied` → `automated`
- * 2. `Precedence: bulk|list|junk` → `automated`
- * 3. `Content-Type: text/calendar` part anywhere → `transactional`
- * 4. From-domain in TRANSACTIONAL_DOMAINS → `transactional`
- * 5. `List-Unsubscribe` AND `List-Id` → `newsletter`
- * 6. `List-Unsubscribe` only → `marketing`
- * 7. DKIM `d=` differs from From domain → `automated`
- * 8. From-domain in SOCIAL_DOMAINS → `social`
- * 9. fallback → `personal`
+ * - A signal that identifies WHO sent the mail (calendar part, allow-listed
+ *   domain) outranks a signal that only says the mail was sent in bulk.
+ * - `Precedence` and `Auto-Submitted` say "a machine sent this", which is true
+ *   of nearly every newsletter, marketing blast, and platform notification.
+ *   Ranking them first collapsed `newsletter`, `marketing`, `social` and
+ *   `transactional` into `automated`, leaving those buckets empty.
+ *
+ * 1. `Content-Type: text/calendar` part anywhere → `transactional`
+ * 2. From-domain in TRANSACTIONAL_DOMAINS → `transactional`
+ * 3. From-domain in SOCIAL_DOMAINS → `social`
+ * 4. `List-Unsubscribe` AND `List-Id` → `newsletter`
+ * 5. `List-Unsubscribe` only → `marketing`
+ * 6. `Auto-Submitted: auto-generated|auto-replied` → `automated`
+ * 7. `Precedence: bulk|list|junk` → `automated`
+ * 8. Machine sender (no-reply local-part, `Feedback-ID`,
+ *    `X-Auto-Response-Suppress`) → `automated`
+ * 9. DKIM `d=` differs from From domain → `automated`
+ * 10. fallback → `personal`
  */
 export const classifyByHeaders = (parsed: ParsedMail): Category => {
 	const headers = parsed.headers;
 	const lines = parsed.headerLines;
 
-	if (matchesAutoSubmitted(headers)) return MessageCategory.automated;
-	if (matchesPrecedence(headers)) return MessageCategory.automated;
 	if (hasCalendarPart(parsed.attachments)) return MessageCategory.transactional;
 
 	const fromDomain = getFromDomain(parsed);
@@ -72,20 +81,24 @@ export const classifyByHeaders = (parsed: ParsedMail): Category => {
 		return MessageCategory.transactional;
 	}
 
+	if (fromDomain && domainMatches(fromDomain, SOCIAL_DOMAINS)) {
+		return MessageCategory.social;
+	}
+
 	const hasListUnsubscribe = hasHeaderLine(lines, "list-unsubscribe");
 	const hasListId = hasHeaderLine(lines, "list-id");
 
 	if (hasListUnsubscribe && hasListId) return MessageCategory.newsletter;
 	if (hasListUnsubscribe) return MessageCategory.marketing;
 
+	if (matchesAutoSubmitted(headers)) return MessageCategory.automated;
+	if (matchesPrecedence(headers)) return MessageCategory.automated;
+	if (isMachineSender(parsed, lines)) return MessageCategory.automated;
+
 	if (fromDomain && dkimMismatchResult(headers, lines, fromDomain).mismatch) {
 		return MessageCategory.automated;
 	}
 
-	if (fromDomain && domainMatches(fromDomain, SOCIAL_DOMAINS)) {
-		return MessageCategory.social;
-	}
-
 	return MessageCategory.personal;
 };
 
@@ -242,6 +255,26 @@ const hasCalendarPart = (attachments: Attachment[] | undefined): boolean => {
 	return false;
 };
 
+/**
+ * Whether the sender is a machine that does not read replies — a no-reply
+ * local-part, or a header only bulk/notification infrastructure sets. This is
+ * what routes platform notifications (npm, CI, password resets) to `automated`
+ * instead of the `personal` fallback; they carry no list or bulk headers.
+ */
+const isMachineSender = (parsed: ParsedMail, lines: HeaderLines): boolean => {
+	if (hasMachineHeader(lines.map((line) => line.key))) return true;
+	const localPart = getFromLocalPart(parsed);
+	return localPart !== null && isMachineLocalPart(localPart);
+};
+
+const getFromLocalPart = (parsed: ParsedMail): string | null => {
+	const address = parsed.from?.value?.[0]?.address;
+	if (!address) return null;
+	const at = address.lastIndexOf("@");
+	if (at <= 0) return null;
+	return address.slice(0, at);
+};
+
 const getFromDomain = (parsed: ParsedMail): string | null => {
 	const from = parsed.from;
 	if (!from || !from.value || from.value.length === 0) return null;