@remit/mailbox-service 0.0.98 → 0.0.100

package/src/body-sync.ts

@@ -23,7 +23,6 @@ import { deriveAddressId, deriveBodyPartId } from "@remit/data-ports/id";
 import { isBulkSender } from "@remit/data-ports/wellknown";
 import {
 	FilterState,
-	MessageCategory,
 	MessageClassificationState,
 	PlacementAction,
 	PlacementConfidence,
@@ -40,6 +39,10 @@ import pMap from "p-map";
 import { BodyParseError, parseMessageBody } from "./body-parse.js";
 import { mapBodyPartsToContent } from "./body-part-mapper.js";
 import { calendarParts } from "./calendar-parts.js";
+import {
+	classifyParsedMessage,
+	extractPrimaryFromEmail,
+} from "./classify-message.js";
 import { extractListId } from "./filters/list-id.js";
 import type { FilterMessage } from "./filters/match.js";
 import {
@@ -48,21 +51,16 @@ import {
 	FilterPipeline,
 } from "./filters/pipeline.js";
 import type { FlagQueueService } from "./flag-queue.js";
-import {
-	classifyByHeaders,
-	extractAuthenticity,
-	extractAuthResult,
-	extractHasListUnsubscribe,
-	extractProviderSpam,
-} from "./heuristics/classifyByHeaders.js";
 import {
 	classifyPlacement,
 	type FolderPlacement,
 	type PlacementVerdict,
 	resolveSenderPlacement,
 } from "./heuristics/classifyPlacement.js";
-import { extractSenderMismatch } from "./heuristics/senderMismatch.js";
-import { denormalizeMessageCategory } from "./message-category.js";
+import {
+	denormalizeMessageCategory,
+	hasDecidedCategory,
+} from "./message-category.js";
 import type { PlacementMoveService } from "./placement-move.js";
 import { type QuarantineService, shapeFromMessageData } from "./quarantine.js";
 import { extractSnippetFromEmail } from "./snippet.js";
@@ -133,14 +131,6 @@ const isConnectionDrop = (error: unknown): boolean => {
 	return code === "EConnectionClosed" || code === "NoConnection";
 };
 
-export const extractPrimaryFromEmail = (parsed: ParsedMail): string | null => {
-	const from = parsed.from;
-	if (!from || !from.value || from.value.length === 0) return null;
-	const address = from.value[0]?.address;
-	if (!address) return null;
-	return address.toLowerCase();
-};
-
 /**
  * Project a parsed message onto the fields a filter matches against (RFC 034) —
  * the literal-clause targets plus the text a semantic anchor embeds. Kept
@@ -158,55 +148,30 @@ const SNIPPET_LENGTH = 256;
 
 /**
  * The snippet the list row shows, from whichever body part carries text.
+ * Exported for the classification backfill (issue #1197), which denormalizes
+ * the same fields the body-store path does, from the same bytes.
  */
-const extractSnippet = (parsed: ParsedMail): string =>
+export const extractSnippet = (parsed: ParsedMail): string =>
 	extractSnippetFromEmail(
 		parsed.text,
 		typeof parsed.html === "string" ? parsed.html : undefined,
 		SNIPPET_LENGTH,
 	);
 
-/**
- * RFC 034 Decision 3.1: `Message.category` is written once and never mutated
- * after — RFC 030's message-list GSI sort key depends on it never churning.
- * "Already decided" is any real category, so a message that carries one is
- * never re-categorized by a re-entrant pass.
- *
- * `uncategorized` fails this test whether or not the classifier has run, which
- * is deliberate: this asks what the row holds, not what was done to it. Whether
- * the classifier has run is {@link Message.classificationState}, and it is the
- * skip guard in {@link BodySyncService.syncBodies} — not this one — that keeps
- * a declined message from being examined twice.
- */
-const hasDecidedCategory = (
-	category: ThreadMessageCategory | undefined,
-): boolean =>
-	category !== undefined && category !== MessageCategory.uncategorized;
-
-/**
- * Issue #383 (RFC 039 Non-goals): whether {@link BodySyncService.computePlacement}
- * has already produced a verdict for this message — moved, left in place, or
- * archived, confident or unsure alike. Absence means placement has genuinely
- * never been evaluated. The same two re-entrant paths `hasDecidedCategory`
- * guards (`fetchAndGetBody`'s `NoSuchKey` fallback, `syncBodies(..., force:
- * true)`) also re-enter `computePlacement`; without this guard a message a
- * user manually rescued (never touched by Remit, so `movedByRemit` never
- * recorded anything) can be silently re-evaluated and moved right back.
- */
-const hasDecidedPlacement = (placementDecidedAt: number | undefined): boolean =>
-	placementDecidedAt !== undefined;
-
 /**
  * Issue #499: whether a completed body-sync pass has already stored this
  * message's body. `bodyStorageKey` is written last, once every derivation of
- * that pass has run, so its presence means that pass finished. The same two
- * re-entrant paths `hasDecidedCategory` and `hasDecidedPlacement` guard
- * (`fetchAndGetBody`'s `NoSuchKey` fallback, `syncBodies(..., force: true)`)
- * are the ones that reach the derivations again with it already set.
+ * that pass has run, so its presence means that pass finished.
  *
  * Key presence, and nothing more. What the classifier decided is
  * `Message.category`; whether it ran at all is `Message.classificationState`.
  * Neither is inferred from here — the field this reads is the storage fact.
+ *
+ * The re-fetch paths that re-enter with the key already set (`syncBodies(...,
+ * force: true)` and `fetchAndGetBody`'s `NoSuchKey` fallback) are re-stores
+ * that skip the derivations entirely (issue #1011), so this guard is what
+ * keeps a first store that lands on an already-stored row from re-deciding
+ * anything.
  */
 const hasStoredBody = (bodyStorageKey: string | undefined): boolean =>
 	Boolean(bodyStorageKey);
@@ -217,9 +182,9 @@ const hasStoredBody = (bodyStorageKey: string | undefined): boolean =>
  * means "nothing confident to say":
  *
  * - `already-moved-by-remit` is {@link classifyPlacement}'s guard against
- *   re-deciding a settled placement, and auto-archive is a re-decision. The
- *   re-entrant paths `hasDecidedPlacement` names reach it on messages Remit
- *   itself placed, or that a user moved back afterwards.
+ *   re-deciding a settled placement, and auto-archive is a re-decision. It
+ *   fires on messages Remit itself placed, or that a user moved back
+ *   afterwards.
  * - A message already in Junk gets an unsure `leave` whatever the sender's
  *   `blocked` flag says, since the demote branch only fires outside Junk.
  *   Moving mail out of Junk is a rescue, which this file reserves for a
@@ -397,6 +362,12 @@ export class BodySyncService {
 		// messageId so we can match FETCH rows back and re-enqueue any UID the
 		// server never returns.
 		const pending = new Map<number, string>();
+		// Message IDs in this batch whose row already carried `bodyStorageKey`
+		// when `force` re-added them to `pending`: a forced re-fetch of an
+		// already-stored body is a re-store (issue #1011) — bytes only, no
+		// decision re-run. A message with no stored body inside a forced batch
+		// is still a first store and takes the full decision pass.
+		const reStoredMessageIds = new Set<string>();
 
 		// One read per round, not per message (issue #72). The list is small by
 		// design and almost always empty, so a lookup per message would put a
@@ -438,6 +409,7 @@ export class BodySyncService {
 				skippedCount++;
 				continue;
 			}
+			if (message.bodyStorageKey) reStoredMessageIds.add(messageId);
 			pending.set(message.uid, messageId);
 		}
 
@@ -468,6 +440,7 @@ export class BodySyncService {
 						accountId,
 						accountConfigId,
 						source,
+						reStoredMessageIds.has(messageId),
 					);
 				} catch (error) {
 					// A per-message store failure (e.g. the parsed-body S3 write) is a
@@ -644,12 +617,18 @@ export class BodySyncService {
 	 * cache, per-part objects). The S3 upload never sees a whole-body concat —
 	 * the storage service streams it — but mailparser still needs the full bytes,
 	 * so we collect them in parallel. Later issues move parsing off the hot path.
+	 *
+	 * `isReStore` marks a re-fetch of a message whose row already carried
+	 * `bodyStorageKey` (the `force` re-arm in {@link syncBodies}): the
+	 * post-store steps then re-materialize the body artifacts without
+	 * re-running any decision (issue #1011).
 	 */
 	private async storeStreamedBody(
 		messageId: string,
 		accountId: string,
 		accountConfigId: string,
 		source: Readable,
+		isReStore: boolean,
 	): Promise<void> {
 		const toStorage = new PassThrough();
 		const chunks: Buffer[] = [];
@@ -693,6 +672,7 @@ export class BodySyncService {
 			{
 				uri: ref.uri,
 			},
+			isReStore,
 		);
 	}
 
@@ -724,6 +704,16 @@ export class BodySyncService {
 		accountConfigId: string,
 		body: Buffer,
 		bodyRef: { uri: string },
+		/**
+		 * Whether this call is a re-store of an already-classified message
+		 * rather than a first store. The two re-entrant paths (`force: true`
+		 * in `syncBodies`, and `fetchAndGetBody`'s `NoSuchKey` IMAP fallback) are
+		 * both body-only re-fetches: a body re-sync fetches and stores bytes and
+		 * runs no decision logic (issue #1011). When `isReStore` is true the
+		 * placement/filter/category machinery is skipped entirely and the
+		 * message's existing category is carried forward unchanged.
+		 */
+		isReStore: boolean,
 	): Promise<ParsedMail> {
 		// The one operation on this path that can fail because of how the message
 		// is built. Its own try block lives in `parseMessageBody`, so this frame —
@@ -733,6 +723,43 @@ export class BodySyncService {
 		// feed happens later, alongside the Message write (see below), once the
 		// write-once category is decided.
 		const parsed = await parseMessageBody(body);
+
+		// Body-only re-fetch (issue #1011): the decision pass already ran when
+		// this message's body first landed, so a re-store re-materializes only
+		// the body-derived artifacts — parsed cache, body parts, the storage
+		// key — and re-decides nothing: no classification, placement, filters,
+		// labels, read state, and no second engagement-counter increment.
+		// Skipping the decisions here is what lets the write-once fields stay
+		// safe without per-derivation re-entrancy guards.
+		if (isReStore) {
+			// Same ordering rule as the first-store path below: the parsed-body
+			// cache must be durable before `bodyStorageKey` — the skip-guard
+			// signal — is written, so a failure here requeues the message and
+			// the retry genuinely re-attempts the artifact writes.
+			await this.storeParsedBodyCache(
+				accountConfigId,
+				accountId,
+				messageId,
+				parsed,
+			);
+			if (!isBodyPartDeferralEnabled()) {
+				await this.storeBodyPartContents(
+					accountConfigId,
+					accountId,
+					messageId,
+					parsed,
+				);
+			}
+			await this.messageService.update(messageId, {
+				bodyStorageKey: bodyRef.uri,
+			});
+			this.log.info(
+				{ messageId, storageKey: bodyRef.uri },
+				"Body re-stored without re-running decisions",
+			);
+			return parsed;
+		}
+
 		const snippet = extractSnippet(parsed);
 		const listId = extractListId(parsed);
 
@@ -892,20 +919,19 @@ export class BodySyncService {
 		// once the parsed cache AND the move (when any) are.
 		//
 		// `category` is RFC 034 D3.1's write-once field (RFC 030's message-list
-		// GSI sort key depends on it never churning). This step re-enters on an
-		// already-classified message through two shipped paths — the `NoSuchKey`
-		// fallback in `fetchAndGetBody` and `syncBodies(..., force: true)` — so a
-		// real, previously-decided category is carried forward unchanged instead
-		// of the just-recomputed one.
-		// This also protects a `flags.category` override (issue #299): without
-		// this guard a re-entrant pass would let a *later* override silently
-		// rewrite a category already decided on an earlier message, which is
-		// exactly the churn RFC 030's GSI-safety argument forbids.
+		// GSI sort key depends on it never churning). The re-entrant re-store
+		// paths (issue #1011) return before this write; the carry-forward keeps
+		// the field write-once even when a first store lands on a row that
+		// already carries a real category — a previously-decided category wins
+		// over the just-recomputed one. It likewise protects a `flags.category`
+		// override (issue #299): recomputing would let a *later* override silently
+		// rewrite a category already decided, which is exactly the churn RFC
+		// 030's GSI-safety argument forbids.
 		//
-		// `resolved.placementDecidedAt` (issue #383) guards the same two
-		// re-entrant paths for placement: `computePlacement` already declined to
-		// recompute a verdict once this field is set, so it is only ever present
-		// here on the pass that first decided it.
+		// `resolved.placementDecidedAt` (issue #383) is only ever present on
+		// the pass that first decided it: the re-store paths skip placement
+		// entirely (issue #1011), so `computePlacement` is reached only on a
+		// first store.
 		const finalCategory = hasDecidedCategory(existingMessage.category)
 			? existingMessage.category
 			: classification.category;
@@ -1026,13 +1052,17 @@ export class BodySyncService {
 
 			// Same shared step the sync path (storeStreamedBody) runs — issue
 			// #1271 unified the two body paths so classification/placement no
-			// longer depends on which one fetched the body first.
+			// longer depends on which one fetched the body first. A store from
+			// here is a re-store exactly when the row already claimed a stored
+			// body — the NoSuchKey fallback above; a row with no bodyStorageKey is
+			// a genuine first store and takes the full decision pass.
 			parsed = await this.applyPostStoreSteps(
 				messageId,
 				accountId,
 				accountConfigId,
 				body,
 				{ uri: ref.uri },
+				message.bodyStorageKey != null,
 			);
 		} else {
 			parsed = await simpleParser(body);
@@ -1059,87 +1089,16 @@ export class BodySyncService {
 	/**
 	 * Header classification, with the sender's `Address.flags.category`
 	 * override (issue #299, RFC 039 Decision 3) substituted for the
-	 * header-derived category when one is set. Returns the subset of the
-	 * Message update that carries the derived fields; the caller folds it into
-	 * a single UpdateItem alongside `bodyStorageKey`. Optional signals are
-	 * omitted when absent so we never overwrite an existing value with
-	 * `undefined`.
-	 *
-	 * The override wins outright rather than blending with the heuristic — RFC
-	 * 039 Decision 3 treats a direct reclassification as final, the same as
-	 * `flags.blocked`/`vip` already override placement. The caller
-	 * (`applyPostStoreSteps`) gates on `hasDecidedCategory` before this result
-	 * reaches a write, so a message that already carries a real category is
-	 * never re-touched regardless of what this returns.
+	 * header-derived category when one is set. Shared with the classification
+	 * backfill (issue #1197) as {@link classifyParsedMessage}, so the one-time
+	 * pass and the body-store path can never classify the same bytes
+	 * differently.
 	 */
 	private async classifyMessage(
 		accountConfigId: string,
 		parsed: ParsedMail,
 	): Promise<UpdateMessageInput & { category: ThreadMessageCategory }> {
-		const headerCategory = classifyByHeaders(parsed);
-		const authenticity = extractAuthenticity(parsed);
-		const authResult = extractAuthResult(parsed);
-		const providerSpam = extractProviderSpam(parsed);
-		const hasListUnsubscribe = extractHasListUnsubscribe(parsed);
-
-		// A passing SPF/DKIM/DMARC check proves the sending domain, not the
-		// identity the message claims — on a shared-tenant host the verified
-		// subdomain belongs to whoever signed up. These two comparisons say
-		// whether the claim holds, and run only over mail the provider already
-		// called spam.
-		const senderMismatch =
-			authenticity === null
-				? {}
-				: extractSenderMismatch(parsed, {
-						fromDomain: authenticity.fromDomain,
-						spamClassified: providerSpam?.classified === true,
-						bulkSender: hasListUnsubscribe,
-					});
-
-		const fromEmail = extractPrimaryFromEmail(parsed);
-		const categoryOverride = fromEmail
-			? await this.resolveCategoryOverride(accountConfigId, fromEmail)
-			: undefined;
-
-		return {
-			category: categoryOverride ?? headerCategory,
-			hasListUnsubscribe,
-			...(authenticity !== null
-				? { authenticity: { ...authenticity, ...senderMismatch } }
-				: {}),
-			...(authResult !== null ? { authResult } : {}),
-			...(providerSpam !== null ? { providerSpam } : {}),
-		};
-	}
-
-	/**
-	 * The one-sided half of `Address.flags.category` (issue #299, RFC 039
-	 * Decision 3): a real value here overrides `classifyByHeaders` outright.
-	 * A second, separate `Address` read from `deriveSenderPlacementSignals`'s
-	 * (issue #300) — sharing one fetch was not practical, because the two have
-	 * different failure contracts: placement's read is best-effort (a failure
-	 * is caught and logged, never failing the sync), while this one feeds the
-	 * write-once `Message.category` at the moment it is decided, where a
-	 * masked infra failure would silently classify by headers when the user
-	 * asked for something else. Only a genuinely-absent Address is "no
-	 * override" — any other failure (throttle, infra) propagates, matching the
-	 * existing `deriveSenderPlacementSignals` convention.
-	 */
-	private async resolveCategoryOverride(
-		accountConfigId: string,
-		fromEmail: string,
-	): Promise<ThreadMessageCategory | undefined> {
-		try {
-			const addressId = deriveAddressId(accountConfigId, fromEmail);
-			const address = await this.addressService.getAddress(
-				accountConfigId,
-				addressId,
-			);
-			return address.flags?.category?.value;
-		} catch (err) {
-			if (!(err instanceof NotFoundError)) throw err;
-			return undefined;
-		}
+		return classifyParsedMessage(this.addressService, accountConfigId, parsed);
 	}
 
 	private async incrementInboundCount(
@@ -1492,14 +1451,12 @@ export class BodySyncService {
 
 		const message = await this.messageService.get(messageId);
 
-		// Issue #383: placement is meant to run once per message (RFC 039
-		// Non-goals). Once a verdict has EVER been decided for this message —
-		// moved, left in place, or archived — a re-entrant pass (the `NoSuchKey`
-		// fallback in `fetchAndGetBody`, `syncBodies(..., force: true)`) must not
-		// recompute it: a message a user has since moved by hand (which never
-		// touches `movedByRemit`) would otherwise be silently re-evaluated
-		// against the same signals that placed it in the first place.
-		if (hasDecidedPlacement(message.placementDecidedAt)) return {};
+		// Issue #383 / #1011: placement still runs once per message (RFC 039
+		// Non-goals), but the once-per-message guarantee now comes from the
+		// caller rather than a guard here — the re-store paths
+		// (`fetchAndGetBody`'s `NoSuchKey` fallback, `syncBodies(..., force:
+		// true)`) skip the decision steps entirely, so only the pass that first
+		// stores a body ever reaches this method.
 
 		const junkMailbox =
 			await mailboxSpecialUseService.findJunkMailbox(accountId);