@remit/mailbox-service 0.0.36 → 0.0.38

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.36",
+  "version": "0.0.38",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/body-sync-placement-writeonce.test.ts

@@ -0,0 +1,323 @@
+/**
+ * RFC 039 Non-goals / issue #383: placement is meant to run once per message.
+ * #378 (issue #355) guarded `Message.category` against the same two re-entrant
+ * paths re-running `applyPostStoreSteps` on an already-processed message —
+ * `fetchAndGetBody`'s `NoSuchKey` fallback and `syncBodies(..., force: true)`
+ * — but did not guard `resolvePlacement`/`computePlacement`. Without a guard,
+ * a message the provider originally junked and a user later rescued by hand
+ * (never touched by Remit, so `movedByRemit` records nothing) gets
+ * `classifyPlacement` re-evaluated against the same demote signals that
+ * junked it in the first place, and can be silently moved right back.
+ *
+ * Each "already decided" test below presets `placementDecidedAt` on the
+ * fixture (as a genuine first pass would have left it) and feeds the
+ * re-entrant pass a body whose headers WOULD trigger a confident demote if
+ * `classifyPlacement` ran fresh — so a regression that drops the guard shows
+ * up as an unwanted move, not as a passing test relying on deterministic
+ * heuristics happening to agree.
+ */
+
+import assert from "node:assert/strict";
+import { Readable } from "node:stream";
+import { describe, it } from "node:test";
+import type {
+	AddressItem,
+	IAddressRepository,
+	IEnvelopeRepository,
+	IMailboxSpecialUseRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+	MessageItem,
+	UpdateMessageInput,
+} from "@remit/data-ports";
+import { MailboxSpecialUse } from "@remit/domain-enums";
+import type { StorageService } from "@remit/storage-service";
+import type { PlacementConfig } from "./body-sync.js";
+import { BodySyncService } from "./body-sync.js";
+import type { PlacementMoveService } from "./placement-move.js";
+import type { IImapConnection } from "./types.js";
+
+const MAILBOXES = {
+	inbox: { mailboxId: "mb-inbox", fullPath: "INBOX" },
+	junk: { mailboxId: "mb-junk", fullPath: "Junk" },
+};
+
+/**
+ * DKIM signing domain mismatches the From domain, dmarc=fail, sender
+ * untrusted, a provider-spam header present (any value) — exactly the
+ * deterministic signal set `classifyPlacement`'s demote branch (inbox → junk,
+ * HIGH bar) confidently acts on when the message currently sits in Inbox.
+ */
+const DEMOTE_EML = Buffer.from(
+	[
+		"From: Support <support@evil-mimic.example>",
+		"To: me@example.com",
+		"Subject: Verify your account",
+		"Authentication-Results: mx.example.com; dmarc=fail",
+		"DKIM-Signature: v=1; a=rsa-sha256; d=relay.example.net; s=sel; b=xxx",
+		"X-Spam-Status: No, score=0.1",
+		"Content-Type: text/plain",
+		"",
+		"body",
+	].join("\r\n"),
+);
+
+interface Harness {
+	service: BodySyncService;
+	message: MessageItem;
+	messageUpdates: Array<{ messageId: string; input: UpdateMessageInput }>;
+	moves: Array<{ messageId: string; destinationMailboxId: string }>;
+}
+
+const buildHarness = (
+	message: Partial<MessageItem> & Pick<MessageItem, "messageId">,
+	retrieve: () => Promise<Buffer>,
+): Harness => {
+	const messageUpdates: Array<{
+		messageId: string;
+		input: UpdateMessageInput;
+	}> = [];
+	const moves: Array<{ messageId: string; destinationMailboxId: string }> = [];
+
+	const messageRow = {
+		uid: 1,
+		mailboxId: MAILBOXES.inbox.mailboxId,
+		...message,
+	} as unknown as MessageItem;
+
+	const messageService = {
+		get: async () => messageRow,
+		update: async (messageId: string, input: UpdateMessageInput) => {
+			messageUpdates.push({ messageId, input });
+			Object.assign(messageRow, input);
+		},
+	} as unknown as IMessageRepository;
+
+	const threadMessageService = {
+		findAllByMessageId: async () => [
+			{
+				threadMessageId: "tm-1",
+				messageId: message.messageId,
+				mailboxId: messageRow.mailboxId,
+				sentDate: 1,
+				isRead: false,
+				isDeleted: false,
+				hasStars: false,
+				hasAttachment: false,
+			},
+		],
+		update: async () => {},
+	} as unknown as IThreadMessageRepository;
+
+	const storageService = {
+		retrieve,
+		storeMessageBody: async () => ({ uri: `s3://bodies/${message.messageId}` }),
+		storeMessageBodyStream: async () => ({
+			uri: `s3://bodies/${message.messageId}`,
+		}),
+		storeParsedBody: async () => {},
+		listBodyParts: async () => [],
+	} as unknown as StorageService;
+
+	const addressService = {
+		getAddress: async () => ({ flags: {} }) as unknown as AddressItem,
+		incrementInboundCount: async () => {},
+	} as unknown as IAddressRepository;
+
+	const envelopeService = {
+		listBodyParts: async () => [],
+	} as unknown as IEnvelopeRepository;
+
+	const mailboxSpecialUseService = {
+		findBySpecialUse: async (_accountId: string, specialUse: string) =>
+			specialUse === MailboxSpecialUse.Junk ? MAILBOXES.junk : null,
+		findInboxMailbox: async () => MAILBOXES.inbox,
+	} as unknown as IMailboxSpecialUseRepository;
+
+	const placementMoveService = {
+		moveMessage: async (
+			_accountConfigId: string,
+			messageId: string,
+			destinationMailboxId: string,
+		) => {
+			moves.push({ messageId, destinationMailboxId });
+			messageRow.mailboxId = destinationMailboxId;
+		},
+	} as unknown as PlacementMoveService;
+
+	const placementConfig: PlacementConfig = {
+		mailboxSpecialUseService,
+		placementMoveService,
+	};
+
+	const service = new BodySyncService(
+		messageService,
+		storageService,
+		threadMessageService,
+		addressService,
+		envelopeService,
+		{ info: () => {}, error: () => {} },
+		placementConfig,
+	);
+
+	return { service, message: messageRow, messageUpdates, moves };
+};
+
+const noSuchKeyError = () =>
+	Object.assign(new Error("missing"), { name: "NoSuchKey" });
+
+describe("placement survives a re-entrant computePlacement pass (issue #383)", () => {
+	it("keeps a user-rescued message in Inbox through the NoSuchKey IMAP re-fetch", async () => {
+		const harness = buildHarness(
+			{
+				messageId: "m-1",
+				mailboxId: MAILBOXES.inbox.mailboxId,
+				bodyStorageKey: "s3://bodies/m-1",
+				movedByRemit: false,
+				placementDecidedAt: 500,
+			},
+			async () => {
+				throw noSuchKeyError();
+			},
+		);
+
+		const connection = {
+			openBox: async () => {},
+			fetchMessageBody: async () => DEMOTE_EML,
+		} as unknown as IImapConnection;
+
+		await harness.service.fetchAndGetBody(
+			"m-1",
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			async () => connection,
+		);
+
+		assert.deepEqual(
+			harness.moves,
+			[],
+			"an already-decided placement must not be re-evaluated, even though these headers would confidently demote if classifyPlacement ran fresh",
+		);
+		assert.equal(harness.message.mailboxId, MAILBOXES.inbox.mailboxId);
+		assert.equal(harness.messageUpdates[0]?.input.movedByRemit, undefined);
+		assert.equal(harness.messageUpdates[0]?.input.placementVerdict, undefined);
+	});
+
+	it("keeps a user-rescued message in Inbox when syncBodies re-fetches with force", async () => {
+		const harness = buildHarness(
+			{
+				messageId: "m-1",
+				mailboxId: MAILBOXES.inbox.mailboxId,
+				bodyStorageKey: "s3://bodies/m-1",
+				movedByRemit: false,
+				placementDecidedAt: 500,
+			},
+			async () => {
+				throw new Error("force path must not retrieve from storage");
+			},
+		);
+
+		const connection = {
+			openBox: async () => {},
+			async *fetchMessageBodies(uids: number[]) {
+				for (const uid of uids) {
+					yield { uid, source: Readable.from([DEMOTE_EML]) };
+				}
+			},
+		} as unknown as IImapConnection;
+
+		const result = await harness.service.syncBodies(
+			["m-1"],
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			async () => connection,
+			true,
+		);
+
+		assert.deepEqual(result.syncedMessageIds, ["m-1"]);
+		assert.deepEqual(
+			harness.moves,
+			[],
+			"a forced re-sync must not re-decide an already-decided placement",
+		);
+		assert.equal(harness.message.mailboxId, MAILBOXES.inbox.mailboxId);
+	});
+
+	it("still evaluates and can confidently move a message that has never been placement-classified", async () => {
+		const harness = buildHarness(
+			{
+				messageId: "m-1",
+				mailboxId: MAILBOXES.inbox.mailboxId,
+				movedByRemit: false,
+			},
+			async () => {
+				throw new Error("no body stored yet; must not retrieve");
+			},
+		);
+
+		const connection = {
+			openBox: async () => {},
+			fetchMessageBody: async () => DEMOTE_EML,
+		} as unknown as IImapConnection;
+
+		await harness.service.fetchAndGetBody(
+			"m-1",
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			async () => connection,
+		);
+
+		assert.deepEqual(harness.moves, [
+			{ messageId: "m-1", destinationMailboxId: MAILBOXES.junk.mailboxId },
+		]);
+		assert.equal(harness.messageUpdates[0]?.input.movedByRemit, true);
+		assert.ok(
+			typeof harness.messageUpdates[0]?.input.placementDecidedAt === "number",
+			"a genuine first evaluation must record placementDecidedAt",
+		);
+	});
+
+	it("continues to protect a message Remit itself already moved (movedByRemit: true), unchanged behavior", async () => {
+		const harness = buildHarness(
+			{
+				messageId: "m-1",
+				mailboxId: MAILBOXES.inbox.mailboxId,
+				bodyStorageKey: "s3://bodies/m-1",
+				movedByRemit: true,
+				// No placementDecidedAt — a legacy row synced before issue #383's
+				// guard existed. `classifyPlacement`'s own `movedByRemit` check
+				// must still hold on its own.
+			},
+			async () => {
+				throw noSuchKeyError();
+			},
+		);
+
+		const connection = {
+			openBox: async () => {},
+			fetchMessageBody: async () => DEMOTE_EML,
+		} as unknown as IImapConnection;
+
+		await harness.service.fetchAndGetBody(
+			"m-1",
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			async () => connection,
+		);
+
+		assert.deepEqual(
+			harness.moves,
+			[],
+			"movedByRemit must keep protecting a legacy row with no placementDecidedAt of its own",
+		);
+		assert.equal(harness.message.mailboxId, MAILBOXES.inbox.mailboxId);
+		assert.ok(
+			typeof harness.messageUpdates[0]?.input.placementDecidedAt === "number",
+			"the legacy row is backfilled with placementDecidedAt going forward, self-healing for future passes",
+		);
+	});
+});

package/src/body-sync.ts

@@ -74,6 +74,7 @@ type ThreadMessageCategory = ThreadMessageItem["category"];
 interface PlacementOutcome {
 	verdict?: MessagePlacementVerdict;
 	move?: { destinationMailboxId: string; destinationPath: string };
+	placementDecidedAt?: number;
 }
 
 /**
@@ -185,6 +186,19 @@ const hasDecidedCategory = (
 ): boolean =>
 	category !== undefined && category !== MessageCategory.uncategorized;
 
+/**
+ * Issue #383 (RFC 039 Non-goals): whether {@link BodySyncService.computePlacement}
+ * has already produced a verdict for this message — moved, left in place, or
+ * archived, confident or unsure alike. Absence means placement has genuinely
+ * never been evaluated. The same two re-entrant paths `hasDecidedCategory`
+ * guards (`fetchAndGetBody`'s `NoSuchKey` fallback, `syncBodies(..., force:
+ * true)`) also re-enter `computePlacement`; without this guard a message a
+ * user manually rescued (never touched by Remit, so `movedByRemit` never
+ * recorded anything) can be silently re-evaluated and moved right back.
+ */
+const hasDecidedPlacement = (placementDecidedAt: number | undefined): boolean =>
+	placementDecidedAt !== undefined;
+
 export const toParsedBody = (parsed: ParsedMail): ParsedBody => ({
 	text: parsed.text ?? null,
 	html: typeof parsed.html === "string" ? parsed.html : null,
@@ -867,6 +881,11 @@ export class BodySyncService {
 		// this guard a re-entrant pass would let a *later* override silently
 		// rewrite a category already decided on an earlier message, which is
 		// exactly the churn RFC 030's GSI-safety argument forbids.
+		//
+		// `resolved.placementDecidedAt` (issue #383) guards the same two
+		// re-entrant paths for placement: `computePlacement` already declined to
+		// recompute a verdict once this field is set, so it is only ever present
+		// here on the pass that first decided it.
 		const existingMessage = await this.messageService.get(messageId);
 		const finalCategory = hasDecidedCategory(existingMessage.category)
 			? existingMessage.category
@@ -889,6 +908,9 @@ export class BodySyncService {
 			...(moved ? { movedByRemit: true } : {}),
 			...(resolved.verdict ? { placementVerdict: resolved.verdict } : {}),
 			...(filterMove ? { filterMove } : {}),
+			...(resolved.placementDecidedAt
+				? { placementDecidedAt: resolved.placementDecidedAt }
+				: {}),
 		};
 		await this.messageService.update(messageId, update);
 		this.log.info({ messageId, storageKey: bodyRef.uri }, "Body stored");
@@ -1320,8 +1342,11 @@ export class BodySyncService {
 	 *
 	 * Returns a {@link PlacementOutcome}: a `verdict` to persist whenever Remit
 	 * decided to act (action != leave), confident and unsure alike, so the
-	 * distribution is queryable on the message; and a `move` to enqueue only for
-	 * a confident verdict.
+	 * distribution is queryable on the message; a `move` to enqueue only for a
+	 * confident verdict; and `placementDecidedAt` whenever a verdict — including
+	 * "leave" — was genuinely computed (issue #383), so a re-entrant call short-
+	 * circuits instead of re-deciding a placement the user may have since
+	 * overridden by hand.
 	 *
 	 * Always logs a structured verdict line for confident, actionable verdicts so
 	 * the real distribution is observable on a live mailbox.
@@ -1334,7 +1359,7 @@ export class BodySyncService {
 	 * alertable field instead of failing the surrounding message store. The empty
 	 * {@link PlacementOutcome} means "no action", whether Remit genuinely decided
 	 * to leave the message alone or placement itself failed; the alert log is
-	 * what distinguishes the latter.
+	 * what distinguishes the latter, and neither case marks the message decided.
 	 */
 	private async resolvePlacement(
 		messageId: string,
@@ -1378,6 +1403,16 @@ export class BodySyncService {
 		const { mailboxSpecialUseService } = placementConfig;
 
 		const message = await this.messageService.get(messageId);
+
+		// Issue #383: placement is meant to run once per message (RFC 039
+		// Non-goals). Once a verdict has EVER been decided for this message —
+		// moved, left in place, or archived — a re-entrant pass (the `NoSuchKey`
+		// fallback in `fetchAndGetBody`, `syncBodies(..., force: true)`) must not
+		// recompute it: a message a user has since moved by hand (which never
+		// touches `movedByRemit`) would otherwise be silently re-evaluated
+		// against the same signals that placed it in the first place.
+		if (hasDecidedPlacement(message.placementDecidedAt)) return {};
+
 		const junkMailbox = await mailboxSpecialUseService.findBySpecialUse(
 			accountId,
 			MailboxSpecialUse.Junk,
@@ -1421,14 +1456,16 @@ export class BodySyncService {
 		// audit record of its own. `flags.autoArchive` (issue #300) is a distinct,
 		// lower-priority filing preference: it only files a message away when
 		// `blocked`/DKIM/DMARC had nothing to say, never overriding a confident
-		// junk/inbox verdict computed above.
+		// junk/inbox verdict computed above. Still marked decided (issue #383):
+		// "leave" is itself a verdict, not "not yet evaluated".
 		if (verdict.action === "leave") {
-			return this.resolveAutoArchive(
+			const outcome = await this.resolveAutoArchive(
 				mailboxSpecialUseService,
 				message,
 				accountId,
 				signals.autoArchive,
 			);
+			return { ...outcome, placementDecidedAt: Date.now() };
 		}
 
 		// Audit verdict — recorded for every actionable verdict (both
@@ -1452,12 +1489,12 @@ export class BodySyncService {
 		// Only a confident verdict moves mail. An unsure verdict is recorded
 		// but never enqueues a move.
 		if (verdict.confidence !== "confident") {
-			return { verdict: audit };
+			return { verdict: audit, placementDecidedAt: audit.decidedAt };
 		}
 
 		const target =
 			verdict.action === "move-to-inbox" ? inboxMailbox : junkMailbox;
-		if (!target) return { verdict: audit };
+		if (!target) return { verdict: audit, placementDecidedAt: audit.decidedAt };
 
 		// Structured verdict line — emitted for confident, actionable verdicts
 		// so the real verdict distribution is observable on a live mailbox.
@@ -1480,6 +1517,7 @@ export class BodySyncService {
 				destinationMailboxId: target.mailboxId,
 				destinationPath: target.fullPath,
 			},
+			placementDecidedAt: audit.decidedAt,
 		};
 	}
 

package/src/filters/match.test.ts

@@ -228,14 +228,17 @@ describe("cosineSimilarity", () => {
 });
 
 describe("selectMoveWinner", () => {
-	const filter = (filterId: string, ruleChangedAt: number): FilterItem =>
-		({ filterId, ruleChangedAt }) as FilterItem;
+	const filter = (
+		filterId: string,
+		actionChangedAt: number,
+		ruleChangedAt = actionChangedAt,
+	): FilterItem => ({ filterId, actionChangedAt, ruleChangedAt }) as FilterItem;
 
 	it("returns undefined with no candidates", () => {
 		assert.equal(selectMoveWinner([]), undefined);
 	});
 
-	it("picks the most-recently-changed filter", () => {
+	it("picks the most-recently action-changed filter", () => {
 		const winner = selectMoveWinner([
 			filter("a", 100),
 			filter("b", 300),
@@ -244,7 +247,7 @@ describe("selectMoveWinner", () => {
 		assert.equal(winner?.filterId, "b");
 	});
 
-	it("tie-breaks on filterId when ruleChangedAt is identical", () => {
+	it("tie-breaks on filterId when actionChangedAt is identical", () => {
 		const winner = selectMoveWinner([
 			filter("a", 100),
 			filter("c", 100),
@@ -252,6 +255,23 @@ describe("selectMoveWinner", () => {
 		]);
 		assert.equal(winner?.filterId, "c");
 	});
+
+	it("ignores a ruleChangedAt-only bump — extending scope/expiry alone must not promote a filter to move-winner (reader #384)", () => {
+		// filter A's predicate/action last changed at 10:00 (actionChangedAt).
+		// filter B was created at 09:00 and never had its predicate/action
+		// touched since, but a user extended its expiry at 11:00 — bumping only
+		// its ruleChangedAt (RFC 034 Decision 3.2 / #294), not what it matches or
+		// does. B must not out-rank A.
+		const filterA = filter("filter-a", 10_00);
+		const filterB = filter("filter-b", 9_00, 11_00);
+
+		const winner = selectMoveWinner([filterA, filterB]);
+		assert.equal(
+			winner?.filterId,
+			"filter-a",
+			"A still wins: its actionChangedAt (10:00) beats B's (09:00), even though B's ruleChangedAt (11:00) is later",
+		);
+	});
 });
 
 describe("buildMatchText", () => {

package/src/filters/match.ts

@@ -134,10 +134,14 @@ export const cosineSimilarity = (
 };
 
 /**
- * The move a message ends in when several filters matched: the most-recently
- * *changed* filter wins (RFC 034 Decision 3.2), tie-broken on `filterId` for the
- * unreachable identical-timestamp case. `ruleChangedAt` — not `updatedAt` — is
- * the signal, so a cosmetic rename never flips an exclusive move.
+ * The move a message ends in when several filters matched: the filter whose
+ * predicate or action was most recently *changed* wins (RFC 034 Decision 3.2),
+ * tie-broken on `filterId` for the unreachable identical-timestamp case.
+ * `actionChangedAt` — not `ruleChangedAt` — is the signal: `ruleChangedAt` also
+ * bumps on a scope/expiry-only edit (reader #266), which changes a filter's
+ * lifecycle, not what it matches or does, and must not reorder exclusive-move
+ * precedence (reader #384). Nor is it `updatedAt`, so a cosmetic rename never
+ * flips an exclusive move either.
  */
 export const selectMoveWinner = (
 	candidates: readonly FilterItem[],
@@ -148,12 +152,12 @@ export const selectMoveWinner = (
 			winner = candidate;
 			continue;
 		}
-		if (candidate.ruleChangedAt > winner.ruleChangedAt) {
+		if (candidate.actionChangedAt > winner.actionChangedAt) {
 			winner = candidate;
 			continue;
 		}
 		if (
-			candidate.ruleChangedAt === winner.ruleChangedAt &&
+			candidate.actionChangedAt === winner.actionChangedAt &&
 			candidate.filterId > winner.filterId
 		) {
 			winner = candidate;