@remit/mailbox-service 0.0.37 → 0.0.39

package/README.md

@@ -1,11 +1,11 @@
 # @remit/mailbox-service
 
-IMAP mailbox synchronization service for Remit. Provides connection management, mailbox discovery, and message sync with DynamoDB persistence.
+IMAP mailbox synchronization service for Remit. Provides connection management, mailbox discovery, and message sync. Persistence is injected: the service takes repository ports, never a database client.
 
 ## Features
 
 - **ImapFlow-based**: Modern async/await IMAP client with native envelope parsing
-- **Mailbox Sync**: Discovers and syncs mailbox metadata from IMAP to DynamoDB
+- **Mailbox Sync**: Discovers and syncs mailbox metadata from IMAP into the mailbox repository
 - **Message Sync**: Newest-first sync strategy with dual-watermark tracking
 - **Address Extraction**: Parses and stores envelope addresses with role tracking
 
@@ -55,10 +55,11 @@ await connection.disconnect();
 ```typescript
 import { MailboxSyncService } from "@remit/mailbox-service";
 
-const syncService = new MailboxSyncService({
-  client: dynamoDBClient,
-  table: "remit-table",
-});
+const syncService = new MailboxSyncService(
+  mailboxRepository,
+  mailboxSpecialUseRepository,
+  logger,
+);
 
 const result = await syncService.syncMailboxes(
   { accountId: "acc-123" },
@@ -103,7 +104,7 @@ const synced = await messageSyncService.syncMessages(
 
 | Export               | Description                               |
 | -------------------- | ----------------------------------------- |
-| `MailboxSyncService` | Syncs mailbox list from IMAP to DynamoDB  |
+| `MailboxSyncService` | Syncs mailbox list from IMAP into the store |
 | `MessageSyncService` | Syncs messages with newest-first strategy |
 
 ### Types

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.37",
+  "version": "0.0.39",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/body-parse.ts

@@ -9,7 +9,7 @@ type FailureCode = QuarantineItem["failureCode"];
  *
  * This type is the whole reason the sync path can quarantine anything. Before
  * it, every catch site on the body path saw one undifferentiated `unknown`
- * covering mailparser, S3, DynamoDB and SQS alike, so "the message is built in
+ * covering mailparser, S3, the database and SQS alike, so "the message is built in
  * a way Remit could not read" was indistinguishable from "S3 returned a 503".
  * Recording the second as the first advances the cursor past mail that is
  * perfectly fine and never fetches it again.

package/src/body-sync-placement-writeonce.test.ts

@@ -0,0 +1,323 @@
+/**
+ * RFC 039 Non-goals / issue #383: placement is meant to run once per message.
+ * #378 (issue #355) guarded `Message.category` against the same two re-entrant
+ * paths re-running `applyPostStoreSteps` on an already-processed message —
+ * `fetchAndGetBody`'s `NoSuchKey` fallback and `syncBodies(..., force: true)`
+ * — but did not guard `resolvePlacement`/`computePlacement`. Without a guard,
+ * a message the provider originally junked and a user later rescued by hand
+ * (never touched by Remit, so `movedByRemit` records nothing) gets
+ * `classifyPlacement` re-evaluated against the same demote signals that
+ * junked it in the first place, and can be silently moved right back.
+ *
+ * Each "already decided" test below presets `placementDecidedAt` on the
+ * fixture (as a genuine first pass would have left it) and feeds the
+ * re-entrant pass a body whose headers WOULD trigger a confident demote if
+ * `classifyPlacement` ran fresh — so a regression that drops the guard shows
+ * up as an unwanted move, not as a passing test relying on deterministic
+ * heuristics happening to agree.
+ */
+
+import assert from "node:assert/strict";
+import { Readable } from "node:stream";
+import { describe, it } from "node:test";
+import type {
+	AddressItem,
+	IAddressRepository,
+	IEnvelopeRepository,
+	IMailboxSpecialUseRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+	MessageItem,
+	UpdateMessageInput,
+} from "@remit/data-ports";
+import { MailboxSpecialUse } from "@remit/domain-enums";
+import type { StorageService } from "@remit/storage-service";
+import type { PlacementConfig } from "./body-sync.js";
+import { BodySyncService } from "./body-sync.js";
+import type { PlacementMoveService } from "./placement-move.js";
+import type { IImapConnection } from "./types.js";
+
+const MAILBOXES = {
+	inbox: { mailboxId: "mb-inbox", fullPath: "INBOX" },
+	junk: { mailboxId: "mb-junk", fullPath: "Junk" },
+};
+
+/**
+ * DKIM signing domain mismatches the From domain, dmarc=fail, sender
+ * untrusted, a provider-spam header present (any value) — exactly the
+ * deterministic signal set `classifyPlacement`'s demote branch (inbox → junk,
+ * HIGH bar) confidently acts on when the message currently sits in Inbox.
+ */
+const DEMOTE_EML = Buffer.from(
+	[
+		"From: Support <support@evil-mimic.example>",
+		"To: me@example.com",
+		"Subject: Verify your account",
+		"Authentication-Results: mx.example.com; dmarc=fail",
+		"DKIM-Signature: v=1; a=rsa-sha256; d=relay.example.net; s=sel; b=xxx",
+		"X-Spam-Status: No, score=0.1",
+		"Content-Type: text/plain",
+		"",
+		"body",
+	].join("\r\n"),
+);
+
+interface Harness {
+	service: BodySyncService;
+	message: MessageItem;
+	messageUpdates: Array<{ messageId: string; input: UpdateMessageInput }>;
+	moves: Array<{ messageId: string; destinationMailboxId: string }>;
+}
+
+const buildHarness = (
+	message: Partial<MessageItem> & Pick<MessageItem, "messageId">,
+	retrieve: () => Promise<Buffer>,
+): Harness => {
+	const messageUpdates: Array<{
+		messageId: string;
+		input: UpdateMessageInput;
+	}> = [];
+	const moves: Array<{ messageId: string; destinationMailboxId: string }> = [];
+
+	const messageRow = {
+		uid: 1,
+		mailboxId: MAILBOXES.inbox.mailboxId,
+		...message,
+	} as unknown as MessageItem;
+
+	const messageService = {
+		get: async () => messageRow,
+		update: async (messageId: string, input: UpdateMessageInput) => {
+			messageUpdates.push({ messageId, input });
+			Object.assign(messageRow, input);
+		},
+	} as unknown as IMessageRepository;
+
+	const threadMessageService = {
+		findAllByMessageId: async () => [
+			{
+				threadMessageId: "tm-1",
+				messageId: message.messageId,
+				mailboxId: messageRow.mailboxId,
+				sentDate: 1,
+				isRead: false,
+				isDeleted: false,
+				hasStars: false,
+				hasAttachment: false,
+			},
+		],
+		update: async () => {},
+	} as unknown as IThreadMessageRepository;
+
+	const storageService = {
+		retrieve,
+		storeMessageBody: async () => ({ uri: `s3://bodies/${message.messageId}` }),
+		storeMessageBodyStream: async () => ({
+			uri: `s3://bodies/${message.messageId}`,
+		}),
+		storeParsedBody: async () => {},
+		listBodyParts: async () => [],
+	} as unknown as StorageService;
+
+	const addressService = {
+		getAddress: async () => ({ flags: {} }) as unknown as AddressItem,
+		incrementInboundCount: async () => {},
+	} as unknown as IAddressRepository;
+
+	const envelopeService = {
+		listBodyParts: async () => [],
+	} as unknown as IEnvelopeRepository;
+
+	const mailboxSpecialUseService = {
+		findBySpecialUse: async (_accountId: string, specialUse: string) =>
+			specialUse === MailboxSpecialUse.Junk ? MAILBOXES.junk : null,
+		findInboxMailbox: async () => MAILBOXES.inbox,
+	} as unknown as IMailboxSpecialUseRepository;
+
+	const placementMoveService = {
+		moveMessage: async (
+			_accountConfigId: string,
+			messageId: string,
+			destinationMailboxId: string,
+		) => {
+			moves.push({ messageId, destinationMailboxId });
+			messageRow.mailboxId = destinationMailboxId;
+		},
+	} as unknown as PlacementMoveService;
+
+	const placementConfig: PlacementConfig = {
+		mailboxSpecialUseService,
+		placementMoveService,
+	};
+
+	const service = new BodySyncService(
+		messageService,
+		storageService,
+		threadMessageService,
+		addressService,
+		envelopeService,
+		{ info: () => {}, error: () => {} },
+		placementConfig,
+	);
+
+	return { service, message: messageRow, messageUpdates, moves };
+};
+
+const noSuchKeyError = () =>
+	Object.assign(new Error("missing"), { name: "NoSuchKey" });
+
+describe("placement survives a re-entrant computePlacement pass (issue #383)", () => {
+	it("keeps a user-rescued message in Inbox through the NoSuchKey IMAP re-fetch", async () => {
+		const harness = buildHarness(
+			{
+				messageId: "m-1",
+				mailboxId: MAILBOXES.inbox.mailboxId,
+				bodyStorageKey: "s3://bodies/m-1",
+				movedByRemit: false,
+				placementDecidedAt: 500,
+			},
+			async () => {
+				throw noSuchKeyError();
+			},
+		);
+
+		const connection = {
+			openBox: async () => {},
+			fetchMessageBody: async () => DEMOTE_EML,
+		} as unknown as IImapConnection;
+
+		await harness.service.fetchAndGetBody(
+			"m-1",
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			async () => connection,
+		);
+
+		assert.deepEqual(
+			harness.moves,
+			[],
+			"an already-decided placement must not be re-evaluated, even though these headers would confidently demote if classifyPlacement ran fresh",
+		);
+		assert.equal(harness.message.mailboxId, MAILBOXES.inbox.mailboxId);
+		assert.equal(harness.messageUpdates[0]?.input.movedByRemit, undefined);
+		assert.equal(harness.messageUpdates[0]?.input.placementVerdict, undefined);
+	});
+
+	it("keeps a user-rescued message in Inbox when syncBodies re-fetches with force", async () => {
+		const harness = buildHarness(
+			{
+				messageId: "m-1",
+				mailboxId: MAILBOXES.inbox.mailboxId,
+				bodyStorageKey: "s3://bodies/m-1",
+				movedByRemit: false,
+				placementDecidedAt: 500,
+			},
+			async () => {
+				throw new Error("force path must not retrieve from storage");
+			},
+		);
+
+		const connection = {
+			openBox: async () => {},
+			async *fetchMessageBodies(uids: number[]) {
+				for (const uid of uids) {
+					yield { uid, source: Readable.from([DEMOTE_EML]) };
+				}
+			},
+		} as unknown as IImapConnection;
+
+		const result = await harness.service.syncBodies(
+			["m-1"],
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			async () => connection,
+			true,
+		);
+
+		assert.deepEqual(result.syncedMessageIds, ["m-1"]);
+		assert.deepEqual(
+			harness.moves,
+			[],
+			"a forced re-sync must not re-decide an already-decided placement",
+		);
+		assert.equal(harness.message.mailboxId, MAILBOXES.inbox.mailboxId);
+	});
+
+	it("still evaluates and can confidently move a message that has never been placement-classified", async () => {
+		const harness = buildHarness(
+			{
+				messageId: "m-1",
+				mailboxId: MAILBOXES.inbox.mailboxId,
+				movedByRemit: false,
+			},
+			async () => {
+				throw new Error("no body stored yet; must not retrieve");
+			},
+		);
+
+		const connection = {
+			openBox: async () => {},
+			fetchMessageBody: async () => DEMOTE_EML,
+		} as unknown as IImapConnection;
+
+		await harness.service.fetchAndGetBody(
+			"m-1",
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			async () => connection,
+		);
+
+		assert.deepEqual(harness.moves, [
+			{ messageId: "m-1", destinationMailboxId: MAILBOXES.junk.mailboxId },
+		]);
+		assert.equal(harness.messageUpdates[0]?.input.movedByRemit, true);
+		assert.ok(
+			typeof harness.messageUpdates[0]?.input.placementDecidedAt === "number",
+			"a genuine first evaluation must record placementDecidedAt",
+		);
+	});
+
+	it("continues to protect a message Remit itself already moved (movedByRemit: true), unchanged behavior", async () => {
+		const harness = buildHarness(
+			{
+				messageId: "m-1",
+				mailboxId: MAILBOXES.inbox.mailboxId,
+				bodyStorageKey: "s3://bodies/m-1",
+				movedByRemit: true,
+				// No placementDecidedAt — a legacy row synced before issue #383's
+				// guard existed. `classifyPlacement`'s own `movedByRemit` check
+				// must still hold on its own.
+			},
+			async () => {
+				throw noSuchKeyError();
+			},
+		);
+
+		const connection = {
+			openBox: async () => {},
+			fetchMessageBody: async () => DEMOTE_EML,
+		} as unknown as IImapConnection;
+
+		await harness.service.fetchAndGetBody(
+			"m-1",
+			"acc-1",
+			"cfg-1",
+			"INBOX",
+			async () => connection,
+		);
+
+		assert.deepEqual(
+			harness.moves,
+			[],
+			"movedByRemit must keep protecting a legacy row with no placementDecidedAt of its own",
+		);
+		assert.equal(harness.message.mailboxId, MAILBOXES.inbox.mailboxId);
+		assert.ok(
+			typeof harness.messageUpdates[0]?.input.placementDecidedAt === "number",
+			"the legacy row is backfilled with placementDecidedAt going forward, self-healing for future passes",
+		);
+	});
+});

package/src/body-sync.ts

@@ -74,6 +74,7 @@ type ThreadMessageCategory = ThreadMessageItem["category"];
 interface PlacementOutcome {
 	verdict?: MessagePlacementVerdict;
 	move?: { destinationMailboxId: string; destinationPath: string };
+	placementDecidedAt?: number;
 }
 
 /**
@@ -185,6 +186,19 @@ const hasDecidedCategory = (
 ): boolean =>
 	category !== undefined && category !== MessageCategory.uncategorized;
 
+/**
+ * Issue #383 (RFC 039 Non-goals): whether {@link BodySyncService.computePlacement}
+ * has already produced a verdict for this message — moved, left in place, or
+ * archived, confident or unsure alike. Absence means placement has genuinely
+ * never been evaluated. The same two re-entrant paths `hasDecidedCategory`
+ * guards (`fetchAndGetBody`'s `NoSuchKey` fallback, `syncBodies(..., force:
+ * true)`) also re-enter `computePlacement`; without this guard a message a
+ * user manually rescued (never touched by Remit, so `movedByRemit` never
+ * recorded anything) can be silently re-evaluated and moved right back.
+ */
+const hasDecidedPlacement = (placementDecidedAt: number | undefined): boolean =>
+	placementDecidedAt !== undefined;
+
 export const toParsedBody = (parsed: ParsedMail): ParsedBody => ({
 	text: parsed.text ?? null,
 	html: typeof parsed.html === "string" ? parsed.html : null,
@@ -867,6 +881,11 @@ export class BodySyncService {
 		// this guard a re-entrant pass would let a *later* override silently
 		// rewrite a category already decided on an earlier message, which is
 		// exactly the churn RFC 030's GSI-safety argument forbids.
+		//
+		// `resolved.placementDecidedAt` (issue #383) guards the same two
+		// re-entrant paths for placement: `computePlacement` already declined to
+		// recompute a verdict once this field is set, so it is only ever present
+		// here on the pass that first decided it.
 		const existingMessage = await this.messageService.get(messageId);
 		const finalCategory = hasDecidedCategory(existingMessage.category)
 			? existingMessage.category
@@ -889,6 +908,9 @@ export class BodySyncService {
 			...(moved ? { movedByRemit: true } : {}),
 			...(resolved.verdict ? { placementVerdict: resolved.verdict } : {}),
 			...(filterMove ? { filterMove } : {}),
+			...(resolved.placementDecidedAt
+				? { placementDecidedAt: resolved.placementDecidedAt }
+				: {}),
 		};
 		await this.messageService.update(messageId, update);
 		this.log.info({ messageId, storageKey: bodyRef.uri }, "Body stored");
@@ -1320,8 +1342,11 @@ export class BodySyncService {
 	 *
 	 * Returns a {@link PlacementOutcome}: a `verdict` to persist whenever Remit
 	 * decided to act (action != leave), confident and unsure alike, so the
-	 * distribution is queryable on the message; and a `move` to enqueue only for
-	 * a confident verdict.
+	 * distribution is queryable on the message; a `move` to enqueue only for a
+	 * confident verdict; and `placementDecidedAt` whenever a verdict — including
+	 * "leave" — was genuinely computed (issue #383), so a re-entrant call short-
+	 * circuits instead of re-deciding a placement the user may have since
+	 * overridden by hand.
 	 *
 	 * Always logs a structured verdict line for confident, actionable verdicts so
 	 * the real distribution is observable on a live mailbox.
@@ -1334,7 +1359,7 @@ export class BodySyncService {
 	 * alertable field instead of failing the surrounding message store. The empty
 	 * {@link PlacementOutcome} means "no action", whether Remit genuinely decided
 	 * to leave the message alone or placement itself failed; the alert log is
-	 * what distinguishes the latter.
+	 * what distinguishes the latter, and neither case marks the message decided.
 	 */
 	private async resolvePlacement(
 		messageId: string,
@@ -1378,6 +1403,16 @@ export class BodySyncService {
 		const { mailboxSpecialUseService } = placementConfig;
 
 		const message = await this.messageService.get(messageId);
+
+		// Issue #383: placement is meant to run once per message (RFC 039
+		// Non-goals). Once a verdict has EVER been decided for this message —
+		// moved, left in place, or archived — a re-entrant pass (the `NoSuchKey`
+		// fallback in `fetchAndGetBody`, `syncBodies(..., force: true)`) must not
+		// recompute it: a message a user has since moved by hand (which never
+		// touches `movedByRemit`) would otherwise be silently re-evaluated
+		// against the same signals that placed it in the first place.
+		if (hasDecidedPlacement(message.placementDecidedAt)) return {};
+
 		const junkMailbox = await mailboxSpecialUseService.findBySpecialUse(
 			accountId,
 			MailboxSpecialUse.Junk,
@@ -1421,14 +1456,16 @@ export class BodySyncService {
 		// audit record of its own. `flags.autoArchive` (issue #300) is a distinct,
 		// lower-priority filing preference: it only files a message away when
 		// `blocked`/DKIM/DMARC had nothing to say, never overriding a confident
-		// junk/inbox verdict computed above.
+		// junk/inbox verdict computed above. Still marked decided (issue #383):
+		// "leave" is itself a verdict, not "not yet evaluated".
 		if (verdict.action === "leave") {
-			return this.resolveAutoArchive(
+			const outcome = await this.resolveAutoArchive(
 				mailboxSpecialUseService,
 				message,
 				accountId,
 				signals.autoArchive,
 			);
+			return { ...outcome, placementDecidedAt: Date.now() };
 		}
 
 		// Audit verdict — recorded for every actionable verdict (both
@@ -1452,12 +1489,12 @@ export class BodySyncService {
 		// Only a confident verdict moves mail. An unsure verdict is recorded
 		// but never enqueues a move.
 		if (verdict.confidence !== "confident") {
-			return { verdict: audit };
+			return { verdict: audit, placementDecidedAt: audit.decidedAt };
 		}
 
 		const target =
 			verdict.action === "move-to-inbox" ? inboxMailbox : junkMailbox;
-		if (!target) return { verdict: audit };
+		if (!target) return { verdict: audit, placementDecidedAt: audit.decidedAt };
 
 		// Structured verdict line — emitted for confident, actionable verdicts
 		// so the real verdict distribution is observable on a live mailbox.
@@ -1480,6 +1517,7 @@ export class BodySyncService {
 				destinationMailboxId: target.mailboxId,
 				destinationPath: target.fullPath,
 			},
+			placementDecidedAt: audit.decidedAt,
 		};
 	}
 

package/src/mailbox-cursor.ts

@@ -33,8 +33,8 @@ export interface MailboxCursorGuardDeps {
 /**
  * `cursorState` is total per RFC 032 (defaults to `normal`) — but that default
  * only applies to rows written after this field existed. A row persisted
- * before this migration (DynamoDB attribute never written / Postgres column
- * genuinely NULL) reads back with the attribute absent despite the type
+ * before this migration (the column genuinely NULL) reads back absent
+ * despite the type
  * saying otherwise, so every consumer here treats `undefined` the same as
  * `normal` defensively rather than trusting the type.
  */

package/src/mailbox-management.ts

@@ -100,7 +100,7 @@ export const validateMailboxOperation = (
  * Service for managing mailbox operations (create, rename, delete).
  *
  * Implements an optimistic local-first pattern:
- * 1. Updates are applied locally first (DynamoDB)
+ * 1. Updates are applied locally first
  * 2. Changes are queued for IMAP sync via SQS
  * 3. Worker processes queue and syncs to IMAP server
  */
@@ -244,7 +244,7 @@ export class MailboxManagementService {
 
 		this.log.info({ mailboxId, path }, "Deleted mailbox on IMAP server");
 
-		// Delete the mailbox entity from DynamoDB
+		// Delete the mailbox entity from the local store
 		await this.mailboxService.delete(accountId, mailboxId);
 
 		return { success: true };

package/src/mailbox-queue.ts

@@ -83,7 +83,7 @@ export interface MailboxQueueConfig {
  * Service for mailbox management with automatic IMAP sync queueing.
  *
  * Implements optimistic local-first pattern:
- * 1. Updates local DynamoDB state immediately
+ * 1. Updates local state immediately
  * 2. Enqueues mailbox management event to SQS for worker to sync to IMAP
  *
  * This follows the same pattern as FlagQueueService (RFC 014).

package/src/message-move.ts

@@ -115,7 +115,7 @@ export interface DeleteOptions {
  * Service for moving, copying, and deleting messages.
  *
  * Implements optimistic local-first pattern:
- * 1. Updates local DynamoDB state immediately (Message + ThreadMessage)
+ * 1. Updates local state immediately (Message + ThreadMessage)
  * 2. Enqueues event to SQS for worker to sync to IMAP
  *
  * Following RFC 016 for message deletion and moving.

package/src/mime-walker.ts

@@ -266,8 +266,8 @@ const toRecord = (
  * **Part-path uniqueness**: some IMAP servers (and the `message/rfc822`
  * inner-body convention) return child nodes with an empty `part` field.
  * Assigning ROOT_PART_PATH to every such node would produce duplicate keys
- * and cause a DynamoDB "multiple operations on one item" error in
- * `upsertBodyParts`. Non-root nodes without a `part` therefore receive a
+ * and make `upsertBodyParts` write the same body part twice. Non-root nodes
+ * without a `part` therefore receive a
  * synthetic path `<parentPath>.<siblingIndex>` that is stable across
  * repeated syncs of the same message.
  */
@@ -287,7 +287,7 @@ export const walkMimeStructure = (root: MimeNode): BodyPartRecord[] => {
 			partPath = ROOT_PART_PATH;
 		} else {
 			// Non-root node without an IMAP part path — synthesise one so
-			// the DynamoDB keys remain unique. This happens most commonly
+			// the body-part keys remain unique. This happens most commonly
 			// for the body of a message/rfc822 attachment, whose inner
 			// structure imapflow attaches as a childNode with part="".
 			partPath = `${parentPath}.${siblingIndex}`;

package/src/pass-through-unit-of-work.ts

@@ -2,9 +2,9 @@ import type { IUnitOfWork, UnitOfWorkRepositories } from "@remit/data-ports";
 
 /**
  * Runs the write set against fixed repositories with no surrounding
- * transaction. For backends (DynamoDB) that have no cross-entity transaction:
- * the writes are not atomic, matching that backend's own guarantees. The
- * Postgres path injects a real transactional unit of work instead.
+ * transaction: the writes are not atomic. For a backend that has no
+ * cross-entity transaction this matches its own guarantees; the SQLite path
+ * injects a real transactional unit of work instead.
  */
 export class PassThroughUnitOfWork implements IUnitOfWork {
 	constructor(private repos: UnitOfWorkRepositories) {}