@remit/mailbox-service 0.0.32 → 0.0.34

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.32",
+  "version": "0.0.34",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/body-sync-unsubscribed-auto-read.test.ts

@@ -0,0 +1,176 @@
+/**
+ * Issue #302 (RFC 039 Decision 3): `Address.flags.unsubscribed` is
+ * documented as "auto-mark-read until sender stops" and settable from
+ * `IntelligencePane.tsx`, but nothing consumed it — every message from an
+ * unsubscribed sender still arrived unread like any other. These tests drive
+ * `BodySyncService` end-to-end (read-path body materialization →
+ * `applyPostStoreSteps`) and assert on the actual `FlagQueueService.markAsRead`
+ * call, so a regression that drops the flag read — not just a helper in
+ * isolation — shows up here.
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	AddressItem,
+	IAddressRepository,
+	IEnvelopeRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+} from "@remit/data-ports";
+import type { StorageService } from "@remit/storage-service";
+import { BodySyncService } from "./body-sync.js";
+import type { FlagQueueService } from "./flag-queue.js";
+import type { IImapConnection } from "./types.js";
+
+const PLAIN_EML = (fromEmail: string) =>
+	Buffer.from(
+		[
+			`From: Sender <${fromEmail}>`,
+			"To: me@example.com",
+			"Subject: Hello",
+			"Content-Type: text/plain",
+			"",
+			"body",
+		].join("\r\n"),
+	);
+
+interface MarkReadCall {
+	accountConfigId: string;
+	messageId: string;
+	accountId: string;
+}
+
+interface Harness {
+	service: BodySyncService;
+	markReadCalls: MarkReadCall[];
+}
+
+const buildHarness = (
+	flags: AddressItem["flags"],
+	withUnsubscribeConfig = true,
+): Harness => {
+	const markReadCalls: MarkReadCall[] = [];
+
+	const messageService = {
+		get: async () => ({
+			messageId: "m-1",
+			mailboxId: "mb-inbox",
+			uid: 1,
+		}),
+		update: async () => {},
+	} as unknown as IMessageRepository;
+
+	const threadMessageService = {
+		findAllByMessageId: async () => [
+			{
+				threadMessageId: "tm-1",
+				sentDate: 1,
+				mailboxId: "mb-inbox",
+				isRead: false,
+				isDeleted: false,
+				hasStars: false,
+				hasAttachment: false,
+			},
+		],
+		update: async () => {},
+	} as unknown as IThreadMessageRepository;
+
+	const storageService = {
+		storeMessageBody: async () => ({ uri: "s3://bodies/m-1" }),
+		storeParsedBody: async () => {},
+		listBodyParts: async () => [],
+	} as unknown as StorageService;
+
+	const addressService = {
+		getAddress: async () => ({ flags }) as unknown as AddressItem,
+		incrementInboundCount: async () => {},
+	} as unknown as IAddressRepository;
+
+	const envelopeService = {
+		listBodyParts: async () => [],
+	} as unknown as IEnvelopeRepository;
+
+	const flagQueueService = {
+		markAsRead: async (
+			accountConfigId: string,
+			messageId: string,
+			accountId: string,
+		) => {
+			markReadCalls.push({ accountConfigId, messageId, accountId });
+		},
+	} as unknown as FlagQueueService;
+
+	const service = new BodySyncService(
+		messageService,
+		storageService,
+		threadMessageService,
+		addressService,
+		envelopeService,
+		{ info: () => {}, error: () => {} },
+		undefined,
+		undefined,
+		undefined,
+		withUnsubscribeConfig ? { flagQueueService } : undefined,
+	);
+
+	return { service, markReadCalls };
+};
+
+const readBody = async (
+	service: BodySyncService,
+	fromEmail = "someone@example.com",
+) => {
+	const connection = {
+		openBox: async () => {},
+		fetchMessageBody: async () => PLAIN_EML(fromEmail),
+	} as unknown as IImapConnection;
+	return service.fetchAndGetBody(
+		"m-1",
+		"acc-1",
+		"cfg-1",
+		"INBOX",
+		async () => connection,
+	);
+};
+
+describe("Address.flags.unsubscribed drives auto-mark-read (issue #302)", () => {
+	it("marks a message from an unsubscribed sender as read at sync time", async () => {
+		const harness = buildHarness({ unsubscribed: { value: true, setAt: 1 } });
+
+		await readBody(harness.service);
+
+		assert.deepEqual(harness.markReadCalls, [
+			{ accountConfigId: "cfg-1", messageId: "m-1", accountId: "acc-1" },
+		]);
+	});
+
+	it("leaves read state alone for a sender without the flag", async () => {
+		const harness = buildHarness({});
+
+		await readBody(harness.service);
+
+		assert.deepEqual(harness.markReadCalls, []);
+	});
+
+	it("leaves read state alone once the flag is unset (no caching of the decision)", async () => {
+		const harness = buildHarness({
+			unsubscribed: { value: false, setAt: 1 },
+		});
+
+		await readBody(harness.service);
+
+		assert.deepEqual(harness.markReadCalls, []);
+	});
+
+	it("is a no-op when body sync was built without an UnsubscribeConfig", async () => {
+		const harness = buildHarness(
+			{ unsubscribed: { value: true, setAt: 1 } },
+			false,
+		);
+
+		await readBody(harness.service);
+
+		assert.deepEqual(harness.markReadCalls, []);
+	});
+});

package/src/body-sync.ts

@@ -37,6 +37,7 @@ import {
 	type FilterDecision,
 	FilterPipeline,
 } from "./filters/pipeline.js";
+import type { FlagQueueService } from "./flag-queue.js";
 import {
 	classifyByHeaders,
 	extractAuthenticity,
@@ -257,6 +258,17 @@ export interface QuarantineConfig {
 	attempts: number;
 }
 
+/**
+ * What body sync needs to auto-mark-read a message from an unsubscribed
+ * sender (issue #302, RFC 039 Decision 3). Reuses the same
+ * `FlagQueueService.markAsRead` a manual mark-as-read already goes through —
+ * local `\Seen` + `ThreadMessage.isRead` + a durable pending IMAP flag-push
+ * marker — so this fires the same round-trip, not a second primitive.
+ */
+export interface UnsubscribeConfig {
+	flagQueueService: FlagQueueService;
+}
+
 export class BodySyncService {
 	private log: BodySyncLogger;
 	private readonly filterPipeline?: FilterPipeline;
@@ -271,6 +283,7 @@ export class BodySyncService {
 		private readonly placementConfig?: PlacementConfig,
 		private readonly filterConfig?: FilterConfig,
 		private readonly quarantineConfig?: QuarantineConfig,
+		private readonly unsubscribeConfig?: UnsubscribeConfig,
 	) {
 		this.log = logger ?? noopLogger;
 		this.filterPipeline = filterConfig
@@ -813,6 +826,18 @@ export class BodySyncService {
 			});
 		}
 
+		// `flags.unsubscribed` (issue #302, RFC 039 Decision 3): auto-mark-read,
+		// reusing the same FlagQueueService.markAsRead round-trip a manual
+		// mark-as-read goes through — idempotent on a retry (flipFlag no-ops when
+		// the message is already \Seen), so a failure here safely re-fires on the
+		// next attempt rather than being lost behind the bodyStorageKey skip guard.
+		await this.applyUnsubscribedAutoRead(
+			messageId,
+			accountId,
+			accountConfigId,
+			parsed,
+		);
+
 		const moved = Boolean(resolved.move || filterMoved);
 
 		// ONE Message UpdateItem per synced message: bodyStorageKey + every
@@ -1136,6 +1161,59 @@ export class BodySyncService {
 		return unknown;
 	}
 
+	/**
+	 * `flags.unsubscribed` (issue #302, RFC 039 Decision 3): "auto-mark-read
+	 * until sender stops" — fires on every new message from that sender for as
+	 * long as the flag stays set, per the flag's own doc comment; there is no
+	 * separate expiry mechanism, and no caching of the decision beyond this
+	 * per-message `Address` read. A no-op when body sync was built without an
+	 * {@link UnsubscribeConfig} or the message carries no `From` address.
+	 */
+	private async applyUnsubscribedAutoRead(
+		messageId: string,
+		accountId: string,
+		accountConfigId: string,
+		parsed: ParsedMail,
+	): Promise<void> {
+		if (!this.unsubscribeConfig) return;
+
+		const fromEmail = extractPrimaryFromEmail(parsed);
+		if (!fromEmail) return;
+
+		const unsubscribed = await this.deriveSenderUnsubscribed(
+			accountConfigId,
+			fromEmail,
+		);
+		if (!unsubscribed) return;
+
+		await this.unsubscribeConfig.flagQueueService.markAsRead(
+			accountConfigId,
+			messageId,
+			accountId,
+		);
+	}
+
+	private async deriveSenderUnsubscribed(
+		accountConfigId: string,
+		fromEmail: string,
+	): Promise<boolean> {
+		try {
+			const addressId = deriveAddressId(accountConfigId, fromEmail);
+			const address = await this.addressService.getAddress(
+				accountConfigId,
+				addressId,
+			);
+			return address.flags?.unsubscribed?.value === true;
+		} catch (err) {
+			// A genuinely-absent address means "not unsubscribed". Any other
+			// failure (AccessDenied, throttle, infra) must NOT be silently
+			// downgraded — let it crash so the read-state decision isn't made on
+			// bad data.
+			if (!(err instanceof NotFoundError)) throw err;
+			return false;
+		}
+	}
+
 	/**
 	 * Evaluate the account's active filters against a synced message (RFC 034),
 	 * BEFORE the single Message update — so a filter's `movedByRemit` flag joins

package/src/filters/pipeline.test.ts

@@ -61,6 +61,7 @@ const buildPipeline = (filter: FilterItem) => {
 				state.embedCalls += 1;
 				return [];
 			},
+			embeddingId: "test-model@0",
 		},
 	};
 	return { pipeline: new FilterPipeline(config, { info: () => {} }), state };
@@ -101,3 +102,193 @@ describe("FilterPipeline — anchorless From/Or filter at index time", () => {
 		assert.equal(state.embedCalls, 0);
 	});
 });
+
+describe("FilterPipeline — anchor lazy re-embed on model drift (reader #295)", () => {
+	const anchoredFilter = (): FilterItem =>
+		({
+			filterId: "flt-semantic",
+			accountConfigId: "cfg-1",
+			name: "receipts",
+			scope: "Standing",
+			state: FilterState.Active,
+			hasAnchor: true,
+			ruleChangedAt: 1,
+			matchOperator: FilterMatchOperator.And,
+			literalClauses: [],
+			actionLabelId: "lbl-1",
+			actionMailboxId: NO_ACTION,
+			createdAt: 1,
+			updatedAt: 1,
+		}) as unknown as FilterItem;
+
+	const message = {
+		from: "billing@stripe.com",
+		fromName: "Stripe",
+		subject: "Your receipt",
+		text: "Thanks for your payment",
+		listId: "",
+	};
+
+	it("re-embeds a stale anchor and matches on the very next evaluation, persisting the refreshed row", async () => {
+		const putCalls: Array<Record<string, unknown>> = [];
+		let embedCalls = 0;
+		const config: FilterConfig = {
+			filterService: {
+				listByAccountAndState: async () => [anchoredFilter()],
+				refreshExpiry: async (f: FilterItem) => f,
+			} as unknown as IFilterRepository,
+			filterAnchorService: {
+				get: async () => ({
+					accountConfigId: "cfg-1",
+					filterId: "flt-semantic",
+					// Stale-model space — orthogonal to what the current model
+					// produces, so a raw comparison against it would not match.
+					anchorEmbedding: [0, 1, 0],
+					anchorEmbeddingId: "old-model@3",
+					anchorSourceText: "your receipt is ready",
+					anchorMessageId: "msg-anchor",
+				}),
+				put: async (input: Record<string, unknown>) => {
+					putCalls.push(input);
+					return { ...input, createdAt: 1, updatedAt: 2 };
+				},
+			} as unknown as IFilterAnchorRepository,
+			messageLabelService: {} as unknown as IMessageLabelRepository,
+			placementMoveService: {} as unknown as PlacementMoveService,
+			embedder: {
+				embed: async () => {
+					embedCalls += 1;
+					return [1, 0, 0];
+				},
+				embeddingId: "new-model@3",
+			},
+		};
+
+		const decision = await new FilterPipeline(config, {
+			info: () => {},
+		}).evaluate("cfg-1", "m-1", message);
+
+		assert.deepEqual(
+			decision.labels,
+			[{ labelId: "lbl-1", filterId: "flt-semantic" }],
+			"the anchor matches once re-embedded into the current model's space",
+		);
+		assert.equal(
+			putCalls.length,
+			1,
+			"the refreshed anchor is written back exactly once",
+		);
+		assert.deepEqual(putCalls[0], {
+			accountConfigId: "cfg-1",
+			filterId: "flt-semantic",
+			anchorEmbedding: [1, 0, 0],
+			anchorEmbeddingId: "new-model@3",
+			anchorSourceText: "your receipt is ready",
+			anchorMessageId: "msg-anchor",
+		});
+		assert.equal(
+			embedCalls,
+			2,
+			"one embed re-embeds the anchor's source text, one embeds the candidate message",
+		);
+	});
+
+	it("never re-embeds when the anchor's embeddingId is already current", async () => {
+		let putCalls = 0;
+		let embedCalls = 0;
+		const config: FilterConfig = {
+			filterService: {
+				listByAccountAndState: async () => [anchoredFilter()],
+				refreshExpiry: async (f: FilterItem) => f,
+			} as unknown as IFilterRepository,
+			filterAnchorService: {
+				get: async () => ({
+					anchorEmbedding: [1, 0, 0],
+					anchorEmbeddingId: "current-model@3",
+					anchorSourceText: "your receipt is ready",
+					anchorMessageId: "msg-anchor",
+				}),
+				put: async () => {
+					putCalls += 1;
+					throw new Error("must not be called");
+				},
+			} as unknown as IFilterAnchorRepository,
+			messageLabelService: {} as unknown as IMessageLabelRepository,
+			placementMoveService: {} as unknown as PlacementMoveService,
+			embedder: {
+				embed: async () => {
+					embedCalls += 1;
+					return [1, 0, 0];
+				},
+				embeddingId: "current-model@3",
+			},
+		};
+
+		const decision = await new FilterPipeline(config, {
+			info: () => {},
+		}).evaluate("cfg-1", "m-1", message);
+
+		assert.deepEqual(decision.labels, [
+			{ labelId: "lbl-1", filterId: "flt-semantic" },
+		]);
+		assert.equal(putCalls, 0, "a current anchor is never rewritten");
+		assert.equal(
+			embedCalls,
+			1,
+			"only the candidate message is embedded — no wasted re-embed call",
+		);
+	});
+
+	it("degrades a re-embed failure to skipping just this filter, not the whole pass", async () => {
+		const literalFilter: FilterItem = {
+			filterId: "flt-literal",
+			accountConfigId: "cfg-1",
+			name: "stripe receipts",
+			scope: "Standing",
+			state: FilterState.Active,
+			hasAnchor: false,
+			ruleChangedAt: 1,
+			matchOperator: FilterMatchOperator.Or,
+			literalClauses: [{ field: FilterClauseField.From, value: "stripe.com" }],
+			actionLabelId: "lbl-literal",
+			actionMailboxId: NO_ACTION,
+			createdAt: 1,
+			updatedAt: 1,
+		} as unknown as FilterItem;
+
+		const config: FilterConfig = {
+			filterService: {
+				listByAccountAndState: async () => [anchoredFilter(), literalFilter],
+				refreshExpiry: async (f: FilterItem) => f,
+			} as unknown as IFilterRepository,
+			filterAnchorService: {
+				get: async () => ({
+					anchorEmbedding: [0, 1, 0],
+					anchorEmbeddingId: "old-model@3",
+					anchorSourceText: "your receipt is ready",
+					anchorMessageId: "msg-anchor",
+				}),
+				put: async () => {
+					throw new Error("SQLITE_BUSY");
+				},
+			} as unknown as IFilterAnchorRepository,
+			messageLabelService: {} as unknown as IMessageLabelRepository,
+			placementMoveService: {} as unknown as PlacementMoveService,
+			embedder: {
+				embed: async () => [1, 0, 0],
+				embeddingId: "new-model@3",
+			},
+		};
+
+		const decision = await new FilterPipeline(config, {
+			info: () => {},
+			error: () => {},
+		}).evaluate("cfg-1", "m-1", message);
+
+		assert.deepEqual(
+			decision.labels,
+			[{ labelId: "lbl-literal", filterId: "flt-literal" }],
+			"the anchored filter's re-embed failure is isolated — the literal filter still applies",
+		);
+	});
+});

package/src/filters/pipeline.ts

@@ -26,6 +26,15 @@ import {
  */
 export interface MessageEmbedder {
 	embed(text: string): Promise<number[]>;
+	/**
+	 * `<modelId>@<dimensions>` identifier of the model currently configured —
+	 * the same scheme `EmbeddingService.embeddingId`
+	 * (`packages/search-service/src/embeddings.ts`) derives. Compared against
+	 * `FilterAnchor.anchorEmbeddingId` to detect a model drift a same-dimension
+	 * swap would otherwise pass through silently (RFC 039 Decision 1a, reader
+	 * #295).
+	 */
+	readonly embeddingId: string;
 }
 
 export interface FilterLogger {
@@ -215,7 +224,7 @@ export class FilterPipeline {
 		// One deterministic point read per semantic candidate that survived the
 		// literal pre-filter — never a scan, never the anchor message itself
 		// (RFC 034 Decision 2.3).
-		const anchor = await this.config.filterAnchorService.get(
+		let anchor = await this.config.filterAnchorService.get(
 			accountConfigId,
 			filter.filterId,
 		);
@@ -227,6 +236,27 @@ export class FilterPipeline {
 			return false;
 		}
 
+		const embedder = this.config.embedder;
+		if (embedder && anchor.anchorEmbeddingId !== embedder.embeddingId) {
+			// The embedding model has drifted since this anchor was last written
+			// (RFC 039 Decision 1a) — re-embed the already-persisted
+			// `anchorSourceText` and write it back in place, lazily, on this read.
+			// No migration job walks these rows proactively; a re-embed failure
+			// here propagates to the per-filter catch in `match()`, which logs and
+			// skips this filter for this one evaluation exactly as an unrecoverable
+			// stale anchor does today — never a terminal state, so the next
+			// message that reaches this filter retries.
+			const anchorEmbedding = await embedder.embed(anchor.anchorSourceText);
+			anchor = await this.config.filterAnchorService.put({
+				accountConfigId,
+				filterId: filter.filterId,
+				anchorEmbedding,
+				anchorEmbeddingId: embedder.embeddingId,
+				anchorSourceText: anchor.anchorSourceText,
+				anchorMessageId: anchor.anchorMessageId,
+			});
+		}
+
 		const vector = await embed();
 		if (!vector) {
 			this.log.debug?.(