@remit/mailbox-service 0.0.111 → 0.0.112

package/src/body-sync-unclassifiable-once.test.ts

@@ -1,260 +0,0 @@
-/**
- * A body the classifier had nothing to say about is examined once (issue #331).
- *
- * `Message.category` is NOT NULL with an `uncategorized` default, so that value
- * means two things at once: a message the classifier has not reached, and one it
- * reached with nothing to say. Keying "needs classifying" on it made every later
- * sync pass re-read the body from storage and rewrite both rows for the same
- * answer — a read amplification that scales with sync frequency and never
- * converges.
- *
- * `Message.classificationState` is the fact itself, written in the same
- * UpdateItem as the answer. It is persistence-only — no response model carries
- * it — and it is what makes the declined cohort addressable later without a
- * sync pass guessing at it.
- *
- * The sender in these tests carries a `flags.category` override of
- * `uncategorized` — the Reclassify dialog's own value for "no category" — which
- * is how a message ends up stored and `uncategorized` at the same time.
- */
-
-import assert from "node:assert/strict";
-import { Readable } from "node:stream";
-import { describe, it } from "node:test";
-import type {
-	AddressItem,
-	IAddressRepository,
-	IEnvelopeRepository,
-	IMessageRepository,
-	IThreadMessageRepository,
-	MessageItem,
-	ThreadMessageItem,
-	UpdateMessageInput,
-	UpdateThreadMessageInput,
-} from "@remit/data-ports";
-import { NotFoundError } from "@remit/data-ports/errors";
-import { MessageCategory } from "@remit/domain-enums";
-import type { StorageService } from "@remit/storage-service";
-import { BodySyncService } from "./body-sync.js";
-import type { IImapConnection } from "./types.js";
-
-const LINKEDIN_EML = Buffer.from(
-	[
-		"From: LinkedIn <messages-noreply@linkedin.com>",
-		"To: me@example.com",
-		"Subject: You have a new invitation",
-		"List-Unsubscribe: <https://www.linkedin.com/e/unsub>",
-		"Content-Type: text/plain",
-		"",
-		"invitation",
-	].join("\r\n"),
-);
-
-interface Harness {
-	service: BodySyncService;
-	message: MessageItem;
-	rows: ThreadMessageItem[];
-	messageUpdates: Array<{ messageId: string; input: UpdateMessageInput }>;
-	threadUpdates: UpdateThreadMessageInput[];
-	retrieved: string[];
-}
-
-const buildHarness = (
-	message: Partial<MessageItem> & Pick<MessageItem, "messageId">,
-	overrideCategory?: (typeof MessageCategory)[keyof typeof MessageCategory],
-): Harness => {
-	const messageUpdates: Array<{
-		messageId: string;
-		input: UpdateMessageInput;
-	}> = [];
-	const threadUpdates: UpdateThreadMessageInput[] = [];
-	const retrieved: string[] = [];
-
-	const messageRow = {
-		uid: 1,
-		mailboxId: "mb-1",
-		category: MessageCategory.uncategorized,
-		...message,
-	} as unknown as MessageItem;
-
-	const rows: ThreadMessageItem[] = [
-		{
-			threadMessageId: "tm-1",
-			messageId: message.messageId,
-			mailboxId: "mb-1",
-			sentDate: 1,
-			isRead: false,
-			isDeleted: false,
-			hasStars: false,
-			hasAttachment: false,
-			category: messageRow.category,
-		} as unknown as ThreadMessageItem,
-	];
-
-	const messageService = {
-		get: async () => messageRow,
-		update: async (messageId: string, input: UpdateMessageInput) => {
-			messageUpdates.push({ messageId, input });
-			Object.assign(messageRow, input);
-		},
-	} as unknown as IMessageRepository;
-
-	const threadMessageService = {
-		findAllByMessageId: async () => rows,
-		update: async (
-			_accountConfigId: string,
-			threadMessageId: string,
-			input: UpdateThreadMessageInput,
-		) => {
-			threadUpdates.push(input);
-			const row = rows.find((r) => r.threadMessageId === threadMessageId);
-			if (row) Object.assign(row, input);
-		},
-	} as unknown as IThreadMessageRepository;
-
-	const storageService = {
-		retrieve: async (key: string) => {
-			retrieved.push(key);
-			return LINKEDIN_EML;
-		},
-		storeMessageBody: async () => ({ uri: `s3://bodies/${message.messageId}` }),
-		storeMessageBodyStream: async () => ({
-			uri: `s3://bodies/${message.messageId}`,
-		}),
-		storeParsedBody: async () => {},
-		listBodyParts: async () => [],
-	} as unknown as StorageService;
-
-	const addressService = {
-		getAddress: async () => {
-			if (overrideCategory === undefined) {
-				throw new NotFoundError("Address not found");
-			}
-			return {
-				flags: { category: { value: overrideCategory, setAt: 1_000 } },
-			} as unknown as AddressItem;
-		},
-		incrementInboundCount: async () => {},
-	} as unknown as IAddressRepository;
-
-	const service = new BodySyncService(
-		messageService,
-		storageService,
-		threadMessageService,
-		addressService,
-		{ listBodyParts: async () => [] } as unknown as IEnvelopeRepository,
-		{ info: () => {}, error: () => {} },
-	);
-
-	return {
-		service,
-		message: messageRow,
-		rows,
-		messageUpdates,
-		threadUpdates,
-		retrieved,
-	};
-};
-
-const imapConnection = () =>
-	({
-		openBox: async () => {},
-		async *fetchMessageBodies(uids: number[]) {
-			for (const uid of uids) {
-				yield { uid, source: Readable.from([LINKEDIN_EML]) };
-			}
-		},
-	}) as unknown as IImapConnection;
-
-const syncPass = (harness: Harness) =>
-	harness.service.syncBodies(
-		["m-1"],
-		"acc-1",
-		"cfg-1",
-		{ fullPath: "INBOX", placement: "inbox" },
-		async () => imapConnection(),
-	);
-
-describe("a body the classifier declined to categorize is examined once", () => {
-	it("does not read or rewrite the stored body on the pass after the one that stored it", async () => {
-		const harness = buildHarness(
-			{ messageId: "m-1" },
-			MessageCategory.uncategorized,
-		);
-
-		const first = await syncPass(harness);
-
-		assert.deepEqual(first.syncedMessageIds, ["m-1"]);
-		assert.equal(harness.message.category, MessageCategory.uncategorized);
-		assert.equal(harness.message.bodyStorageKey, "s3://bodies/m-1");
-		assert.equal(harness.messageUpdates.length, 1);
-		// The answer and the record that it was given, in one write.
-		assert.equal(
-			harness.messageUpdates[0].input.classificationState,
-			"Examined",
-		);
-
-		const second = await syncPass(harness);
-
-		assert.equal(second.skippedCount, 1);
-		assert.deepEqual(second.failedMessageIds, []);
-		assert.deepEqual(harness.retrieved, []);
-		assert.equal(harness.messageUpdates.length, 1);
-		assert.equal(harness.threadUpdates.length, 1);
-	});
-
-	it("leaves a message that is already stored and examined alone", async () => {
-		const harness = buildHarness(
-			{
-				messageId: "m-1",
-				bodyStorageKey: "s3://bodies/m-1",
-				category: MessageCategory.uncategorized,
-				classificationState: "Examined",
-			} as Partial<MessageItem> & Pick<MessageItem, "messageId">,
-			MessageCategory.uncategorized,
-		);
-
-		const result = await syncPass(harness);
-
-		assert.equal(result.skippedCount, 1);
-		assert.deepEqual(harness.retrieved, []);
-		assert.deepEqual(harness.messageUpdates, []);
-		assert.deepEqual(harness.threadUpdates, []);
-	});
-
-	it("leaves mail stored before the classifier ran marked as never examined", async () => {
-		const harness = buildHarness(
-			{
-				messageId: "m-1",
-				bodyStorageKey: "s3://bodies/m-1",
-				category: MessageCategory.uncategorized,
-				classificationState: "NotExamined",
-			} as Partial<MessageItem> & Pick<MessageItem, "messageId">,
-			MessageCategory.uncategorized,
-		);
-
-		const result = await syncPass(harness);
-
-		// A sync pass does not re-derive this cohort — that is the amplification
-		// #331 removes. It leaves the row saying so, which is what lets a
-		// deliberate backfill select it later.
-		assert.equal(result.skippedCount, 1);
-		assert.deepEqual(harness.retrieved, []);
-		assert.deepEqual(harness.messageUpdates, []);
-		assert.equal(harness.message.classificationState, "NotExamined");
-	});
-
-	it("still fetches and classifies a message whose body has never been stored", async () => {
-		const harness = buildHarness({ messageId: "m-1" });
-
-		const result = await syncPass(harness);
-
-		assert.deepEqual(result.syncedMessageIds, ["m-1"]);
-		assert.equal(harness.messageUpdates.length, 1);
-		assert.equal(
-			harness.messageUpdates[0].input.category,
-			MessageCategory.social,
-		);
-		assert.equal(harness.rows[0].category, MessageCategory.social);
-	});
-});

package/src/body-sync-unsubscribed-auto-read.test.ts

@@ -1,176 +0,0 @@
-/**
- * Issue #302 (RFC 039 Decision 3): `Address.flags.unsubscribed` is
- * documented as "auto-mark-read until sender stops" and settable from
- * `IntelligencePane.tsx`, but nothing consumed it — every message from an
- * unsubscribed sender still arrived unread like any other. These tests drive
- * `BodySyncService` end-to-end (read-path body materialization →
- * `applyPostStoreSteps`) and assert on the actual `FlagQueueService.markAsRead`
- * call, so a regression that drops the flag read — not just a helper in
- * isolation — shows up here.
- */
-
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import type {
-	AddressItem,
-	IAddressRepository,
-	IEnvelopeRepository,
-	IMessageRepository,
-	IThreadMessageRepository,
-} from "@remit/data-ports";
-import type { StorageService } from "@remit/storage-service";
-import { BodySyncService } from "./body-sync.js";
-import type { FlagQueueService } from "./flag-queue.js";
-import type { IImapConnection } from "./types.js";
-
-const PLAIN_EML = (fromEmail: string) =>
-	Buffer.from(
-		[
-			`From: Sender <${fromEmail}>`,
-			"To: me@example.com",
-			"Subject: Hello",
-			"Content-Type: text/plain",
-			"",
-			"body",
-		].join("\r\n"),
-	);
-
-interface MarkReadCall {
-	accountConfigId: string;
-	messageId: string;
-	accountId: string;
-}
-
-interface Harness {
-	service: BodySyncService;
-	markReadCalls: MarkReadCall[];
-}
-
-const buildHarness = (
-	flags: AddressItem["flags"],
-	withUnsubscribeConfig = true,
-): Harness => {
-	const markReadCalls: MarkReadCall[] = [];
-
-	const messageService = {
-		get: async () => ({
-			messageId: "m-1",
-			mailboxId: "mb-inbox",
-			uid: 1,
-		}),
-		update: async () => {},
-	} as unknown as IMessageRepository;
-
-	const threadMessageService = {
-		findAllByMessageId: async () => [
-			{
-				threadMessageId: "tm-1",
-				sentDate: 1,
-				mailboxId: "mb-inbox",
-				isRead: false,
-				isDeleted: false,
-				hasStars: false,
-				hasAttachment: false,
-			},
-		],
-		update: async () => {},
-	} as unknown as IThreadMessageRepository;
-
-	const storageService = {
-		storeMessageBody: async () => ({ uri: "s3://bodies/m-1" }),
-		storeParsedBody: async () => {},
-		listBodyParts: async () => [],
-	} as unknown as StorageService;
-
-	const addressService = {
-		getAddress: async () => ({ flags }) as unknown as AddressItem,
-		incrementInboundCount: async () => {},
-	} as unknown as IAddressRepository;
-
-	const envelopeService = {
-		listBodyParts: async () => [],
-	} as unknown as IEnvelopeRepository;
-
-	const flagQueueService = {
-		markAsRead: async (
-			accountConfigId: string,
-			messageId: string,
-			accountId: string,
-		) => {
-			markReadCalls.push({ accountConfigId, messageId, accountId });
-		},
-	} as unknown as FlagQueueService;
-
-	const service = new BodySyncService(
-		messageService,
-		storageService,
-		threadMessageService,
-		addressService,
-		envelopeService,
-		{ info: () => {}, error: () => {} },
-		undefined,
-		undefined,
-		undefined,
-		withUnsubscribeConfig ? { flagQueueService } : undefined,
-	);
-
-	return { service, markReadCalls };
-};
-
-const readBody = async (
-	service: BodySyncService,
-	fromEmail = "someone@example.com",
-) => {
-	const connection = {
-		openBox: async () => {},
-		fetchMessageBody: async () => PLAIN_EML(fromEmail),
-	} as unknown as IImapConnection;
-	return service.fetchAndGetBody(
-		"m-1",
-		"acc-1",
-		"cfg-1",
-		{ fullPath: "INBOX", placement: "inbox" },
-		async () => connection,
-	);
-};
-
-describe("Address.flags.unsubscribed drives auto-mark-read (issue #302)", () => {
-	it("marks a message from an unsubscribed sender as read at sync time", async () => {
-		const harness = buildHarness({ unsubscribed: { value: true, setAt: 1 } });
-
-		await readBody(harness.service);
-
-		assert.deepEqual(harness.markReadCalls, [
-			{ accountConfigId: "cfg-1", messageId: "m-1", accountId: "acc-1" },
-		]);
-	});
-
-	it("leaves read state alone for a sender without the flag", async () => {
-		const harness = buildHarness({});
-
-		await readBody(harness.service);
-
-		assert.deepEqual(harness.markReadCalls, []);
-	});
-
-	it("leaves read state alone once the flag is unset (no caching of the decision)", async () => {
-		const harness = buildHarness({
-			unsubscribed: { value: false, setAt: 1 },
-		});
-
-		await readBody(harness.service);
-
-		assert.deepEqual(harness.markReadCalls, []);
-	});
-
-	it("is a no-op when body sync was built without an UnsubscribeConfig", async () => {
-		const harness = buildHarness(
-			{ unsubscribed: { value: true, setAt: 1 } },
-			false,
-		);
-
-		await readBody(harness.service);
-
-		assert.deepEqual(harness.markReadCalls, []);
-	});
-});

package/src/filters/anchor-drift.ts

@@ -1,55 +0,0 @@
-import type {
-	FilterAnchorItem,
-	IFilterAnchorRepository,
-} from "@remit/data-ports";
-
-/**
- * The half of an embedding service an anchor refresh needs: the text-to-vector
- * call and the `<modelId>@<dimensions>` identifier of the model behind it (the
- * same scheme `EmbeddingService.embeddingId` derives). Compared against
- * `FilterAnchor.anchorEmbeddingId` to detect a model drift a same-dimension
- * swap would otherwise pass through silently (RFC 039 Decision 1a, reader
- * #295).
- */
-export interface AnchorEmbedder {
-	embed(text: string): Promise<number[]>;
-	readonly embeddingId: string;
-}
-
-export interface AnchorDriftDeps {
-	anchorRepository: Pick<IFilterAnchorRepository, "put">;
-	/** Absent on a deployment with no embedder wired; then nothing can drift. */
-	embedder: AnchorEmbedder | undefined;
-}
-
-/**
- * The anchor to score against, re-embedded in place when the embedding model
- * has drifted since it was written (RFC 039 Decision 1a). No migration job
- * walks these rows proactively, so the refresh is lazy: the first read that
- * notices the stamp no longer matches the configured model re-embeds the
- * already-persisted `anchorSourceText` and writes it back under the current
- * id. A failure here is never a terminal state — the row is left as it was,
- * so the next read that reaches this anchor retries.
- *
- * The single mechanism behind both index-time matching
- * ({@link FilterPipeline}) and the back-apply pass's cross-filter precedence
- * check, which must agree on what a filter currently matches (reader #399).
- */
-export const refreshAnchorForEmbedder = async (
-	deps: AnchorDriftDeps,
-	anchor: FilterAnchorItem,
-): Promise<FilterAnchorItem> => {
-	const { embedder } = deps;
-	if (!embedder || anchor.anchorEmbeddingId === embedder.embeddingId) {
-		return anchor;
-	}
-	const anchorEmbedding = await embedder.embed(anchor.anchorSourceText);
-	return deps.anchorRepository.put({
-		accountConfigId: anchor.accountConfigId,
-		filterId: anchor.filterId,
-		anchorEmbedding,
-		anchorEmbeddingId: embedder.embeddingId,
-		anchorSourceText: anchor.anchorSourceText,
-		anchorMessageId: anchor.anchorMessageId,
-	});
-};

package/src/filters/message-embedder.test.ts

@@ -1,39 +0,0 @@
-import assert from "node:assert/strict";
-import { afterEach, describe, it } from "node:test";
-import {
-	_resetMessageEmbedderForTest,
-	getMessageEmbedder,
-} from "./message-embedder.js";
-
-describe("getMessageEmbedder", () => {
-	afterEach(() => {
-		_resetMessageEmbedderForTest();
-	});
-
-	it("adapts the batch embedding service into a single-text message embedder", async () => {
-		const vector = await getMessageEmbedder().embed("invoice from stripe");
-
-		assert.ok(Array.isArray(vector));
-		assert.ok(vector.length > 0);
-		assert.ok(vector.every((value) => typeof value === "number"));
-	});
-
-	it("embeds the same text deterministically", async () => {
-		const embedder = getMessageEmbedder();
-		const a = await embedder.embed("your monthly receipt");
-		const b = await embedder.embed("your monthly receipt");
-
-		assert.deepEqual(a, b);
-	});
-
-	it("memoizes the embedder across calls", () => {
-		assert.strictEqual(getMessageEmbedder(), getMessageEmbedder());
-	});
-
-	it("rebuilds after a reset", () => {
-		const first = getMessageEmbedder();
-		_resetMessageEmbedderForTest();
-
-		assert.notStrictEqual(getMessageEmbedder(), first);
-	});
-});

package/src/filters/message-embedder.ts

@@ -1,33 +0,0 @@
-import { buildEmbeddingServiceFromEnv } from "@remit/search-service/from-env";
-import type { MessageEmbedder } from "./pipeline.js";
-
-/**
- * Adapt the batch {@link EmbeddingService} the rest of the stack composes from
- * env into the single-text {@link MessageEmbedder} the filter pipeline needs: one
- * incoming message becomes one message-level vector to compare against a filter's
- * persisted anchor (RFC 034 Decision 2.1/2.3).
- *
- * The embedder is selected by the same `SEARCH_EMBEDDING_*` env the search-index
- * worker reads, so every filter pass embeds under the identical model the anchors
- * were built with. The instance is memoized across warm invocations so a local
- * Transformers.js model loads once per container rather than per event.
- */
-let cached: MessageEmbedder | undefined;
-
-export const getMessageEmbedder = (): MessageEmbedder => {
-	if (cached) return cached;
-	const service = buildEmbeddingServiceFromEnv();
-	cached = {
-		embed: async (text: string): Promise<number[]> => {
-			const [vector] = await service.embed([text]);
-			return vector;
-		},
-		embeddingId: service.embeddingId,
-	};
-	return cached;
-};
-
-/** Reset the singleton — test use only. */
-export const _resetMessageEmbedderForTest = (): void => {
-	cached = undefined;
-};