@remit/mailbox-service 0.0.111 → 0.0.112

package/src/filters/match.test.ts

@@ -3,9 +3,7 @@ import { describe, it } from "node:test";
 import type { FilterItem } from "@remit/data-ports";
 import { FilterClauseField, FilterMatchOperator } from "@remit/domain-enums";
 import {
-	buildMatchText,
 	clauseMatches,
-	cosineSimilarity,
 	type FilterMessage,
 	literalClausesMatch,
 	literalClauseTerms,
@@ -197,7 +195,7 @@ describe("clauseMatches", () => {
 });
 
 describe("literalClausesMatch", () => {
-	it("passes vacuously with no clauses (a purely-semantic filter)", () => {
+	it("passes vacuously with no clauses", () => {
 		assert.equal(
 			literalClausesMatch([], FilterMatchOperator.And, message()),
 			true,
@@ -243,27 +241,6 @@ describe("literalClausesMatch", () => {
 	});
 });
 
-describe("cosineSimilarity", () => {
-	it("scores identical vectors as 1", () => {
-		assert.equal(cosineSimilarity([1, 2, 3], [1, 2, 3]), 1);
-	});
-
-	it("scores orthogonal vectors as 0", () => {
-		assert.equal(cosineSimilarity([1, 0], [0, 1]), 0);
-	});
-
-	it("scores a zero vector as 0 rather than dividing by zero", () => {
-		assert.equal(cosineSimilarity([0, 0], [1, 1]), 0);
-	});
-
-	it("throws on a dimension mismatch instead of scoring incomparable vectors", () => {
-		assert.throws(
-			() => cosineSimilarity([1, 2], [1, 2, 3]),
-			/dimension mismatch/,
-		);
-	});
-});
-
 describe("selectMoveWinner", () => {
 	const filter = (
 		filterId: string,
@@ -311,20 +288,6 @@ describe("selectMoveWinner", () => {
 	});
 });
 
-describe("buildMatchText", () => {
-	it("joins subject and body and bounds the length", () => {
-		const text = buildMatchText(message({ subject: "hello", text: "world" }));
-		assert.equal(text, "hello\nworld");
-	});
-
-	it("caps at 512 characters", () => {
-		const text = buildMatchText(
-			message({ subject: "", text: "x".repeat(1000) }),
-		);
-		assert.equal(text.length, 512);
-	});
-});
-
 describe("literalClauseTerms (reader #459)", () => {
 	it("narrows From and Subject onto the columns that carry them", () => {
 		assert.deepEqual(

package/src/filters/match.ts

@@ -16,25 +16,9 @@ type FilterClause = FilterItem["literalClauses"][number];
  */
 export const NO_ACTION = FILTER_NO_ACTION;
 
-/**
- * Default cosine cut-off for a semantic-anchor match (RFC 031 "the semantic
- * anchor evaluates as a kNN threshold against the new message's embedding").
- * Tunable per pipeline via {@link FilterConfig.similarityThreshold}.
- */
-export const DEFAULT_SEMANTIC_MATCH_THRESHOLD = 0.75;
-
-/**
- * Bound on the text embedded for a semantic match — the same 512-char budget
- * `buildTextPreview` applies to chunk vectors and `FilterAnchor.anchorSourceText`
- * persists, so the candidate side of the comparison is derived on the same
- * footing as the anchor side.
- */
-const MATCH_TEXT_LIMIT = 512;
-
 /**
  * The message fields a filter evaluates against — the literal-clause targets
- * (from / subject / body) plus the text embedded for a semantic anchor. A plain
- * value object so the matcher stays independent of the `ParsedMail` shape and is
+ * (from / subject / body / list). A plain value object so the matcher stays independent of the `ParsedMail` shape and is
  * trivially constructed in a test.
  */
 export interface FilterMessage {
@@ -112,8 +96,8 @@ export const clauseMatches = (
 
 /**
  * Whether the literal clauses match under their combine operator. Empty clauses
- * carry no literal constraint and pass vacuously — a purely-semantic filter (no
- * clauses, only an anchor) is gated by its anchor alone, not by this.
+ * carry no literal constraint and pass vacuously; a caller that needs a
+ * predicate checks for clauses itself.
  */
 export const literalClausesMatch = (
 	clauses: readonly FilterClause[],
@@ -207,31 +191,6 @@ export const literalClauseTerms = (
 	return { terms, operator: "or" };
 };
 
-/**
- * Cosine similarity of two equal-length vectors. Throws on a dimension mismatch
- * rather than silently scoring incomparable vectors — a mismatch means a stale
- * anchor embedded under a different model (`anchorEmbeddingId`), which is a
- * migration concern, not a match to guess at.
- */
-export const cosineSimilarity = (
-	a: readonly number[],
-	b: readonly number[],
-): number => {
-	if (a.length !== b.length) {
-		throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
-	}
-	let dot = 0;
-	let normA = 0;
-	let normB = 0;
-	for (let i = 0; i < a.length; i++) {
-		dot += a[i] * b[i];
-		normA += a[i] * a[i];
-		normB += b[i] * b[i];
-	}
-	if (normA === 0 || normB === 0) return 0;
-	return dot / (Math.sqrt(normA) * Math.sqrt(normB));
-};
-
 /**
  * The move a message ends in when several filters matched: the filter whose
  * predicate or action was most recently *changed* wins (RFC 034 Decision 3.2),
@@ -264,11 +223,3 @@ export const selectMoveWinner = (
 	}
 	return winner;
 };
-
-/**
- * The candidate text embedded once for a semantic match — subject then body,
- * bounded to the anchor's `anchorSourceText` budget so both sides of the cosine
- * comparison are derived from comparable inputs.
- */
-export const buildMatchText = (msg: FilterMessage): string =>
-	`${msg.subject}\n${msg.text}`.slice(0, MATCH_TEXT_LIMIT);

package/src/filters/pipeline.test.ts

@@ -1,330 +1,199 @@
-/**
- * The index-time filter pass over an anchorless literal filter — the shape the
- * organize widen creates on a deployment without the vector pipeline: `From`
- * clauses combined with `Or`, `hasAnchor: false`. This is what keeps a standing
- * "move all mail from these senders" filter working on future mail with no
- * vectors, so it is pinned directly: a matching sender resolves the move, a
- * non-matching one resolves nothing, and the embedder is never touched.
- */
-
 import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import type {
-	FilterItem,
-	IFilterAnchorRepository,
-	IFilterRepository,
-	IMessageLabelRepository,
-} from "@remit/data-ports";
+import { afterEach, beforeEach, describe, it, mock } from "node:test";
+import type { AccountItem, CreateFilterInput } from "@remit/data-ports";
 import {
+	AccountAuthType,
+	ConnectionState,
 	FilterClauseField,
 	FilterMatchOperator,
-	FilterState,
+	FilterScope,
 } from "@remit/domain-enums";
-import type { PlacementMoveService } from "../placement-move.js";
-import { NO_ACTION } from "./match.js";
-import { type FilterConfig, FilterPipeline } from "./pipeline.js";
+import { PlacementMoveService } from "../placement-move.js";
+import {
+	openStore,
+	type Store,
+	seedAccount,
+	seedMailbox,
+} from "../test-helpers/test-store.js";
+import type { FilterMessage } from "./match.js";
+import { FilterPipeline } from "./pipeline.js";
+
+const ACCOUNT = "cfg-1";
+
+let store: Store;
+let account: AccountItem;
+let archive: string;
+let projects: string;
+
+beforeEach(async () => {
+	store = await openStore();
+	account = await seedAccount(store, { accountConfigId: ACCOUNT });
+	archive = (await seedMailbox(store, account, "Archive")).mailboxId;
+	projects = (await seedMailbox(store, account, "Projects")).mailboxId;
+});
+
+afterEach(async () => {
+	mock.timers.reset();
+	await store.close();
+});
 
-const senderFilter = (destinationMailboxId: string): FilterItem =>
-	({
-		filterId: "flt-senders",
-		accountConfigId: "cfg-1",
-		name: "npm barrage",
-		scope: "Standing",
-		state: FilterState.Active,
-		hasAnchor: false,
-		ruleChangedAt: 1,
+const fromGithub: FilterMessage = {
+	from: "npm@github.com",
+	fromName: "npm",
+	subject: "A new version of left-pad is available",
+	text: "body",
+	listId: "",
+};
+
+const receipt: FilterMessage = {
+	from: "billing@stripe.com",
+	fromName: "Stripe",
+	subject: "Your receipt",
+	text: "Thanks for your payment",
+	listId: "",
+};
+
+const senders = (actionMailboxId: string) =>
+	store.filter.create({
+		accountConfigId: ACCOUNT,
+		name: "rule",
+		scope: FilterScope.Standing,
 		matchOperator: FilterMatchOperator.Or,
 		literalClauses: [
 			{ field: FilterClauseField.From, value: "npm@github.com" },
 			{ field: FilterClauseField.From, value: "notifications@github.com" },
 		],
-		actionLabelId: NO_ACTION,
-		actionMailboxId: destinationMailboxId,
-		createdAt: 1,
-		updatedAt: 1,
-	}) as unknown as FilterItem;
+		actionMailboxId,
+	});
 
-const buildPipeline = (...filters: FilterItem[]) => {
-	const state = { embedCalls: 0 };
-	const config: FilterConfig = {
-		filterService: {
-			listByAccountAndState: async () => filters,
-			refreshExpiry: async (f: FilterItem) => f,
-		} as unknown as IFilterRepository,
-		filterAnchorService: {
-			get: async () => undefined,
-		} as unknown as IFilterAnchorRepository,
-		messageLabelService: {} as unknown as IMessageLabelRepository,
-		placementMoveService: {} as unknown as PlacementMoveService,
-		embedder: {
-			embed: async () => {
-				state.embedCalls += 1;
-				return [];
-			},
-			embeddingId: "test-model@0",
+const pipeline = () =>
+	new FilterPipeline(
+		{
+			filterService: store.filter,
+			messageLabelService: store.messageLabel,
+			mailboxService: store.mailbox,
+			placementMoveService: new PlacementMoveService({
+				messageService: store.message,
+				threadMessageService: store.threadMessage,
+				markerService: store.placementMove,
+				addressService: store.address,
+				mailboxSpecialUseService: store.mailboxSpecialUse,
+				mailboxService: store.mailbox,
+				sqsQueueUrl: "https://sqs.test/message-mgmt",
+			}),
 		},
-	};
-	return { pipeline: new FilterPipeline(config, { info: () => {} }), state };
-};
-
-describe("FilterPipeline — anchorless From/Or filter at index time", () => {
-	it("resolves the move for a matching sender and never embeds", async () => {
-		const { pipeline, state } = buildPipeline(senderFilter("mbx-archive"));
-		const decision = await pipeline.evaluate("cfg-1", "m-1", {
-			from: "npm@github.com",
-			fromName: "npm",
-			subject: "A new version of left-pad is available",
-			text: "body",
-			listId: "",
-		});
+		{ info: () => {}, error: () => {} },
+	);
+
+describe("FilterPipeline over literal filters", () => {
+	it("resolves the move for a matching sender", async () => {
+		const filter = await senders(archive);
+
+		const decision = await pipeline().evaluate(
+			ACCOUNT,
+			account.accountId,
+			"m-1",
+			fromGithub,
+		);
 
 		assert.deepEqual(decision.move, {
-			destinationMailboxId: "mbx-archive",
-			filterId: "flt-senders",
+			destinationMailboxId: archive,
+			filterId: filter.filterId,
 		});
-		// A literal-only filter is decided by its clauses alone — the embedder is
-		// never constructed or called.
-		assert.equal(state.embedCalls, 0);
 	});
 
-	it("resolves nothing for a sender none of the clauses match", async () => {
-		const { pipeline, state } = buildPipeline(senderFilter("mbx-archive"));
-		const decision = await pipeline.evaluate("cfg-1", "m-2", {
-			from: "hello@stripe.com",
-			fromName: "Stripe",
-			subject: "Your receipt",
-			text: "body",
-			listId: "",
-		});
+	it("resolves nothing for a sender no clause names", async () => {
+		await senders(archive);
+
+		const decision = await pipeline().evaluate(
+			ACCOUNT,
+			account.accountId,
+			"m-2",
+			{
+				...fromGithub,
+				from: "hello@stripe.com",
+			},
+		);
 
 		assert.equal(decision.move, undefined);
 		assert.deepEqual(decision.labels, []);
-		assert.equal(state.embedCalls, 0);
 	});
-});
 
-/**
- * Arbitration survives exactly here: two standing filters colliding over the
- * same message at the same trigger. A move is exclusive, so the pass resolves
- * one destination — the more-recently-changed action wins (reader #497 leaves
- * this untouched while removing arbitration from the user-initiated apply).
- */
-describe("FilterPipeline — two standing filters colliding on one message", () => {
-	const rivalFilter = (): FilterItem =>
-		({
-			...senderFilter("mbx-projects"),
-			filterId: "flt-rival",
-			name: "github to projects",
-			actionChangedAt: 2,
-		}) as unknown as FilterItem;
-
-	it("resolves a single destination from the more-recently-changed action", async () => {
-		const older = {
-			...senderFilter("mbx-archive"),
-			actionChangedAt: 1,
-		} as unknown as FilterItem;
-		const { pipeline } = buildPipeline(older, rivalFilter());
-		const decision = await pipeline.evaluate("cfg-1", "m-3", {
-			from: "npm@github.com",
-			fromName: "npm",
-			subject: "A new version of left-pad is available",
-			text: "body",
-			listId: "",
-		});
+	it("gives a contested message to the more recently changed action", async () => {
+		mock.timers.enable({ apis: ["Date"], now: 1_000_000 });
+		await senders(archive);
+		mock.timers.setTime(9_000_000);
+		const rival = await senders(projects);
+
+		const decision = await pipeline().evaluate(
+			ACCOUNT,
+			account.accountId,
+			"m-3",
+			fromGithub,
+		);
 
 		assert.deepEqual(decision.move, {
-			destinationMailboxId: "mbx-projects",
-			filterId: "flt-rival",
+			destinationMailboxId: projects,
+			filterId: rival.filterId,
 		});
 	});
 });
 
-describe("FilterPipeline — anchor lazy re-embed on model drift (reader #295)", () => {
-	const anchoredFilter = (): FilterItem =>
-		({
-			filterId: "flt-semantic",
-			accountConfigId: "cfg-1",
-			name: "receipts",
-			scope: "Standing",
-			state: FilterState.Active,
-			hasAnchor: true,
-			ruleChangedAt: 1,
-			matchOperator: FilterMatchOperator.And,
-			literalClauses: [],
-			actionLabelId: "lbl-1",
-			actionMailboxId: NO_ACTION,
-			createdAt: 1,
-			updatedAt: 1,
-		}) as unknown as FilterItem;
-
-	const message = {
-		from: "billing@stripe.com",
-		fromName: "Stripe",
-		subject: "Your receipt",
-		text: "Thanks for your payment",
-		listId: "",
-	};
-
-	it("re-embeds a stale anchor and matches on the very next evaluation, persisting the refreshed row", async () => {
-		const putCalls: Array<Record<string, unknown>> = [];
-		let embedCalls = 0;
-		const config: FilterConfig = {
-			filterService: {
-				listByAccountAndState: async () => [anchoredFilter()],
-				refreshExpiry: async (f: FilterItem) => f,
-			} as unknown as IFilterRepository,
-			filterAnchorService: {
-				get: async () => ({
-					accountConfigId: "cfg-1",
-					filterId: "flt-semantic",
-					// Stale-model space — orthogonal to what the current model
-					// produces, so a raw comparison against it would not match.
-					anchorEmbedding: [0, 1, 0],
-					anchorEmbeddingId: "old-model@3",
-					anchorSourceText: "your receipt is ready",
-					anchorMessageId: "msg-anchor",
-				}),
-				put: async (input: Record<string, unknown>) => {
-					putCalls.push(input);
-					return { ...input, createdAt: 1, updatedAt: 2 };
-				},
-			} as unknown as IFilterAnchorRepository,
-			messageLabelService: {} as unknown as IMessageLabelRepository,
-			placementMoveService: {} as unknown as PlacementMoveService,
-			embedder: {
-				embed: async () => {
-					embedCalls += 1;
-					return [1, 0, 0];
-				},
-				embeddingId: "new-model@3",
-			},
-		};
-
-		const decision = await new FilterPipeline(config, {
-			info: () => {},
-		}).evaluate("cfg-1", "m-1", message);
-
-		assert.deepEqual(
-			decision.labels,
-			[{ labelId: "lbl-1", filterId: "flt-semantic" }],
-			"the anchor matches once re-embedded into the current model's space",
+describe("FilterPipeline scopes a filter to its folder's account", () => {
+	it("leaves another account's mail alone, as if the filter had not matched", async () => {
+		const sibling = await store.account.create({
+			accountConfigId: ACCOUNT,
+			username: "me-too@example.com",
+			email: "me-too@example.com",
+			authType: AccountAuthType.Password,
+			passwordHash: '{"ciphertext":"x"}',
+			imapHost: "imap.example.com",
+			imapPort: 993,
+			imapTls: true,
+			imapStartTls: false,
+			isActive: true,
+			connectionState: ConnectionState.Authenticated,
+		});
+		const travel = await seedMailbox(store, sibling, "Travel");
+		const filter = await senders(travel.mailboxId);
+
+		const mine = await pipeline().evaluate(
+			ACCOUNT,
+			account.accountId,
+			"m-1",
+			fromGithub,
 		);
-		assert.equal(
-			putCalls.length,
-			1,
-			"the refreshed anchor is written back exactly once",
+		const theirs = await pipeline().evaluate(
+			ACCOUNT,
+			sibling.accountId,
+			"m-2",
+			fromGithub,
 		);
-		assert.deepEqual(putCalls[0], {
-			accountConfigId: "cfg-1",
-			filterId: "flt-semantic",
-			anchorEmbedding: [1, 0, 0],
-			anchorEmbeddingId: "new-model@3",
-			anchorSourceText: "your receipt is ready",
-			anchorMessageId: "msg-anchor",
+
+		assert.deepEqual(mine, { labels: [] });
+		assert.deepEqual(theirs.move, {
+			destinationMailboxId: travel.mailboxId,
+			filterId: filter.filterId,
 		});
 		assert.equal(
-			embedCalls,
-			2,
-			"one embed re-embeds the anchor's source text, one embeds the candidate message",
+			(await store.filter.get(ACCOUNT, filter.filterId)).state,
+			"Active",
 		);
 	});
 
-	it("never re-embeds when the anchor's embeddingId is already current", async () => {
-		let putCalls = 0;
-		let embedCalls = 0;
-		const config: FilterConfig = {
-			filterService: {
-				listByAccountAndState: async () => [anchoredFilter()],
-				refreshExpiry: async (f: FilterItem) => f,
-			} as unknown as IFilterRepository,
-			filterAnchorService: {
-				get: async () => ({
-					anchorEmbedding: [1, 0, 0],
-					anchorEmbeddingId: "current-model@3",
-					anchorSourceText: "your receipt is ready",
-					anchorMessageId: "msg-anchor",
-				}),
-				put: async () => {
-					putCalls += 1;
-					throw new Error("must not be called");
-				},
-			} as unknown as IFilterAnchorRepository,
-			messageLabelService: {} as unknown as IMessageLabelRepository,
-			placementMoveService: {} as unknown as PlacementMoveService,
-			embedder: {
-				embed: async () => {
-					embedCalls += 1;
-					return [1, 0, 0];
-				},
-				embeddingId: "current-model@3",
-			},
-		};
-
-		const decision = await new FilterPipeline(config, {
-			info: () => {},
-		}).evaluate("cfg-1", "m-1", message);
+	it("turns the filter off when no account holds its folder any more", async () => {
+		const filter = await senders("mbx-gone");
 
-		assert.deepEqual(decision.labels, [
-			{ labelId: "lbl-1", filterId: "flt-semantic" },
-		]);
-		assert.equal(putCalls, 0, "a current anchor is never rewritten");
-		assert.equal(
-			embedCalls,
-			1,
-			"only the candidate message is embedded — no wasted re-embed call",
+		const decision = await pipeline().evaluate(
+			ACCOUNT,
+			account.accountId,
+			"m-1",
+			fromGithub,
 		);
-	});
 
-	it("degrades a re-embed failure to skipping just this filter, not the whole pass", async () => {
-		const literalFilter: FilterItem = {
-			filterId: "flt-literal",
-			accountConfigId: "cfg-1",
-			name: "stripe receipts",
-			scope: "Standing",
-			state: FilterState.Active,
-			hasAnchor: false,
-			ruleChangedAt: 1,
-			matchOperator: FilterMatchOperator.Or,
-			literalClauses: [{ field: FilterClauseField.From, value: "stripe.com" }],
-			actionLabelId: "lbl-literal",
-			actionMailboxId: NO_ACTION,
-			createdAt: 1,
-			updatedAt: 1,
-		} as unknown as FilterItem;
-
-		const config: FilterConfig = {
-			filterService: {
-				listByAccountAndState: async () => [anchoredFilter(), literalFilter],
-				refreshExpiry: async (f: FilterItem) => f,
-			} as unknown as IFilterRepository,
-			filterAnchorService: {
-				get: async () => ({
-					anchorEmbedding: [0, 1, 0],
-					anchorEmbeddingId: "old-model@3",
-					anchorSourceText: "your receipt is ready",
-					anchorMessageId: "msg-anchor",
-				}),
-				put: async () => {
-					throw new Error("SQLITE_BUSY");
-				},
-			} as unknown as IFilterAnchorRepository,
-			messageLabelService: {} as unknown as IMessageLabelRepository,
-			placementMoveService: {} as unknown as PlacementMoveService,
-			embedder: {
-				embed: async () => [1, 0, 0],
-				embeddingId: "new-model@3",
-			},
-		};
-
-		const decision = await new FilterPipeline(config, {
-			info: () => {},
-			error: () => {},
-		}).evaluate("cfg-1", "m-1", message);
-
-		assert.deepEqual(
-			decision.labels,
-			[{ labelId: "lbl-literal", filterId: "flt-literal" }],
-			"the anchored filter's re-embed failure is isolated — the literal filter still applies",
-		);
+		assert.deepEqual(decision, { labels: [] });
+		const stored = await store.filter.get(ACCOUNT, filter.filterId);
+		assert.equal(stored.state, "Disabled");
+		assert.equal(stored.disabledReason, "FolderMissing");
 	});
 });