@remit/account-worker 0.0.25 → 0.0.26

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/account-worker",
-  "version": "0.0.25",
+  "version": "0.0.26",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/handlers/organize-job.test.ts

@@ -1,10 +1,23 @@
 import assert from "node:assert/strict";
-import { describe, it } from "node:test";
+import { randomUUID } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { afterEach, describe, it } from "node:test";
 import type { RemitClient } from "@remit/backend/client";
-import type { OrganizeMatchDeps } from "@remit/backend/organize";
+import {
+	buildOrganizeMatchDeps,
+	type OrganizeMatchDeps,
+	type OrganizeSemanticDeps,
+} from "@remit/backend/organize";
+import type { FilterAnchorItem, FilterItem } from "@remit/data-ports";
+import { NotFoundError } from "@remit/data-ports/errors";
+import { DrizzleThreadMessageRepository } from "@remit/drizzle-service";
+import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
 import { noopLogger } from "@remit/logger-lambda/noop-logger";
 import type { OrganizeJobEvent } from "../events.js";
-import { processOrganizeJob } from "./organize-job.js";
+import {
+	type ProcessOrganizeJobDeps,
+	processOrganizeJob,
+} from "./organize-job.js";
 
 interface Update {
 	state: string;
@@ -29,6 +42,7 @@ const jobClient = (
 			get: async () => ({
 				organizeJobId: "job-1",
 				accountConfigId: "cfg-1",
+				filterId: "None",
 				anchorMessageId: "None",
 				matchOperator: "And",
 				literalClauses,
@@ -56,6 +70,7 @@ const explodingMatchDeps = (): OrganizeMatchDeps =>
 			throw new Error("SQLITE_BUSY");
 		},
 		filterAnchors: {
+			get: async () => null,
 			listByAccountConfig: async () => [],
 			put: async () => {
 				throw new Error("unreachable");
@@ -104,3 +119,323 @@ describe("processOrganizeJob (reader #463)", () => {
 		assert.equal(updates[1]?.errorMessage, "SQLITE_BUSY");
 	});
 });
+
+interface Move {
+	messageId: string;
+	destinationMailboxId: string;
+}
+
+const CONFIG = "cfg-1";
+const INBOX = "mbx-inbox";
+const ARCHIVE = "mbx-archive";
+
+const standingFilter = (over: Partial<FilterItem> = {}): FilterItem => ({
+	filterId: "flt-1",
+	accountConfigId: CONFIG,
+	name: "Invoices",
+	scope: "Standing",
+	state: "Active",
+	disabledReason: "None",
+	hasAnchor: false,
+	ruleChangedAt: 1,
+	actionChangedAt: 1,
+	matchOperator: "And",
+	literalClauses: [{ field: "From", value: "billing@example.com" }],
+	actionLabelId: "None",
+	actionMailboxId: "mbx-invoices",
+	createdAt: 0,
+	updatedAt: 0,
+	...over,
+});
+
+/**
+ * The thread rows live in the shipped SQLite schema, read through the real
+ * repository, so the inbox narrowing is the store's own query. One invoice
+ * sits in the inbox and one in the archive, both from the filter's sender
+ * and both saying "overdue" in their preview.
+ */
+const seedThreads = async (
+	threadMessage: DrizzleThreadMessageRepository,
+): Promise<{ inbox: string; archive: string }> => {
+	const seed = async (mailboxId: string, uid: number): Promise<string> => {
+		const messageId = randomUUID();
+		await threadMessage.create({
+			accountConfigId: CONFIG,
+			threadId: randomUUID(),
+			messageId,
+			mailboxId,
+			uid,
+			referenceOrder: 0,
+			internalDate: uid,
+			sentDate: uid,
+			subject: "Your invoice",
+			fromEmail: "billing@example.com",
+			fromName: "Billing",
+			snippet: "This invoice is overdue",
+			isRead: false,
+			isDeleted: false,
+			hasAttachment: false,
+			hasStars: false,
+		});
+		return messageId;
+	};
+	return { inbox: await seed(INBOX, 1), archive: await seed(ARCHIVE, 2) };
+};
+
+interface World {
+	client: RemitClient;
+	inbox: string;
+	archive: string;
+	updates: Update[];
+	moves: Move[];
+	labeled: string[];
+	close: () => void;
+}
+
+const world = async (
+	filters: FilterItem[],
+	anchors: FilterAnchorItem[] = [],
+): Promise<World> => {
+	const store = createShippedSqliteDb();
+	store.sqlite.exec(
+		readFileSync(
+			new URL(
+				"../../../../npm-scripts/sqlite-search-index.sql",
+				import.meta.url,
+			),
+			"utf8",
+		),
+	);
+	const threadMessage = new DrizzleThreadMessageRepository(store.db as never);
+	const { inbox, archive } = await seedThreads(threadMessage);
+	const updates: Update[] = [];
+	const labeled: string[] = [];
+	const mailboxOf: Record<string, string> = {
+		[inbox]: INBOX,
+		[archive]: ARCHIVE,
+	};
+	const client = {
+		threadMessage,
+		organizeJobRequest: {
+			get: async () => ({
+				organizeJobId: "job-1",
+				accountConfigId: CONFIG,
+				filterId: "flt-1",
+				anchorMessageId: "None",
+				matchOperator: "And",
+				literalClauses: [],
+				similarityThreshold: 0.75,
+				actionLabelId: "None",
+				actionMailboxId: "None",
+			}),
+			update: async (_id: string, patch: Update) => {
+				updates.push(patch);
+			},
+		},
+		filter: {
+			get: async (_cfg: string, filterId: string) => {
+				const found = filters.find((f) => f.filterId === filterId);
+				if (!found) throw new NotFoundError(`Filter not found: ${filterId}`);
+				return found;
+			},
+			refreshExpiry: async (filter: FilterItem) => filter,
+			listByAccountAndState: async (_cfg: string, state: string) =>
+				filters.filter((f) => f.state === state),
+		},
+		filterAnchor: {
+			get: async (_cfg: string, filterId: string) =>
+				anchors.find((anchor) => anchor.filterId === filterId) ?? null,
+		},
+		account: {
+			listMailSourcesByAccountConfig: async () => [{ accountId: "acct-1" }],
+		},
+		mailboxSpecialUse: {
+			findInboxMailbox: async () => ({ mailboxId: INBOX, fullPath: "INBOX" }),
+		},
+		message: {
+			get: async (messageId: string) => ({
+				messageId,
+				mailboxId: mailboxOf[messageId],
+			}),
+		},
+		mailbox: { resolveAccountId: async () => "acct-1" },
+		messageLabel: {
+			apply: async (input: { messageId: string }) => {
+				labeled.push(input.messageId);
+				return {};
+			},
+		},
+	} as unknown as RemitClient;
+	return {
+		client,
+		inbox,
+		archive,
+		updates,
+		moves: [],
+		labeled,
+		close: store.close,
+	};
+};
+
+const recordingMoves = (moves: Move[]) =>
+	({
+		moveMessage: async (
+			_cfg: string,
+			messageId: string,
+			destinationMailboxId: string,
+		) => {
+			moves.push({ messageId, destinationMailboxId });
+		},
+	}) as unknown as NonNullable<ProcessOrganizeJobDeps["moveService"]>;
+
+const runFilterJob = (w: World) =>
+	processOrganizeJob(event, noopLogger, {
+		client: w.client,
+		matchDeps: buildOrganizeMatchDeps(w.client),
+		moveService: recordingMoves(w.moves),
+	});
+
+const EMBEDDING_ID = "test-model@4";
+const ANCHOR_VECTOR = [1, 0, 0, 0];
+const ORTHOGONAL_VECTOR = [0, 1, 0, 0];
+
+const persistedAnchor: FilterAnchorItem = {
+	filterId: "flt-1",
+	accountConfigId: CONFIG,
+	anchorMessageId: "msg-anchor",
+	anchorEmbedding: ANCHOR_VECTOR,
+	anchorEmbeddingId: EMBEDDING_ID,
+	anchorSourceText: "Your invoice",
+	createdAt: 0,
+	updatedAt: 0,
+};
+
+/**
+ * The vector index answers the anchor with the inbox invoice, and only when
+ * asked about the inbox: the kNN read is where a similarity filter finds its
+ * candidates.
+ */
+const vectorMatchDeps = (w: World): OrganizeMatchDeps => ({
+	...buildOrganizeMatchDeps(w.client),
+	semantic: () => ({
+		buildAnchor: async () => {
+			throw new Error("a filter widens on its persisted anchor");
+		},
+		vectorStore: {
+			query: async (query: { filter?: { mailboxId?: string } }) =>
+				query.filter?.mailboxId === INBOX
+					? [
+							{
+								chunkId: `${w.inbox}#body`,
+								score: 0.9,
+								metadata: { messageId: w.inbox, mailboxIds: [INBOX] },
+							},
+						]
+					: [],
+			getByMessage: async () => [],
+		} as unknown as OrganizeSemanticDeps["vectorStore"],
+		embed: async () => ANCHOR_VECTOR,
+		embeddingId: EMBEDDING_ID,
+	}),
+});
+
+describe("processOrganizeJob for a filter job (#1354)", () => {
+	let current: World | undefined;
+	afterEach(() => current?.close());
+
+	it("moves the pre-existing matching inbox message and leaves the archive alone", async () => {
+		current = await world([standingFilter()]);
+
+		await runFilterJob(current);
+
+		assert.deepEqual(current.moves, [
+			{ messageId: current.inbox, destinationMailboxId: "mbx-invoices" },
+		]);
+		const done = current.updates.at(-1);
+		assert.equal(done?.state, "Complete");
+		assert.equal(done?.matchedCount, 1);
+		assert.equal(done?.appliedCount, 1);
+	});
+
+	it("matches a free-text rule from search on the stored preview", async () => {
+		current = await world([
+			standingFilter({
+				literalClauses: [{ field: "HasWords", value: "overdue" }],
+			}),
+		]);
+
+		await runFilterJob(current);
+
+		assert.deepEqual(current.moves, [
+			{ messageId: current.inbox, destinationMailboxId: "mbx-invoices" },
+		]);
+		assert.equal(current.updates.at(-1)?.state, "Complete");
+	});
+
+	it("leaves the move to a newer filter that also matches, and still labels", async () => {
+		current = await world([
+			standingFilter({ actionLabelId: "lbl-bills" }),
+			standingFilter({
+				filterId: "flt-2",
+				name: "Receipts",
+				actionChangedAt: 2,
+				actionMailboxId: "mbx-receipts",
+			}),
+		]);
+
+		await runFilterJob(current);
+
+		assert.deepEqual(current.moves, []);
+		assert.deepEqual(current.labeled, [current.inbox]);
+		assert.equal(current.updates.at(-1)?.state, "Complete");
+	});
+
+	it("fails the job without a retry when the filter is turned off", async () => {
+		current = await world([
+			standingFilter({ state: "Disabled", disabledReason: "UserDisabled" }),
+		]);
+
+		await runFilterJob(current);
+
+		assert.deepEqual(current.moves, []);
+		assert.equal(current.updates.at(-1)?.state, "Failed");
+		assert.match(String(current.updates.at(-1)?.errorMessage), /turned off/);
+	});
+
+	it("fails the job without a retry when the filter was deleted", async () => {
+		current = await world([]);
+
+		await runFilterJob(current);
+
+		assert.equal(current.updates.at(-1)?.state, "Failed");
+		assert.match(String(current.updates.at(-1)?.errorMessage), /deleted/);
+	});
+
+	it("moves a message the vector query found, though the preview scores below the threshold", async () => {
+		current = await world(
+			[standingFilter({ hasAnchor: true, literalClauses: [] })],
+			[persistedAnchor],
+		);
+
+		// The index-time evaluator embeds subject and preview; this embedder puts
+		// that text orthogonal to the anchor, so evaluate alone would call the
+		// message a miss. It only orders the move, so the vector match stands.
+		await processOrganizeJob(event, noopLogger, {
+			client: current.client,
+			matchDeps: vectorMatchDeps(current),
+			moveService: recordingMoves(current.moves),
+			embedder: {
+				embed: async () => ORTHOGONAL_VECTOR,
+				embeddingId: EMBEDDING_ID,
+			},
+		});
+
+		assert.deepEqual(current.moves, [
+			{ messageId: current.inbox, destinationMailboxId: "mbx-invoices" },
+		]);
+		const done = current.updates.at(-1);
+		assert.equal(done?.state, "Complete");
+		assert.equal(done?.matchedCount, 1);
+		assert.equal(done?.appliedCount, 1);
+	});
+});

package/src/handlers/organize-job.ts

@@ -1,11 +1,16 @@
 import { getClient, type RemitClient } from "@remit/backend/client";
 import {
+	type ApplyOrganizeDeps,
 	applyOrganize,
+	backApplyFilter,
 	buildOrganizeMatchDeps,
 	buildOrganizeMoveService,
+	type FilterBackApplyDeps,
+	isFilterJob,
 	matchOrganize,
 	ORGANIZE_MATCH_LIMIT,
 	type OrganizeMatchDeps,
+	type OrganizeRejection,
 	predicateFromJob,
 } from "@remit/backend/organize";
 import type { Logger } from "@remit/logger-lambda";
@@ -14,12 +19,64 @@ import type { OrganizeJobEvent } from "../events.js";
 export interface ProcessOrganizeJobDeps {
 	client?: RemitClient;
 	matchDeps?: OrganizeMatchDeps;
+	moveService?: ApplyOrganizeDeps["moveService"];
+	embedder?: FilterBackApplyDeps["embedder"];
 }
 
+interface OrganizeOutcome {
+	rejected: OrganizeRejection | null;
+	matched: number;
+	applied: number;
+	failed: number;
+	semanticUnavailable: boolean;
+}
+
+const rejectedOutcome = (rejected: OrganizeRejection): OrganizeOutcome => ({
+	rejected,
+	matched: 0,
+	applied: 0,
+	failed: 0,
+	semanticUnavailable: false,
+});
+
+/**
+ * The "all like these" pass: the predicate snapshotted on the row, over every
+ * folder, every requested move applied (reader #497).
+ */
+const runSnapshot = async (
+	client: RemitClient,
+	matchDeps: OrganizeMatchDeps,
+	moveService: ApplyOrganizeDeps["moveService"],
+	job: Awaited<ReturnType<RemitClient["organizeJobRequest"]["get"]>>,
+): Promise<OrganizeOutcome> => {
+	const predicate = predicateFromJob(job);
+	const match = await matchOrganize(
+		matchDeps,
+		job.accountConfigId,
+		predicate,
+		ORGANIZE_MATCH_LIMIT,
+	);
+	if (match.rejected) return rejectedOutcome(match.rejected);
+	const { applied, failed } = await applyOrganize(
+		{ client, moveService },
+		job.accountConfigId,
+		match.messageIds,
+		predicate,
+	);
+	return {
+		rejected: null,
+		matched: match.messageIds.length,
+		applied,
+		failed,
+		semanticUnavailable: match.semanticUnavailable,
+	};
+};
+
 /**
- * Run a "all like these" back-apply job (RFC 034, #1278): match the corpus
- * against the job's snapshotted predicate and apply the action to every match,
- * in one pass, then record the counts. Mirrors the export job's lifecycle —
+ * Run a back-apply job (RFC 034, #1278): an "all like these" pass matches the
+ * corpus against the job's snapshotted predicate; a filter job (#1354) runs the
+ * named standing filter over the inbox with index-time move precedence. Either
+ * applies the action to every match in one pass, then records the counts. Mirrors the export job's lifecycle —
  * Running, then Complete/Failed — on the same fanout seam.
  *
  * Reuses the shared matcher (so the applied set equals what preview returned)
@@ -53,41 +110,37 @@ export const processOrganizeJob = async (
 	);
 
 	try {
-		const predicate = predicateFromJob(job);
 		const matchDeps = deps.matchDeps ?? buildOrganizeMatchDeps(client);
-		const match = await matchOrganize(
-			matchDeps,
-			accountConfigId,
-			predicate,
-			ORGANIZE_MATCH_LIMIT,
-		);
+		const moveService = deps.moveService ?? buildOrganizeMoveService(client);
+		const outcome = isFilterJob(job)
+			? await backApplyFilter(
+					{ client, matchDeps, moveService, embedder: deps.embedder },
+					accountConfigId,
+					job.filterId,
+				).then((result) =>
+					result.rejected ? rejectedOutcome(result.rejected) : result,
+				)
+			: await runSnapshot(client, matchDeps, moveService, job);
 
-		if (match.rejected) {
+		if (outcome.rejected) {
 			await client.organizeJobRequest.update(organizeJobId, {
 				state: "Failed",
 				matchedCount: 0,
 				appliedCount: 0,
 				failedCount: 0,
-				errorMessage: match.rejected.message,
+				errorMessage: outcome.rejected.message,
 			});
 			log.warn(
-				{ accountConfigId, organizeJobId, reason: match.rejected.reason },
+				{ accountConfigId, organizeJobId, reason: outcome.rejected.reason },
 				"Organize back-apply refused the job's rule; failing the job without a retry",
 			);
 			return;
 		}
 
-		const { messageIds, semanticUnavailable } = match;
-		const { applied, failed } = await applyOrganize(
-			{ client, moveService: buildOrganizeMoveService(client) },
-			accountConfigId,
-			messageIds,
-			predicate,
-		);
-
+		const { matched, applied, failed, semanticUnavailable } = outcome;
 		await client.organizeJobRequest.update(organizeJobId, {
 			state: "Complete",
-			matchedCount: messageIds.length,
+			matchedCount: matched,
 			appliedCount: applied,
 			failedCount: failed,
 		});
@@ -95,7 +148,8 @@ export const processOrganizeJob = async (
 			{
 				accountConfigId,
 				organizeJobId,
-				matched: messageIds.length,
+				filterId: job.filterId,
+				matched,
 				applied,
 				failed,
 				semanticUnavailable,