@remit/backend 0.0.114 → 0.0.115

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/backend",
-  "version": "0.0.114",
+  "version": "0.0.115",
   "description": "Remit Mail Inspector API backend",
   "license": "MIT",
   "author": "",
@@ -27,6 +27,10 @@
       "types": "./src/service/organize.ts",
       "default": "./src/service/organize.ts"
     },
+    "./sender-category": {
+      "types": "./src/service/sender-category.ts",
+      "default": "./src/service/sender-category.ts"
+    },
     "./trigger-sync": {
       "types": "./src/service/trigger-sync.ts",
       "default": "./src/service/trigger-sync.ts"

package/src/handlers/address.test.ts

@@ -1,5 +1,6 @@
 import assert from "node:assert/strict";
-import { afterEach, describe, it } from "node:test";
+import { afterEach, describe, it, mock } from "node:test";
+import type { SendMessageCommand } from "@aws-sdk/client-sqs";
 import type {
 	AddressResponse,
 	UpdateAddressInput,
@@ -17,6 +18,7 @@ import {
 	type RemitClient,
 	setClient,
 } from "../service/data-client.js";
+import { sqsClient } from "../service/sqs.js";
 import { AddressDetailOperations, AddressOperations } from "./address.js";
 
 const searchAddresses =
@@ -81,6 +83,7 @@ const clientReturning = (items: AddressItem[], seen: Listing[]): RemitClient =>
 	}) as unknown as RemitClient;
 
 afterEach(() => {
+	mock.restoreAll();
 	_resetForTest();
 });
 
@@ -243,3 +246,98 @@ describe("AddressDetailOperations_updateAddress removing a flag (#615)", () => {
 		assert.equal(response.flags.muted, undefined);
 	});
 });
+
+/**
+ * The retroactive half of a sender-category override (#415). Setting the flag
+ * enqueues one back-apply carrying the category it was set to; clearing it —
+ * the revert-to-auto route — enqueues nothing, because the classifier's own
+ * answer for mail already filed cannot be re-derived without every body.
+ */
+describe("AddressDetailOperations_updateAddress category back-apply (#415)", () => {
+	const withStubbedQueue = (enqueued: SendMessageCommand[]): void => {
+		process.env.SQS_QUEUE_URL_ACCOUNT_FANOUT =
+			"http://localhost:9324/queue/account-fanout-test";
+		mock.method(sqsClient, "send", async (command: SendMessageCommand) => {
+			enqueued.push(command);
+			return {};
+		});
+	};
+
+	it("enqueues a back-apply naming the sender and the set it was fired for", async () => {
+		const enqueued: SendMessageCommand[] = [];
+		withStubbedQueue(enqueued);
+		setClient(clientHolding(address({ flags: {} }), []));
+
+		await updateAddress(
+			updateContextFor({
+				flags: { category: { value: "newsletter", setAt: 20 } },
+			}),
+			eventFor(SUB),
+		);
+
+		assert.equal(enqueued.length, 1);
+		assert.deepEqual(JSON.parse(String(enqueued[0]?.input.MessageBody)), {
+			type: "SenderCategoryBackApply",
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			addressId: "addr-1",
+			normalizedEmail: "amsterdam@pocahondas.nl",
+			category: "newsletter",
+			categorySetAt: 20,
+		});
+	});
+
+	it("says the override is saved and names the retry when the queue refuses", async () => {
+		process.env.SQS_QUEUE_URL_ACCOUNT_FANOUT =
+			"http://localhost:9324/queue/account-fanout-test";
+		mock.method(sqsClient, "send", async () => {
+			throw new Error("AWS.SimpleQueueService.NonExistentQueue");
+		});
+		setClient(clientHolding(address({ flags: {} }), []));
+
+		await assert.rejects(
+			updateAddress(
+				updateContextFor({
+					flags: { category: { value: "newsletter", setAt: 20 } },
+				}),
+				eventFor(SUB),
+			),
+			(error: Error) => {
+				assert.match(error.message, /saved/);
+				assert.match(error.message, /same category again/);
+				return true;
+			},
+		);
+	});
+
+	it("enqueues nothing when the override is cleared back to auto", async () => {
+		const enqueued: SendMessageCommand[] = [];
+		withStubbedQueue(enqueued);
+		setClient(
+			clientHolding(
+				address({ flags: { category: { value: "newsletter", setAt: 10 } } }),
+				[],
+			),
+		);
+
+		const response = await updateAddress(
+			updateContextFor({ clearFlags: ["category"] }),
+			eventFor(SUB),
+		);
+
+		assert.equal(response.flags.category, undefined);
+		assert.deepEqual(enqueued, []);
+	});
+
+	it("enqueues nothing for a flag that has no bearing on classification", async () => {
+		const enqueued: SendMessageCommand[] = [];
+		withStubbedQueue(enqueued);
+		setClient(clientHolding(address({ flags: {} }), []));
+
+		await updateAddress(
+			updateContextFor({ flags: { muted: { value: true, setAt: 20 } } }),
+			eventFor(SUB),
+		);
+
+		assert.deepEqual(enqueued, []);
+	});
+});

package/src/handlers/address.ts

@@ -1,14 +1,21 @@
+import { SendMessageCommand } from "@aws-sdk/client-sqs";
 import type {
 	AddressResponse,
 	UpdateAddressInput,
 } from "@remit/api-openapi-types";
-import type { AddressItem, FlagsMergePatch } from "@remit/data-ports";
+import type {
+	AddressFlags,
+	AddressItem,
+	FlagsMergePatch,
+} from "@remit/data-ports";
 import { ForbiddenError } from "@remit/data-ports/errors";
 import { AddressFlagKey } from "@remit/domain-enums";
 import type { APIGatewayProxyEvent } from "aws-lambda";
+import { env } from "expect-env";
 import type { Context } from "openapi-backend";
 import { getAccountConfigIdFromEvent } from "../auth.js";
 import { getClient } from "../service/data-client.js";
+import { sqsClient } from "../service/sqs.js";
 import type {
 	AddressDetailOperationIds,
 	AddressOperationIds,
@@ -73,6 +80,29 @@ export const buildFlagsPatch = (
 	return patch as FlagsMergePatch;
 };
 
+/**
+ * The category flag a back-apply should be enqueued for, or `undefined` when
+ * this patch asks for none (issue #415).
+ *
+ * A SET fires one; a CLEAR — `null` in the patch, whether it came from
+ * `clearFlags` or from the nullable flag value — never does. Reverting a sender
+ * to auto-classification says nothing about what the classifier would have
+ * answered on mail already filed, and re-deriving that would need every
+ * message's body back; the revert takes effect on the sender's next message,
+ * exactly as the override itself did before this.
+ *
+ * The whole flag, not just its value: `setAt` identifies WHICH set the job was
+ * fired for, and the worker refuses to apply a job whose set is no longer the
+ * one standing on the Address.
+ */
+export const backApplyCategoryFlag = (
+	patch: FlagsMergePatch,
+): NonNullable<AddressFlags["category"]> | undefined => {
+	const flag = patch.category;
+	if (flag === null || flag === undefined) return undefined;
+	return flag;
+};
+
 export const AddressOperations: Record<
 	AddressOperationIds,
 	OperationHandler<AddressOperationIds>
@@ -137,6 +167,34 @@ export const AddressDetailOperations: Record<
 			addressId,
 			patch,
 		);
+
+		const flag = backApplyCategoryFlag(patch);
+		if (flag) {
+			// The flag itself is already durable. A queue failure here costs the
+			// user the retroactive pass and nothing else, so the 500 says so and
+			// names the retry — re-sending the same category enqueues a fresh job.
+			await sqsClient
+				.send(
+					new SendMessageCommand({
+						QueueUrl: env.SQS_QUEUE_URL_ACCOUNT_FANOUT,
+						MessageBody: JSON.stringify({
+							type: "SenderCategoryBackApply",
+							accountConfigId,
+							addressId,
+							normalizedEmail: updated.normalizedEmail,
+							category: flag.value,
+							categorySetAt: flag.setAt,
+						}),
+					}),
+				)
+				.catch((cause: unknown) => {
+					throw new Error(
+						`The ${flag.value} override is saved and applies to this sender's next message, but the pass over their existing mail could not be started. Setting the same category again retries it.`,
+						{ cause },
+					);
+				});
+		}
+
 		return toAddressResponse(updated);
 	},
 };

package/src/service/sender-category.test.ts

@@ -0,0 +1,235 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	ThreadMessageFieldTerm,
+	ThreadMessageItem,
+	UpdateMessageInput,
+	UpdateThreadMessageInput,
+} from "@remit/data-ports";
+import {
+	backApplySenderCategory,
+	SENDER_CATEGORY_BACKAPPLY_LIMIT,
+	type SenderCategoryBackApplyDeps,
+} from "./sender-category.js";
+
+const ACCOUNT_CONFIG_ID = "cfg-415";
+const SENDER = "post@ns.nl";
+
+const row = (n: number, over: Partial<ThreadMessageItem> = {}) =>
+	({
+		threadMessageId: `tm-${n}`,
+		threadId: `th-${n}`,
+		messageId: `msg-${n}`,
+		accountConfigId: ACCOUNT_CONFIG_ID,
+		mailboxId: "mbx-inbox",
+		fromEmail: SENDER,
+		fromName: "NS Reizigers",
+		subject: `reisoverzicht ${n}`,
+		category: "personal",
+		sentDate: 2_000 - n,
+		isRead: false,
+		isDeleted: false,
+		hasStars: false,
+		hasAttachment: false,
+		...over,
+	}) as unknown as ThreadMessageItem;
+
+interface Writes {
+	messages: Array<{ messageId: string; input: UpdateMessageInput }>;
+	threadRows: Array<{
+		threadMessageId: string;
+		input: UpdateThreadMessageInput;
+	}>;
+	terms: ThreadMessageFieldTerm[][];
+	pageSizes: Array<number | undefined>;
+}
+
+/**
+ * A store that pages its rows the way the real listing does — newest first,
+ * `limit` rows per page, a continuation token while more remain — and that
+ * answers `findAllByMessageId` from the same rows, so a re-label is provably
+ * written to both halves of the pair.
+ */
+const storeOf = (
+	rows: ThreadMessageItem[],
+	writes: Writes,
+): SenderCategoryBackApplyDeps => ({
+	client: {
+		message: {
+			update: async (messageId: string, input: UpdateMessageInput) => {
+				writes.messages.push({ messageId, input });
+				return {} as never;
+			},
+		},
+		threadMessage: {
+			listByFieldTerms: async (
+				_accountConfigId: string,
+				terms: readonly ThreadMessageFieldTerm[],
+				options?: { limit?: number; continuationToken?: string },
+			) => {
+				writes.terms.push([...terms]);
+				writes.pageSizes.push(options?.limit);
+				const from = Number(options?.continuationToken ?? 0);
+				const size = options?.limit ?? rows.length;
+				const page = rows.slice(from, from + size);
+				const next = from + page.length;
+				return {
+					items: page,
+					continuationToken: next < rows.length ? String(next) : undefined,
+				};
+			},
+			findAllByMessageId: async (_accountConfigId: string, messageId: string) =>
+				rows.filter((r) => r.messageId === messageId),
+			update: async (
+				_accountConfigId: string,
+				threadMessageId: string,
+				input: UpdateThreadMessageInput,
+			) => {
+				writes.threadRows.push({ threadMessageId, input });
+				return {} as never;
+			},
+		},
+	} as unknown as SenderCategoryBackApplyDeps["client"],
+});
+
+const emptyWrites = (): Writes => ({
+	messages: [],
+	threadRows: [],
+	terms: [],
+	pageSizes: [],
+});
+
+describe("backApplySenderCategory (#415)", () => {
+	it("re-labels mail the classifier already decided, on both halves of the pair", async () => {
+		const writes = emptyWrites();
+		const rows = [row(1), row(2), row(3)];
+
+		const result = await backApplySenderCategory(
+			storeOf(rows, writes),
+			ACCOUNT_CONFIG_ID,
+			SENDER,
+			"newsletter",
+		);
+
+		assert.deepEqual(result, { matched: 3, applied: 3, failed: 0 });
+		assert.deepEqual(
+			writes.messages.map((w) => w.messageId),
+			["msg-1", "msg-2", "msg-3"],
+		);
+		assert.deepEqual(
+			writes.messages.map((w) => w.input.category),
+			["newsletter", "newsletter", "newsletter"],
+		);
+		assert.deepEqual(
+			writes.threadRows.map((w) => w.threadMessageId),
+			["tm-1", "tm-2", "tm-3"],
+		);
+		assert.deepEqual(
+			writes.threadRows.map((w) => w.input.category),
+			["newsletter", "newsletter", "newsletter"],
+		);
+	});
+
+	it("stops at the bound, keeping the newest of the sender's mail", async () => {
+		const writes = emptyWrites();
+		const rows = Array.from(
+			{ length: SENDER_CATEGORY_BACKAPPLY_LIMIT + 25 },
+			(_, i) => row(i + 1),
+		);
+
+		const result = await backApplySenderCategory(
+			storeOf(rows, writes),
+			ACCOUNT_CONFIG_ID,
+			SENDER,
+			"newsletter",
+		);
+
+		assert.equal(result.matched, SENDER_CATEGORY_BACKAPPLY_LIMIT);
+		assert.equal(result.applied, SENDER_CATEGORY_BACKAPPLY_LIMIT);
+		assert.equal(writes.messages.length, SENDER_CATEGORY_BACKAPPLY_LIMIT);
+		assert.equal(writes.messages[0]?.messageId, "msg-1");
+		assert.equal(
+			writes.messages.at(-1)?.messageId,
+			`msg-${SENDER_CATEGORY_BACKAPPLY_LIMIT}`,
+		);
+	});
+
+	it("refines the store's substring narrowing to an exact sender match", async () => {
+		const writes = emptyWrites();
+		const rows = [
+			row(1),
+			row(2, { messageId: "msg-lookalike", fromEmail: "no-post@ns.nl" }),
+			row(3, { messageId: "msg-name-only", fromEmail: "spoof@example.com" }),
+		];
+
+		const result = await backApplySenderCategory(
+			storeOf(rows, writes),
+			ACCOUNT_CONFIG_ID,
+			SENDER,
+			"newsletter",
+		);
+
+		assert.equal(result.matched, 1);
+		assert.deepEqual(
+			writes.messages.map((w) => w.messageId),
+			["msg-1"],
+		);
+		assert.deepEqual(writes.terms[0], [{ field: "sender", contains: SENDER }]);
+	});
+
+	it("pages the narrowed listing until the bound fills", async () => {
+		const writes = emptyWrites();
+		const rows = Array.from({ length: 12 }, (_, i) =>
+			i % 2 === 0
+				? row(i + 1)
+				: row(i + 1, {
+						messageId: `msg-other-${i}`,
+						fromEmail: "someone@example.com",
+					}),
+		);
+
+		const result = await backApplySenderCategory(
+			storeOf(rows, writes),
+			ACCOUNT_CONFIG_ID,
+			SENDER,
+			"newsletter",
+			4,
+		);
+
+		assert.equal(result.matched, 4);
+		assert.ok(
+			writes.terms.length > 1,
+			"a page of narrowed rows that yields fewer matches than the bound must be followed by the next page",
+		);
+		assert.deepEqual(writes.pageSizes, Array(writes.terms.length).fill(4));
+	});
+
+	it("counts a poisoned message as failed and re-labels the rest", async () => {
+		const writes = emptyWrites();
+		const rows = [row(1), row(2), row(3)];
+		const store = storeOf(rows, writes);
+		const threadMessage = store.client.threadMessage as unknown as Record<
+			string,
+			unknown
+		>;
+		const findAll = threadMessage.findAllByMessageId as (
+			a: string,
+			b: string,
+		) => Promise<ThreadMessageItem[]>;
+		threadMessage.findAllByMessageId = async (a: string, b: string) =>
+			b === "msg-2" ? [] : findAll(a, b);
+
+		const result = await backApplySenderCategory(
+			store,
+			ACCOUNT_CONFIG_ID,
+			SENDER,
+			"newsletter",
+		);
+
+		assert.deepEqual(result, { matched: 3, applied: 2, failed: 1 });
+		assert.deepEqual(
+			writes.messages.map((w) => w.messageId),
+			["msg-1", "msg-3"],
+		);
+	});
+});

package/src/service/sender-category.ts

@@ -0,0 +1,135 @@
+import type { MessageCategoryValue } from "@remit/mailbox-service";
+import { denormalizeMessageCategory } from "@remit/mailbox-service";
+import type { RemitClient } from "./data-client.js";
+
+/**
+ * How much of a sender's already-classified mail one override reaches: the
+ * newest 500 of their messages held in the local store. A back-apply is a
+ * bounded recent batch, never the sender's whole history — the store-side
+ * narrowing pages until this many MATCHES, so a quiet sender's mail is reached
+ * however old it is, and a bulk sender's oldest mail is deliberately left as
+ * the classifier decided it. The same constant the organize back-apply bounds
+ * its match set with (`ORGANIZE_MATCH_LIMIT`), for the same reason.
+ */
+export const SENDER_CATEGORY_BACKAPPLY_LIMIT = 500;
+
+export interface SenderCategoryBackApplyDeps {
+	client: Pick<RemitClient, "message" | "threadMessage">;
+}
+
+export interface SenderCategoryBackApplyResult {
+	matched: number;
+	applied: number;
+	failed: number;
+}
+
+/**
+ * The sender's newest messages in the local store, bounded at `limit`, newest
+ * first and deduplicated by messageId.
+ *
+ * The store evaluates `sender` as an accent- and case-insensitive SUBSTRING
+ * over the From address and display name together, so it is a narrowing and
+ * not the verdict: every page is refined here to an exact `fromEmail` match
+ * before it counts. Paging continues until the cap fills or the store runs
+ * out, which is what makes the bound a bound on the sender's messages rather
+ * than on how far back the read reached (#459).
+ */
+const listSenderMessageIds = async (
+	deps: SenderCategoryBackApplyDeps,
+	accountConfigId: string,
+	normalizedEmail: string,
+	limit: number,
+): Promise<string[]> => {
+	const wanted = normalizedEmail.toLowerCase();
+	const messageIds: string[] = [];
+	const seen = new Set<string>();
+	let continuationToken: string | undefined;
+
+	do {
+		const page = await deps.client.threadMessage.listByFieldTerms(
+			accountConfigId,
+			[{ field: "sender", contains: wanted }],
+			{ limit, continuationToken, excludeDeleted: true },
+		);
+		for (const row of page.items) {
+			if (row.fromEmail?.toLowerCase() !== wanted) continue;
+			if (seen.has(row.messageId)) continue;
+			seen.add(row.messageId);
+			messageIds.push(row.messageId);
+			if (messageIds.length >= limit) return messageIds;
+		}
+		continuationToken = page.continuationToken;
+	} while (continuationToken);
+
+	return messageIds;
+};
+
+/**
+ * Back-apply a sender's `Address.flags.category` override to mail that was
+ * already classified before the override was set (#415). Issue #299 gave the
+ * override full effect on the sender's NEXT message only; this is the
+ * retroactive half, deliberately bounded to
+ * {@link SENDER_CATEGORY_BACKAPPLY_LIMIT}.
+ *
+ * This is the one path that intentionally rewrites an already-decided
+ * `Message.category`, so it does not — and must not — reuse body sync's
+ * `hasDecidedCategory` write-once guard. Its consistency story is that
+ * `category` is a key attribute of no provisioned index: the DynamoDB LSI that
+ * would have sorted on it was never provisioned and cannot be (#516), and the
+ * filtered-mailbox read path is served by the SQL-only
+ * `tm_by_mailbox_category_date` index, where an update simply moves the row.
+ * What a rewrite can still break is the Message/ThreadMessage pair disagreeing
+ * — the badge is served from the thread row and so is the category filter — so
+ * both are written per message, thread rows first, and a redelivered job
+ * re-runs the same idempotent writes.
+ *
+ * Nothing here touches placement: `category` is not an input to
+ * `classifyPlacement`, so an override re-labels mail where it already sits and
+ * files nothing. No remote mail-server state changes, which is why this is not
+ * an IMAP mutation and does not take the mutator pattern.
+ *
+ * Per-message isolation, exactly as `applyOrganize` has it: one poisoned
+ * message is counted as failed and the pass continues, so a single missing
+ * thread row never costs the other 499 their re-label. Unlike a job row, this
+ * count is not the end of the story — the caller is expected to leave a pass
+ * with failures unacknowledged so redelivery converges the rest.
+ */
+export const backApplySenderCategory = async (
+	deps: SenderCategoryBackApplyDeps,
+	accountConfigId: string,
+	normalizedEmail: string,
+	category: MessageCategoryValue,
+	limit: number = SENDER_CATEGORY_BACKAPPLY_LIMIT,
+): Promise<SenderCategoryBackApplyResult> => {
+	const messageIds = await listSenderMessageIds(
+		deps,
+		accountConfigId,
+		normalizedEmail,
+		limit,
+	);
+
+	const relabel = async (messageId: string): Promise<void> => {
+		await denormalizeMessageCategory(
+			{ threadMessageService: deps.client.threadMessage },
+			accountConfigId,
+			messageId,
+			{ category },
+		);
+		await deps.client.message.update(messageId, { category });
+	};
+
+	let applied = 0;
+	let failed = 0;
+	for (const messageId of messageIds) {
+		const ok = await relabel(messageId)
+			.then(() => true)
+			.catch(() => false);
+		if (ok) {
+			applied += 1;
+		} else {
+			failed += 1;
+		}
+	}
+
+	return { matched: messageIds.length, applied, failed };
+};