@remit/mailbox-service 0.0.102 → 0.0.104

package/src/classification-backfill.ts

@@ -1,5 +1,6 @@
 import type {
 	IAccountConfigRepository,
+	IAccountRepository,
 	IAddressRepository,
 	IMessageRepository,
 	IThreadMessageRepository,
@@ -11,6 +12,11 @@ import { parseMessageBody } from "./body-parse.js";
 import { extractSnippet } from "./body-sync.js";
 import { classifyParsedMessage } from "./classify-message.js";
 import { extractListId } from "./filters/list-id.js";
+import {
+	type FolderPlacementLookup,
+	resolveAccountFolderPlacements,
+} from "./folder-placement.js";
+import type { FolderPlacement } from "./heuristics/classifyPlacement.js";
 import {
 	denormalizeMessageCategory,
 	hasDecidedCategory,
@@ -44,6 +50,8 @@ export interface ClassificationBackfillCheckpointStore {
 
 export interface ClassificationBackfillDeps {
 	accountConfigService: Pick<IAccountConfigRepository, "listAll">;
+	accountService: Pick<IAccountRepository, "listAllByAccountConfig">;
+	mailboxSpecialUseService: FolderPlacementLookup;
 	/** Reads the sender's `Address.flags.category` override (issue #299). */
 	addressService: Pick<IAddressRepository, "getAddress">;
 	threadMessageService: Pick<
@@ -116,6 +124,7 @@ const classifyAndApply = async (
 	accountConfigId: string,
 	row: ThreadMessageItem,
 	bodyStorageKey: string,
+	placement: FolderPlacement,
 ): Promise<void> => {
 	const body = await deps.storageService.retrieve(bodyStorageKey);
 	const parsed = await parseMessageBody(body);
@@ -123,6 +132,7 @@ const classifyAndApply = async (
 		deps.addressService,
 		accountConfigId,
 		parsed,
+		placement,
 	);
 
 	// The same denormalized fields the body-store path writes (see
@@ -223,6 +233,13 @@ export const backfillClassifications = async (
 		const account = accounts[accountIndex];
 		let continuationToken: string | undefined =
 			accountIndex === startIndex ? startContinuationToken : undefined;
+		const mailAccounts = await deps.accountService.listAllByAccountConfig(
+			account.accountConfigId,
+		);
+		const placements = await resolveAccountFolderPlacements(
+			deps.mailboxSpecialUseService,
+			mailAccounts.map((mailAccount) => mailAccount.accountId),
+		);
 
 		do {
 			const page = await deps.threadMessageService.listByAccount(
@@ -297,6 +314,7 @@ export const backfillClassifications = async (
 						account.accountConfigId,
 						row,
 						message.bodyStorageKey,
+						placements.get(row.mailboxId) ?? "other",
 					).then(
 						() => null,
 						(error: unknown) => error,

package/src/classify-message.test.ts

@@ -13,6 +13,7 @@ import assert from "node:assert/strict";
 import { describe, it } from "node:test";
 import type { AddressItem, IAddressRepository } from "@remit/data-ports";
 import { NotFoundError } from "@remit/data-ports/errors";
+import { DisplayNameCorrespondence } from "@remit/domain-enums";
 import { parseMessageBody } from "./body-parse.js";
 import {
 	classifyParsedMessage,
@@ -43,6 +44,18 @@ const LIST_EML = Buffer.from(
 	].join("\r\n"),
 );
 
+const IMPERSONATION_EML = Buffer.from(
+	[
+		"From: InfoMedics <billing@serviceupdatebank.atlassian.net>",
+		"To: me@example.com",
+		"Subject: Openstaande factuur",
+		"DKIM-Signature: v=1; a=rsa-sha256; d=atlassian.net; s=sel; b=xxx",
+		"Content-Type: text/html",
+		"",
+		'<a href="https://betaal-vordering.example.org/pay">Betaal uw factuur</a>',
+	].join("\r\n"),
+);
+
 const addressService = (
 	behavior: (accountConfigId: string, addressId: string) => unknown,
 ): Pick<IAddressRepository, "getAddress"> => ({
@@ -68,6 +81,7 @@ describe("classifyParsedMessage", () => {
 			}),
 			"acc-1",
 			parsed,
+			"inbox",
 		);
 
 		assert.equal(classification.category, "newsletter");
@@ -81,6 +95,7 @@ describe("classifyParsedMessage", () => {
 			addressService(withFlags({ category: { value: "marketing", setAt: 1 } })),
 			"acc-1",
 			parsed,
+			"inbox",
 		);
 
 		// The header table alone would answer `personal`; the override wins
@@ -98,6 +113,7 @@ describe("classifyParsedMessage", () => {
 				}),
 				"acc-1",
 				parsed,
+				"inbox",
 			),
 			/throttled/,
 		);
@@ -124,6 +140,7 @@ describe("classifyParsedMessage", () => {
 			}),
 			"acc-1",
 			parsed,
+			"inbox",
 		);
 
 		assert.equal(reads, 0);
@@ -131,6 +148,48 @@ describe("classifyParsedMessage", () => {
 	});
 });
 
+describe("classifyParsedMessage sender mismatch", () => {
+	const noAddress = addressService(async () => {
+		throw new NotFoundError("no Address row");
+	});
+
+	it("compares the sender of mail in Junk the provider rated clean", async () => {
+		const parsed = await parseMessageBody(IMPERSONATION_EML);
+
+		const classification = await classifyParsedMessage(
+			noAddress,
+			"acc-1",
+			parsed,
+			"junk",
+		);
+
+		assert.equal(
+			classification.authenticity?.displayNameCorrespondence,
+			DisplayNameCorrespondence.Unrelated,
+		);
+		assert.deepEqual(classification.authenticity?.offDomainLinkDomains, [
+			"example.org",
+		]);
+	});
+
+	it("compares nothing for the same mail outside Junk without a provider verdict", async () => {
+		const parsed = await parseMessageBody(IMPERSONATION_EML);
+
+		const classification = await classifyParsedMessage(
+			noAddress,
+			"acc-1",
+			parsed,
+			"inbox",
+		);
+
+		assert.equal(
+			classification.authenticity?.displayNameCorrespondence,
+			undefined,
+		);
+		assert.equal(classification.authenticity?.offDomainLinkDomains, undefined);
+	});
+});
+
 describe("extractPrimaryFromEmail", () => {
 	it("reads the first From address", async () => {
 		const parsed = await parseMessageBody(PLAIN_EML);

package/src/classify-message.ts

@@ -13,6 +13,7 @@ import {
 	extractHasListUnsubscribe,
 	extractProviderSpam,
 } from "./heuristics/classifyByHeaders.js";
+import type { FolderPlacement } from "./heuristics/classifyPlacement.js";
 import { extractSenderMismatch } from "./heuristics/senderMismatch.js";
 
 type ThreadMessageCategory = ThreadMessageItem["category"];
@@ -75,6 +76,7 @@ export const classifyParsedMessage = async (
 	addressService: Pick<IAddressRepository, "getAddress">,
 	accountConfigId: string,
 	parsed: ParsedMail,
+	placement: FolderPlacement,
 ): Promise<UpdateMessageInput & { category: ThreadMessageCategory }> => {
 	const headerCategory = classifyByHeaders(parsed);
 	const authenticity = extractAuthenticity(parsed);
@@ -86,13 +88,14 @@ export const classifyParsedMessage = async (
 	// identity the message claims — on a shared-tenant host the verified
 	// subdomain belongs to whoever signed up. These two comparisons say
 	// whether the claim holds, and run only over mail the provider already
-	// called spam.
+	// called spam or that sits in the Junk folder.
 	const senderMismatch =
 		authenticity === null
 			? {}
 			: extractSenderMismatch(parsed, {
 					fromDomain: authenticity.fromDomain,
-					spamClassified: providerSpam?.classified === true,
+					providerSpamClassified: providerSpam?.classified === true,
+					placement,
 					bulkSender: hasListUnsubscribe,
 				});
 

package/src/folder-placement.ts

@@ -0,0 +1,56 @@
+import type { IMailboxSpecialUseRepository } from "@remit/data-ports";
+import type { FolderPlacement } from "./heuristics/classifyPlacement.js";
+
+export type FolderPlacementLookup = Pick<
+	IMailboxSpecialUseRepository,
+	"findJunkMailbox" | "findInboxMailbox"
+>;
+
+type RoleMailbox = Awaited<
+	ReturnType<IMailboxSpecialUseRepository["findJunkMailbox"]>
+>;
+
+export interface FolderRoleMailboxes {
+	junkMailbox: RoleMailbox;
+	inboxMailbox: RoleMailbox;
+}
+
+export const resolveFolderRoleMailboxes = async (
+	lookup: FolderPlacementLookup,
+	accountId: string,
+): Promise<FolderRoleMailboxes> => ({
+	junkMailbox: await lookup.findJunkMailbox(accountId),
+	inboxMailbox: await lookup.findInboxMailbox(accountId),
+});
+
+export const placementOf = (
+	mailboxId: string,
+	{ junkMailbox, inboxMailbox }: FolderRoleMailboxes,
+): FolderPlacement => {
+	if (junkMailbox?.mailboxId === mailboxId) return "junk";
+	if (inboxMailbox?.mailboxId === mailboxId) return "inbox";
+	return "other";
+};
+
+export const resolveFolderPlacement = async (
+	lookup: FolderPlacementLookup,
+	accountId: string,
+	mailboxId: string,
+): Promise<FolderPlacement> =>
+	placementOf(mailboxId, await resolveFolderRoleMailboxes(lookup, accountId));
+
+export const resolveAccountFolderPlacements = async (
+	lookup: FolderPlacementLookup,
+	accountIds: readonly string[],
+): Promise<ReadonlyMap<string, FolderPlacement>> => {
+	const placements = new Map<string, FolderPlacement>();
+	for (const accountId of accountIds) {
+		const { junkMailbox, inboxMailbox } = await resolveFolderRoleMailboxes(
+			lookup,
+			accountId,
+		);
+		if (inboxMailbox) placements.set(inboxMailbox.mailboxId, "inbox");
+		if (junkMailbox) placements.set(junkMailbox.mailboxId, "junk");
+	}
+	return placements;
+};

package/src/heuristics/senderMismatch.test.ts

@@ -245,7 +245,8 @@ describe("extractSenderMismatch", () => {
 		assert.deepEqual(
 			extractSenderMismatch(parsed, {
 				fromDomain: "serviceupdatebank.atlassian.net",
-				spamClassified: true,
+				providerSpamClassified: true,
+				placement: "other",
 				bulkSender: false,
 			}),
 			{
@@ -255,18 +256,53 @@ describe("extractSenderMismatch", () => {
 		);
 	});
 
-	it("compares nothing when the provider did not call the message spam", async () => {
+	it("flags a message sitting in Junk that the provider rated clean", async () => {
 		const parsed = await parse(infoMedicsPhish);
 		assert.deepEqual(
 			extractSenderMismatch(parsed, {
 				fromDomain: "serviceupdatebank.atlassian.net",
-				spamClassified: false,
+				providerSpamClassified: false,
+				placement: "junk",
 				bulkSender: false,
 			}),
-			{},
+			{
+				displayNameCorrespondence: DisplayNameCorrespondence.Unrelated,
+				offDomainLinkDomains: ["example.org"],
+			},
 		);
 	});
 
+	it("flags a provider-classified message outside Junk", async () => {
+		const parsed = await parse(infoMedicsPhish);
+		assert.deepEqual(
+			extractSenderMismatch(parsed, {
+				fromDomain: "serviceupdatebank.atlassian.net",
+				providerSpamClassified: true,
+				placement: "inbox",
+				bulkSender: false,
+			}),
+			{
+				displayNameCorrespondence: DisplayNameCorrespondence.Unrelated,
+				offDomainLinkDomains: ["example.org"],
+			},
+		);
+	});
+
+	for (const placement of ["inbox", "other"] as const) {
+		it(`compares nothing in ${placement} when the provider did not call the message spam`, async () => {
+			const parsed = await parse(infoMedicsPhish);
+			assert.deepEqual(
+				extractSenderMismatch(parsed, {
+					fromDomain: "serviceupdatebank.atlassian.net",
+					providerSpamClassified: false,
+					placement,
+					bulkSender: false,
+				}),
+				{},
+			);
+		});
+	}
+
 	it("flags a display name spelling the recipient's own address over a foreign domain", async () => {
 		const parsed = await parse([
 			"From: matthijs@ischen.nl <aramirez@secresaludguaviare.gov.co>",
@@ -279,7 +315,8 @@ describe("extractSenderMismatch", () => {
 		assert.equal(
 			extractSenderMismatch(parsed, {
 				fromDomain: "secresaludguaviare.gov.co",
-				spamClassified: true,
+				providerSpamClassified: true,
+				placement: "other",
 				bulkSender: false,
 			}).displayNameCorrespondence,
 			DisplayNameCorrespondence.ForeignAddress,
@@ -299,7 +336,8 @@ describe("extractSenderMismatch", () => {
 		]);
 		const signals = extractSenderMismatch(parsed, {
 			fromDomain: "mailer.esp.example",
-			spamClassified: true,
+			providerSpamClassified: true,
+			placement: "other",
 			bulkSender: true,
 		});
 		assert.equal(signals.displayNameCorrespondence, undefined);

package/src/heuristics/senderMismatch.ts

@@ -1,6 +1,7 @@
 import { DisplayNameCorrespondence } from "@remit/domain-enums";
 import type { ParsedMail } from "mailparser";
 import { getDomain, parse as parseHost } from "tldts";
+import type { FolderPlacement } from "./classifyPlacement.js";
 
 type CorrespondenceValue =
 	(typeof DisplayNameCorrespondence)[keyof typeof DisplayNameCorrespondence];
@@ -10,7 +11,8 @@ type CorrespondenceValue =
  * that belongs to nobody at the sending domain, and body links that leave it.
  *
  * Both are deliberately aggressive, and both are computed only for mail the
- * provider's own filter already called spam (see {@link extractSenderMismatch}).
+ * provider's own filter already called spam or that sits in the Junk folder
+ * (see {@link extractSenderMismatch}).
  * On a shared-tenant host — a free Atlassian, Salesforce or Zendesk instance —
  * the verified subdomain is chosen by whoever signed up, so a passing signature
  * proves the domain and nothing about the identity the message claims.
@@ -22,7 +24,8 @@ export interface SenderMismatchSignals {
 
 export interface SenderMismatchContext {
 	fromDomain: string;
-	spamClassified: boolean;
+	providerSpamClassified: boolean;
+	placement: FolderPlacement;
 	/**
 	 * List-Unsubscribe present. Bulk mail routinely shows a brand name over an
 	 * ESP's sending domain, so the display-name comparison says nothing there and
@@ -223,7 +226,8 @@ export const extractOffDomainLinkDomains = (
 };
 
 /**
- * Both signals, gated on the provider's own spam verdict.
+ * Both signals, gated on a spam verdict: the provider's own, or the message
+ * sitting in the Junk folder.
  *
  * The gate is what lets the checks be this aggressive: ordinary mail never
  * reaches them, so a brand name over an ESP domain or a newsletter full of
@@ -234,7 +238,9 @@ export const extractSenderMismatch = (
 	parsed: ParsedMail,
 	context: SenderMismatchContext,
 ): SenderMismatchSignals => {
-	if (!context.spamClassified) return {};
+	if (!context.providerSpamClassified && context.placement !== "junk") {
+		return {};
+	}
 
 	const offDomainLinkDomains = extractOffDomainLinkDomains(
 		parsed,

package/src/index.ts

@@ -25,6 +25,7 @@ export {
 	parseMessageBody,
 } from "./body-parse.js";
 export {
+	type BodySyncFolder,
 	type BodySyncLogger,
 	BodySyncService,
 	type ConnectionGetter,
@@ -125,6 +126,11 @@ export {
 	type UpdateFlagsInput,
 	type UpdateFlagsResult,
 } from "./flag-queue.js";
+export {
+	type FolderPlacementLookup,
+	resolveAccountFolderPlacements,
+	resolveFolderPlacement,
+} from "./folder-placement.js";
 export {
 	classifyByHeaders,
 	extractAuthenticity,

package/src/outbox-attachment-content.test.ts

@@ -0,0 +1,145 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type { OutboxAttachmentItem } from "@remit/data-ports";
+import { createMockStorageService } from "@remit/storage-service";
+import {
+	loadOutboxAttachmentContents,
+	OutboxAttachmentUnavailableError,
+} from "./outbox-attachment-content.js";
+
+const OWNER = {
+	accountConfigId: "cfg-1",
+	accountId: "acc-1",
+	outboxMessageId: "ob-1",
+};
+const NOW = 1_000;
+
+const row = (
+	outboxAttachmentId: string,
+	overrides: Partial<OutboxAttachmentItem> = {},
+): OutboxAttachmentItem => ({
+	outboxAttachmentId,
+	outboxMessageId: OWNER.outboxMessageId,
+	accountId: OWNER.accountId,
+	accountConfigId: OWNER.accountConfigId,
+	filename: `${outboxAttachmentId}.txt`,
+	contentType: "text/plain",
+	sizeBytes: 5,
+	state: "Stored",
+	storageKey: `key/${outboxAttachmentId}`,
+	reservationExpiresAt: 0,
+	createdAt: 0,
+	updatedAt: 0,
+	...overrides,
+});
+
+const setup = async (rows: OutboxAttachmentItem[], stored: string[]) => {
+	const storage = createMockStorageService();
+	for (const outboxAttachmentId of stored) {
+		await storage.storeOutboxAttachment({
+			...OWNER,
+			outboxAttachmentId,
+			content: Buffer.from(`bytes of ${outboxAttachmentId}`),
+		});
+	}
+	return {
+		attachments: { listByOutboxMessage: async () => rows },
+		storage,
+	};
+};
+
+describe("loadOutboxAttachmentContents", () => {
+	it("returns every stored file with its bytes, in the order it was attached", async () => {
+		const deps = await setup(
+			[row("second", { createdAt: 20 }), row("first", { createdAt: 10 })],
+			["first", "second"],
+		);
+
+		const contents = await loadOutboxAttachmentContents(deps, OWNER, NOW);
+
+		assert.deepEqual(
+			contents.map((item) => [
+				item.filename,
+				item.contentType,
+				item.content.toString(),
+			]),
+			[
+				["first.txt", "text/plain", "bytes of first"],
+				["second.txt", "text/plain", "bytes of second"],
+			],
+		);
+	});
+
+	it("leaves out a reservation that lapsed without its bytes", async () => {
+		const deps = await setup(
+			[
+				row("kept"),
+				row("lapsed", { state: "Pending", reservationExpiresAt: NOW - 1 }),
+			],
+			["kept"],
+		);
+
+		const contents = await loadOutboxAttachmentContents(deps, OWNER, NOW);
+
+		assert.deepEqual(
+			contents.map((item) => item.filename),
+			["kept.txt"],
+		);
+	});
+
+	it("refuses to build a message while an upload is still open", async () => {
+		const deps = await setup(
+			[row("open", { state: "Pending", reservationExpiresAt: NOW + 60 })],
+			[],
+		);
+
+		await assert.rejects(
+			() => loadOutboxAttachmentContents(deps, OWNER, NOW),
+			(error: unknown) => {
+				assert.ok(error instanceof OutboxAttachmentUnavailableError);
+				assert.equal(error.filename, "open.txt");
+				assert.match(error.message, /never finished uploading/);
+				return true;
+			},
+		);
+	});
+
+	it("refuses to build a message whose stored file has no bytes behind it", async () => {
+		const deps = await setup([row("gone")], []);
+
+		await assert.rejects(
+			() => loadOutboxAttachmentContents(deps, OWNER, NOW),
+			(error: unknown) => {
+				assert.ok(error instanceof OutboxAttachmentUnavailableError);
+				assert.equal(
+					error.message,
+					'"gone.txt" could not be read: nothing is stored at key/gone',
+				);
+				return true;
+			},
+		);
+	});
+
+	it("names the file when storage itself fails", async () => {
+		const deps = {
+			attachments: { listByOutboxMessage: async () => [row("payroll")] },
+			storage: {
+				retrieveOutboxAttachment: async (): Promise<Buffer | null> => {
+					throw new Error("S3 is down");
+				},
+			},
+		};
+
+		await assert.rejects(
+			() => loadOutboxAttachmentContents(deps, OWNER, NOW),
+			(error: unknown) => {
+				assert.ok(error instanceof OutboxAttachmentUnavailableError);
+				assert.equal(
+					error.message,
+					'"payroll.txt" could not be read: storage failed (S3 is down)',
+				);
+				return true;
+			},
+		);
+	});
+});

package/src/outbox-attachment-content.ts

@@ -0,0 +1,99 @@
+import type { IOutboxAttachmentRepository } from "@remit/data-ports";
+import { holdsRoom } from "@remit/data-ports";
+import type { StorageService } from "@remit/storage-service";
+
+export interface OutboxAttachmentContent {
+	filename: string;
+	contentType: string;
+	content: Buffer;
+}
+
+/**
+ * A file on a message that cannot be put into it. Carries the file's name so
+ * whoever settles the message can tell the sender which file to fix.
+ */
+export class OutboxAttachmentUnavailableError extends Error {
+	constructor(
+		readonly filename: string,
+		readonly detail: string,
+		options?: { cause: unknown },
+	) {
+		super(`"${filename}" could not be read: ${detail}`, options);
+		this.name = "OutboxAttachmentUnavailableError";
+	}
+}
+
+const describeCause = (error: unknown): string =>
+	error instanceof Error ? error.message : String(error);
+
+export interface OutboxAttachmentContentDeps {
+	attachments: Pick<IOutboxAttachmentRepository, "listByOutboxMessage">;
+	storage: Pick<StorageService, "retrieveOutboxAttachment">;
+}
+
+export interface OutboxAttachmentOwner {
+	accountConfigId: string;
+	accountId: string;
+	outboxMessageId: string;
+}
+
+/**
+ * Every file a message goes out with, bytes included, in the order they were
+ * attached. The send op refuses a draft with an upload still open, so a live
+ * `Pending` row here, or a `Stored` row with nothing behind it, is a message
+ * that cannot be built as the sender saw it — and sending it without the file
+ * would be worse than not sending it.
+ */
+export const loadOutboxAttachmentContents = async (
+	deps: OutboxAttachmentContentDeps,
+	owner: OutboxAttachmentOwner,
+	nowSeconds: number,
+): Promise<OutboxAttachmentContent[]> => {
+	const held = await deps.attachments.listByOutboxMessage(
+		owner.accountConfigId,
+		owner.outboxMessageId,
+	);
+	const live = held
+		.filter((item) => holdsRoom(item, nowSeconds))
+		.sort((a, b) => a.createdAt - b.createdAt);
+
+	const unfinished = live.find((item) => item.state !== "Stored");
+	if (unfinished) {
+		throw new OutboxAttachmentUnavailableError(
+			unfinished.filename,
+			`attachment ${unfinished.outboxAttachmentId} never finished uploading`,
+		);
+	}
+
+	return Promise.all(
+		live.map(async (item) => {
+			const content = await Promise.resolve()
+				.then(() =>
+					deps.storage.retrieveOutboxAttachment(
+						owner.accountConfigId,
+						owner.accountId,
+						owner.outboxMessageId,
+						item.outboxAttachmentId,
+					),
+				)
+				.catch((error: unknown) => {
+					throw new OutboxAttachmentUnavailableError(
+						item.filename,
+						`storage failed (${describeCause(error)})`,
+						{ cause: error },
+					);
+				});
+			if (content === null) {
+				throw new OutboxAttachmentUnavailableError(
+					item.filename,
+					`nothing is stored at ${item.storageKey}`,
+				);
+			}
+			return {
+				filename: item.filename,
+				contentType: item.contentType,
+				content,
+			};
+		}),
+	);
+};