@remit/mailbox-service 0.0.112 → 0.0.114

package/src/external-move.ts

@@ -1,285 +0,0 @@
-import type {
-	IMailboxRepository,
-	IMessageRepository,
-	MailboxItem,
-	MessageItem,
-} from "@remit/data-ports";
-import { isVirtualCopyMailbox } from "@remit/data-ports/virtual-copy";
-import { MessageStatus } from "@remit/domain-enums";
-import type { IImapConnection } from "./types.js";
-
-export interface ExternalMoveLogger {
-	info(obj: Record<string, unknown>, msg: string): void;
-	warn(obj: Record<string, unknown>, msg: string): void;
-}
-
-/**
- * Whether a sighting of an already-stored message in this mailbox CONTESTS the
- * placement the row holds (#859). Contesting is not deciding: the sighting is a
- * candidate move, and {@link confirmDepartures} is what turns it into one.
- *
- * Three sightings do not even contest. The same mailbox is the row standing
- * where it already is. Gmail's All Mail, Starred and Important hold every
- * message that also lives in a real folder, so they are views rather than filing
- * locations; following one would empty every other folder's listing. And a row
- * whose own mutation has not settled is in flight, not stale
- * (imap-mutations R1): the server's answer predates the move reader is still
- * waiting on, so taking it would undo the optimistic write. The sync path
- * reconciles rather than waits (imap-mutations R2): it never blocks on a
- * mutation, it re-reads the row on the next sighting after the move settles.
- *
- * In flight is `status`, and `status` alone (#1096, imap-mutations R3). Every
- * outbound mutation writes `moving` or `deleting` and only settling returns the
- * row to `active`, so `active` is precisely the set nothing is coming for —
- * whatever `syncStatus` says about how the row got there. A row put back by
- * `abandonDelete` carries `abandoned`, and it needs this repair more than any
- * other row does: nothing routine settles it, and reader shares its mailboxes,
- * so refusing it left a message whose delete was refused unable to follow a
- * move the user then made in another client.
- */
-export const sightingContestsPlacement = (
-	mailbox: MailboxItem,
-	message: Pick<MessageItem, "mailboxId" | "status">,
-): boolean => {
-	if (message.mailboxId === mailbox.mailboxId) return false;
-	if (isVirtualCopyMailbox(mailbox)) return false;
-	return message.status === MessageStatus.active;
-};
-
-/**
- * The exact placement a verdict was reached for. The probe reads the row outside
- * the transaction that acts on it, so the verdict names the mailbox and uid it
- * was reached for; a row that moved in between matches no verdict and is left
- * unsettled.
- */
-export const placementKey = (
-	messageId: string,
-	mailboxId: string,
-	uid: number,
-): string => `${messageId}:${mailboxId}:${uid}`;
-
-export interface DepartureVerdicts {
-	/**
-	 * Placements the source folder no longer holds. A sighting elsewhere of one
-	 * of these is a move, and re-points the row.
-	 */
-	departed: Set<string>;
-	/**
-	 * Placements a verdict was reached for at all, departed or still held. A
-	 * contested sighting outside this set was not decided — the source could not
-	 * be asked, or the row has moved since it was — and its uid is held back from
-	 * the watermark so the next round serves the sighting again. Without that a
-	 * single unanswerable round would leave the row mispointed until a cursor
-	 * rebuild, which is the silent never-repair this whole change is about.
-	 */
-	settled: Set<string>;
-	/**
-	 * Message ids whose sighting has to go through the save pipeline: departed,
-	 * or contested with no verdict. Every other sighting changes no placement.
-	 */
-	needsSave: Set<string>;
-}
-
-export interface DepartureProbeDeps {
-	connection: Pick<IImapConnection, "openBox" | "search">;
-	mailboxService: Pick<IMailboxRepository, "get">;
-	messageService: Pick<IMessageRepository, "get">;
-	log: ExternalMoveLogger;
-}
-
-/**
- * Which of this batch's messages have really left the folder their row points at
- * (#1146).
- *
- * A message sighted in a second real folder has not necessarily moved. Living in
- * two folders at once is ordinary mail: a Gmail user label, a Sieve `fileinto`
- * beside a `keep`, a Sent copy a mailing list echoes back. Reading every such
- * sighting as a move takes the message out of the folder it is still in, and
- * whichever folder the sync round enumerates last wins — which is how a labelled
- * message disappeared from the Inbox on the next sync.
- *
- * What separates a move from a copy is the source folder: a move empties it, a
- * copy leaves it alone. So the source is asked, and only a source that no longer
- * holds the uid the row claims makes the sighting a move. A move another client
- * made is still followed — that is the case where the source really is empty —
- * and reader stops fighting a second client over mail neither of them moved.
- *
- * The question is asked with a UID SEARCH rather than a FETCH, for the reason
- * `isMessageGoneFromOpenMailbox` documents: imapflow drops rows on back-to-back
- * FETCHes (#408), and absence read off a FETCH would file live mail as departed.
- * `IImapConnection.search` throws on a failed SEARCH, so an empty answer means
- * the server matched nothing. The SEARCH names the contested uids rather than
- * ALL: the answer needed is about those uids, and a folder's whole uid set is a
- * page of traffic per batch for an account with a large Inbox.
- *
- * One SEARCH per source folder per batch, and no IMAP at all when nothing is
- * contested — which is every round on an account whose folders do not overlap.
- */
-export const confirmDepartures = async (
-	deps: DepartureProbeDeps,
-	accountId: string,
-	sightedIn: MailboxItem,
-	messageIds: string[],
-): Promise<DepartureVerdicts> => {
-	const verdicts: DepartureVerdicts = {
-		departed: new Set<string>(),
-		settled: new Set<string>(),
-		needsSave: new Set<string>(),
-	};
-	if (messageIds.length === 0) return verdicts;
-
-	const stored = await deps.messageService.get(messageIds);
-	// A row carrying no uid names no placement the source can be asked about, so
-	// it reaches no verdict here and its sighting is served again next round. It
-	// is not reachable from a sync — `upsertWithStatus` writes the uid the FETCH
-	// returned — and a folder that somehow produces one stalls its cursor and
-	// raises the alert rather than repairing a row against nothing.
-	const contested = stored.filter(
-		(row) => row.uid > 0 && sightingContestsPlacement(sightedIn, row),
-	);
-	if (contested.length === 0) return verdicts;
-
-	const bySource = new Map<string, MessageItem[]>();
-	for (const row of contested) {
-		const group = bySource.get(row.mailboxId);
-		if (group === undefined) {
-			bySource.set(row.mailboxId, [row]);
-			continue;
-		}
-		group.push(row);
-	}
-
-	for (const [sourceMailboxId, rows] of bySource) {
-		await askSource(
-			deps,
-			accountId,
-			sightedIn,
-			sourceMailboxId,
-			rows,
-			verdicts,
-		);
-	}
-
-	for (const row of contested) {
-		const key = placementKey(row.messageId, row.mailboxId, row.uid);
-		if (verdicts.departed.has(key) || !verdicts.settled.has(key)) {
-			verdicts.needsSave.add(row.messageId);
-		}
-	}
-
-	return verdicts;
-};
-
-/**
- * What one source folder said about the rows pointed at it. Only `holds` decides
- * anything; the other three are the folder declining to be evidence, each for
- * its own reason and each leaving the sightings to come back next round.
- */
-type SourceAnswer =
-	| { kind: "holds"; path: string; held: Set<number> }
-	| { kind: "copiesEverything"; path: string }
-	| { kind: "rekeyed"; path: string; stored: number; served: number }
-	| { kind: "unreachable"; reason: string };
-
-const readSource = async (
-	deps: DepartureProbeDeps,
-	accountId: string,
-	sourceMailboxId: string,
-	rows: MessageItem[],
-): Promise<SourceAnswer> => {
-	const source = await deps.mailboxService.get(accountId, sourceMailboxId);
-
-	// A row whose source is All Mail, Starred or Important is not filed there:
-	// those folders hold a copy of everything and never release anything, so
-	// asking them would answer "still here" forever and freeze the row on a view.
-	if (isVirtualCopyMailbox(source)) {
-		return { kind: "copiesEverything", path: source.fullPath };
-	}
-
-	const box = await deps.connection.openBox(source.fullPath, true);
-
-	// A source folder on a different UIDVALIDITY axis holds no uid these rows can
-	// be compared against, so its message set is evidence of nothing (RFC 9051
-	// 2.3.1.1). The cursor rebuild re-keys that folder; until it has, the
-	// sightings stay unsettled and come back next round.
-	if (box.uidvalidity !== source.uidValidity) {
-		return {
-			kind: "rekeyed",
-			path: source.fullPath,
-			stored: source.uidValidity,
-			served: box.uidvalidity,
-		};
-	}
-
-	const held = await deps.connection.search([
-		["UID", rows.map((row) => row.uid).join(",")],
-	]);
-	return { kind: "holds", path: source.fullPath, held: new Set(held) };
-};
-
-const askSource = async (
-	deps: DepartureProbeDeps,
-	accountId: string,
-	sightedIn: MailboxItem,
-	sourceMailboxId: string,
-	rows: MessageItem[],
-	verdicts: DepartureVerdicts,
-): Promise<void> => {
-	const settle = (row: MessageItem, departed: boolean): void => {
-		const key = placementKey(row.messageId, row.mailboxId, row.uid);
-		verdicts.settled.add(key);
-		if (departed) verdicts.departed.add(key);
-	};
-
-	// A source folder reader cannot reach decides nothing, and does not fail the
-	// round either. Another client renaming or deleting it between the row being
-	// written and now is ordinary — reader shares its mailboxes — and the sync of
-	// the unrelated folder that sighted these messages must survive it.
-	const answer = await readSource(
-		deps,
-		accountId,
-		sourceMailboxId,
-		rows,
-	).catch<SourceAnswer>((error: unknown) => ({
-		kind: "unreachable",
-		reason: error instanceof Error ? error.message : String(error),
-	}));
-
-	const observed = {
-		sourceMailboxId,
-		sightedMailboxId: sightedIn.mailboxId,
-		contested: rows.length,
-	};
-
-	if (answer.kind === "unreachable") {
-		deps.log.warn(
-			{ ...observed, error: answer.reason },
-			"Could not ask the source folder whether it still holds these messages; leaving the sightings unsettled this round",
-		);
-		return;
-	}
-
-	if (answer.kind === "rekeyed") {
-		deps.log.warn(
-			{
-				...observed,
-				sourceMailboxPath: answer.path,
-				storedUidValidity: answer.stored,
-				servedUidValidity: answer.served,
-			},
-			"Source folder is on a different UIDVALIDITY axis; leaving these sightings unsettled this round",
-		);
-		return;
-	}
-
-	if (answer.kind === "copiesEverything") {
-		for (const row of rows) settle(row, true);
-		deps.log.info(
-			{ ...observed, sourceMailboxPath: answer.path },
-			"Row points at a folder that copies every message; the sighting in a real folder is where it lives",
-		);
-		return;
-	}
-
-	for (const row of rows) settle(row, !answer.held.has(row.uid));
-};

package/src/filter-operations.test.ts

@@ -1,319 +0,0 @@
-import assert from "node:assert/strict";
-import { afterEach, beforeEach, describe, it } from "node:test";
-import type { AccountItem, FilterItem, MailboxItem } from "@remit/data-ports";
-import {
-	FilterDisabledReason,
-	FilterMatchOperator,
-	FilterScope,
-	FilterState,
-	MailboxSyncStatus,
-} from "@remit/domain-enums";
-import {
-	createFilter,
-	type FilterPatch,
-	listFilters,
-	type NewFilter,
-	updateFilter,
-} from "./filter-operations.js";
-import type { Refusable } from "./refusal.js";
-import {
-	captureQueue,
-	openStore,
-	type QueueCapture,
-	type Store,
-	seedAccount,
-	seedMailbox,
-} from "./test-helpers/test-store.js";
-
-let store: Store;
-let account: AccountItem;
-let capture: QueueCapture;
-
-const FANOUT_QUEUE = "http://localhost:4566/000000000000/account-fanout";
-
-beforeEach(async () => {
-	capture = captureQueue();
-	store = await openStore();
-	account = await seedAccount(store);
-});
-
-afterEach(async () => {
-	capture.restore();
-	await store.close();
-});
-
-const deps = () => ({
-	account: store.account,
-	mailbox: store.mailbox,
-	filter: store.filter,
-	organizeJobRequest: store.organizeJobRequest,
-	sqsQueueUrl: FANOUT_QUEUE,
-	logger: { info: () => {} },
-});
-
-const rule = (actionMailboxId: string): NewFilter => ({
-	name: "Receipts",
-	scope: FilterScope.Standing,
-	expiresAt: undefined,
-	matchOperator: FilterMatchOperator.And,
-	literalClauses: [{ field: "From", value: "billing@example.com" }],
-	actionLabelId: "None",
-	actionMailboxId,
-});
-
-const refusalOf = <T>(outcome: Refusable<T>) => {
-	assert.equal(outcome.ok, false, "expected a refusal");
-	return outcome.ok ? undefined : outcome.refusal;
-};
-
-const pendingFolder = async (fullPath: string): Promise<MailboxItem> => {
-	const created = await store.mailbox.create({
-		accountId: account.accountId,
-		namespacePrefix: "",
-		hierarchyDelimiter: "/",
-		fullPath,
-		uidValidity: 0,
-		uidNext: 1,
-		highestModseq: "0",
-		messageCount: 0,
-		unseenCount: 0,
-		deletedCount: 0,
-		totalSize: 0,
-		lastSyncUid: 0,
-		highWaterMarkUid: 0,
-		lastMessageSyncAt: 0,
-		syncStatus: MailboxSyncStatus.pending,
-	});
-	if (created.outcome !== "Created") throw new Error(`${fullPath} exists`);
-	return created.mailbox;
-};
-
-const storedFilter = (over: Partial<FilterItem>): Promise<FilterItem> =>
-	store.filter.create({
-		accountConfigId: account.accountConfigId,
-		...rule("None"),
-		...over,
-	});
-
-describe("createFilter", () => {
-	it("stores the filter and queues its pass over existing mail", async () => {
-		const folder = await seedMailbox(store, account, "Receipts");
-
-		const outcome = await createFilter(
-			deps(),
-			{ accountConfigId: account.accountConfigId, userId: "user-1" },
-			{ accountId: account.accountId, filter: rule(folder.mailboxId) },
-		);
-
-		assert.equal(outcome.ok, true);
-		if (!outcome.ok) return;
-		const job = await store.organizeJobRequest.get(
-			outcome.value.organizeJob.organizeJobId,
-		);
-		assert.equal(job.filterId, outcome.value.filter.filterId);
-		assert.equal(job.userId, "user-1");
-		assert.deepEqual(job.literalClauses, []);
-		assert.equal(job.actionMailboxId, "None");
-		assert.deepEqual(capture.sent, [
-			{
-				queueUrl: FANOUT_QUEUE,
-				body: {
-					type: "OrganizeJob",
-					accountConfigId: account.accountConfigId,
-					organizeJobId: job.organizeJobId,
-				},
-			},
-		]);
-	});
-
-	it("waits for the folder it moves mail into, storing nothing", async () => {
-		const folder = await pendingFolder("Receipts");
-
-		const outcome = await createFilter(
-			deps(),
-			{ accountConfigId: account.accountConfigId, userId: "user-1" },
-			{ accountId: account.accountId, filter: rule(folder.mailboxId) },
-		);
-
-		assert.deepEqual(refusalOf(outcome), {
-			kind: "NotSettled",
-			message:
-				"“Receipts” isn’t ready yet — the mail server hasn’t confirmed it.",
-			mailboxId: folder.mailboxId,
-			syncStatus: MailboxSyncStatus.pending,
-		});
-		assert.deepEqual(
-			await store.filter.listByAccountConfig(account.accountConfigId),
-			[],
-		);
-		assert.deepEqual(capture.sent, []);
-	});
-
-	it("refuses a Temporary filter without an expiry, storing nothing", async () => {
-		const folder = await seedMailbox(store, account, "Receipts");
-
-		const outcome = await createFilter(
-			deps(),
-			{ accountConfigId: account.accountConfigId, userId: "user-1" },
-			{
-				accountId: account.accountId,
-				filter: { ...rule(folder.mailboxId), scope: FilterScope.Temporary },
-			},
-		);
-
-		assert.deepEqual(refusalOf(outcome), {
-			kind: "Invalid",
-			message:
-				"A Temporary filter needs expiresAt — pick a date, or switch scope to Standing.",
-		});
-		assert.deepEqual(
-			await store.filter.listByAccountConfig(account.accountConfigId),
-			[],
-		);
-	});
-});
-
-describe("updateFilter", () => {
-	it("writes only the fields a patch may set, dropping server-derived ones", async () => {
-		const filter = await storedFilter({});
-		const smuggled: Record<string, unknown> = {
-			name: "Bills",
-			ttl: 123,
-			disabledReason: FilterDisabledReason.FolderMissing,
-			ruleChangedAt: 999,
-			filterId: "sneaky",
-		};
-
-		const outcome = await updateFilter(deps(), account.accountConfigId, {
-			accountId: account.accountId,
-			filterId: filter.filterId,
-			patch: smuggled as FilterPatch,
-		});
-
-		assert.equal(outcome.ok, true);
-		const stored = await store.filter.get(
-			account.accountConfigId,
-			filter.filterId,
-		);
-		assert.equal(stored.name, "Bills");
-		assert.equal(stored.ttl, undefined);
-		assert.equal(stored.disabledReason, FilterDisabledReason.None);
-		assert.equal(stored.ruleChangedAt, filter.ruleChangedAt);
-	});
-
-	it("records UserDisabled when the user turns a filter off", async () => {
-		const filter = await storedFilter({});
-
-		await updateFilter(deps(), account.accountConfigId, {
-			accountId: account.accountId,
-			filterId: filter.filterId,
-			patch: { state: FilterState.Disabled },
-		});
-
-		const stored = await store.filter.get(
-			account.accountConfigId,
-			filter.filterId,
-		);
-		assert.equal(stored.state, FilterState.Disabled);
-		assert.equal(stored.disabledReason, FilterDisabledReason.UserDisabled);
-	});
-
-	it("refuses to bind to a folder the mail server has not confirmed", async () => {
-		const filter = await storedFilter({});
-		const folder = await pendingFolder("Receipts");
-
-		const outcome = await updateFilter(deps(), account.accountConfigId, {
-			accountId: account.accountId,
-			filterId: filter.filterId,
-			patch: { actionMailboxId: folder.mailboxId },
-		});
-
-		assert.equal(refusalOf(outcome)?.kind, "NotSettled");
-		const stored = await store.filter.get(
-			account.accountConfigId,
-			filter.filterId,
-		);
-		assert.equal(stored.actionMailboxId, "None");
-	});
-
-	it("will not turn on a filter whose folder is gone", async () => {
-		const filter = await storedFilter({
-			actionMailboxId: "mbx-gone",
-			state: FilterState.Disabled,
-			disabledReason: FilterDisabledReason.UserDisabled,
-		});
-
-		const outcome = await updateFilter(deps(), account.accountConfigId, {
-			accountId: account.accountId,
-			filterId: filter.filterId,
-			patch: { state: FilterState.Active },
-		});
-
-		assert.deepEqual(refusalOf(outcome), {
-			kind: "Invalid",
-			message:
-				"The folder this filter moves mail into no longer exists. Pick another folder, then turn the filter on.",
-		});
-	});
-
-	it("refuses a Temporary filter without an expiry", async () => {
-		const filter = await storedFilter({});
-
-		const outcome = await updateFilter(deps(), account.accountConfigId, {
-			accountId: account.accountId,
-			filterId: filter.filterId,
-			patch: { scope: FilterScope.Temporary },
-		});
-
-		assert.deepEqual(refusalOf(outcome), {
-			kind: "Invalid",
-			message:
-				"A Temporary filter needs expiresAt — pick a date, or switch scope to Standing.",
-		});
-	});
-
-	it("renames a filter", async () => {
-		const filter = await storedFilter({});
-
-		const outcome = await updateFilter(deps(), account.accountConfigId, {
-			accountId: account.accountId,
-			filterId: filter.filterId,
-			patch: { name: "Bills" },
-		});
-
-		assert.equal(outcome.ok && outcome.value.name, "Bills");
-	});
-});
-
-describe("listFilters", () => {
-	it("expires a lapsed Temporary filter as it reads it", async () => {
-		await storedFilter({
-			scope: FilterScope.Temporary,
-			expiresAt: "2020-01-01T00:00:00+00:00",
-		});
-
-		const outcome = await listFilters(deps(), account.accountConfigId, {
-			accountId: account.accountId,
-			continuationToken: undefined,
-		});
-
-		assert.deepEqual(
-			outcome.ok && outcome.value.items.map((item) => item.state),
-			[FilterState.Expired],
-		);
-	});
-
-	it("answers another tenant's account as not found", async () => {
-		const other = await seedAccount(store, { accountConfigId: "cfg-other" });
-
-		const outcome = await listFilters(deps(), account.accountConfigId, {
-			accountId: other.accountId,
-			continuationToken: undefined,
-		});
-
-		assert.deepEqual(refusalOf(outcome), {
-			kind: "NotFound",
-			message: `Account not found: ${other.accountId}`,
-		});
-	});
-});