@remit/config-transfer 0.0.1 → 0.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/config-transfer",
-  "version": "0.0.1",
+  "version": "0.0.3",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/binder.ts

@@ -0,0 +1,184 @@
+import { readConfigDocument } from "@remit/config-format";
+import type {
+	ConfigImportItem,
+	ConfigImportUnresolvedRefItem,
+	IAccountSettingRepository,
+	IConfigImportRepository,
+	IFilterRepository,
+	IMailboxRepository,
+} from "@remit/data-ports";
+import { composeSettingName } from "@remit/data-ports/account-settings";
+import {
+	AccountSettingName,
+	ConfigImportRefKind,
+	ConfigImportState,
+} from "@remit/domain-enums";
+import type { AppointFolderRole } from "./import-repositories.js";
+
+export interface ConfigBinderRepositories {
+	configImport: Pick<IConfigImportRepository, "listByAccountConfig" | "update">;
+	accountSetting: Pick<IAccountSettingRepository, "upsert">;
+	filter: Pick<IFilterRepository, "update">;
+	mailbox: Pick<IMailboxRepository, "listAllByAccount">;
+}
+
+export interface ConfigBinderDeps {
+	repositories: ConfigBinderRepositories;
+	appointFolderRole: AppointFolderRole;
+	now?: () => number;
+}
+
+/**
+ * What one folder path an import was waiting for is still waiting for, told as
+ * a count so a caller can log it without walking the refs again.
+ */
+export interface BindResult {
+	bound: number;
+	stillPending: number;
+}
+
+/**
+ * Bind the folder references an import could not resolve, now that discovery
+ * has produced this account's mailboxes.
+ *
+ * Every reference names a folder by IMAP path, because the ids are new on every
+ * discovery. This runs where discovery ends, resolves what it can, writes the
+ * row each reference belongs to, and drops it. An import whose last reference
+ * is gone is Complete; the rest stay, and surface on `GET /config` until the
+ * folder they name shows up or the person removes the expectation.
+ *
+ * Idempotent and replayable: every write is an upsert or an update onto a row
+ * the import already created, so a second discovery finds nothing to do rather
+ * than doing it again differently.
+ */
+export const bindImportedFolders = async (
+	deps: ConfigBinderDeps,
+	accountConfigId: string,
+	accountId: string,
+): Promise<BindResult> => {
+	const { repositories } = deps;
+	const now = deps.now ?? Date.now;
+
+	const imports = (
+		await repositories.configImport.listByAccountConfig(accountConfigId)
+	).filter((row) => row.state === ConfigImportState.Pending);
+	if (imports.length === 0) return { bound: 0, stillPending: 0 };
+
+	const byPath = new Map(
+		(await repositories.mailbox.listAllByAccount(accountId)).map(
+			(mailbox) => [mailbox.fullPath, mailbox.mailboxId] as const,
+		),
+	);
+
+	let bound = 0;
+	let stillPending = 0;
+
+	for (const row of imports) {
+		const document = readConfigDocument(row.document);
+		const remaining: ConfigImportUnresolvedRefItem[] = [];
+
+		for (const ref of row.unresolvedRefs) {
+			const mailboxId =
+				ref.accountId === accountId ? byPath.get(ref.folderPath) : undefined;
+			if (mailboxId === undefined) {
+				remaining.push(ref);
+				continue;
+			}
+			await bindRef(deps, accountConfigId, ref, mailboxId, document);
+			bound++;
+		}
+
+		stillPending += remaining.length;
+		if (remaining.length === row.unresolvedRefs.length) continue;
+
+		await repositories.configImport.update(row.importId, {
+			unresolvedRefs: remaining,
+			...(remaining.length === 0
+				? { state: ConfigImportState.Complete, completedAt: now() }
+				: {}),
+		});
+	}
+
+	return { bound, stillPending };
+};
+
+type BoundDocument = ReturnType<typeof readConfigDocument>;
+
+const bindRef = async (
+	deps: ConfigBinderDeps,
+	accountConfigId: string,
+	ref: ConfigImportUnresolvedRefItem,
+	mailboxId: string,
+	document: BoundDocument,
+): Promise<void> => {
+	const { repositories } = deps;
+	if (ref.kind === ConfigImportRefKind.FilterAction) {
+		await repositories.filter.update(accountConfigId, ref.target, {
+			actionMailboxId: mailboxId,
+		});
+		return;
+	}
+
+	if (ref.kind === ConfigImportRefKind.FolderRole) {
+		await deps.appointFolderRole(
+			accountConfigId,
+			ref.accountId,
+			ref.target,
+			mailboxId,
+			ref.folderPath,
+		);
+		return;
+	}
+
+	// The account the file named, when this instance still holds it under that
+	// id — it does for every account the import created, because the id crosses
+	// verbatim. A merge onto an account already here can have renumbered it, and
+	// then the path is the only thing left to match on.
+	const named = document.accounts.find(
+		(account) => account.accountId === ref.accountId,
+	);
+	const override = (
+		named
+			? named.folderOverrides
+			: document.accounts.flatMap((a) => a.folderOverrides)
+	).find((candidate) => candidate.folderPath === ref.folderPath);
+	if (!override) return;
+
+	if (override.displayName !== "") {
+		await repositories.accountSetting.upsert({
+			accountConfigId,
+			name: composeSettingName(
+				AccountSettingName.MailboxDisplayName,
+				mailboxId,
+			),
+			value: { kind: "String", value: override.displayName },
+		});
+	}
+	if (override.muted !== null) {
+		await repositories.accountSetting.upsert({
+			accountConfigId,
+			name: composeSettingName(AccountSettingName.MailboxMuted, mailboxId),
+			value: { kind: "MutedFlag", value: override.muted },
+		});
+	}
+};
+
+/**
+ * The folder paths an import is still waiting on, for `GET /config`. The newest
+ * pending import only: an older one waiting for the same folder is waiting for
+ * the same thing, and naming it twice tells nobody anything.
+ */
+export const pendingImportOf = (
+	imports: readonly ConfigImportItem[],
+): { importId: string; folderPaths: string[] } | undefined => {
+	const pending = imports
+		.filter((row) => row.state === ConfigImportState.Pending)
+		.sort((a, b) => b.createdAt - a.createdAt)[0];
+	if (!pending || pending.unresolvedRefs.length === 0) return undefined;
+	return {
+		importId: pending.importId,
+		folderPaths: [
+			...new Set(pending.unresolvedRefs.map((ref) => ref.folderPath)),
+		],
+	};
+};

package/src/export.test.ts

@@ -7,7 +7,7 @@ import {
 	composeFolderRoleAppointmentLabelName,
 	composeFolderRoleAppointmentName,
 } from "@remit/data-ports/folder-role";
-import { readConfigForExport } from "./export.js";
+import { ConfigExportRefusedError, readConfigForExport } from "./export.js";
 import {
 	ACCOUNT_CONFIG_ID,
 	asRepositories,
@@ -18,6 +18,7 @@ import {
 	makeAnchor,
 	makeFilter,
 	makeLabel,
+	makeLegacyFlaggedAddress,
 	makeMailbox,
 	makeSetting,
 	OAUTH_ACCOUNT_ID,
@@ -491,6 +492,55 @@ test("an exported address carries its whole flag payload, and no derived flag",
 	});
 });
 
+test("a flag set before the timestamp was recorded exports at the sentinel", async () => {
+	const fixture = fullFixture();
+	fixture.addresses = [makeLegacyFlaggedAddress()];
+
+	const document = await exportFixture(fixture);
+
+	assert.equal(ReaderConfigDocumentSchema.safeParse(document).success, true);
+	assert.deepEqual(document.addressFlags, [
+		{
+			normalizedEmail: "old@legacy.example",
+			displayName: "Legacy",
+			flags: {
+				vip: { value: true, setAt: 0, setBy: "web-client" },
+				category: { value: "newsletter", setAt: 0 },
+			},
+		},
+	]);
+});
+
+test("a flag the format cannot carry is refused by address and by flag", async () => {
+	const fixture = fullFixture();
+	fixture.addresses = [
+		...fixture.addresses,
+		makeAddress({
+			addressId: "adr-broken",
+			normalizedEmail: "broken@example.com",
+			flags: {
+				category: { value: "Nonsense", setAt: 1750000000000 },
+			} as unknown as AddressItem["flags"],
+		}),
+	];
+
+	await assert.rejects(exportFixture(fixture), (error: unknown) => {
+		assert.ok(error instanceof ConfigExportRefusedError);
+		assert.match(
+			error.message,
+			/addressFlags\[broken@example\.com\]\.flags\.category\.value/,
+		);
+		return true;
+	});
+});
+
+test("a refusal outside the address list keeps the path the schema gave it", async () => {
+	const fixture = fullFixture();
+	fixture.labels = [makeLabel({ labelId: "lbl-blank", name: "" })];
+
+	await assert.rejects(exportFixture(fixture), /labels\[0\]\.name/);
+});
+
 test("a configuration that holds no row at all still exports a valid document", async () => {
 	const repositories = asRepositories(fullFixture());
 	const document = await readConfigForExport(

package/src/export.ts

@@ -9,6 +9,7 @@ import {
 	type ReaderConfigDocument,
 	ReaderConfigDocumentSchema,
 } from "@remit/config-format";
+import { formatPath } from "@remit/config-format/errors";
 import type {
 	AccountItem,
 	AddressFlags,
@@ -39,12 +40,72 @@ import {
  */
 const NO_ACTION = "None";
 
+/**
+ * The instance's own data does not fit the document format. Nothing is written:
+ * a file the import would refuse is worse than no file. Every problem names the
+ * row it came from, because the operator reading this has to go and fix one.
+ */
+export class ConfigExportRefusedError extends Error {
+	readonly problems: readonly string[];
+
+	constructor(problems: readonly string[]) {
+		super(
+			`This configuration cannot be exported as a v${CURRENT_SCHEMA_VERSION} document: ${problems.join("; ")}`,
+		);
+		this.name = "ConfigExportRefusedError";
+		this.problems = problems;
+	}
+}
+
 /** Which credential the account needs before it can sync again. */
 const credentialsFor = (account: AccountItem): ConfigAccount["credentials"] =>
 	account.authType === AccountAuthType.OauthMicrosoft
 		? { required: "oauth", provider: "microsoft" }
 		: { required: "password" };
 
+/**
+ * A flag as the rows actually hold it. `setAt` arrived after the first flags
+ * were written, so rows from before it carry none; the stored type says
+ * otherwise because it describes the shape being written today.
+ */
+interface StoredFlag<TValue> {
+	value: TValue;
+	setAt?: number;
+	setBy?: string;
+	expiresAt?: number;
+	reason?: string;
+}
+
+/** The same flag as the document spells it: `setAt` always present. */
+interface CarriedFlag<TValue> {
+	value: TValue;
+	setAt: number;
+	setBy?: string;
+	expiresAt?: number;
+	reason?: string;
+}
+
+/** The format's sentinel for a flag set before the timestamp was recorded. */
+const SET_AT_UNKNOWN = 0;
+
+/**
+ * One flag, field by field rather than as a copy of the stored object, so no
+ * attribute a later build adds to the row travels without a line here.
+ *
+ * A row with no `setAt` exports at the sentinel. Dropping the flag would lose a
+ * decision its owner made, and stamping the export's own clock would claim it
+ * was made today.
+ */
+const carriedFlag = <TValue>(
+	flag: StoredFlag<TValue>,
+): CarriedFlag<TValue> => ({
+	value: flag.value,
+	setAt: flag.setAt ?? SET_AT_UNKNOWN,
+	...(flag.setBy === undefined ? {} : { setBy: flag.setBy }),
+	...(flag.expiresAt === undefined ? {} : { expiresAt: flag.expiresAt }),
+	...(flag.reason === undefined ? {} : { reason: flag.reason }),
+});
+
 /**
  * Every mailbox in the configuration, by id, plus which account holds it.
  * Folder-referencing config is stored against mailbox ids, and every one of
@@ -111,7 +172,7 @@ const folderOverridesOf = (
 		overrides.push({
 			folderPath,
 			displayName: override.displayName,
-			muted: override.muted,
+			muted: override.muted === null ? null : carriedFlag(override.muted),
 		});
 	}
 	return overrides;
@@ -163,7 +224,7 @@ const toConfigAccount = (
 		username: account.smtpUsername,
 	},
 	displayName: settings.displayName,
-	muted: settings.muted,
+	muted: settings.muted === null ? null : carriedFlag(settings.muted),
 	composeLanguages: settings.composeLanguages,
 	signature: {
 		plainText: settings.signaturePlainText,
@@ -234,13 +295,13 @@ const toConfigFilter = (
 const userFlagsOf = (flags: AddressFlags | undefined): ConfigAddressFlags => {
 	const stored = flags ?? {};
 	const user: ConfigAddressFlags = {};
-	if (stored.trusted) user.trusted = stored.trusted;
-	if (stored.blocked) user.blocked = stored.blocked;
-	if (stored.muted) user.muted = stored.muted;
-	if (stored.vip) user.vip = stored.vip;
-	if (stored.category) user.category = stored.category;
-	if (stored.autoArchive) user.autoArchive = stored.autoArchive;
-	if (stored.unsubscribed) user.unsubscribed = stored.unsubscribed;
+	if (stored.trusted) user.trusted = carriedFlag(stored.trusted);
+	if (stored.blocked) user.blocked = carriedFlag(stored.blocked);
+	if (stored.muted) user.muted = carriedFlag(stored.muted);
+	if (stored.vip) user.vip = carriedFlag(stored.vip);
+	if (stored.category) user.category = carriedFlag(stored.category);
+	if (stored.autoArchive) user.autoArchive = carriedFlag(stored.autoArchive);
+	if (stored.unsubscribed) user.unsubscribed = carriedFlag(stored.unsubscribed);
 	return user;
 };
 
@@ -278,6 +339,25 @@ const readAddresses = async (
 	return addresses;
 };
 
+/**
+ * Where a refusal landed, said so an operator can act on it. A path into
+ * `addressFlags` names an array index of a list nobody can look up, so the
+ * index is replaced by the address itself; the flag is already the next segment.
+ */
+const refusalLocation = (
+	path: readonly PropertyKey[],
+	document: ReaderConfigDocument,
+): string => {
+	const [head, index, ...rest] = path;
+	if (head !== "addressFlags" || typeof index !== "number")
+		return formatPath(path);
+	const email = document.addressFlags[index]?.normalizedEmail;
+	if (email === undefined) return formatPath(path);
+	return rest.length === 0
+		? `addressFlags[${email}]`
+		: `addressFlags[${email}].${formatPath(rest)}`;
+};
+
 const accountSettingsOf = (
 	settings: SettingsView,
 	accountId: string,
@@ -380,5 +460,11 @@ export const readConfigForExport = async (
 			.map(toConfigAddressFlags),
 	};
 
-	return ReaderConfigDocumentSchema.parse(document);
+	const parsed = ReaderConfigDocumentSchema.safeParse(document);
+	if (parsed.success) return parsed.data;
+	throw new ConfigExportRefusedError(
+		parsed.error.issues.map(
+			(issue) => `${refusalLocation(issue.path, document)}: ${issue.message}`,
+		),
+	);
 };

package/src/fixtures.ts

@@ -168,6 +168,23 @@ export const makeAddress = (
 	...overrides,
 });
 
+/**
+ * An address flagged before `setAt` was recorded, as v0.6 left it in the
+ * database. The stored type cannot spell the row — it describes what is written
+ * today — so the fixture asserts the shape rather than deriving it, which is
+ * the whole point: this is what the live instance actually holds.
+ */
+export const makeLegacyFlaggedAddress = (): AddressItem =>
+	makeAddress({
+		addressId: "adr-legacy",
+		normalizedEmail: "old@legacy.example",
+		displayName: "Legacy",
+		flags: {
+			vip: { value: true, setBy: "web-client" },
+			category: { value: "newsletter" },
+		} as unknown as AddressItem["flags"],
+	});
+
 /** The fixture, behind the repository interfaces the export takes. */
 export const asRepositories = (
 	fixture: ConfigFixture,

package/src/import-repositories.ts

@@ -0,0 +1,97 @@
+import type {
+	AccountConfigItem,
+	CreateAccountConfigInput,
+	IAccountRepository,
+	IAccountSettingRepository,
+	IAddressRepository,
+	IConfigImportRepository,
+	IFilterAnchorRepository,
+	IFilterRepository,
+	ILabelRepository,
+	IMailboxRepository,
+	UpdateAccountConfigInput,
+} from "@remit/data-ports";
+
+/**
+ * Everything the import reads and writes, and nothing else. Repositories rather
+ * than a client or an HTTP surface, for the same reason the export takes them:
+ * what a configuration contains is decided here, not by whichever process
+ * happens to be holding the file.
+ */
+export interface ConfigImportRepositories {
+	accountConfig: {
+		get: (accountConfigId: string) => Promise<AccountConfigItem>;
+		create: (input: CreateAccountConfigInput) => Promise<AccountConfigItem>;
+		update: (
+			accountConfigId: string,
+			input: UpdateAccountConfigInput,
+		) => Promise<AccountConfigItem>;
+	};
+	account: Pick<
+		IAccountRepository,
+		"listAllByAccountConfig" | "create" | "update"
+	>;
+	accountSetting: Pick<
+		IAccountSettingRepository,
+		"listByAccountConfig" | "upsert"
+	>;
+	mailbox: Pick<IMailboxRepository, "listAllByAccount">;
+	label: Pick<ILabelRepository, "listByAccountConfig" | "create" | "update">;
+	filter: Pick<IFilterRepository, "listByAccountConfig" | "create" | "update">;
+	filterAnchor: Pick<IFilterAnchorRepository, "put">;
+	address: Pick<
+		IAddressRepository,
+		"listByAccountConfig" | "upsertAddress" | "mergeFlags"
+	>;
+	configImport: Pick<
+		IConfigImportRepository,
+		"create" | "update" | "listByAccountConfig"
+	>;
+}
+
+/**
+ * A filter anchor, re-embedded from the source text the file carries. The
+ * stored vector is a function of the model that produced it, so it never
+ * travels; this is the one thing an import has to compute rather than copy.
+ *
+ * Optional, and free to answer `undefined`, because a configuration file has to
+ * import on a deployment that ships no embedding model at all. Either way the
+ * anchor lands stamped `ANCHOR_EMBEDDING_PENDING`, which the existing lazy
+ * repair (`refreshAnchorForEmbedder`) treats as needing a fresh vector the
+ * first time the filter is matched.
+ */
+export type EmbedAnchor = (
+	sourceText: string,
+) => Promise<{ embedding: number[]; embeddingId: string } | undefined>;
+
+/**
+ * Persist one role's folder appointment. Injected rather than composed here:
+ * an appointment row means a person decided, and it keeps that meaning only
+ * while one writer produces it — the same one every other appointment in the
+ * repo goes through.
+ */
+export type AppointFolderRole = (
+	accountConfigId: string,
+	accountId: string,
+	role: string,
+	mailboxId: string,
+	lastKnownPath: string,
+) => Promise<void>;
+
+export interface ConfigImportDeps {
+	repositories: ConfigImportRepositories;
+	appointFolderRole: AppointFolderRole;
+	/**
+	 * Runs the whole apply as one write set, rolling back on any throw.
+	 *
+	 * Optional, and its presence is the fact the report words itself from: a
+	 * backend that supplies this rolls a failed apply back to nothing, and one
+	 * that does not leaves behind whatever landed before the failure. Only the
+	 * caller knows which, so it says so by passing this or omitting it rather
+	 * than by handing over a pass-through that would claim atomicity it has not
+	 * got.
+	 */
+	transaction?: <T>(run: () => Promise<T>) => Promise<T>;
+	embedAnchor?: EmbedAnchor;
+	now?: () => number;
+}