@remit/config-transfer 0.0.2 → 0.0.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/config-transfer",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/export.test.ts

@@ -7,9 +7,10 @@ import {
 	composeFolderRoleAppointmentLabelName,
 	composeFolderRoleAppointmentName,
 } from "@remit/data-ports/folder-role";
-import { readConfigForExport } from "./export.js";
+import { ConfigExportRefusedError, readConfigForExport } from "./export.js";
 import {
 	ACCOUNT_CONFIG_ID,
+	asAddressRepository,
 	asRepositories,
 	type ConfigFixture,
 	makeAccount,
@@ -17,7 +18,9 @@ import {
 	makeAddress,
 	makeAnchor,
 	makeFilter,
+	makeJunkOnlyFlaggedAddress,
 	makeLabel,
+	makeLegacyFlaggedAddress,
 	makeMailbox,
 	makeSetting,
 	OAUTH_ACCOUNT_ID,
@@ -462,13 +465,48 @@ test("an address exports only when the user decided something about it", async (
 			},
 		}),
 	];
-	fixture.addresses = [...derivedOnly, ...fixture.addresses];
+	fixture.addresses = [
+		...derivedOnly,
+		...fixture.addresses,
+		makeJunkOnlyFlaggedAddress(),
+	];
 
 	const document = await exportFixture(fixture);
 
 	assert.deepEqual(
 		document.addressFlags.map((entry) => entry.normalizedEmail),
-		["post@bank.example", "noreply@newsletter.example"],
+		["post@bank.example", "sale@junk.example", "noreply@newsletter.example"],
+	);
+});
+
+test("an address the machine marked junk-only still exports the decision on it", async () => {
+	const fixture = fullFixture();
+	fixture.addresses = [...fixture.addresses, makeJunkOnlyFlaggedAddress()];
+
+	const suggested = await asAddressRepository(fixture).listByAccountConfig({
+		accountConfigId: ACCOUNT_CONFIG_ID,
+	});
+	assert.equal(
+		suggested.items.some((item) => item.addressId === "adr-junk-news"),
+		false,
+		"the suggest listing is right to hide it, and is not the export's read",
+	);
+
+	const document = await exportFixture(fixture);
+
+	const exported = document.addressFlags.find(
+		(entry) => entry.normalizedEmail === "sale@junk.example",
+	);
+	assert.deepEqual(exported?.flags.unsubscribed, {
+		value: true,
+		setAt: 1755000000000,
+	});
+	assert.equal(exported?.flags.autoArchive?.value, true);
+	assert.equal(exported?.flags.category?.value, "newsletter");
+	assert.equal(
+		"junkOnly" in (exported?.flags ?? {}),
+		false,
+		"the machine's own flag is still not carried",
 	);
 });
 
@@ -491,6 +529,55 @@ test("an exported address carries its whole flag payload, and no derived flag",
 	});
 });
 
+test("a flag set before the timestamp was recorded exports at the sentinel", async () => {
+	const fixture = fullFixture();
+	fixture.addresses = [makeLegacyFlaggedAddress()];
+
+	const document = await exportFixture(fixture);
+
+	assert.equal(ReaderConfigDocumentSchema.safeParse(document).success, true);
+	assert.deepEqual(document.addressFlags, [
+		{
+			normalizedEmail: "old@legacy.example",
+			displayName: "Legacy",
+			flags: {
+				vip: { value: true, setAt: 0, setBy: "web-client" },
+				category: { value: "newsletter", setAt: 0 },
+			},
+		},
+	]);
+});
+
+test("a flag the format cannot carry is refused by address and by flag", async () => {
+	const fixture = fullFixture();
+	fixture.addresses = [
+		...fixture.addresses,
+		makeAddress({
+			addressId: "adr-broken",
+			normalizedEmail: "broken@example.com",
+			flags: {
+				category: { value: "Nonsense", setAt: 1750000000000 },
+			} as unknown as AddressItem["flags"],
+		}),
+	];
+
+	await assert.rejects(exportFixture(fixture), (error: unknown) => {
+		assert.ok(error instanceof ConfigExportRefusedError);
+		assert.match(
+			error.message,
+			/addressFlags\[broken@example\.com\]\.flags\.category\.value/,
+		);
+		return true;
+	});
+});
+
+test("a refusal outside the address list keeps the path the schema gave it", async () => {
+	const fixture = fullFixture();
+	fixture.labels = [makeLabel({ labelId: "lbl-blank", name: "" })];
+
+	await assert.rejects(exportFixture(fixture), /labels\[0\]\.name/);
+});
+
 test("a configuration that holds no row at all still exports a valid document", async () => {
 	const repositories = asRepositories(fullFixture());
 	const document = await readConfigForExport(

package/src/export.ts

@@ -9,6 +9,7 @@ import {
 	type ReaderConfigDocument,
 	ReaderConfigDocumentSchema,
 } from "@remit/config-format";
+import { formatPath } from "@remit/config-format/errors";
 import type {
 	AccountItem,
 	AddressFlags,
@@ -39,12 +40,72 @@ import {
  */
 const NO_ACTION = "None";
 
+/**
+ * The instance's own data does not fit the document format. Nothing is written:
+ * a file the import would refuse is worse than no file. Every problem names the
+ * row it came from, because the operator reading this has to go and fix one.
+ */
+export class ConfigExportRefusedError extends Error {
+	readonly problems: readonly string[];
+
+	constructor(problems: readonly string[]) {
+		super(
+			`This configuration cannot be exported as a v${CURRENT_SCHEMA_VERSION} document: ${problems.join("; ")}`,
+		);
+		this.name = "ConfigExportRefusedError";
+		this.problems = problems;
+	}
+}
+
 /** Which credential the account needs before it can sync again. */
 const credentialsFor = (account: AccountItem): ConfigAccount["credentials"] =>
 	account.authType === AccountAuthType.OauthMicrosoft
 		? { required: "oauth", provider: "microsoft" }
 		: { required: "password" };
 
+/**
+ * A flag as the rows actually hold it. `setAt` arrived after the first flags
+ * were written, so rows from before it carry none; the stored type says
+ * otherwise because it describes the shape being written today.
+ */
+interface StoredFlag<TValue> {
+	value: TValue;
+	setAt?: number;
+	setBy?: string;
+	expiresAt?: number;
+	reason?: string;
+}
+
+/** The same flag as the document spells it: `setAt` always present. */
+interface CarriedFlag<TValue> {
+	value: TValue;
+	setAt: number;
+	setBy?: string;
+	expiresAt?: number;
+	reason?: string;
+}
+
+/** The format's sentinel for a flag set before the timestamp was recorded. */
+const SET_AT_UNKNOWN = 0;
+
+/**
+ * One flag, field by field rather than as a copy of the stored object, so no
+ * attribute a later build adds to the row travels without a line here.
+ *
+ * A row with no `setAt` exports at the sentinel. Dropping the flag would lose a
+ * decision its owner made, and stamping the export's own clock would claim it
+ * was made today.
+ */
+const carriedFlag = <TValue>(
+	flag: StoredFlag<TValue>,
+): CarriedFlag<TValue> => ({
+	value: flag.value,
+	setAt: flag.setAt ?? SET_AT_UNKNOWN,
+	...(flag.setBy === undefined ? {} : { setBy: flag.setBy }),
+	...(flag.expiresAt === undefined ? {} : { expiresAt: flag.expiresAt }),
+	...(flag.reason === undefined ? {} : { reason: flag.reason }),
+});
+
 /**
  * Every mailbox in the configuration, by id, plus which account holds it.
  * Folder-referencing config is stored against mailbox ids, and every one of
@@ -111,7 +172,7 @@ const folderOverridesOf = (
 		overrides.push({
 			folderPath,
 			displayName: override.displayName,
-			muted: override.muted,
+			muted: override.muted === null ? null : carriedFlag(override.muted),
 		});
 	}
 	return overrides;
@@ -163,7 +224,7 @@ const toConfigAccount = (
 		username: account.smtpUsername,
 	},
 	displayName: settings.displayName,
-	muted: settings.muted,
+	muted: settings.muted === null ? null : carriedFlag(settings.muted),
 	composeLanguages: settings.composeLanguages,
 	signature: {
 		plainText: settings.signaturePlainText,
@@ -234,13 +295,13 @@ const toConfigFilter = (
 const userFlagsOf = (flags: AddressFlags | undefined): ConfigAddressFlags => {
 	const stored = flags ?? {};
 	const user: ConfigAddressFlags = {};
-	if (stored.trusted) user.trusted = stored.trusted;
-	if (stored.blocked) user.blocked = stored.blocked;
-	if (stored.muted) user.muted = stored.muted;
-	if (stored.vip) user.vip = stored.vip;
-	if (stored.category) user.category = stored.category;
-	if (stored.autoArchive) user.autoArchive = stored.autoArchive;
-	if (stored.unsubscribed) user.unsubscribed = stored.unsubscribed;
+	if (stored.trusted) user.trusted = carriedFlag(stored.trusted);
+	if (stored.blocked) user.blocked = carriedFlag(stored.blocked);
+	if (stored.muted) user.muted = carriedFlag(stored.muted);
+	if (stored.vip) user.vip = carriedFlag(stored.vip);
+	if (stored.category) user.category = carriedFlag(stored.category);
+	if (stored.autoArchive) user.autoArchive = carriedFlag(stored.autoArchive);
+	if (stored.unsubscribed) user.unsubscribed = carriedFlag(stored.unsubscribed);
 	return user;
 };
 
@@ -268,7 +329,7 @@ const readAddresses = async (
 	const addresses: AddressItem[] = [];
 	let cursor: string | undefined;
 	do {
-		const page = await repositories.address.listByAccountConfig({
+		const page = await repositories.address.listAllByAccountConfigPage({
 			accountConfigId,
 			cursor,
 		});
@@ -278,6 +339,25 @@ const readAddresses = async (
 	return addresses;
 };
 
+/**
+ * Where a refusal landed, said so an operator can act on it. A path into
+ * `addressFlags` names an array index of a list nobody can look up, so the
+ * index is replaced by the address itself; the flag is already the next segment.
+ */
+const refusalLocation = (
+	path: readonly PropertyKey[],
+	document: ReaderConfigDocument,
+): string => {
+	const [head, index, ...rest] = path;
+	if (head !== "addressFlags" || typeof index !== "number")
+		return formatPath(path);
+	const email = document.addressFlags[index]?.normalizedEmail;
+	if (email === undefined) return formatPath(path);
+	return rest.length === 0
+		? `addressFlags[${email}]`
+		: `addressFlags[${email}].${formatPath(rest)}`;
+};
+
 const accountSettingsOf = (
 	settings: SettingsView,
 	accountId: string,
@@ -380,5 +460,11 @@ export const readConfigForExport = async (
 			.map(toConfigAddressFlags),
 	};
 
-	return ReaderConfigDocumentSchema.parse(document);
+	const parsed = ReaderConfigDocumentSchema.safeParse(document);
+	if (parsed.success) return parsed.data;
+	throw new ConfigExportRefusedError(
+		parsed.error.issues.map(
+			(issue) => `${refusalLocation(issue.path, document)}: ${issue.message}`,
+		),
+	);
 };

package/src/fixtures.ts

@@ -5,6 +5,7 @@ import type {
 	AddressItem,
 	FilterAnchorItem,
 	FilterItem,
+	IAddressRepository,
 	LabelItem,
 	MailboxItem,
 } from "@remit/data-ports";
@@ -168,6 +169,96 @@ export const makeAddress = (
 	...overrides,
 });
 
+/**
+ * An address flagged before `setAt` was recorded, as v0.6 left it in the
+ * database. The stored type cannot spell the row — it describes what is written
+ * today — so the fixture asserts the shape rather than deriving it, which is
+ * the whole point: this is what the live instance actually holds.
+ */
+export const makeLegacyFlaggedAddress = (): AddressItem =>
+	makeAddress({
+		addressId: "adr-legacy",
+		normalizedEmail: "old@legacy.example",
+		displayName: "Legacy",
+		flags: {
+			vip: { value: true, setBy: "web-client" },
+			category: { value: "newsletter" },
+		} as unknown as AddressItem["flags"],
+	});
+
+/**
+ * The shape #1029 was about: mail the reader unsubscribed from and never
+ * replied to, landing only in Junk, so sync marks the row `junkOnly`. The
+ * autocomplete listing is right to hide it; neither half of the transfer is.
+ */
+export const makeJunkOnlyFlaggedAddress = (): AddressItem =>
+	makeAddress({
+		addressId: "adr-junk-news",
+		normalizedEmail: "sale@junk.example",
+		flags: {
+			junkOnly: { value: true, setAt: 1750000000000 },
+			unsubscribed: { value: true, setAt: 1755000000000 },
+			autoArchive: { value: true, setAt: 1755000000000 },
+			category: { value: "newsletter", setAt: 1755000000000 },
+		},
+	});
+
+/**
+ * The suggest predicate the stored listing applies, spelled in JavaScript: an
+ * address is hidden exactly when the machine marked it `junkOnly` and the
+ * account has neither corresponded with the sender nor flagged it. Modelled
+ * rather than ignored, because a stub that filters nothing cannot tell a read
+ * that hides decisions apart from one that does (#1029).
+ */
+export const suggestable = (address: AddressItem): boolean => {
+	const flags = address.flags ?? {};
+	if (flags.junkOnly?.value !== true) return true;
+	const corresponded =
+		address.outboundCount + address.replyCount > 0 ||
+		flags.vip?.value === true ||
+		flags.trusted?.value === true;
+	return (
+		corresponded || flags.blocked?.value === true || flags.muted?.value === true
+	);
+};
+
+const byAddressId = (rows: AddressItem[]): AddressItem[] =>
+	[...rows].sort((left, right) =>
+		left.addressId < right.addressId
+			? -1
+			: left.addressId > right.addressId
+				? 1
+				: 0,
+	);
+
+/**
+ * Both address reads, so a test can hold them against each other: the suggest
+ * listing hides a junk-only row, the export's listing carries every row in
+ * `addressId` order.
+ */
+export const asAddressRepository = (
+	fixture: ConfigFixture,
+): Pick<
+	IAddressRepository,
+	"listByAccountConfig" | "listAllByAccountConfigPage"
+> => {
+	const pageOf = (rows: AddressItem[], cursor: string | undefined) => {
+		const size = fixture.addressPageSize ?? rows.length + 1;
+		const offset = cursor ? Number(cursor) : 0;
+		const next = offset + size;
+		return {
+			items: rows.slice(offset, next),
+			continuationToken: next < rows.length ? String(next) : undefined,
+		};
+	};
+	return {
+		listByAccountConfig: async ({ cursor }) =>
+			pageOf(fixture.addresses.filter(suggestable), cursor),
+		listAllByAccountConfigPage: async ({ cursor }) =>
+			pageOf(byAddressId(fixture.addresses), cursor),
+	};
+};
+
 /** The fixture, behind the repository interfaces the export takes. */
 export const asRepositories = (
 	fixture: ConfigFixture,
@@ -194,17 +285,5 @@ export const asRepositories = (
 	filterAnchor: {
 		listByAccountConfig: async () => fixture.anchors,
 	},
-	address: {
-		listByAccountConfig: async ({ cursor }) => {
-			const size = fixture.addressPageSize ?? fixture.addresses.length + 1;
-			const offset = cursor ? Number(cursor) : 0;
-			const items = fixture.addresses.slice(offset, offset + size);
-			const next = offset + size;
-			return {
-				items,
-				continuationToken:
-					next < fixture.addresses.length ? String(next) : undefined,
-			};
-		},
-	},
+	address: asAddressRepository(fixture),
 });

package/src/import-repositories.ts

@@ -39,9 +39,14 @@ export interface ConfigImportRepositories {
 	label: Pick<ILabelRepository, "listByAccountConfig" | "create" | "update">;
 	filter: Pick<IFilterRepository, "listByAccountConfig" | "create" | "update">;
 	filterAnchor: Pick<IFilterAnchorRepository, "put">;
+	/**
+	 * Not the suggest listing, for the reason the export refuses it too: what a
+	 * configuration already holds is decided over every stored row, junk-only
+	 * ones included (#1029).
+	 */
 	address: Pick<
 		IAddressRepository,
-		"listByAccountConfig" | "upsertAddress" | "mergeFlags"
+		"listAllByAccountConfigPage" | "upsertAddress" | "mergeFlags"
 	>;
 	configImport: Pick<
 		IConfigImportRepository,

package/src/import.test.ts

@@ -28,11 +28,14 @@ import {
 	makeAddress,
 	makeAnchor,
 	makeFilter,
+	makeJunkOnlyFlaggedAddress,
 	makeLabel,
+	makeLegacyFlaggedAddress,
 	makeMailbox,
 	makeSetting,
 	OAUTH_ACCOUNT_ID,
 	PASSWORD_ACCOUNT_ID,
+	suggestable,
 } from "./fixtures.js";
 import { ANCHOR_EMBEDDING_PENDING, importConfig } from "./import.js";
 import type { ConfigImportDeps } from "./import-repositories.js";
@@ -245,7 +248,13 @@ const repositoriesOf = (store: Store, accountConfigId: string): any => {
 			listByAccountConfig: async () => [...store.anchors],
 		},
 		address: {
+			// Both reads, so the stub can tell them apart: the suggest listing
+			// hides a junk-only row, and the import must not be reading it (#1029).
 			listByAccountConfig: async () => ({
+				items: store.addresses.filter(suggestable),
+				continuationToken: undefined,
+			}),
+			listAllByAccountConfigPage: async () => ({
 				items: [...store.addresses],
 				continuationToken: undefined,
 			}),
@@ -524,12 +533,20 @@ const sourceFixture = (): ConfigFixture => ({
 	],
 });
 
-const exportSource = () =>
-	readConfigForExport(
-		asRepositories(sourceFixture()),
-		ACCOUNT_CONFIG_ID,
-		IDENTITY,
-	);
+/**
+ * The same configuration with one sender flagged before `setAt` existed. Held
+ * apart from the source fixture rather than folded into it: half the tests here
+ * count what the file carries, and a legacy row is a fact about the database
+ * this instance happens to hold, not about every configuration.
+ */
+const sourceWithLegacyFlagsFixture = (): ConfigFixture => {
+	const fixture = sourceFixture();
+	fixture.addresses = [...fixture.addresses, makeLegacyFlaggedAddress()];
+	return fixture;
+};
+
+const exportSource = (fixture: ConfigFixture = sourceFixture()) =>
+	readConfigForExport(asRepositories(fixture), ACCOUNT_CONFIG_ID, IDENTITY);
 
 const apply = (
 	store: Store,
@@ -564,7 +581,7 @@ const discover = (
 };
 
 test("a configuration survives an export, an import and a discovery unchanged", async () => {
-	const document = await exportSource();
+	const document = await exportSource(sourceWithLegacyFlagsFixture());
 	const store = emptyStore();
 
 	const report = reportOf(await apply(store, document));
@@ -685,6 +702,31 @@ test("a configuration that already holds something refuses the default import",
 	assert.equal(outcome.conflict.details.addressFlags, "2");
 });
 
+test("a junk-only sender the user decided about still holds the configuration", async () => {
+	const document = await exportSource();
+	const store = emptyStore();
+	store.addresses.push(makeJunkOnlyFlaggedAddress());
+
+	assert.equal(
+		store.addresses.filter(suggestable).length,
+		0,
+		"the suggest listing hides it, and is not the read the import may use",
+	);
+
+	const outcome = await importConfig(depsOf(store, ACCOUNT_CONFIG_ID), {
+		accountConfigId: ACCOUNT_CONFIG_ID,
+		userId: TARGET_USER_ID,
+		document,
+		mode: "apply",
+		onExisting: "abort",
+	});
+
+	assert.equal(outcome.outcome, "conflict");
+	if (outcome.outcome !== "conflict") throw new Error("unreachable");
+	assert.equal(outcome.conflict.code, "config_not_empty");
+	assert.equal(outcome.conflict.details.addressFlags, "1");
+});
+
 test("merge folds the file in and deletes nothing the file left out", async () => {
 	const document = await exportSource();
 	const store = emptyStore();
@@ -770,6 +812,19 @@ test("a flagged sender arrives ahead of its mail, keyed on the email string", as
 	assert.equal(news?.flags?.unsubscribed?.value, true);
 });
 
+test("a flag that arrives at the sentinel is stored at the sentinel", async () => {
+	const document = await exportSource(sourceWithLegacyFlagsFixture());
+	const store = emptyStore();
+
+	await apply(store, document);
+
+	const legacy = store.addresses.find(
+		(address) => address.normalizedEmail === "old@legacy.example",
+	);
+	assert.equal(legacy?.flags?.vip?.setAt, 0);
+	assert.equal(legacy?.flags?.category?.setAt, 0);
+});
+
 test("a folder no mailbox answers to yet is kept, then bound when discovery finds it", async () => {
 	const document = await exportSource();
 	const store = emptyStore();

package/src/import.ts

@@ -118,11 +118,14 @@ const readExisting = async (
 	]);
 	const live = accounts.filter((account) => account.deletedAt === undefined);
 
+	// Not the suggest listing: it hides an address the machine marked `junkOnly`,
+	// so a decision the user made about a junk-only sender would read as absent
+	// and be re-created or left out of the conflict count (#1029).
 	let flaggedAddresses = 0;
 	const addressEmails = new Set<string>();
 	let cursor: string | undefined;
 	do {
-		const page = await repositories.address.listByAccountConfig({
+		const page = await repositories.address.listAllByAccountConfigPage({
 			accountConfigId,
 			cursor,
 		});

package/src/index.ts

@@ -5,7 +5,11 @@ export {
 	type ConfigBinderRepositories,
 	pendingImportOf,
 } from "./binder.js";
-export { carriesUserFlag, readConfigForExport } from "./export.js";
+export {
+	ConfigExportRefusedError,
+	carriesUserFlag,
+	readConfigForExport,
+} from "./export.js";
 export {
 	ANCHOR_EMBEDDING_PENDING,
 	type ImportConfigInput,

package/src/repositories.ts

@@ -25,7 +25,13 @@ export interface ConfigExportRepositories {
 	label: Pick<ILabelRepository, "listByAccountConfig">;
 	filter: Pick<IFilterRepository, "listByAccountConfig">;
 	filterAnchor: Pick<IFilterAnchorRepository, "listByAccountConfig">;
-	address: Pick<IAddressRepository, "listByAccountConfig">;
+	/**
+	 * Not the suggest listing: that one hides an address the machine marked
+	 * `junkOnly` unless the account has corresponded with it, which is right for
+	 * autocomplete and wrong for a file whose whole job is to keep the reader's
+	 * decisions (#1029).
+	 */
+	address: Pick<IAddressRepository, "listAllByAccountConfigPage">;
 }
 
 /** Who wrote the file, and where from. Recorded; never authoritative on import. */