@remit/config-transfer 0.0.10 → 0.0.12

package/src/import.ts

@@ -26,6 +26,8 @@ import {
 	ConfigImportRefKind,
 	ConfigImportState,
 	ConnectionState,
+	FilterDisabledReason,
+	FilterState,
 } from "@remit/domain-enums";
 import { carriesUserFlag } from "./export.js";
 import type { ConfigImportDeps } from "./import-repositories.js";
@@ -37,17 +39,6 @@ import {
 	ConfigImportVerdict,
 } from "./report.js";
 
-/**
- * The `<modelId>@<dimensions>` stamp an anchor carries when this deployment
- * could not embed its source text at import time. It matches no real embedder,
- * which is what makes the existing lazy repair (`refreshAnchorForEmbedder`)
- * treat the anchor as needing a fresh vector the first time it is matched —
- * the same route a model migration takes. Deliberately the same literal as
- * `UNKNOWN_CHUNK_EMBEDDING_ID` in `@remit/search-service`, which config
- * transfer cannot import without dragging the vector stack into the CLI.
- */
-export const ANCHOR_EMBEDDING_PENDING = "unknown";
-
 /** The named sentinel a Filter carries for "no action of this kind". */
 const NO_ACTION = "None";
 
@@ -62,6 +53,15 @@ export interface ImportConfigInput {
 
 const normalize = (value: string): string => value.trim().toLowerCase();
 
+const filterFolderPendingMessage = (name: string, folderPath: string): string =>
+	`Filter "${name}" files mail into "${folderPath}", which this account does not hold yet. The folder is created on the mail server when the account next syncs, and the filter stays off until the server confirms it.`;
+
+const folderPendingMessage = (folderPath: string): string =>
+	`"${folderPath}" is not a folder this account holds yet. It is created on the mail server when the account next syncs, and the setting is bound once the server confirms it.`;
+
+const awaitingFolderReason = (folderPath: string): string =>
+	`Off until "${folderPath}" exists on the mail server.`;
+
 /**
  * The version a document claims, read straight off the raw input. The parse
  * failed, so this is the only honest answer the report can give — and a report
@@ -259,9 +259,9 @@ const landingState = (
 
 /**
  * What the apply has decided so far. Owned by `importConfig` rather than by
- * `applyDocument` so a write that fails part-way still has it: on a backend
- * without a transaction those items are what actually landed, and a report that
- * threw them away would name a stopping point without naming the state.
+ * `applyDocument` so a write that fails part-way still has it: those items are
+ * what actually landed, and a report that threw them away would name a stopping
+ * point without naming the state.
  */
 interface ApplyProgress {
 	items: ConfigImportItemReport[];
@@ -277,56 +277,6 @@ const emptyProgress = (): ApplyProgress => ({
 	unresolvedRefs: [],
 });
 
-interface AnchorVector {
-	embedding: number[];
-	embeddingId: string;
-}
-
-/**
- * Every anchor's vector, computed before the transaction opens.
- *
- * Two reasons it happens here and not at the point of write. An embedder is a
- * network round trip and the sqlite write queue is process-wide, so embedding
- * inside the transaction holds that queue against every other writer for the
- * length of the call. And a failed embed is not a failed import: it degrades
- * exactly the way a deployment that ships no embedder does, landing the anchor
- * stamped `ANCHOR_EMBEDDING_PENDING` for the lazy repair to rebuild the first
- * time the filter runs. A 503 from the embedder must not cost a person their
- * configuration.
- */
-const embedAnchors = async (
-	deps: ConfigImportDeps,
-	document: ReaderConfigDocument,
-	warnings: ConfigImportProblem[],
-): Promise<Map<string, AnchorVector>> => {
-	const vectors = new Map<string, AnchorVector>();
-	const { embedAnchor } = deps;
-
-	for (const filter of document.filters) {
-		const { anchor } = filter;
-		if (anchor === null) continue;
-
-		const embedded = embedAnchor
-			? await embedAnchor(anchor.sourceText).catch(() => undefined)
-			: undefined;
-
-		if (embedded) {
-			vectors.set(normalize(filter.name), embedded);
-			continue;
-		}
-
-		warnings.push(
-			problem(
-				"anchor_not_embedded",
-				`The example behind filter "${filter.name}" is carried as text; its vector is rebuilt the first time the filter runs.`,
-				{ filter: filter.name },
-			),
-		);
-	}
-
-	return vectors;
-};
-
 /**
  * The id this instance will hold each account in the file under: the one it
  * already has for that id or email, or the file's own, which `create` preserves
@@ -367,8 +317,7 @@ const foldersFor = (
 
 /**
  * A write that failed after validation. It carries the item it stopped on so
- * the report can name it: the import is atomic, so nothing landed, and where it
- * stopped is the only fact worth recovering from.
+ * the report can name it beside what landed before it.
  */
 class ImportWriteError extends Error {
 	constructor(
@@ -388,7 +337,6 @@ const applyDocument = async (
 	input: ImportConfigInput,
 	document: ReaderConfigDocument,
 	existing: ExistingConfig,
-	anchorVectors: ReadonlyMap<string, AnchorVector>,
 	progress: ApplyProgress,
 ): Promise<string> => {
 	const { repositories } = deps;
@@ -474,8 +422,7 @@ const applyDocument = async (
 			account.email,
 			() =>
 				repositories.account.create({
-					// Verbatim: a resynced message id hashes from it, so preserving it is
-					// what keeps every filter anchor's provenance valid across the drop.
+					// Verbatim: a resynced message id hashes from it.
 					accountId: account.accountId,
 					accountConfigId,
 					...endpoints,
@@ -534,7 +481,6 @@ const applyDocument = async (
 	const existingFilterByName = new Map(
 		existing.filters.map((filter) => [normalize(filter.name), filter] as const),
 	);
-	const filterIdByName = new Map<string, string>();
 
 	const resolveFilterFolder = (
 		filter: ConfigFilter,
@@ -563,7 +509,12 @@ const applyDocument = async (
 					? NO_ACTION
 					: (labelIdByName.get(normalize(filter.actionLabelName)) ?? NO_ACTION),
 			actionMailboxId: folder.actionMailboxId,
-			hasAnchor: filter.anchor !== null,
+			...(folder.pending
+				? {
+						state: FilterState.Disabled,
+						disabledReason: FilterDisabledReason.AwaitingFolder,
+					}
+				: {}),
 		};
 
 		const held = existingFilterByName.get(key);
@@ -586,12 +537,14 @@ const applyDocument = async (
 					)
 				).filterId;
 
-		filterIdByName.set(key, filterId);
-		record(
-			ConfigImportSection.Filters,
-			filter.name,
-			held ? ConfigImportVerdict.Updated : ConfigImportVerdict.Created,
-		);
+		items.push({
+			section: ConfigImportSection.Filters,
+			key: filter.name,
+			verdict: held ? ConfigImportVerdict.Updated : ConfigImportVerdict.Created,
+			...(folder.pending && filter.actionFolder
+				? { reason: awaitingFolderReason(filter.actionFolder.folderPath) }
+				: {}),
+		});
 
 		if (folder.pending && filter.actionFolder) {
 			unresolvedRefs.push({
@@ -601,45 +554,21 @@ const applyDocument = async (
 					filter.actionFolder.accountId,
 				folderPath: filter.actionFolder.folderPath,
 				target: filterId,
+				mailboxId: NO_ACTION,
 			});
 			warnings.push(
 				problem(
 					"folder_not_found_yet",
-					`Filter "${filter.name}" files mail into "${filter.actionFolder.folderPath}", which this account does not hold yet. It is bound once the folder list has been read.`,
+					filterFolderPendingMessage(
+						filter.name,
+						filter.actionFolder.folderPath,
+					),
 					{ filter: filter.name, folderPath: filter.actionFolder.folderPath },
 				),
 			);
 		}
 	}
 
-	// ---- anchors --------------------------------------------------------
-	// Vectors were computed before the transaction opened; a missing one is an
-	// anchor carried as text, already warned about there.
-	for (const filter of document.filters) {
-		const { anchor } = filter;
-		if (anchor === null) continue;
-		const key = normalize(filter.name);
-		const filterId = filterIdByName.get(key);
-		if (filterId === undefined) {
-			throw new Error(
-				`Filter "${filter.name}" carries an anchor but the filter loop minted no id for it. The two loops read the same documents.filters, so this cannot happen without one of them changing.`,
-			);
-		}
-
-		const embedded = anchorVectors.get(key);
-
-		await write(ConfigImportSection.Filters, filter.name, () =>
-			repositories.filterAnchor.put({
-				accountConfigId,
-				filterId,
-				anchorEmbedding: embedded?.embedding ?? [],
-				anchorEmbeddingId: embedded?.embeddingId ?? ANCHOR_EMBEDDING_PENDING,
-				anchorSourceText: anchor.sourceText,
-				anchorMessageId: anchor.sourceMessageId,
-			}),
-		);
-	}
-
 	// ---- address flags --------------------------------------------------
 	for (const entry of document.addressFlags) {
 		await write(ConfigImportSection.AddressFlags, entry.normalizedEmail, () =>
@@ -699,11 +628,10 @@ const applyDocument = async (
 	for (const ref of unresolvedRefs) {
 		if (ref.kind === ConfigImportRefKind.FilterAction) continue;
 		warnings.push(
-			problem(
-				"folder_not_found_yet",
-				`"${ref.folderPath}" is not a folder this account holds yet. It is bound once the folder list has been read.`,
-				{ folderPath: ref.folderPath, accountId: ref.accountId },
-			),
+			problem("folder_not_found_yet", folderPendingMessage(ref.folderPath), {
+				folderPath: ref.folderPath,
+				accountId: ref.accountId,
+			}),
 		);
 	}
 
@@ -861,6 +789,7 @@ class SettingWriter {
 					accountId,
 					folderPath: role.folderPath,
 					target: role.role,
+					mailboxId: NO_ACTION,
 				});
 				continue;
 			}
@@ -881,6 +810,7 @@ class SettingWriter {
 					accountId,
 					folderPath: override.folderPath,
 					target: override.folderPath,
+					mailboxId: NO_ACTION,
 				});
 				continue;
 			}
@@ -914,8 +844,8 @@ class SettingWriter {
  *
  * The order is fixed: parse and migrate, check every reference and every
  * natural key against what this configuration already holds, and only then
- * write — inside one transaction, accounts before labels before filters before
- * their anchors before addresses before settings, because each of those reads
+ * write — accounts before labels before filters before
+ * addresses before settings, because each of those reads
  * an id the one before it minted.
  *
  * A document this reader will not import is the operation's expected outcome,
@@ -979,19 +909,16 @@ export const importConfig = async (
 		};
 	}
 
-	// Before the transaction: an embed is a network round trip, and the write
-	// queue it would otherwise hold is process-wide.
-	const embedWarnings: ConfigImportProblem[] = [];
-	const anchorVectors = await embedAnchors(deps, document, embedWarnings);
-
-	const { transaction } = deps;
-	const atomic = transaction !== undefined;
 	const progress = emptyProgress();
-	const apply = (): Promise<string> =>
-		applyDocument(deps, input, document, existing, anchorVectors, progress);
 
 	try {
-		const importId = await (transaction ? transaction(apply) : apply());
+		const importId = await applyDocument(
+			deps,
+			input,
+			document,
+			existing,
+			progress,
+		);
 		return {
 			outcome: "report",
 			report: {
@@ -1001,16 +928,17 @@ export const importConfig = async (
 				applied: true,
 				items: progress.items,
 				errors: [],
-				warnings: [...embedWarnings, ...progress.warnings],
+				warnings: progress.warnings,
 				accountsNeedingCredentials: progress.accountsNeedingCredentials,
 			},
 		};
 	} catch (error) {
 		if (!(error instanceof ImportWriteError)) throw error;
 		// What the report owes the reader is where it stopped and what is now in
-		// the store. Under a transaction the answer to the second is "nothing";
-		// without one it is everything recorded up to the failure.
-		const landed = atomic ? emptyProgress() : progress;
+		// the store: everything recorded up to the failure. Every write is keyed on
+		// a natural key, so importing the same file again with merge folds the
+		// rest in over what landed.
+		const landed = progress;
 		return {
 			outcome: "report",
 			report: {
@@ -1029,15 +957,11 @@ export const importConfig = async (
 				errors: [
 					problem(
 						"import_write_failed",
-						`${error.message} ${
-							atomic
-								? "Nothing was written; the import stopped here."
-								: "The items above it were written and remain; the import stopped here."
-						}`,
+						`${error.message} The items above it were written and remain; import the same file again with merge to finish.`,
 						{ section: error.section, key: error.key },
 					),
 				],
-				warnings: [...embedWarnings, ...landed.warnings],
+				warnings: landed.warnings,
 				accountsNeedingCredentials: landed.accountsNeedingCredentials,
 			},
 		};
@@ -1102,10 +1026,19 @@ const plan = (
 		});
 	}
 	for (const filter of document.filters) {
+		const { actionFolder } = filter;
+		const awaiting =
+			actionFolder !== null &&
+			!foldersFor(existing, liveAccountId, actionFolder.accountId).has(
+				actionFolder.folderPath,
+			);
 		items.push({
 			section: ConfigImportSection.Filters,
 			key: filter.name,
 			verdict: verdict(heldFilters.has(normalize(filter.name))),
+			...(awaiting && actionFolder
+				? { reason: awaitingFolderReason(actionFolder.folderPath) }
+				: {}),
 		});
 	}
 	for (const entry of document.addressFlags) {
@@ -1139,7 +1072,7 @@ const plan = (
 		warnings.push(
 			problem(
 				"folder_not_found_yet",
-				`Filter "${filter.name}" files mail into "${actionFolder.folderPath}", which this account does not hold yet. It is bound once the folder list has been read.`,
+				filterFolderPendingMessage(filter.name, actionFolder.folderPath),
 				{ filter: filter.name, folderPath: actionFolder.folderPath },
 			),
 		);
@@ -1155,11 +1088,10 @@ const plan = (
 		for (const folderPath of paths) {
 			if (folders.has(folderPath)) continue;
 			warnings.push(
-				problem(
-					"folder_not_found_yet",
-					`"${folderPath}" is not a folder this account holds yet. It is bound once the folder list has been read.`,
-					{ folderPath, accountId },
-				),
+				problem("folder_not_found_yet", folderPendingMessage(folderPath), {
+					folderPath,
+					accountId,
+				}),
 			);
 		}
 	}

package/src/index.ts

@@ -3,6 +3,8 @@ export {
 	bindImportedFolders,
 	type ConfigBinderDeps,
 	type ConfigBinderRepositories,
+	type CreateImportedFolder,
+	disableFiltersMissingFolders,
 	pendingImportOf,
 } from "./binder.js";
 export {
@@ -11,7 +13,6 @@ export {
 	readConfigForExport,
 } from "./export.js";
 export {
-	ANCHOR_EMBEDDING_PENDING,
 	type ImportConfigInput,
 	importConfig,
 } from "./import.js";
@@ -19,7 +20,6 @@ export type {
 	AppointFolderRole,
 	ConfigImportDeps,
 	ConfigImportRepositories,
-	EmbedAnchor,
 } from "./import-repositories.js";
 export {
 	type ConfigImportConflict,

package/src/repositories.ts

@@ -3,7 +3,6 @@ import type {
 	IAccountRepository,
 	IAccountSettingRepository,
 	IAddressRepository,
-	IFilterAnchorRepository,
 	IFilterRepository,
 	ILabelRepository,
 	IMailboxRepository,
@@ -24,7 +23,6 @@ export interface ConfigExportRepositories {
 	mailbox: Pick<IMailboxRepository, "listAllByAccount">;
 	label: Pick<ILabelRepository, "listByAccountConfig">;
 	filter: Pick<IFilterRepository, "listByAccountConfig">;
-	filterAnchor: Pick<IFilterAnchorRepository, "listByAccountConfig">;
 	/**
 	 * Not the suggest listing: that one hides an address the machine marked
 	 * `junkOnly` unless the account has corresponded with it, which is right for