emdash 0.39.0 → 0.40.0

package/src/transfer/import/rebuild.ts

@@ -0,0 +1,300 @@
+/**
+ * The `rebuild` stage: derived state the importer does not write row by row.
+ *
+ * - `search`: full-text indexes for collections whose package search config
+ *   is enabled, populated in rowid ranges, then the collection's final
+ *   `search_config` (SQLite only; the plan declares `search_unsupported`
+ *   elsewhere).
+ * - `media_usage`: every imported collection is marked stale so the
+ *   media-usage maintenance engine re-indexes it.
+ * - `options`: `POST_IMPORT_OPTION_RESETS`.
+ * - `taxonomies`: each taxonomy's structure group, rebuilt from its imported
+ *   definitions; a group whose name no definition has any more is deleted.
+ * - `caches`: isolate caches and every object-cache namespace.
+ */
+
+import { sql } from "kysely";
+
+import { isSqlite } from "../../database/dialect-helpers.js";
+import { parseTaxonomyCollections } from "../../database/repositories/taxonomy-def.js";
+import { markContentMediaUsageCollectionStale } from "../../media/usage/content-refresh.js";
+import {
+	CacheNamespace,
+	invalidateCollectionCache,
+	invalidateObjectCache,
+} from "../../object-cache/index.js";
+import { resetRegisteredCollectionsCache } from "../../schema/collection-slugs-cache.js";
+import { invalidateSchemaCache } from "../../schema/zod-generator.js";
+import { FTSManager } from "../../search/fts-manager.js";
+import type { SearchTokenizer } from "../../search/types.js";
+import { SEARCH_TOKENIZERS } from "../../search/types.js";
+import { invalidateSiteSettingsCache } from "../../settings/index.js";
+import { TransferError } from "../errors.js";
+import { compareIds, type CollectionRecord } from "../format/kinds.js";
+import { POST_IMPORT_OPTION_RESETS } from "../format/settings.js";
+import { applyTransformations } from "../format/transformations.js";
+import { REBUILD_STEPS, type RebuildStep } from "../ops/states.js";
+import type { ImportContext } from "./context.js";
+import { ulidFromHash } from "./ids.js";
+
+const FTS_ROWS_PER_UNIT = 500;
+const FTS_SETUP_STATEMENTS = 20;
+const FTS_POPULATE_STATEMENTS = 4;
+const STALE_STATEMENTS = 4;
+const OPTION_STATEMENTS = 4;
+const TAXONOMY_STATEMENTS = 4;
+/** Cache invalidation runs no queries; its checkpoint does. */
+const CACHE_STATEMENTS = 1;
+
+export interface RebuildPosition {
+	step: RebuildStep;
+	after: string | null;
+}
+
+export type RebuildCheckpoint = (position: RebuildPosition) => Promise<void>;
+
+/** Run rebuild units from `position`; returns null when the stage is done. */
+export async function runRebuild(
+	context: ImportContext,
+	start: RebuildPosition,
+	checkpoint: RebuildCheckpoint,
+): Promise<RebuildPosition | null> {
+	let position: RebuildPosition | null = start;
+	while (position) {
+		const next: RebuildPosition | null | "budget" = await runUnit(context, position);
+		if (next === "budget") return position;
+		if (next) await checkpoint(next);
+		position = next;
+	}
+	return null;
+}
+
+function nextStep(step: RebuildStep): RebuildPosition | null {
+	const following = REBUILD_STEPS[REBUILD_STEPS.indexOf(step) + 1];
+	return following ? { step: following, after: null } : null;
+}
+
+async function collections(context: ImportContext): Promise<CollectionRecord[]> {
+	return (await context.allRecords("collection")).toSorted((a, b) => compareIds(a.slug, b.slug));
+}
+
+async function runUnit(
+	context: ImportContext,
+	position: RebuildPosition,
+): Promise<RebuildPosition | null | "budget"> {
+	switch (position.step) {
+		case "search":
+			return searchUnit(context, position.after);
+		case "media_usage": {
+			const collection = (await collections(context)).find(
+				(candidate) => position.after === null || compareIds(candidate.slug, position.after) > 0,
+			);
+			if (!collection) return nextStep("media_usage");
+			if (!context.budget.canStart({ queries: STALE_STATEMENTS })) return "budget";
+			context.budget.start();
+			await markContentMediaUsageCollectionStale(
+				context.db,
+				collection.slug,
+				"CONTENT_USAGE_STALE",
+			);
+			return { step: "media_usage", after: collection.slug };
+		}
+		case "options": {
+			if (!context.budget.canStart({ queries: OPTION_STATEMENTS })) return "budget";
+			context.budget.start();
+			for (const name of POST_IMPORT_OPTION_RESETS.bumpVersion) {
+				await sql`
+					INSERT INTO options (name, value)
+					VALUES (${name}, '2')
+					ON CONFLICT(name) DO UPDATE SET value = CASE
+						WHEN CAST(options.value AS INTEGER) % 2 = 0
+							THEN CAST(CAST(options.value AS INTEGER) + 2 AS TEXT)
+						ELSE CAST(CAST(options.value AS INTEGER) + 1 AS TEXT)
+					END
+				`.execute(context.db);
+			}
+			await context.db
+				.deleteFrom("options")
+				.where("name", "in", [...POST_IMPORT_OPTION_RESETS.delete])
+				.execute();
+			return nextStep("options");
+		}
+		case "taxonomies":
+			return taxonomyUnit(context, position.after);
+		case "caches":
+			if (!context.budget.canStart({ queries: CACHE_STATEMENTS })) return "budget";
+			context.budget.start();
+			await invalidateCaches(context);
+			return nextStep("caches");
+	}
+}
+
+function isTokenizer(value: unknown): value is SearchTokenizer {
+	return SEARCH_TOKENIZERS.some((tokenizer) => tokenizer === value);
+}
+
+interface SearchTarget {
+	record: CollectionRecord;
+	config: Record<string, unknown>;
+}
+
+async function searchTargets(context: ImportContext): Promise<SearchTarget[]> {
+	const targets: SearchTarget[] = [];
+	for (const record of await collections(context)) {
+		const final = applyTransformations(record, context.plan, { fieldColumnTypes: new Map() });
+		const config = final.searchConfig;
+		if (typeof config !== "object" || config === null || Array.isArray(config)) continue;
+		if (config.enabled !== true) continue;
+		if (!isSqlite(context.db)) {
+			throw new TransferError(
+				"TRANSFER_IMPORT_ERROR",
+				"Full-text search is not available on this target",
+				{ detail: { kind: "collection", id: record.id } },
+			);
+		}
+		targets.push({ record, config });
+	}
+	return targets;
+}
+
+/** `after`: null, `slug` (done), or `slug:rowid` (index created, rows ≤ rowid indexed). */
+async function searchUnit(
+	context: ImportContext,
+	after: string | null,
+): Promise<RebuildPosition | null | "budget"> {
+	const [doneSlug, rowidText] = after?.split(":") ?? [];
+	const inProgress = rowidText === undefined ? null : Number(rowidText);
+	const target = (await searchTargets(context)).find((candidate) =>
+		inProgress === null
+			? doneSlug === undefined || compareIds(candidate.record.slug, doneSlug) > 0
+			: candidate.record.slug === doneSlug,
+	);
+	if (!target) return nextStep("search");
+	const slug = target.record.slug;
+	const fts = new FTSManager(context.db);
+
+	if (inProgress === null) {
+		if (!context.budget.canStart({ queries: FTS_SETUP_STATEMENTS })) return "budget";
+		context.budget.start();
+		const fields = await fts.getSearchableFields(slug);
+		if (fields.length > 0) {
+			const weights = target.config.weights;
+			await fts.dropFtsTable(slug);
+			await fts.createFtsTable(
+				slug,
+				fields,
+				typeof weights === "object" && weights !== null
+					? Object.fromEntries(
+							Object.entries(weights).filter(
+								(entry): entry is [string, number] => typeof entry[1] === "number",
+							),
+						)
+					: undefined,
+				isTokenizer(target.config.tokenize) ? target.config.tokenize : undefined,
+			);
+		}
+		return { step: "search", after: `${slug}:0` };
+	}
+
+	if (!context.budget.canStart({ queries: FTS_POPULATE_STATEMENTS })) return "budget";
+	context.budget.start();
+	const fields = await fts.getSearchableFields(slug);
+	const last =
+		fields.length > 0 ? await fts.populateRange(slug, fields, inProgress, FTS_ROWS_PER_UNIT) : null;
+	if (last !== null) return { step: "search", after: `${slug}:${last}` };
+	await context.db
+		.updateTable("_emdash_collections")
+		.set({ search_config: JSON.stringify(target.config) })
+		.where("id", "=", target.record.id)
+		.execute();
+	return { step: "search", after: slug };
+}
+
+/**
+ * `after`: the last taxonomy name whose group was rebuilt, or null.
+ *
+ * The group becomes hierarchical if any definition is and lists every collection
+ * any definition lists. The definitions keep the package's values, which the
+ * verify stage compares; they differ from the group only where the exporting
+ * site's locales disagreed.
+ */
+async function taxonomyUnit(
+	context: ImportContext,
+	after: string | null,
+): Promise<RebuildPosition | null | "budget"> {
+	if (!context.budget.canStart({ queries: TAXONOMY_STATEMENTS })) return "budget";
+	context.budget.start();
+	const db = context.db;
+	let nextName = db
+		.selectFrom("_emdash_taxonomy_defs")
+		.select((eb) => eb.fn.min("name").as("name"));
+	if (after !== null) nextName = nextName.where("name", ">", after);
+	const defs = await db
+		.selectFrom("_emdash_taxonomy_defs")
+		.select(["id", "name", "hierarchical", "collections", "translation_group"])
+		.where("name", "=", nextName)
+		.orderBy("id")
+		.execute();
+	const name = defs[0]?.name;
+	if (name === undefined) {
+		await db
+			.deleteFrom("_emdash_taxonomy_def_groups")
+			.where((eb) =>
+				eb.not(
+					eb.exists(
+						eb
+							.selectFrom("_emdash_taxonomy_defs")
+							.select(sql`1`.as("one"))
+							.whereRef("_emdash_taxonomy_defs.name", "=", "_emdash_taxonomy_def_groups.name"),
+					),
+				),
+			)
+			.execute();
+		return nextStep("taxonomies");
+	}
+
+	const groupId = defs
+		.map((def) => def.translation_group ?? def.id)
+		.reduce((lowest, candidate) => (candidate < lowest ? candidate : lowest));
+	const collectionSlugs = [
+		...new Set(defs.flatMap((def) => parseTaxonomyCollections(def.collections))),
+	];
+	const hierarchical = defs.some((def) => def.hierarchical === 1) ? 1 : 0;
+	const taken = await db
+		.selectFrom("_emdash_taxonomy_def_groups")
+		.select("id")
+		.where("id", "=", groupId)
+		.where("name", "!=", name)
+		.executeTakeFirst();
+	const id = taken ? await ulidFromHash(context.operationId, "taxonomy_group", name) : groupId;
+	await db
+		.insertInto("_emdash_taxonomy_def_groups")
+		.values({ id, name, hierarchical, collections: JSON.stringify(collectionSlugs) })
+		.onConflict((oc) =>
+			oc
+				.column("name")
+				.doUpdateSet({ id, hierarchical, collections: JSON.stringify(collectionSlugs) }),
+		)
+		.execute();
+	return { step: "taxonomies", after: name };
+}
+
+async function invalidateCaches(context: ImportContext): Promise<void> {
+	for (const namespace of Object.values(CacheNamespace)) invalidateObjectCache(namespace);
+	for (const collection of await collections(context)) {
+		invalidateCollectionCache(collection.slug);
+		invalidateSchemaCache(collection.slug);
+	}
+	invalidateSiteSettingsCache();
+	resetRegisteredCollectionsCache();
+	const { invalidateTermCache, invalidateTaxonomyDefsCache } =
+		await import("../../taxonomies/index.js");
+	const { invalidateBylineCache } = await import("../../bylines/index.js");
+	const { invalidateRedirectCache } = await import("../../redirects/cache.js");
+	const { invalidateUrlPatternCache } = await import("../../query.js");
+	invalidateTermCache();
+	invalidateTaxonomyDefsCache();
+	invalidateBylineCache();
+	invalidateRedirectCache();
+	invalidateUrlPatternCache();
+}

package/src/transfer/import/rows.ts

@@ -0,0 +1,114 @@
+/**
+ * Record ↔ row helpers for the importer over the shared column codec
+ * (`format/column-codec.ts`).
+ *
+ * Stored rows are compared with the rows the importer meant to write after
+ * decoding both sides with the same codec, so dialect differences (Postgres
+ * `json` columns returning parsed values, `bigint` strings) do not register as
+ * mismatches. Postgres stores REAL columns as float4, so REAL values are
+ * compared after `Math.fround` there.
+ */
+
+import type { Kysely } from "kysely";
+
+import { isPostgres } from "../../database/dialect-helpers.js";
+import type { Database } from "../../database/types.js";
+import { isTransferError, TransferError } from "../errors.js";
+import { canonicalJson } from "../format/canonical.js";
+import { decodeColumn, encodeColumn } from "../format/column-codec.js";
+import type { ColumnCodec, ColumnSpec } from "../format/columns.js";
+
+export type Row = Record<string, unknown>;
+
+/** Codec per column, for the columns a comparison covers. */
+export type RowCodecs = Readonly<Record<string, ColumnCodec>>;
+
+/** Codecs of the columns a record carries (`field`, `principal`, `mediaRef`). */
+export function recordCodecs(columns: Readonly<Record<string, ColumnSpec>>): RowCodecs {
+	const codecs: Record<string, ColumnCodec> = {};
+	for (const [column, spec] of Object.entries(columns)) {
+		if ("property" in spec) codecs[column] = spec.codec;
+	}
+	return codecs;
+}
+
+/**
+ * The row for `record` over every column the record carries. `record` must
+ * already be in target form: placeholders resolved and principals mapped.
+ */
+export function encodeRecord(
+	db: Kysely<Database>,
+	columns: Readonly<Record<string, ColumnSpec>>,
+	record: Readonly<Record<string, unknown>>,
+	location: { kind: string; id: string },
+): Row {
+	const row: Row = {};
+	for (const [column, spec] of Object.entries(columns)) {
+		if (!("property" in spec)) continue;
+		row[column] = encodeValue(
+			db,
+			spec.codec,
+			Object.hasOwn(record, spec.property) ? record[spec.property] : undefined,
+			{ ...location, column },
+		);
+	}
+	return row;
+}
+
+/** `encodeColumn`, with the record and column named in a failure. */
+export function encodeValue(
+	db: Kysely<Database>,
+	codec: ColumnCodec,
+	value: unknown,
+	location: { kind: string; id: string; column: string },
+	options: { notNull?: boolean } = {},
+): unknown {
+	try {
+		return encodeColumn(db, codec, value, options);
+	} catch (error) {
+		if (!isTransferError(error)) throw error;
+		throw new TransferError(error.code, "Record value does not fit its target column", {
+			detail: { ...location, ...error.detail },
+			cause: error,
+		});
+	}
+}
+
+function comparable(
+	db: Kysely<Database>,
+	codec: ColumnCodec,
+	raw: unknown,
+	options: { float4: boolean; encoded: boolean },
+): string | undefined {
+	const stored =
+		options.encoded && codec === "nativeJson" && isPostgres(db) && typeof raw === "string"
+			? JSON.parse(raw)
+			: raw;
+	const value = decodeColumn(db, codec, stored);
+	if (value === undefined) return undefined;
+	if (options.float4 && codec === "real" && typeof value === "number") {
+		return canonicalJson(Math.fround(value));
+	}
+	return canonicalJson(value);
+}
+
+/**
+ * The first column (in `codecs` order) where the stored row differs from the
+ * row the importer encoded, or null when they are equal over those columns.
+ * Postgres returns `json` columns parsed, so an encoded `nativeJson` value is
+ * parsed before it is compared.
+ */
+export function firstDifference(
+	db: Kysely<Database>,
+	codecs: RowCodecs,
+	encoded: Row,
+	stored: Row,
+	options: { float4: boolean },
+): string | null {
+	for (const [column, codec] of Object.entries(codecs)) {
+		const want = comparable(db, codec, encoded[column], { ...options, encoded: true });
+		const have = comparable(db, codec, stored[column], { ...options, encoded: false });
+		if (want !== have) return column;
+	}
+	return null;
+}

package/src/transfer/import/scaffold.ts

@@ -0,0 +1,221 @@
+/**
+ * The `clear_scaffold` stage: delete the seeded scaffold the plan declared
+ * (`seeded_scaffold_removed`), one item per unit, then wait until
+ * media-usage collection deletions of removed collections have finished so
+ * the package may reuse their slugs.
+ */
+
+import { sql } from "kysely";
+
+import { processDueMediaUsageCollectionDeletions } from "../../media/usage/collection-deletion-processor.js";
+import { SchemaError, SchemaRegistry } from "../../schema/registry.js";
+import { TransferError } from "../errors.js";
+import type { ImportTransformation } from "../format/transformations.js";
+import type { ImportContext } from "./context.js";
+
+export type ScaffoldItemRef = Extract<
+	ImportTransformation,
+	{ code: "seeded_scaffold_removed" }
+>["items"][number];
+
+const ITEM_STATEMENTS = 40;
+
+export function declaredScaffold(context: ImportContext): readonly ScaffoldItemRef[] {
+	return context.plan.transformations.flatMap((transformation) =>
+		transformation.code === "seeded_scaffold_removed" ? transformation.items : [],
+	);
+}
+
+export type ScaffoldProgress =
+	| { state: "continue"; step: number }
+	| { state: "wait"; step: number }
+	| { state: "done" };
+
+/**
+ * Delete declared items from `step` on. Each item is removed only if it is
+ * still the scaffold it was when analyzed (an unassigned term, an empty menu,
+ * an empty widget area, a theme section, a seeded block type); deleting an already-deleted item
+ * is a no-op.
+ */
+export async function clearScaffold(
+	context: ImportContext,
+	step: number,
+	checkpoint: (step: number) => Promise<void>,
+): Promise<ScaffoldProgress> {
+	const items = declaredScaffold(context);
+	let next = step;
+	while (next < items.length) {
+		const item = items[next];
+		if (!item) break;
+		if (!context.budget.canStart({ queries: ITEM_STATEMENTS })) {
+			return { state: "continue", step: next };
+		}
+		context.budget.start();
+		await removeItem(context, item);
+		next++;
+		await checkpoint(next);
+	}
+
+	const collectionIds = items.flatMap((item) => (item.type === "collection" ? [item.id] : []));
+	while (collectionIds.length > 0) {
+		if (!context.budget.canStart({ queries: ITEM_STATEMENTS })) {
+			return { state: "continue", step: next };
+		}
+		context.budget.start();
+		const pending = await context.db
+			.selectFrom("_emdash_media_usage_collection_deletions")
+			.select("collection_id")
+			.where("collection_id", "in", collectionIds)
+			.execute();
+		if (pending.length === 0) break;
+		const tick = await processDueMediaUsageCollectionDeletions(context.db);
+		if (tick.outcome === "idle" || tick.outcome === "claim_lost" || tick.outcome === "retry") {
+			return { state: "wait", step: next };
+		}
+		if (tick.outcome === "failed") {
+			throw new TransferError(
+				"TRANSFER_IMPORT_ERROR",
+				"Removing the seeded scaffold collections failed",
+			);
+		}
+	}
+	return { state: "done" };
+}
+
+async function removeItem(context: ImportContext, item: ScaffoldItemRef): Promise<void> {
+	const db = context.db;
+	switch (item.type) {
+		case "collection": {
+			const row =
+				(await db
+					.selectFrom("_emdash_collections")
+					.select("slug")
+					.where("id", "=", item.id)
+					.executeTakeFirst()) ??
+				(await db
+					.selectFrom("_emdash_media_usage_collection_deletions")
+					.select("collection_slug as slug")
+					.where("collection_id", "=", item.id)
+					.executeTakeFirst());
+			if (!row) return;
+			try {
+				await new SchemaRegistry(db).deleteCollection(row.slug);
+			} catch (error) {
+				if (error instanceof SchemaError && error.code === "COLLECTION_NOT_FOUND") return;
+				if (error instanceof SchemaError) {
+					throw new TransferError("TRANSFER_TARGET_NOT_EMPTY", "Scaffold collection is in use", {
+						detail: { type: item.type, id: item.id, reason: error.code },
+						cause: error,
+					});
+				}
+				throw error;
+			}
+			return;
+		}
+		case "term":
+			await db
+				.deleteFrom("taxonomies")
+				.where("id", "=", item.id)
+				.where((eb) =>
+					eb.not(
+						eb.exists(
+							eb
+								.selectFrom("content_taxonomies")
+								.select(sql`1`.as("one"))
+								.where(
+									"content_taxonomies.taxonomy_id",
+									"=",
+									eb.fn.coalesce("taxonomies.translation_group", "taxonomies.id"),
+								),
+						),
+					),
+				)
+				.execute();
+			return;
+		case "menu_item":
+			await db.deleteFrom("_emdash_menu_items").where("id", "=", item.id).execute();
+			return;
+		case "widget":
+			await db.deleteFrom("_emdash_widgets").where("id", "=", item.id).execute();
+			return;
+		case "taxonomy_def":
+			await db
+				.deleteFrom("_emdash_taxonomy_defs")
+				.where("id", "=", item.id)
+				.where((eb) =>
+					eb.not(
+						eb.exists(
+							eb
+								.selectFrom("taxonomies")
+								.select(sql`1`.as("one"))
+								.whereRef("taxonomies.name", "=", "_emdash_taxonomy_defs.name"),
+						),
+					),
+				)
+				.execute();
+			return;
+		case "menu":
+			await db
+				.deleteFrom("_emdash_menus")
+				.where("id", "=", item.id)
+				.where((eb) =>
+					eb.not(
+						eb.exists(
+							eb
+								.selectFrom("_emdash_menu_items")
+								.select(sql`1`.as("one"))
+								.whereRef("_emdash_menu_items.menu_id", "=", "_emdash_menus.id"),
+						),
+					),
+				)
+				.execute();
+			return;
+		case "widget_area":
+			await db
+				.deleteFrom("_emdash_widget_areas")
+				.where("id", "=", item.id)
+				.where((eb) =>
+					eb.not(
+						eb.exists(
+							eb
+								.selectFrom("_emdash_widgets")
+								.select(sql`1`.as("one"))
+								.whereRef("_emdash_widgets.area_id", "=", "_emdash_widget_areas.id"),
+						),
+					),
+				)
+				.execute();
+			return;
+		case "section":
+			await db
+				.deleteFrom("_emdash_sections")
+				.where("id", "=", item.id)
+				.where("source", "=", "theme")
+				.execute();
+			return;
+		case "block_type":
+			await db
+				.deleteFrom("_emdash_block_type_versions")
+				.where("block_type_id", "=", item.id)
+				.where((eb) =>
+					eb.exists(
+						eb
+							.selectFrom("_emdash_block_types")
+							.select(sql`1`.as("one"))
+							.where("_emdash_block_types.id", "=", item.id)
+							.where("_emdash_block_types.source", "=", "seed"),
+					),
+				)
+				.execute();
+			await db
+				.deleteFrom("_emdash_block_types")
+				.where("id", "=", item.id)
+				.where("source", "=", "seed")
+				.execute();
+			return;
+		default:
+			throw new TransferError("TRANSFER_IMPORT_ERROR", "Unknown scaffold item type", {
+				detail: { type: item.type, id: item.id },
+			});
+	}
+}