emdash 0.40.0 → 0.40.1

package/src/seed/apply.ts

@@ -6,7 +6,15 @@
  */
 
 import { imageSize } from "image-size";
-import type { Kysely } from "kysely";
+import type {
+	Kysely,
+	KyselyPlugin,
+	PluginTransformQueryArgs,
+	PluginTransformResultArgs,
+	QueryResult,
+	RootOperationNode,
+	UnknownRow,
+} from "kysely";
 import mime from "mime/lite";
 import { ulid } from "ulidx";
 
@@ -23,6 +31,7 @@ import {
 	saveTaxonomyStructure,
 } from "../database/repositories/taxonomy-def.js";
 import { TaxonomyRepository } from "../database/repositories/taxonomy.js";
+import type { ContentItem } from "../database/repositories/types.js";
 import { withTransaction } from "../database/transaction.js";
 import type { Database } from "../database/types.js";
 import type { MediaValue } from "../fields/types.js";
@@ -37,10 +46,12 @@ import { FTSManager } from "../search/fts-manager.js";
 import { invalidateSiteSettingsCache, setSiteSettings } from "../settings/index.js";
 import type { SiteSettings } from "../settings/types.js";
 import type { Storage } from "../storage/types.js";
+import { chunks, SQL_BATCH_SIZE } from "../utils/chunks.js";
 import type {
 	SeedFile,
 	SeedApplyOptions,
 	SeedApplyResult,
+	SeedContentEntry,
 	SeedCollection,
 	SeedTaxonomy,
 	SeedTaxonomyTerm,
@@ -158,14 +169,90 @@ export async function applySeed(
 	seed: SeedFile,
 	options: SeedApplyOptions = {},
 ): Promise<SeedApplyResult> {
-	return coalesceObjectCacheWrites(() => applySeedWrites(db, seed, options));
+	const { result } = await coalesceObjectCacheWrites(() =>
+		applySeedWrites(db, seed, options, null),
+	);
+	return result;
+}
+
+/** Per-call limits for `applySeedWithinBudget`. A limit that is left out is not checked. */
+export interface SeedApplyBudget {
+	queries?: number;
+	mediaDownloads?: number;
+}
+
+/** How many of a seed's items (taxonomy terms, bylines, content entries) exist after a call. */
+export interface SeedApplyProgress {
+	done: number;
+	total: number;
+}
+
+/**
+ * Apply a seed over several calls, for a platform that caps the queries or
+ * fetches of one request.
+ *
+ * The limits are thresholds, checked before each taxonomy term, byline and
+ * content entry is created and before the phases that follow content. Schema,
+ * taxonomy definitions and settings are applied in every call, the item that
+ * crosses a threshold finishes, and the phases after content (menus,
+ * redirects, widgets, sections) run in one call, so set the limits well below
+ * the platform's. A call that stops returns `complete: false`; applying the
+ * same seed again continues with the first item not yet created. A call
+ * creates at least one item before it stops, so repeated calls finish. A
+ * `$media` URL used by entries in different calls is downloaded and stored
+ * again, as a new media row, in each of those calls. Requires
+ * `onConflict: "skip"`.
+ */
+export async function applySeedWithinBudget(
+	db: Kysely<Database>,
+	seed: SeedFile,
+	options: SeedApplyOptions,
+	limits: SeedApplyBudget,
+): Promise<{ result: SeedApplyResult; complete: boolean; progress: SeedApplyProgress }> {
+	const budget = new SeedBudget(limits);
+	return coalesceObjectCacheWrites(() =>
+		applySeedWrites(db.withPlugin(budget), seed, options, budget),
+	);
+}
+
+/**
+ * Counts what a budgeted call spends. As a Kysely plugin it sees every query
+ * the call runs, including those inside transactions.
+ */
+class SeedBudget implements KyselyPlugin {
+	#queries = 0;
+	#mediaDownloads = 0;
+
+	constructor(private readonly limits: SeedApplyBudget) {}
+
+	transformQuery(args: PluginTransformQueryArgs): RootOperationNode {
+		this.#queries++;
+		return args.node;
+	}
+
+	transformResult(args: PluginTransformResultArgs): Promise<QueryResult<UnknownRow>> {
+		return Promise.resolve(args.result);
+	}
+
+	countMediaDownload(): void {
+		this.#mediaDownloads++;
+	}
+
+	isSpent(): boolean {
+		const { queries, mediaDownloads } = this.limits;
+		return (
+			(queries !== undefined && this.#queries >= queries) ||
+			(mediaDownloads !== undefined && this.#mediaDownloads >= mediaDownloads)
+		);
+	}
 }
 
 async function applySeedWrites(
 	db: Kysely<Database>,
 	seed: SeedFile,
 	options: SeedApplyOptions,
-): Promise<SeedApplyResult> {
+	budget: SeedBudget | null,
+): Promise<{ result: SeedApplyResult; complete: boolean; progress: SeedApplyProgress }> {
 	// Validate seed first
 	const validation = validateSeed(seed);
 	if (!validation.valid) {
@@ -179,6 +266,10 @@ async function applySeedWrites(
 		onConflict = "skip",
 	} = options;
 
+	if (budget && onConflict !== "skip") {
+		throw new Error('A seed budget requires onConflict: "skip"');
+	}
+
 	// Result counters
 	const result: SeedApplyResult = {
 		blockTypes: { created: 0, skipped: 0, updated: 0 },
@@ -194,6 +285,13 @@ async function applySeedWrites(
 		content: { created: 0, skipped: 0, updated: 0 },
 		media: { created: 0, skipped: 0 },
 	};
+	let complete = true;
+	const progress: SeedApplyProgress = { done: 0, total: countSeedItems(seed, includeContent) };
+	// With a budget, `onConflict` is "skip", so these counters count creations only.
+	const mayCreateItem = (): boolean =>
+		!budget ||
+		result.taxonomies.terms + result.bylines.created + result.content.created === 0 ||
+		!budget.isSpent();
 
 	// Media context for $media resolution
 	const mediaContext: MediaContext = {
@@ -201,6 +299,7 @@ async function applySeedWrites(
 		storage: storage ?? null,
 		skipMediaDownload,
 		mediaCache: new Map(), // Cache downloaded media by URL to avoid re-downloading
+		budget,
 	};
 
 	// Apply order (critical for foreign keys and references):
@@ -482,36 +581,47 @@ async function applySeedWrites(
 			// Create terms (if provided)
 			if (includeContent && taxonomy.terms && taxonomy.terms.length > 0) {
 				const termRepo = new TaxonomyRepository(db);
+				const existingTerms = await findExistingSeedTerms(db, taxonomy.name, taxonomy.terms);
 
 				if (structure.hierarchical) {
-					await applyHierarchicalTerms(
+					const applied = await applyHierarchicalTerms(
 						termRepo,
 						taxonomy.name,
 						defLocale,
 						taxonomy.terms,
+						existingTerms,
 						termSeedIdMap,
 						result,
 						onConflict,
+						mayCreateItem,
 					);
+					progress.done += applied.processed;
+					complete = applied.complete;
 				} else {
 					for (const term of taxonomy.terms) {
 						const termLocale = resolveConfiguredLocale(term.locale ?? defLocale);
-						const existing = await termRepo.findBySlug(taxonomy.name, term.slug, termLocale);
-						if (existing) {
+						const termKey = seedTermKey(termLocale, term.slug);
+						const existingId = existingTerms.get(termKey);
+						if (existingId) {
 							if (onConflict === "error") {
 								throw new Error(
 									`Conflict: taxonomy term "${term.slug}" in "${taxonomy.name}" (${termLocale}) already exists`,
 								);
 							}
 							if (onConflict === "update") {
-								await termRepo.update(existing.id, {
+								await termRepo.update(existingId, {
 									label: term.label,
 									data: term.description ? { description: term.description } : {},
 								});
 								result.taxonomies.terms++;
 							}
-							if (term.id) termSeedIdMap.set(term.id, existing.id);
+							if (term.id) termSeedIdMap.set(term.id, existingId);
+							progress.done++;
 						} else {
+							if (!mayCreateItem()) {
+								complete = false;
+								break;
+							}
 							const translationOf = term.translationOf
 								? termSeedIdMap.get(term.translationOf)
 								: undefined;
@@ -523,12 +633,15 @@ async function applySeedWrites(
 								locale: termLocale,
 								translationOf,
 							});
+							existingTerms.set(termKey, created.id);
 							if (term.id) termSeedIdMap.set(term.id, created.id);
 							result.taxonomies.terms++;
+							progress.done++;
 						}
 					}
 				}
 			}
+			if (!complete) break;
 		}
 
 		// Seeded/updated defs change which taxonomies exist — clear the
@@ -539,11 +652,15 @@ async function applySeedWrites(
 	}
 
 	// 6. Bylines
-	if (includeContent && seed.bylines) {
+	if (complete && includeContent && seed.bylines) {
 		const bylineRepo = new BylineRepository(db);
+		const existingBylines = await findExistingSeedBylines(
+			db,
+			seed.bylines.map((byline) => byline.slug),
+		);
 		for (const byline of seed.bylines) {
-			const existing = await bylineRepo.findBySlug(byline.slug);
-			if (existing) {
+			const existingId = existingBylines.get(byline.slug);
+			if (existingId) {
 				if (onConflict === "error") {
 					throw new Error(`Conflict: byline "${byline.slug}" already exists`);
 				}
@@ -555,7 +672,7 @@ async function applySeedWrites(
 					// one untouched.
 					const avatar = byline.avatar ? await resolveSeedBylineAvatar(db, byline.avatar) : null;
 					try {
-						const updated = await bylineRepo.update(existing.id, {
+						const updated = await bylineRepo.update(existingId, {
 							displayName: byline.displayName,
 							bio: byline.bio ?? null,
 							websiteUrl: byline.websiteUrl ?? null,
@@ -563,7 +680,7 @@ async function applySeedWrites(
 							...(avatar ? { avatarMediaId: avatar.id } : {}),
 						});
 						// update() returns null (no throw) if the row vanished between
-						// findBySlug and here; treat that as a failure so the catch
+						// the lookup and here; treat that as a failure so the catch
 						// cleans up any freshly-created avatar media instead of leaking it.
 						if (!updated) {
 							throw new Error(`Byline "${byline.slug}" disappeared during update`);
@@ -574,18 +691,24 @@ async function applySeedWrites(
 						if (avatar?.created) await deleteMediaRow(db, avatar.id);
 						throw error;
 					}
-					seedBylineIdMap.set(byline.id, existing.id);
+					seedBylineIdMap.set(byline.id, existingId);
 					result.bylines.updated++;
 					if (avatar?.created) result.media.created++;
+					progress.done++;
 					continue;
 				}
 
 				// skip
-				seedBylineIdMap.set(byline.id, existing.id);
+				seedBylineIdMap.set(byline.id, existingId);
 				result.bylines.skipped++;
+				progress.done++;
 				continue;
 			}
 
+			if (!mayCreateItem()) {
+				complete = false;
+				break;
+			}
 			const avatar = byline.avatar ? await resolveSeedBylineAvatar(db, byline.avatar) : null;
 			let createdId: string;
 			try {
@@ -602,14 +725,16 @@ async function applySeedWrites(
 				if (avatar?.created) await deleteMediaRow(db, avatar.id);
 				throw error;
 			}
+			existingBylines.set(byline.slug, createdId);
 			seedBylineIdMap.set(byline.id, createdId);
 			result.bylines.created++;
 			if (avatar?.created) result.media.created++;
+			progress.done++;
 		}
 	}
 
 	// 7. Content (created before menus so refs can resolve)
-	if (includeContent && seed.content) {
+	if (complete && includeContent && seed.content) {
 		const contentRepo = new ContentRepository(db);
 		const schemaRegistry = new SchemaRegistry(db);
 		// Settings and fields are all written above, so every entry can share the
@@ -624,19 +749,16 @@ async function applySeedWrites(
 				const resolvedBlockTypes = collectionInfo?.fields.some((field) => field.type === "blocks")
 					? await resolveBlockTypes(db)
 					: undefined;
+				const existingEntries = await findExistingSeedEntries(
+					contentRepo,
+					collectionSlug,
+					entries,
+					defaultLocale,
+				);
 				for (const entry of entries) {
-					const entrySlug =
-						typeof entry.slug === "string" && entry.slug.trim().length > 0 ? entry.slug : null;
-					// Resolve the entry's locale up front so a non-`en` single-locale
-					// export (which omits `locale`) is filed under the project default
-					// rather than `en` (#1421).
-					const entryLocale = resolveConfiguredLocale(entry.locale ?? defaultLocale);
-
-					// Slugful entries use the existing locale-aware key. Slugless seed
-					// entries persist their seed ID, which keeps re-application idempotent.
-					const existing = entrySlug
-						? await contentRepo.findBySlug(collectionSlug, entrySlug, entryLocale)
-						: await contentRepo.findById(collectionSlug, entry.id);
+					const { slug: entrySlug, locale: entryLocale } = seedEntryIdentity(entry, defaultLocale);
+					const entryKey = seedEntryKey(entry, entrySlug, entryLocale);
+					const existing = existingEntries.get(entryKey);
 
 					if (existing) {
 						if (onConflict === "error") {
@@ -738,6 +860,7 @@ async function applySeedWrites(
 
 							seedIdMap.set(entry.id, existing.id);
 							result.content.updated++;
+							progress.done++;
 							await markSeedContentCollectionStale(collectionSlug);
 							continue;
 						}
@@ -745,9 +868,15 @@ async function applySeedWrites(
 						// skip
 						result.content.skipped++;
 						seedIdMap.set(entry.id, existing.id);
+						progress.done++;
 						continue;
 					}
 
+					if (!mayCreateItem()) {
+						complete = false;
+						break;
+					}
+
 					// Resolve $ref and $media in data
 					let resolvedData = await resolveReferences(entry.data, seedIdMap, mediaContext, result);
 					if (collectionInfo) {
@@ -828,15 +957,26 @@ async function applySeedWrites(
 					}
 
 					seedIdMap.set(entry.id, created.id);
+					existingEntries.set(entryKey, created);
 					result.content.created++;
+					progress.done++;
 					await markSeedContentCollectionStale(collectionSlug);
 				}
+				if (!complete) break;
 			}
 		} finally {
 			await retryFailedSeedContentStaleMarks();
 		}
 	}
 
+	if (complete && !mayCreateItem()) {
+		complete = false;
+	}
+	if (!complete) {
+		await invalidateSeedCaches();
+		return { result, complete, progress };
+	}
+
 	// 8. Menus and Menu Items (after content so refs can resolve)
 	if (seed.menus) {
 		// seed-local id -> resolved info, used to wire `translationOf` refs.
@@ -1064,31 +1204,165 @@ async function applySeedWrites(
 		}
 	}
 
-	// Invalidate caches that may have been affected by seed data.
-	// Seed creates bylines, redirects, and collections, all of which
-	// have module-level caches in the hot path.
+	await invalidateSeedCaches();
+
+	return { result, complete, progress };
+}
+
+function countSeedItems(seed: SeedFile, includeContent: boolean): number {
+	if (!includeContent) return 0;
+	const terms = (seed.taxonomies ?? []).reduce(
+		(total, taxonomy) => total + (taxonomy.terms?.length ?? 0),
+		0,
+	);
+	const entries = Object.values(seed.content ?? {}).reduce(
+		(total, collectionEntries) => total + collectionEntries.length,
+		0,
+	);
+	return terms + (seed.bylines?.length ?? 0) + entries;
+}
+
+function seedTermKey(locale: string, slug: string): string {
+	return `${locale}::${slug}`;
+}
+
+/**
+ * Find which of a taxonomy's seed terms already exist, as term ids keyed by
+ * `seedTermKey`. Each query covers a batch of slugs.
+ */
+async function findExistingSeedTerms(
+	db: Kysely<Database>,
+	taxonomyName: string,
+	terms: SeedTaxonomyTerm[],
+): Promise<Map<string, string>> {
+	const existing = new Map<string, string>();
+	for (const batch of chunks([...new Set(terms.map((term) => term.slug))], SQL_BATCH_SIZE)) {
+		const rows = await db
+			.selectFrom("taxonomies")
+			.select(["id", "slug", "locale"])
+			.where("name", "=", taxonomyName)
+			.where("slug", "in", batch)
+			.execute();
+		for (const row of rows) existing.set(seedTermKey(row.locale, row.slug), row.id);
+	}
+	return existing;
+}
+
+/**
+ * Find which seed bylines already exist, as byline ids keyed by slug. A slug
+ * in several locales resolves to the lowest locale code, as
+ * `BylineRepository.findBySlug` does. Each query covers a batch of slugs.
+ */
+async function findExistingSeedBylines(
+	db: Kysely<Database>,
+	slugs: string[],
+): Promise<Map<string, string>> {
+	const existing = new Map<string, string>();
+	for (const batch of chunks([...new Set(slugs)], SQL_BATCH_SIZE)) {
+		const rows = await db
+			.selectFrom("_emdash_bylines")
+			.select(["id", "slug"])
+			.where("slug", "in", batch)
+			.orderBy("locale", "asc")
+			.execute();
+		for (const row of rows) {
+			if (!existing.has(row.slug)) existing.set(row.slug, row.id);
+		}
+	}
+	return existing;
+}
+
+/**
+ * Invalidate caches that may have been affected by seed data.
+ * Seed creates bylines, redirects, and collections, all of which
+ * have module-level caches in the hot path.
+ */
+async function invalidateSeedCaches(): Promise<void> {
 	const { invalidateBylineCache } = await import("../bylines/index.js");
 	const { invalidateRedirectCache } = await import("../redirects/cache.js");
 	const { invalidateUrlPatternCache } = await import("../query.js");
 	invalidateBylineCache();
 	invalidateRedirectCache();
 	invalidateUrlPatternCache();
+}
 
-	return result;
+function seedEntryIdentity(
+	entry: SeedContentEntry,
+	defaultLocale: string,
+): { slug: string | null; locale: string } {
+	return {
+		slug: typeof entry.slug === "string" && entry.slug.trim().length > 0 ? entry.slug : null,
+		// Resolve the entry's locale up front so a non-`en` single-locale
+		// export (which omits `locale`) is filed under the project default
+		// rather than `en`.
+		locale: resolveConfiguredLocale(entry.locale ?? defaultLocale),
+	};
 }
 
 /**
- * Apply hierarchical taxonomy terms (parents before children)
+ * Slugful entries use the existing locale-aware key. Slugless seed entries
+ * persist their seed ID, which keeps re-application idempotent.
+ */
+function seedEntryKey(entry: SeedContentEntry, slug: string | null, locale: string): string {
+	return slug === null ? `id:${entry.id}` : `slug:${locale}:${slug}`;
+}
+
+/**
+ * Find which of a collection's seed entries already exist, keyed by
+ * `seedEntryKey`. Each query covers a batch of entries.
+ */
+async function findExistingSeedEntries(
+	repo: ContentRepository,
+	collectionSlug: string,
+	entries: SeedContentEntry[],
+	defaultLocale: string,
+): Promise<Map<string, ContentItem>> {
+	const identities = entries.map((entry) => ({
+		entry,
+		...seedEntryIdentity(entry, defaultLocale),
+	}));
+	const slugsByLocale = new Map<string, string[]>();
+	for (const { slug, locale } of identities) {
+		if (slug === null) continue;
+		const slugs = slugsByLocale.get(locale);
+		if (slugs) slugs.push(slug);
+		else slugsByLocale.set(locale, [slug]);
+	}
+
+	const bySlug = new Map<string, Map<string, ContentItem>>();
+	for (const [locale, slugs] of slugsByLocale) {
+		bySlug.set(locale, await repo.findManyBySlugsInLocale(collectionSlug, slugs, locale));
+	}
+	const byId = await repo.findManyByIds(
+		collectionSlug,
+		identities.filter(({ slug }) => slug === null).map(({ entry }) => entry.id),
+	);
+
+	const existing = new Map<string, ContentItem>();
+	for (const { entry, slug, locale } of identities) {
+		const item = slug === null ? byId.get(entry.id) : bySlug.get(locale)?.get(slug);
+		if (item) existing.set(seedEntryKey(entry, slug, locale), item);
+	}
+	return existing;
+}
+
+/**
+ * Apply hierarchical taxonomy terms (parents before children). Stops before
+ * creating a term once `mayCreate` returns false, and returns how many terms
+ * were found or created.
  */
 async function applyHierarchicalTerms(
 	termRepo: TaxonomyRepository,
 	taxonomyName: string,
 	defLocale: string,
 	terms: SeedTaxonomyTerm[],
+	existingTerms: Map<string, string>,
 	termSeedIdMap: Map<string, string>,
 	result: SeedApplyResult,
-	onConflict: "skip" | "update" | "error" = "skip",
-): Promise<void> {
+	onConflict: "skip" | "update" | "error",
+	mayCreate: () => boolean,
+): Promise<{ processed: number; complete: boolean }> {
+	let processed = 0;
 	// "locale::slug" -> id, so the same slug can resolve per locale.
 	const slugToId = new Map<string, string>();
 	const resolveTermLocale = (term: SeedTaxonomyTerm) =>
@@ -1111,24 +1385,26 @@ async function applyHierarchicalTerms(
 			const parentId = term.parent ? slugToId.get(`${termLocale}::${term.parent}`) : undefined;
 			const translationOf = term.translationOf ? termSeedIdMap.get(term.translationOf) : undefined;
 
-			const existing = await termRepo.findBySlug(taxonomyName, term.slug, termLocale);
-			if (existing) {
+			const termKey = seedTermKey(termLocale, term.slug);
+			const existingId = existingTerms.get(termKey);
+			if (existingId) {
 				if (onConflict === "error") {
 					throw new Error(
 						`Conflict: taxonomy term "${term.slug}" in "${taxonomyName}" (${termLocale}) already exists`,
 					);
 				}
 				if (onConflict === "update") {
-					await termRepo.update(existing.id, {
+					await termRepo.update(existingId, {
 						label: term.label,
 						parentId,
 						data: term.description ? { description: term.description } : {},
 					});
 					result.taxonomies.terms++;
 				}
-				slugToId.set(`${termLocale}::${term.slug}`, existing.id);
-				if (term.id) termSeedIdMap.set(term.id, existing.id);
+				slugToId.set(termKey, existingId);
+				if (term.id) termSeedIdMap.set(term.id, existingId);
 			} else {
+				if (!mayCreate()) return { processed, complete: false };
 				const created = await termRepo.create({
 					name: taxonomyName,
 					slug: term.slug,
@@ -1138,10 +1414,12 @@ async function applyHierarchicalTerms(
 					locale: termLocale,
 					translationOf,
 				});
-				slugToId.set(`${termLocale}::${term.slug}`, created.id);
+				existingTerms.set(termKey, created.id);
+				slugToId.set(termKey, created.id);
 				if (term.id) termSeedIdMap.set(term.id, created.id);
 				result.taxonomies.terms++;
 			}
+			processed++;
 
 			processedThisPass.push(term.slug + "::" + termLocale);
 		}
@@ -1155,6 +1433,7 @@ async function applyHierarchicalTerms(
 	if (remaining.length > 0) {
 		console.warn(`Could not process ${remaining.length} terms due to missing parents/translations`);
 	}
+	return { processed, complete: true };
 }
 
 /**
@@ -1377,6 +1656,7 @@ interface MediaContext {
 	storage: Storage | null;
 	skipMediaDownload: boolean;
 	mediaCache: Map<string, MediaValue>; // URL -> resolved MediaValue
+	budget: SeedBudget | null;
 }
 
 /**
@@ -1563,6 +1843,7 @@ async function resolveMedia(
 
 		// Download the media (ssrfSafeFetch re-validates redirect targets)
 		console.log(`  📥 Downloading: ${url}`);
+		ctx.budget?.countMediaDownload();
 		const response = await ssrfSafeFetch(url, {
 			headers: {
 				// Some services like Unsplash require a user-agent