emdash 0.27.0 → 0.28.1

package/src/import/utils.ts

@@ -4,8 +4,10 @@
  * Common constants and functions used across all WordPress import sources.
  */
 
+import type { PortableTextBlock } from "@emdash-cms/gutenberg-to-portable-text";
 import mime from "mime/lite";
 
+import { RESERVED_FIELD_SLUGS } from "../schema/types.js";
 import type { ImportFieldDef, CollectionSchemaStatus } from "./types.js";
 
 // =============================================================================
@@ -183,6 +185,211 @@ export function inferMetaType(
 	return "string";
 }
 
+// =============================================================================
+// Plugin Bookkeeping Meta
+// =============================================================================
+
+/**
+ * Meta prefixes written by well-known WordPress plugins as operational
+ * bookkeeping (sync state, counters, cache keys) — not content. Without
+ * this filter, a mature site's analysis suggests dozens of junk fields
+ * per post type and the real content fields drown in them.
+ *
+ * ponytail: curated list of the plugins we've seen in the wild, not a
+ * taxonomy of the WP ecosystem. Unknown plugins' meta still gets through;
+ * extend the list as real sites surface new offenders.
+ */
+const PLUGIN_META_PREFIXES = [
+	"aawp_", // AAWP (Amazon affiliate)
+	"algolia_", // Algolia / WP Search with Algolia
+	"amazon_polly_", // Amazon Polly
+	"ampforwp_", // AMP for WP
+	"classifai_", // ClassifAI
+	"essb_", // Easy Social Share Buttons
+	"eg_", // Essential Grid
+	"gnpub_", // Google News publisher tools
+	"jetpack_", // Jetpack
+	"mashsb_", // MashShare
+	"monsterinsights_", // MonsterInsights
+	"onesignal_", // OneSignal push
+	"penci_", // Penci themes
+	"perfmatters_", // Perfmatters
+	"pys_", // PixelYourSite
+	"rank_math_", // Rank Math internals (title/description go through the SEO pass)
+	"rp4wp_", // Related Posts for WP
+	"saswp_", // Schema & Structured Data for WP
+	"sbg_", // Simple Blog Grid
+	"snap_", // SNAP auto-poster
+	"spay_", // Simple Pay
+	"tie_", // TieLabs themes
+	"wl_", // WordLift
+	"wpil_", // Link Whisper
+	"wprm_", // WP Recipe Maker internals
+	"wpswa_", // WP Search with Algolia
+	"wpuf_", // WP User Frontend
+	"yarpp_", // YARPP
+];
+
+/** Exact meta keys that are plugin/core bookkeeping, not content. */
+const PLUGIN_META_KEYS = new Set([
+	"entity_same_as", // WordLift
+	"exclude_from_search", // search exclusion plugins
+	"footnotes", // Gutenberg core footnotes store
+	"inline_featured_image", // inline featured image plugin
+	"os_meta", // theme option stores
+	"thirstydata", // ThirstyAffiliates
+]);
+
+/**
+ * Check whether a meta key is well-known plugin bookkeeping that should
+ * not become a content field. Hyphens are normalized to underscores
+ * before matching (e.g. `ampforwp-amp-on-off`).
+ */
+export function isPluginBookkeepingMeta(key: string): boolean {
+	const normalized = key.replaceAll("-", "_");
+	if (PLUGIN_META_KEYS.has(normalized)) return true;
+	return PLUGIN_META_PREFIXES.some((prefix) => normalized.startsWith(prefix));
+}
+
+// =============================================================================
+// Field Slug Sanitization
+// =============================================================================
+
+const INVALID_FIELD_SLUG_CHARS = /[^a-z0-9_]+/g;
+const LEADING_NON_ALPHA_CHARS = /^[^a-z]+/;
+
+/**
+ * Sanitize a WordPress meta/ACF key into a valid EmDash field slug
+ * (`/^[a-z][a-z0-9_]*$/`, max 63 chars, not reserved).
+ *
+ * Must be applied consistently on both sides of an import: once when
+ * creating fields from the analysis, and again when matching incoming
+ * meta keys onto schema fields — otherwise keys like `my-field` create
+ * `my_field` but never receive values.
+ */
+export function sanitizeFieldSlug(key: string): string {
+	const sanitized = key
+		.toLowerCase()
+		.replace(INVALID_FIELD_SLUG_CHARS, "_")
+		.replace(LEADING_NON_ALPHA_CHARS, "")
+		.slice(0, 63);
+	if (!sanitized) return "field";
+	if (RESERVED_FIELD_SLUGS.includes(sanitized)) return `wp_${sanitized}`;
+	return sanitized;
+}
+
+// =============================================================================
+// Internal Link Relativization
+// =============================================================================
+
+const REGEX_SPECIALS = /[.*+?^${}()|[\]\\]/g;
+const LEADING_WWW = /^www\./;
+
+/**
+ * Turn an absolute URL into a root-relative one when it points at the
+ * source site (www-insensitive). Returns null when the URL should be
+ * left alone: external links, non-http(s) schemes, and `/wp-content/`
+ * media files — those stay absolute so the later media pass can match
+ * them against its old-URL -> new-URL map.
+ */
+function relativizeUrl(url: string, sourceHost: string): string | null {
+	if (!url.startsWith("http://") && !url.startsWith("https://")) return null;
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname.replace(LEADING_WWW, "") !== sourceHost) return null;
+		if (parsed.pathname.startsWith("/wp-content/")) return null;
+		return `${parsed.pathname}${parsed.search}${parsed.hash}` || "/";
+	} catch {
+		return null;
+	}
+}
+
+function relativizeMarkDefs(
+	markDefs: Array<{ _type: string; [key: string]: unknown }> | undefined,
+	sourceHost: string,
+): void {
+	for (const def of markDefs ?? []) {
+		if (def._type === "link" && typeof def.href === "string") {
+			def.href = relativizeUrl(def.href, sourceHost) ?? def.href;
+		}
+	}
+}
+
+/**
+ * Rewrite internal links in imported content to root-relative URLs, in
+ * place. Without this, imported posts keep linking back to the old
+ * WordPress domain (e.g. `https://oldsite.com/companies/google/`)
+ * instead of staying on the new site.
+ *
+ * ponytail: path structures are kept as-is (WP permalink /2024/05/slug/
+ * stays /2024/05/slug/) — mapping old paths onto the new site's routes
+ * is the planned permalink->redirect-map feature.
+ */
+export function relativizeContentLinks(blocks: PortableTextBlock[], siteUrl: string): void {
+	let sourceHost: string;
+	try {
+		sourceHost = new URL(siteUrl).hostname.replace(LEADING_WWW, "");
+	} catch {
+		return;
+	}
+	// Raw HTML in the wild uses double-quoted, single-quoted, and unquoted
+	// href values; the backreference \1 matches the closing quote (or
+	// nothing, for unquoted). Rewritten links are normalized to href="...".
+	const hrefPattern = new RegExp(
+		`href=(["']?)https?://(?:www\\.)?${sourceHost.replace(REGEX_SPECIALS, "\\$&")}(/[^"'\\s>]*)?\\1`,
+		"gi",
+	);
+
+	for (const block of blocks) {
+		switch (block._type) {
+			case "block":
+				relativizeMarkDefs(block.markDefs, sourceHost);
+				break;
+			case "image":
+				// asset.url stays absolute (media pass), only the click-through link
+				if (block.link) block.link = relativizeUrl(block.link, sourceHost) ?? block.link;
+				break;
+			case "table":
+				for (const row of block.rows) {
+					for (const cell of row.cells) relativizeMarkDefs(cell.markDefs, sourceHost);
+				}
+				break;
+			case "columns":
+				for (const column of block.columns) relativizeContentLinks(column.content, siteUrl);
+				break;
+			case "cover":
+				relativizeContentLinks(block.content, siteUrl);
+				break;
+			case "button":
+				if (block.url) block.url = relativizeUrl(block.url, sourceHost) ?? block.url;
+				break;
+			case "buttons":
+				for (const button of block.buttons) {
+					if (button.url) button.url = relativizeUrl(button.url, sourceHost) ?? button.url;
+				}
+				break;
+			case "htmlBlock":
+				block.html = block.html.replace(
+					hrefPattern,
+					(_m, _quote: string, path: string | undefined) => {
+						return `href="${path || "/"}"`;
+					},
+				);
+				break;
+			// URL-less or media-only blocks: media URLs are the media pass's job
+			case "code":
+			case "embed":
+			case "gallery":
+			case "break":
+			case "file":
+			case "pullquote":
+				break;
+			default:
+				block satisfies never;
+		}
+	}
+}
+
 // =============================================================================
 // String Utilities
 // =============================================================================

package/src/import/wxr-taxonomies.ts

@@ -198,6 +198,45 @@ async function findTaxonomyDef(
 	return null;
 }
 
+/**
+ * Rebuild the taxonomy lookup maps from the database without creating
+ * anything. The chunked WP import runs term creation once (first content
+ * chunk); every later chunk only needs the maps to attach per-post
+ * assignments, and two SELECTs are far cheaper than re-walking every term
+ * through the ensure path.
+ *
+ * ponytail: first row wins per (name, slug), ordered by locale asc — the
+ * same row `findBySlug` without a locale resolves to. Per-locale term
+ * variants beyond that are the mirror pass's job, not the importer's.
+ */
+export async function loadTaxonomyPlanFromDb(db: Kysely<Database>): Promise<TaxonomyImportPlan> {
+	const state = makeState();
+
+	const defs = await db
+		.selectFrom("_emdash_taxonomy_defs")
+		.select(["name", "collections"])
+		.orderBy("locale", "asc")
+		.execute();
+	for (const def of defs) {
+		if (!state.plan.collectionsByTaxonomy.has(def.name)) {
+			state.plan.collectionsByTaxonomy.set(def.name, new Set(parseDefCollections(def.collections)));
+		}
+	}
+
+	const terms = await db
+		.selectFrom("taxonomies")
+		.select(["name", "slug", "id"])
+		.orderBy("locale", "asc")
+		.execute();
+	for (const term of terms) {
+		if (!state.plan.termIdByNameAndSlug.get(term.name)?.has(term.slug)) {
+			rememberTerm(state, term.name, term.slug, term.id);
+		}
+	}
+
+	return state.plan;
+}
+
 /**
  * Find or create a term in the given taxonomy. Returns the term id. Callers
  * must verify the taxonomy def exists before calling — this helper assumes

package/src/loader.ts

@@ -116,12 +116,23 @@ function foldedHydrationSelects(db: Kysely<any>, type: string, outer: string) {
 	const agg = (inner: RawBuilder<unknown>) =>
 		pg ? sql`coalesce(json_agg(${inner}), '[]'::json)` : sql`json_group_array(${inner})`;
 
+	// Pin the join order for the per-entry hydration subqueries on SQLite (#1722).
+	// SQLite honours `CROSS JOIN` ordering, forcing the join to drive from the
+	// pivot (`content_taxonomies` / `_emdash_content_bylines`) by
+	// `(collection, entry_id)` and probe the term/byline table by
+	// `translation_group`. Without it, a stats-blind D1 planner (D1 never runs
+	// ANALYZE / maintains `sqlite_stat1`) is free to drive the correlated
+	// subquery from `taxonomies`/`_emdash_bylines` by `locale`, scanning every
+	// row in the locale per emitted entry. Postgres keeps statistics and rejects
+	// `CROSS JOIN … ON`, so it stays a plain `JOIN` there.
+	const foldJoin = pg ? sql`JOIN` : sql`CROSS JOIN`;
+
 	const termObj = obj(
 		"'id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group",
 	);
 	// Filter terms to the entry's own locale (matches #1441: terms render in the
 	// entry's resolved locale, not all locale variants of the attached group).
-	const terms = sql`(SELECT ${agg(termObj)} FROM ${sql.ref("content_taxonomies")} AS ct JOIN ${sql.ref("taxonomies")} AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ${type} AND ct.entry_id = ${o}.id AND t.locale = ${o}.locale) AS ${sql.ref("_emdash_terms")}`;
+	const terms = sql`(SELECT ${agg(termObj)} FROM ${sql.ref("content_taxonomies")} AS ct ${foldJoin} ${sql.ref("taxonomies")} AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ${type} AND ct.entry_id = ${o}.id AND t.locale = ${o}.locale) AS ${sql.ref("_emdash_terms")}`;
 
 	const bylineInner = obj(
 		"'id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group",
@@ -132,7 +143,7 @@ function foldedHydrationSelects(db: Kysely<any>, type: string, outer: string) {
 			)
 		: sql.raw("json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', ");
 	const credit = sql`${creditObj}${bylineInner})`;
-	const bylines = sql`(SELECT ${agg(credit)} FROM ${sql.ref("_emdash_content_bylines")} AS cb JOIN ${sql.ref("_emdash_bylines")} AS b ON b.translation_group = cb.byline_id LEFT JOIN ${sql.ref("media")} AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ${type} AND cb.content_id = ${o}.id AND b.locale = ${o}.locale) AS ${sql.ref("_emdash_bylines")}`;
+	const bylines = sql`(SELECT ${agg(credit)} FROM ${sql.ref("_emdash_content_bylines")} AS cb ${foldJoin} ${sql.ref("_emdash_bylines")} AS b ON b.translation_group = cb.byline_id LEFT JOIN ${sql.ref("media")} AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ${type} AND cb.content_id = ${o}.id AND b.locale = ${o}.locale) AS ${sql.ref("_emdash_bylines")}`;
 	return { terms, bylines };
 }
 

package/src/media/usage/content-fields.ts

@@ -0,0 +1,119 @@
+import type { Kysely } from "kysely";
+
+import type { Database } from "../../database/types.js";
+import { validateIdentifier } from "../../database/validate.js";
+import type { MediaUsageExtractionField, MediaUsageExtractionSubField } from "./types.js";
+
+export type ContentMediaUsageField = MediaUsageExtractionField;
+
+export interface ContentMediaUsageFieldDiscovery {
+	extractionFields: ContentMediaUsageField[];
+	displayFieldSlugs: string[];
+}
+
+export class MediaUsageFieldDiscoveryError extends Error {
+	constructor(
+		message: string,
+		public code: "INVALID_REPEATER_VALIDATION",
+	) {
+		super(message);
+		this.name = "MediaUsageFieldDiscoveryError";
+	}
+}
+
+interface FieldDiscoveryRow {
+	slug: string;
+	type: string;
+	validation: string | null;
+}
+
+const DISPLAY_FIELD_SLUGS = ["title", "name"] as const;
+const SUPPORTED_TOP_LEVEL_TYPES = ["file", "image", "portableText"] as const;
+
+type SupportedTopLevelType = (typeof SUPPORTED_TOP_LEVEL_TYPES)[number];
+
+export async function loadContentMediaUsageFields(
+	db: Kysely<Database>,
+	collectionSlug: string,
+): Promise<ContentMediaUsageFieldDiscovery> {
+	validateIdentifier(collectionSlug, "collection slug");
+
+	const rows = await db
+		.selectFrom("_emdash_fields")
+		.innerJoin("_emdash_collections", "_emdash_collections.id", "_emdash_fields.collection_id")
+		.select(["_emdash_fields.slug", "_emdash_fields.type", "_emdash_fields.validation"])
+		.where("_emdash_collections.slug", "=", collectionSlug)
+		.execute();
+
+	const extractionFields: ContentMediaUsageField[] = [];
+	const rowBySlug = new Map<string, FieldDiscoveryRow>();
+
+	for (const row of rows) {
+		rowBySlug.set(row.slug, row);
+		if (isSupportedTopLevelType(row.type)) {
+			validateIdentifier(row.slug, "media usage field slug");
+			extractionFields.push({ slug: row.slug, type: row.type });
+			continue;
+		}
+
+		if (row.type === "repeater") {
+			validateIdentifier(row.slug, "media usage field slug");
+			const subFields = normalizeRepeaterImageSubFields(row.validation);
+			if (subFields.length > 0) {
+				extractionFields.push({
+					slug: row.slug,
+					type: "repeater",
+					validation: { subFields },
+				});
+			}
+		}
+	}
+
+	extractionFields.sort((a, b) => a.slug.localeCompare(b.slug));
+
+	return {
+		extractionFields,
+		displayFieldSlugs: DISPLAY_FIELD_SLUGS.filter((slug) => {
+			if (!rowBySlug.has(slug)) return false;
+			validateIdentifier(slug, "media usage display field slug");
+			return true;
+		}),
+	};
+}
+
+function normalizeRepeaterImageSubFields(
+	rawValidation: string | null,
+): MediaUsageExtractionSubField[] {
+	const validation = parseValidation(rawValidation);
+	if (!isRecord(validation) || !Array.isArray(validation.subFields)) return [];
+
+	const subFields: MediaUsageExtractionSubField[] = [];
+	for (const subField of validation.subFields) {
+		if (!isRecord(subField) || subField.type !== "image") continue;
+		if (typeof subField.slug !== "string") continue;
+		validateIdentifier(subField.slug, "media usage repeater sub-field slug");
+		subFields.push({ slug: subField.slug, type: "image" });
+	}
+
+	return subFields.toSorted((a, b) => a.slug.localeCompare(b.slug));
+}
+
+function parseValidation(rawValidation: string | null): unknown {
+	if (!rawValidation) return null;
+	try {
+		return JSON.parse(rawValidation);
+	} catch {
+		throw new MediaUsageFieldDiscoveryError(
+			"Repeater field validation must be valid JSON before media usage can be discovered",
+			"INVALID_REPEATER_VALIDATION",
+		);
+	}
+}
+
+function isSupportedTopLevelType(value: string): value is SupportedTopLevelType {
+	return (SUPPORTED_TOP_LEVEL_TYPES as readonly string[]).includes(value);
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}