emdash 1.0.0 → 1.0.1-rc.1

package/src/import/sources/wordpress-plugin.ts

@@ -8,6 +8,8 @@
 import { gutenbergToPortableText } from "@emdash-cms/gutenberg-to-portable-text";
 
 import { encodeBase64 } from "../../utils/base64.js";
+import type { PluginComment } from "../comments.js";
+import type { PluginMenu } from "../menus.js";
 import { ssrfSafeFetch, validateExternalUrl } from "../ssrf.js";
 import type {
 	ImportSource,
@@ -28,6 +30,9 @@ import {
 	mapWpStatus,
 	normalizeUrl,
 	checkSchemaCompatibility,
+	isPluginBookkeepingMeta,
+	relativizeContentLinks,
+	sanitizeFieldSlug,
 } from "../utils.js";
 
 // =============================================================================
@@ -216,6 +221,67 @@ interface PluginMediaItem {
 /** Pattern to remove spaces from application passwords */
 const SPACE_PATTERN = /\s/g;
 
+/**
+ * Build the REST API URL for a plugin endpoint.
+ *
+ * `restRoute: false` uses the pretty form (`/wp-json/emdash/v1/...`),
+ * `restRoute: true` uses the `?rest_route=` form that works on sites with
+ * plain permalinks (where `/wp-json/` doesn't exist).
+ */
+function pluginApiUrl(
+	siteUrl: string,
+	path: string,
+	params: Record<string, string> = {},
+	restRoute = false,
+): string {
+	if (restRoute) {
+		const url = new URL(siteUrl + "/");
+		url.searchParams.set("rest_route", `/emdash/v1/${path}`);
+		for (const [key, value] of Object.entries(params)) {
+			url.searchParams.set(key, value);
+		}
+		return url.toString();
+	}
+	const url = new URL(`${siteUrl}/wp-json/emdash/v1/${path}`);
+	for (const [key, value] of Object.entries(params)) {
+		url.searchParams.set(key, value);
+	}
+	return url.toString();
+}
+
+/**
+ * Fetch a plugin API endpoint, falling back to the `?rest_route=` form when
+ * the pretty `/wp-json/` route 404s or is unreachable. Sites with "Plain"
+ * permalinks have no `/wp-json/` rewrite, so without this fallback they
+ * always fail with a misleading 404.
+ */
+async function fetchPluginApi(
+	siteUrl: string,
+	path: string,
+	params: Record<string, string>,
+	headers: HeadersInit,
+	timeoutMs: number,
+): Promise<Response> {
+	let pretty: Response | null = null;
+	try {
+		pretty = await ssrfSafeFetch(pluginApiUrl(siteUrl, path, params), {
+			headers,
+			signal: AbortSignal.timeout(timeoutMs),
+		});
+	} catch {
+		// Network-level failure -- try the rest_route form before giving up.
+	}
+	// Any response other than 404 (including 401/403/500) is authoritative:
+	// the route exists, so don't mask the real error with a fallback attempt.
+	if (pretty && pretty.status !== 404) {
+		return pretty;
+	}
+	return ssrfSafeFetch(pluginApiUrl(siteUrl, path, params, true), {
+		headers,
+		signal: AbortSignal.timeout(timeoutMs),
+	});
+}
+
 // =============================================================================
 // Import Source
 // =============================================================================
@@ -235,12 +301,13 @@ export const wordpressPluginSource: ImportSource = {
 			// SSRF protection: validate URL before any outbound requests
 			validateExternalUrl(siteUrl);
 
-			const probeUrl = `${siteUrl}/wp-json/emdash/v1/probe`;
-
-			const response = await ssrfSafeFetch(probeUrl, {
-				headers: { Accept: "application/json" },
-				signal: AbortSignal.timeout(10000),
-			});
+			const response = await fetchPluginApi(
+				siteUrl,
+				"probe",
+				{},
+				{ Accept: "application/json" },
+				10000,
+			);
 
 			if (!response.ok) {
 				return null;
@@ -293,14 +360,18 @@ export const wordpressPluginSource: ImportSource = {
 	async analyze(input: SourceInput, context: ImportContext): Promise<ImportAnalysis> {
 		const { siteUrl, headers } = getRequestConfig(input);
 
-		const response = await ssrfSafeFetch(`${siteUrl}/wp-json/emdash/v1/analyze`, {
-			headers,
-			signal: AbortSignal.timeout(30000),
-		});
+		const response = await fetchPluginApi(siteUrl, "analyze", {}, headers, 30000);
 
 		if (!response.ok) {
-			const error = await response.json().catch(() => ({}));
-			throw new Error(error.message || `Failed to analyze site: ${response.statusText}`);
+			const body: unknown = await response.json().catch(() => undefined);
+			const message =
+				typeof body === "object" &&
+				body !== null &&
+				"message" in body &&
+				typeof body.message === "string"
+					? body.message
+					: "";
+			throw new Error(message || `Failed to analyze site: ${response.statusText}`);
 		}
 
 		const data: PluginAnalyzeResponse = await response.json();
@@ -323,6 +394,23 @@ export const wordpressPluginSource: ImportSource = {
 					? [...BASE_REQUIRED_FIELDS, FEATURED_IMAGE_FIELD]
 					: [...BASE_REQUIRED_FIELDS];
 
+				// Surface the post type's custom fields (ACF and plain meta) so
+				// the prepare step creates them — without this, execute() has no
+				// matching schema fields and silently drops the values.
+				const knownSlugs = new Set(requiredFields.map((f) => f.slug));
+				for (const customField of pt.custom_fields ?? []) {
+					if (isPluginBookkeepingMeta(customField.key)) continue;
+					const slug = sanitizeFieldSlug(customField.key);
+					if (knownSlugs.has(slug)) continue;
+					knownSlugs.add(slug);
+					requiredFields.push({
+						slug,
+						label: fieldLabelFromKey(customField.key),
+						type: mapInferredFieldType(customField.inferred_type),
+						required: false,
+					});
+				}
+
 				return {
 					name: pt.name,
 					count: pt.total,
@@ -332,20 +420,24 @@ export const wordpressPluginSource: ImportSource = {
 				};
 			});
 
-		// Fetch media list for attachment info
+		// Fetch the full media list, paginated. Stopping after the first page
+		// silently capped imports at 500 attachments (wp-emdash #1).
 		const attachments: AttachmentInfo[] = [];
 		if (data.attachments.count > 0) {
 			try {
-				// Fetch first page of media to populate attachment info
-				const mediaResponse = await ssrfSafeFetch(
-					`${siteUrl}/wp-json/emdash/v1/media?per_page=500`,
-					{
+				let page = 1;
+				let totalPages = 1;
+				while (page <= totalPages) {
+					const mediaResponse = await fetchPluginApi(
+						siteUrl,
+						"media",
+						{ per_page: "500", page: String(page) },
 						headers,
-						signal: AbortSignal.timeout(30000),
-					},
-				);
-				if (mediaResponse.ok) {
+						30000,
+					);
+					if (!mediaResponse.ok) break;
 					const mediaData: PluginMediaResponse = await mediaResponse.json();
+					totalPages = mediaData.pages;
 					for (const item of mediaData.items) {
 						attachments.push({
 							id: item.id,
@@ -359,6 +451,7 @@ export const wordpressPluginSource: ImportSource = {
 							height: item.height,
 						});
 					}
+					page++;
 				}
 			} catch (e) {
 				console.warn("Failed to fetch media list:", e);
@@ -403,12 +496,13 @@ export const wordpressPluginSource: ImportSource = {
 
 			while (page <= totalPages) {
 				const status = options.includeDrafts ? "any" : "publish";
-				const url = `${siteUrl}/wp-json/emdash/v1/content?post_type=${postType}&status=${status}&per_page=100&page=${page}`;
-
-				const response = await ssrfSafeFetch(url, {
+				const response = await fetchPluginApi(
+					siteUrl,
+					"content",
+					{ post_type: postType, status, per_page: "100", page: String(page) },
 					headers,
-					signal: AbortSignal.timeout(60000),
-				});
+					60000,
+				);
 
 				if (!response.ok) {
 					throw new Error(`Failed to fetch ${postType}: ${response.statusText}`);
@@ -418,7 +512,7 @@ export const wordpressPluginSource: ImportSource = {
 				totalPages = data.pages;
 
 				for (const post of data.items) {
-					yield pluginPostToNormalizedItem(post);
+					yield pluginPostToNormalizedItem(post, siteUrl);
 					yielded++;
 
 					if (options.limit && yielded >= options.limit) {
@@ -444,10 +538,85 @@ export const wordpressPluginSource: ImportSource = {
 	},
 };
 
+/**
+ * Fetch a single page of content for one post type. This is the unit of
+ * work for the chunked import: one Worker invocation imports one page,
+ * keeping each request far below Cloudflare's CPU and subrequest limits
+ * (see issue #475).
+ */
+export async function fetchPluginContentPage(options: {
+	siteUrl: string;
+	token: string;
+	postType: string;
+	page: number;
+	perPage: number;
+	includeDrafts: boolean;
+}): Promise<{ items: NormalizedItem[]; totalPages: number }> {
+	const { siteUrl, headers } = getRequestConfig({
+		type: "url",
+		url: options.siteUrl,
+		token: options.token,
+	});
+
+	const response = await fetchPluginApi(
+		siteUrl,
+		"content",
+		{
+			post_type: options.postType,
+			status: options.includeDrafts ? "any" : "publish",
+			per_page: String(options.perPage),
+			page: String(options.page),
+		},
+		headers,
+		60000,
+	);
+
+	if (!response.ok) {
+		throw new Error(`Failed to fetch ${options.postType}: ${response.statusText}`);
+	}
+
+	const data: PluginContentResponse = await response.json();
+	return {
+		items: data.items.map((post) => pluginPostToNormalizedItem(post, siteUrl)),
+		totalPages: data.pages,
+	};
+}
+
 // =============================================================================
 // Helper Functions
 // =============================================================================
 
+/** Plugin `inferred_type` values that are valid EmDash field types as-is */
+const VALID_INFERRED_TYPES = new Set([
+	"string",
+	"text",
+	"number",
+	"integer",
+	"boolean",
+	"datetime",
+	"json",
+	"reference",
+]);
+
+/**
+ * Map the plugin's inferred custom-field type to an EmDash field type.
+ * Unknown values fall back to string (always safe for TEXT storage).
+ */
+function mapInferredFieldType(inferredType: string): string {
+	return VALID_INFERRED_TYPES.has(inferredType) ? inferredType : "string";
+}
+
+const FIELD_KEY_SEPARATORS = /[_-]+/;
+
+/** Derive a human label from a meta key: "event_start-date" -> "Event Start Date" */
+function fieldLabelFromKey(key: string): string {
+	return key
+		.split(FIELD_KEY_SEPARATORS)
+		.filter(Boolean)
+		.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
+		.join(" ");
+}
+
 /**
  * Convert plugin i18n info to the shared I18nDetection type.
  * Returns undefined when no multilingual plugin is detected.
@@ -506,8 +675,9 @@ function getRequestConfig(input: SourceInput): {
 /**
  * Convert plugin post to normalized item
  */
-function pluginPostToNormalizedItem(post: PluginPost): NormalizedItem {
+function pluginPostToNormalizedItem(post: PluginPost, siteUrl: string): NormalizedItem {
 	const content = post.content ? gutenbergToPortableText(post.content) : [];
+	relativizeContentLinks(content, siteUrl);
 
 	// Extract categories and tags from taxonomies
 	const categories =
@@ -519,6 +689,15 @@ function pluginPostToNormalizedItem(post: PluginPost): NormalizedItem {
 		post.taxonomies?.tags?.map((t) => t.slug) ??
 		[];
 
+	// Everything else is a custom taxonomy assignment (genre, product_cat, ...)
+	const customTaxonomies: Record<string, string[]> = {};
+	for (const [name, terms] of Object.entries(post.taxonomies ?? {})) {
+		if (["category", "categories", "post_tag", "tags"].includes(name)) continue;
+		if (Array.isArray(terms) && terms.length > 0) {
+			customTaxonomies[name] = terms.map((t) => t.slug);
+		}
+	}
+
 	// Build meta from various sources
 	const meta: Record<string, unknown> = { ...post.meta };
 
@@ -548,6 +727,7 @@ function pluginPostToNormalizedItem(post: PluginPost): NormalizedItem {
 		author: post.author?.login,
 		categories,
 		tags,
+		customTaxonomies: Object.keys(customTaxonomies).length > 0 ? customTaxonomies : undefined,
 		meta,
 		featuredImage: post.featured_image?.url,
 		locale: post.locale,
@@ -582,14 +762,13 @@ export async function fetchPluginMedia(
 	// SSRF protection: validate URL before any outbound requests
 	validateExternalUrl(normalizedSiteUrl);
 
-	const url = `${normalizedSiteUrl}/wp-json/emdash/v1/media?per_page=${perPage}&page=${page}`;
-
-	const response = await ssrfSafeFetch(url, {
-		headers: {
-			Accept: "application/json",
-			Authorization: `Basic ${authToken}`,
-		},
-	});
+	const response = await fetchPluginApi(
+		normalizedSiteUrl,
+		"media",
+		{ per_page: String(perPage), page: String(page) },
+		{ Accept: "application/json", Authorization: `Basic ${authToken}` },
+		30000,
+	);
 
 	if (!response.ok) {
 		throw new Error(`Failed to fetch media: ${response.statusText}`);
@@ -608,7 +787,11 @@ export async function fetchPluginTaxonomies(
 	Array<{
 		name: string;
 		label: string;
+		/** Singular label (added in emdash-exporter 1.2.0) */
+		label_singular?: string;
 		hierarchical: boolean;
+		/** WP post types this taxonomy is registered for (added in emdash-exporter 1.2.0) */
+		post_types?: string[];
 		terms: Array<{
 			id: number;
 			name: string;
@@ -624,14 +807,13 @@ export async function fetchPluginTaxonomies(
 	// SSRF protection: validate URL before any outbound requests
 	validateExternalUrl(normalizedSiteUrl);
 
-	const url = `${normalizedSiteUrl}/wp-json/emdash/v1/taxonomies`;
-
-	const response = await ssrfSafeFetch(url, {
-		headers: {
-			Accept: "application/json",
-			Authorization: `Basic ${authToken}`,
-		},
-	});
+	const response = await fetchPluginApi(
+		normalizedSiteUrl,
+		"taxonomies",
+		{},
+		{ Accept: "application/json", Authorization: `Basic ${authToken}` },
+		30000,
+	);
 
 	if (!response.ok) {
 		throw new Error(`Failed to fetch taxonomies: ${response.statusText}`);
@@ -639,3 +821,124 @@ export async function fetchPluginTaxonomies(
 
 	return response.json();
 }
+
+/**
+ * Fetch navigation menus from plugin API (added in emdash-exporter 1.1.0).
+ * Returns an empty array when the endpoint doesn't exist (older plugin).
+ */
+export async function fetchPluginMenus(siteUrl: string, authToken: string): Promise<PluginMenu[]> {
+	const normalizedSiteUrl = normalizeUrl(siteUrl);
+
+	// SSRF protection: validate URL before any outbound requests
+	validateExternalUrl(normalizedSiteUrl);
+
+	const response = await fetchPluginApi(
+		normalizedSiteUrl,
+		"menus",
+		{},
+		{ Accept: "application/json", Authorization: `Basic ${authToken}` },
+		30000,
+	);
+
+	if (response.status === 404) {
+		return [];
+	}
+	if (!response.ok) {
+		throw new Error(`Failed to fetch menus: ${response.statusText}`);
+	}
+
+	return response.json();
+}
+
+/**
+ * Fetch site options from plugin API (title, tagline, logo, favicon, ...)
+ */
+export async function fetchPluginOptions(
+	siteUrl: string,
+	authToken: string,
+): Promise<Record<string, unknown>> {
+	const normalizedSiteUrl = normalizeUrl(siteUrl);
+
+	// SSRF protection: validate URL before any outbound requests
+	validateExternalUrl(normalizedSiteUrl);
+
+	const response = await fetchPluginApi(
+		normalizedSiteUrl,
+		"options",
+		{},
+		{ Accept: "application/json", Authorization: `Basic ${authToken}` },
+		30000,
+	);
+
+	if (!response.ok) {
+		throw new Error(`Failed to fetch options: ${response.statusText}`);
+	}
+
+	return response.json();
+}
+
+/** Comments response from /emdash/v1/comments */
+interface PluginCommentsResponse {
+	items: PluginComment[];
+	total: number;
+	pages: number;
+	page: number;
+	per_page: number;
+}
+
+/**
+ * Fetch a single page of comments from the plugin API (added in
+ * emdash-exporter 1.2.0). The exporter orders by comment ID ascending, so
+ * parents always appear before their children across pages. Returns
+ * `totalPages: 0` when the endpoint doesn't exist (older plugin).
+ */
+export async function fetchPluginCommentsPage(
+	siteUrl: string,
+	authToken: string,
+	page: number,
+): Promise<{ items: PluginComment[]; totalPages: number }> {
+	const normalizedSiteUrl = normalizeUrl(siteUrl);
+
+	// SSRF protection: validate URL before any outbound requests
+	validateExternalUrl(normalizedSiteUrl);
+
+	const response = await fetchPluginApi(
+		normalizedSiteUrl,
+		"comments",
+		{ per_page: "500", page: String(page) },
+		{ Accept: "application/json", Authorization: `Basic ${authToken}` },
+		30000,
+	);
+
+	if (response.status === 404) {
+		return { items: [], totalPages: 0 };
+	}
+	if (!response.ok) {
+		throw new Error(`Failed to fetch comments: ${response.statusText}`);
+	}
+
+	const data: PluginCommentsResponse = await response.json();
+	return { items: data.items, totalPages: data.pages };
+}
+
+/**
+ * Fetch all comments from plugin API, paginating through every page.
+ * Returns an empty array when the endpoint doesn't exist (older plugin).
+ */
+export async function fetchPluginComments(
+	siteUrl: string,
+	authToken: string,
+): Promise<PluginComment[]> {
+	const comments: PluginComment[] = [];
+	let page = 1;
+	let totalPages = 1;
+
+	while (page <= totalPages) {
+		const result = await fetchPluginCommentsPage(siteUrl, authToken, page);
+		totalPages = result.totalPages;
+		comments.push(...result.items);
+		page++;
+	}
+
+	return comments;
+}

package/src/import/sources/wxr.ts

@@ -35,6 +35,7 @@ import {
 	getFilenameFromUrl,
 	guessMimeType,
 	checkSchemaCompatibility,
+	relativizeContentLinks,
 } from "../utils.js";
 
 export const wxrSource: ImportSource = {
@@ -92,7 +93,7 @@ export const wxrSource: ImportSource = {
 			}
 
 			// Convert to normalized item
-			yield wxrPostToNormalizedItem(post, attachmentMap);
+			yield wxrPostToNormalizedItem(post, attachmentMap, wxr.site.link || "");
 
 			count++;
 			if (options.limit && count >= options.limit) {
@@ -281,17 +282,28 @@ function analyzeWxrData(
 function wxrPostToNormalizedItem(
 	post: WxrPost,
 	attachmentMap: Map<string, string>,
+	siteUrl: string,
 ): NormalizedItem {
 	const content = post.content ? gutenbergToPortableText(post.content) : [];
+	if (siteUrl) relativizeContentLinks(content, siteUrl);
 
 	// Resolve featured image: _thumbnail_id is the attachment ID, look up the URL
 	const thumbnailId = post.meta.get("_thumbnail_id");
 	const featuredImage = thumbnailId ? attachmentMap.get(String(thumbnailId)) : undefined;
 
-	// Convert custom taxonomies Map to Record
+	// Convert custom taxonomies Map to Record.
+	// Strip Polylang's `language` taxonomy -- the parser already promoted it
+	// to `post.locale`, and exposing it as a content taxonomy here would
+	// (a) try to insert a `language` taxonomy on EmDash that doesn't exist
+	// and (b) duplicate the locale signal in two places.
 	let customTaxonomies: Record<string, string[]> | undefined;
 	if (post.customTaxonomies && post.customTaxonomies.size > 0) {
-		customTaxonomies = Object.fromEntries(post.customTaxonomies);
+		const filtered = Object.fromEntries(
+			[...post.customTaxonomies].filter(([taxonomy]) => taxonomy !== "language"),
+		);
+		if (Object.keys(filtered).length > 0) {
+			customTaxonomies = filtered;
+		}
 	}
 
 	return {
@@ -314,6 +326,11 @@ function wxrPostToNormalizedItem(
 		menuOrder: post.menuOrder,
 		// Custom taxonomy assignments
 		customTaxonomies,
+		// Multilingual plugin metadata (WPML / Polylang) promoted by the parser.
+		// When absent, the caller (e.g. execute route) falls back to the
+		// upload-wide locale.
+		locale: post.locale,
+		translationGroup: post.translationGroup,
 	};
 }