emdash 0.0.0-b → 0.0.1

package/src/astro/routes/api/import/wordpress/execute.ts

@@ -0,0 +1,283 @@
+/**
+ * WordPress WXR execute import endpoint
+ *
+ * POST /_emdash/api/import/wordpress/execute
+ *
+ * Accepts WXR file and import configuration, imports content into the database.
+ */
+
+import { gutenbergToPortableText } from "@emdash-cms/gutenberg-to-portable-text";
+import type { APIRoute } from "astro";
+import {
+	parseWxrString,
+	ContentRepository,
+	importReusableBlocksAsSections,
+	type WxrPost,
+} from "emdash";
+
+import { requirePerm } from "#api/authorize.js";
+import { apiError, apiSuccess, handleError } from "#api/error.js";
+import { BylineRepository } from "#db/repositories/byline.js";
+import { resolveImportByline } from "#import/utils.js";
+import type { EmDashHandlers, EmDashManifest } from "#types";
+import { slugify } from "#utils/slugify.js";
+
+export const prerender = false;
+
+export interface ImportConfig {
+	/** Map WordPress post types to EmDash collections */
+	postTypeMappings: Record<
+		string,
+		{
+			collection: string;
+			enabled: boolean;
+		}
+	>;
+	/** Whether to skip items that already exist (by slug) */
+	skipExisting: boolean;
+	/** Whether to import reusable blocks (wp_block) as sections */
+	importSections?: boolean;
+	/** Author mappings (WP author login -> EmDash user ID) */
+	authorMappings?: Record<string, string | null>;
+	/** BCP 47 locale for all imported items. When omitted, defaults to defaultLocale. */
+	locale?: string;
+}
+
+export interface ImportResult {
+	success: boolean;
+	imported: number;
+	skipped: number;
+	errors: Array<{ title: string; error: string }>;
+	byCollection: Record<string, number>;
+	/** Sections import results (if enabled) */
+	sections?: {
+		created: number;
+		skipped: number;
+	};
+}
+
+export const POST: APIRoute = async ({ request, locals }) => {
+	const { emdash, emdashManifest, user } = locals;
+
+	const denied = requirePerm(user, "import:execute");
+	if (denied) return denied;
+
+	if (!emdash?.handleContentCreate) {
+		return apiError("NOT_CONFIGURED", "EmDash not configured", 500);
+	}
+
+	try {
+		const formData = await request.formData();
+		const fileEntry = formData.get("file");
+		const file = fileEntry instanceof File ? fileEntry : null;
+		const configEntry = formData.get("config");
+		const configJson = typeof configEntry === "string" ? configEntry : null;
+
+		if (!file) {
+			return apiError("VALIDATION_ERROR", "No file provided", 400);
+		}
+
+		if (!configJson) {
+			return apiError("VALIDATION_ERROR", "No config provided", 400);
+		}
+
+		const config: ImportConfig = JSON.parse(configJson);
+
+		// Parse WXR
+		const text = await file.text();
+		const wxr = await parseWxrString(text);
+
+		// Build attachment ID -> URL map for featured images
+		const attachmentMap = new Map<string, string>();
+		for (const att of wxr.attachments) {
+			if (att.id && att.url) {
+				attachmentMap.set(String(att.id), att.url);
+			}
+		}
+
+		// Build author login -> display name map
+		const authorDisplayNames = new Map<string, string>();
+		for (const author of wxr.authors) {
+			if (!author.login) continue;
+			authorDisplayNames.set(author.login, author.displayName || author.login);
+		}
+
+		// Import content (locale from config scopes all items)
+		const result = await importContent(
+			wxr.posts,
+			config,
+			emdash,
+			emdashManifest,
+			attachmentMap,
+			config.locale,
+			authorDisplayNames,
+		);
+
+		// Import reusable blocks as sections (if enabled)
+		if (config.importSections !== false) {
+			const sectionsResult = await importReusableBlocksAsSections(wxr.posts, emdash.db);
+			result.sections = {
+				created: sectionsResult.sectionsCreated,
+				skipped: sectionsResult.sectionsSkipped,
+			};
+			// Add section errors to main errors array
+			result.errors.push(...sectionsResult.errors);
+			if (sectionsResult.errors.length > 0) {
+				result.success = false;
+			}
+		}
+
+		return apiSuccess(result);
+	} catch (error) {
+		return handleError(error, "Failed to import content", "WXR_IMPORT_ERROR");
+	}
+};
+
+async function importContent(
+	posts: WxrPost[],
+	config: ImportConfig,
+	emdash: EmDashHandlers,
+	manifest: EmDashManifest,
+	attachmentMap: Map<string, string>,
+	locale?: string,
+	authorDisplayNames?: Map<string, string>,
+): Promise<ImportResult> {
+	const result: ImportResult = {
+		success: true,
+		imported: 0,
+		skipped: 0,
+		errors: [],
+		byCollection: {},
+	};
+
+	// Create content repository for checking existing items
+	const contentRepo = new ContentRepository(emdash.db);
+	const bylineRepo = new BylineRepository(emdash.db);
+	const bylineCache = new Map<string, string>();
+
+	for (const post of posts) {
+		const postType = post.postType || "post";
+		const mapping = config.postTypeMappings[postType];
+
+		// Skip if not mapped or disabled
+		if (!mapping || !mapping.enabled) {
+			result.skipped++;
+			continue;
+		}
+
+		const collection = mapping.collection;
+
+		// Check if collection exists in manifest
+		if (!manifest?.collections[collection]) {
+			result.errors.push({
+				title: post.title || "Untitled",
+				error: `Collection "${collection}" does not exist`,
+			});
+			continue;
+		}
+
+		try {
+			// Convert content to Portable Text
+			const content = post.content ? gutenbergToPortableText(post.content) : [];
+
+			// Generate slug from post name or title
+			const slug = post.postName || slugify(post.title || `post-${post.id || Date.now()}`);
+
+			// Check if already exists (idempotency)
+			if (config.skipExisting) {
+				const existing = await contentRepo.findBySlug(collection, slug);
+				if (existing) {
+					result.skipped++;
+					continue;
+				}
+			}
+
+			// Map WordPress status to EmDash status
+			const status = mapStatus(post.status);
+
+			// Build data object with required fields
+			const data: Record<string, unknown> = {
+				title: post.title || "Untitled",
+				content,
+				excerpt: post.excerpt || undefined,
+			};
+
+			// Only add featured_image if the collection has this field and we have a value
+			const collectionSchema = manifest.collections[collection];
+			const hasFeaturedImageField = collectionSchema?.fields
+				? "featured_image" in collectionSchema.fields
+				: false;
+			if (hasFeaturedImageField) {
+				const thumbnailId = post.meta.get("_thumbnail_id");
+				const featuredImage = thumbnailId ? attachmentMap.get(String(thumbnailId)) : undefined;
+				if (featuredImage) {
+					data.featured_image = featuredImage;
+				}
+			}
+
+			// Resolve author ID from mappings
+			let authorId: string | undefined;
+			if (config.authorMappings && post.creator) {
+				const mappedUserId = config.authorMappings[post.creator];
+				if (mappedUserId !== undefined && mappedUserId !== null) {
+					authorId = mappedUserId;
+				}
+			}
+
+			const bylineId = await resolveImportByline(
+				post.creator,
+				authorDisplayNames?.get(post.creator ?? "") ?? post.creator,
+				authorId,
+				bylineRepo,
+				bylineCache,
+			);
+
+			// Create the content item
+			const createResult = await emdash.handleContentCreate(collection, {
+				data,
+				slug,
+				status,
+				authorId,
+				bylines: bylineId ? [{ bylineId }] : undefined,
+				locale,
+			});
+
+			if (createResult.success) {
+				result.imported++;
+				result.byCollection[collection] = (result.byCollection[collection] || 0) + 1;
+			} else {
+				result.errors.push({
+					title: post.title || "Untitled",
+					error:
+						typeof createResult.error === "object" && createResult.error !== null
+							? (createResult.error as { message?: string }).message || "Unknown error"
+							: String(createResult.error),
+				});
+			}
+		} catch (error) {
+			console.error(`Import error for "${post.title || "Untitled"}":`, error);
+			result.errors.push({
+				title: post.title || "Untitled",
+				error: "Failed to import item",
+			});
+		}
+	}
+
+	result.success = result.errors.length === 0;
+	return result;
+}
+
+function mapStatus(wpStatus: string | undefined): string {
+	switch (wpStatus) {
+		case "publish":
+			return "published";
+		case "draft":
+			return "draft";
+		case "pending":
+			return "draft";
+		case "private":
+			return "draft";
+		default:
+			return "draft";
+	}
+}

package/src/astro/routes/api/import/wordpress/media.ts

@@ -0,0 +1,338 @@
+/**
+ * WordPress media import endpoint
+ *
+ * POST /_emdash/api/import/wordpress/media
+ *
+ * Downloads media attachments from WordPress URLs and uploads to EmDash storage.
+ * Streams progress updates as newline-delimited JSON (NDJSON).
+ * Each line is either a progress update or the final result.
+ */
+
+import * as path from "node:path";
+
+import type { APIRoute } from "astro";
+import mime from "mime/lite";
+import { MediaRepository, computeContentHash } from "emdash";
+import { ulid } from "ulidx";
+
+import { requirePerm } from "#api/authorize.js";
+import { apiError, apiSuccess, handleError } from "#api/error.js";
+import { isParseError, parseBody } from "#api/parse.js";
+import { wpMediaImportBody } from "#api/schemas.js";
+import { validateExternalUrl, ssrfSafeFetch, SsrfError } from "#import/ssrf.js";
+import type { EmDashHandlers } from "#types";
+
+import type { AttachmentInfo } from "./analyze.js";
+
+export const prerender = false;
+
+/** Progress update sent during streaming */
+export interface MediaImportProgress {
+	type: "progress";
+	current: number;
+	total: number;
+	filename?: string;
+	status: "downloading" | "uploading" | "done" | "skipped" | "failed";
+	error?: string;
+}
+
+/** Final result sent at end of stream */
+export interface MediaImportResult {
+	type?: "result";
+	/** Successfully imported items */
+	imported: Array<{
+		wpId?: number;
+		originalUrl: string;
+		newUrl: string;
+		mediaId: string;
+	}>;
+	/** Failed items */
+	failed: Array<{
+		wpId?: number;
+		originalUrl: string;
+		error: string;
+	}>;
+	/** Map of old URLs to new URLs (for content rewriting) */
+	urlMap: Record<string, string>;
+}
+
+export const POST: APIRoute = async ({ request, locals }) => {
+	const { emdash, user } = locals;
+
+	const denied = requirePerm(user, "import:execute");
+	if (denied) return denied;
+
+	if (!emdash?.storage) {
+		return apiError("NO_STORAGE", "Storage not configured. Media import requires storage.", 501);
+	}
+
+	if (!emdash?.db) {
+		return apiError("NO_DB", "Database not initialized", 500);
+	}
+
+	try {
+		const body = await parseBody(request, wpMediaImportBody);
+		if (isParseError(body)) return body;
+
+		const attachments = body.attachments as AttachmentInfo[];
+
+		// Check if streaming is requested (default: true)
+		const shouldStream = body.stream !== false;
+
+		if (shouldStream) {
+			// Stream progress updates as NDJSON
+			const stream = new ReadableStream({
+				async start(controller) {
+					const encoder = new TextEncoder();
+					const sendProgress = (progress: MediaImportProgress) => {
+						controller.enqueue(encoder.encode(JSON.stringify(progress) + "\n"));
+					};
+
+					const result = await importMediaWithProgress(
+						attachments,
+						emdash.db,
+						emdash.storage,
+						request.url,
+						sendProgress,
+					);
+
+					// Send final result
+					controller.enqueue(encoder.encode(JSON.stringify({ ...result, type: "result" }) + "\n"));
+					controller.close();
+				},
+			});
+
+			return new Response(stream, {
+				status: 200,
+				headers: {
+					"Content-Type": "application/x-ndjson",
+					"Cache-Control": "private, no-store",
+					"Transfer-Encoding": "chunked",
+				},
+			});
+		}
+
+		// Non-streaming mode
+		const result = await importMediaWithProgress(
+			attachments,
+			emdash.db,
+			emdash.storage,
+			request.url,
+			() => {}, // No-op progress callback
+		);
+
+		return apiSuccess(result);
+	} catch (error) {
+		return handleError(error, "Failed to import media", "IMPORT_ERROR");
+	}
+};
+
+async function importMediaWithProgress(
+	attachments: AttachmentInfo[],
+	db: NonNullable<EmDashHandlers["db"]>,
+	storage: NonNullable<EmDashHandlers["storage"]>,
+	requestUrl: string,
+	onProgress: (progress: MediaImportProgress) => void,
+): Promise<MediaImportResult> {
+	const repo = new MediaRepository(db);
+	const url = new URL(requestUrl);
+	const baseUrl = `${url.protocol}//${url.host}`;
+	const total = attachments.length;
+
+	const result: MediaImportResult = {
+		imported: [],
+		failed: [],
+		urlMap: {},
+	};
+
+	for (let i = 0; i < attachments.length; i++) {
+		const attachment = attachments[i];
+		const current = i + 1;
+		const filename = attachment.filename || `file-${attachment.id}`;
+
+		if (!attachment.url) {
+			result.failed.push({
+				wpId: attachment.id,
+				originalUrl: "",
+				error: "No URL provided",
+			});
+			onProgress({
+				type: "progress",
+				current,
+				total,
+				filename,
+				status: "failed",
+				error: "No URL provided",
+			});
+			continue;
+		}
+
+		try {
+			// SSRF: validate URL before fetching
+			try {
+				validateExternalUrl(attachment.url);
+			} catch (e) {
+				const msg = e instanceof SsrfError ? e.message : "Invalid URL";
+				result.failed.push({
+					wpId: attachment.id,
+					originalUrl: attachment.url,
+					error: `Blocked: ${msg}`,
+				});
+				onProgress({
+					type: "progress",
+					current,
+					total,
+					filename,
+					status: "failed",
+					error: `Blocked: ${msg}`,
+				});
+				continue;
+			}
+
+			// Report downloading
+			onProgress({
+				type: "progress",
+				current,
+				total,
+				filename,
+				status: "downloading",
+			});
+
+			// Download from WordPress (ssrfSafeFetch re-validates redirect targets)
+			const response = await ssrfSafeFetch(attachment.url, {
+				headers: {
+					"User-Agent": "EmDash-Importer/1.0",
+				},
+			});
+
+			if (!response.ok) {
+				result.failed.push({
+					wpId: attachment.id,
+					originalUrl: attachment.url,
+					error: `HTTP ${response.status}: ${response.statusText}`,
+				});
+				onProgress({
+					type: "progress",
+					current,
+					total,
+					filename,
+					status: "failed",
+					error: `HTTP ${response.status}`,
+				});
+				continue;
+			}
+
+			// Get content type from response or guess from filename
+			const contentType =
+				response.headers.get("content-type") || attachment.mimeType || "application/octet-stream";
+
+			// Get the file data
+			const buffer = await response.arrayBuffer();
+			const size = buffer.byteLength;
+
+			// Compute content hash for deduplication
+			const contentHash = await computeContentHash(buffer);
+
+			// Check if we already have this exact content
+			const existing = await repo.findByContentHash(contentHash);
+			if (existing) {
+				// Same content already exists - reuse it
+				const existingUrl = `${baseUrl}/_emdash/api/media/file/${existing.storageKey}`;
+				result.urlMap[attachment.url] = existingUrl;
+				result.imported.push({
+					wpId: attachment.id,
+					originalUrl: attachment.url,
+					newUrl: existingUrl,
+					mediaId: existing.id,
+				});
+				onProgress({
+					type: "progress",
+					current,
+					total,
+					filename,
+					status: "skipped",
+				});
+				continue;
+			}
+
+			// Report uploading
+			onProgress({
+				type: "progress",
+				current,
+				total,
+				filename,
+				status: "uploading",
+			});
+
+			// Generate storage key
+			const id = ulid();
+			const ext = attachment.filename
+				? path.extname(attachment.filename)
+				: getExtensionFromMimeType(contentType);
+			const storageKey = `${id}${ext}`;
+
+			// Upload to storage
+			await storage.upload({
+				key: storageKey,
+				body: new Uint8Array(buffer),
+				contentType,
+			});
+
+			// Create media record with content hash
+			const mediaItem = await repo.create({
+				filename: attachment.filename || `media-${attachment.id}${ext}`,
+				mimeType: contentType,
+				size,
+				storageKey,
+				contentHash,
+				width: undefined,
+				height: undefined,
+			});
+
+			// Build the new URL
+			const newUrl = `${baseUrl}/_emdash/api/media/file/${storageKey}`;
+
+			result.imported.push({
+				wpId: attachment.id,
+				originalUrl: attachment.url,
+				newUrl,
+				mediaId: mediaItem.id,
+			});
+
+			// Add to URL map
+			result.urlMap[attachment.url] = newUrl;
+
+			// Report done
+			onProgress({
+				type: "progress",
+				current,
+				total,
+				filename,
+				status: "done",
+			});
+		} catch (error) {
+			console.error(`Media import error for "${filename}":`, error);
+			const errorMsg = "Failed to import media";
+			result.failed.push({
+				wpId: attachment.id,
+				originalUrl: attachment.url,
+				error: errorMsg,
+			});
+			onProgress({
+				type: "progress",
+				current,
+				total,
+				filename,
+				status: "failed",
+				error: errorMsg,
+			});
+		}
+	}
+
+	return result;
+}
+
+function getExtensionFromMimeType(mimeType: string): string {
+	const ext = mime.getExtension(mimeType);
+	return ext ? `.${ext}` : "";
+}