dineway 0.1.9 → 0.1.12

package/src/astro/routes/api/import/wordpress/execute.ts

@@ -1,330 +0,0 @@
-/**
- * WordPress WXR execute import endpoint
- *
- * POST /_dineway/api/import/wordpress/execute
- *
- * Accepts WXR file and import configuration, imports content into the database.
- */
-
-import { gutenbergToPortableText } from "@dineway-ai/gutenberg-to-portable-text";
-import type { APIRoute } from "astro";
-import {
-	parseWxrString,
-	ContentRepository,
-	importReusableBlocksAsSections,
-	type WxrPost,
-	parseWxrDate,
-} from "dineway";
-
-import { requirePerm } from "#api/authorize.js";
-import { apiError, apiSuccess, handleError } from "#api/error.js";
-import {
-	ensureWorkflowHitlRouteRequest,
-	hitlRequiredRouteError,
-	resolveHitlRouteActor,
-} from "#api/hitl-route-helpers.js";
-import { BylineRepository } from "#db/repositories/byline.js";
-import { resolveImportByline } from "#import/utils.js";
-import { sanitizeWordPressImportSlug } from "#import/wordpress-slugs.js";
-import { RiskPolicyEvaluator, WordPressImportHitlPayloadBuilder } from "#site-context/index.js";
-import type { DinewayHandlers, DinewayManifest } from "#types";
-import { slugify } from "#utils/slugify.js";
-
-export const prerender = false;
-
-export interface ImportConfig {
-	/** Map WordPress post types to Dineway collections */
-	postTypeMappings: Record<
-		string,
-		{
-			collection: string;
-			enabled: boolean;
-		}
-	>;
-	/** Whether to skip items that already exist (by slug) */
-	skipExisting: boolean;
-	/** Whether to import reusable blocks (wp_block) as sections */
-	importSections?: boolean;
-	/** Author mappings (WP author login -> Dineway user ID) */
-	authorMappings?: Record<string, string | null>;
-	/** BCP 47 locale for all imported items. When omitted, defaults to defaultLocale. */
-	locale?: string;
-}
-
-export interface ImportResult {
-	success: boolean;
-	imported: number;
-	skipped: number;
-	errors: Array<{ title: string; error: string }>;
-	byCollection: Record<string, number>;
-	/** Sections import results (if enabled) */
-	sections?: {
-		created: number;
-		skipped: number;
-	};
-}
-
-export const POST: APIRoute = async ({ request, locals }) => {
-	const { dineway, dinewayManifest, user } = locals;
-
-	const denied = requirePerm(user, "import:execute");
-	if (denied) return denied;
-
-	if (!dineway?.handleContentCreate) {
-		return apiError("NOT_CONFIGURED", "Dineway is not configured", 500);
-	}
-
-	try {
-		const formData = await request.formData();
-		const fileEntry = formData.get("file");
-		const file = fileEntry instanceof File ? fileEntry : null;
-		const configEntry = formData.get("config");
-		const configJson = typeof configEntry === "string" ? configEntry : null;
-		const hitlRequestEntry = formData.get("hitlRequestId");
-		const hitlRequestId =
-			typeof hitlRequestEntry === "string" && hitlRequestEntry.length > 0
-				? hitlRequestEntry
-				: undefined;
-
-		if (!file) {
-			return apiError("VALIDATION_ERROR", "No file provided", 400);
-		}
-
-		if (!configJson) {
-			return apiError("VALIDATION_ERROR", "No config provided", 400);
-		}
-
-		const config: ImportConfig = JSON.parse(configJson);
-		const actor = resolveHitlRouteActor(locals);
-		const evaluator = new RiskPolicyEvaluator({
-			db: dineway.db,
-			handlers: dineway,
-		});
-		let text: string;
-		if (evaluator.requiresWorkflowHitl(actor.identity)) {
-			const fileBuffer = await file.arrayBuffer();
-			const action = await new WordPressImportHitlPayloadBuilder().buildExecuteRequest({
-				fileName: file.name,
-				fileBuffer,
-				config,
-			});
-			const decision = await evaluator.evaluateWorkflowHitl({
-				actor: actor.identity,
-				hitlRequestId,
-				action,
-			});
-			if (!decision.allowed) {
-				const ensured = await ensureWorkflowHitlRouteRequest(dineway.db, locals, decision.action);
-				return hitlRequiredRouteError(decision, ensured);
-			}
-			text = new TextDecoder().decode(fileBuffer);
-		} else {
-			text = await file.text();
-		}
-
-		const wxr = await parseWxrString(text);
-
-		// Build attachment ID -> URL map for featured images
-		const attachmentMap = new Map<string, string>();
-		for (const att of wxr.attachments) {
-			if (att.id && att.url) {
-				attachmentMap.set(String(att.id), att.url);
-			}
-		}
-
-		// Build author login -> display name map
-		const authorDisplayNames = new Map<string, string>();
-		for (const author of wxr.authors) {
-			if (!author.login) continue;
-			authorDisplayNames.set(author.login, author.displayName || author.login);
-		}
-
-		// Import content (locale from config scopes all items)
-		const result = await importContent(
-			wxr.posts,
-			config,
-			dineway,
-			dinewayManifest,
-			attachmentMap,
-			config.locale,
-			authorDisplayNames,
-		);
-
-		// Import reusable blocks as sections (if enabled)
-		if (config.importSections !== false) {
-			const sectionsResult = await importReusableBlocksAsSections(wxr.posts, dineway.db);
-			result.sections = {
-				created: sectionsResult.sectionsCreated,
-				skipped: sectionsResult.sectionsSkipped,
-			};
-			// Add section errors to main errors array
-			result.errors.push(...sectionsResult.errors);
-			if (sectionsResult.errors.length > 0) {
-				result.success = false;
-			}
-		}
-
-		return apiSuccess(result);
-	} catch (error) {
-		return handleError(error, "Failed to import content", "WXR_IMPORT_ERROR");
-	}
-};
-
-async function importContent(
-	posts: WxrPost[],
-	config: ImportConfig,
-	dineway: DinewayHandlers,
-	manifest: DinewayManifest,
-	attachmentMap: Map<string, string>,
-	locale?: string,
-	authorDisplayNames?: Map<string, string>,
-): Promise<ImportResult> {
-	const result: ImportResult = {
-		success: true,
-		imported: 0,
-		skipped: 0,
-		errors: [],
-		byCollection: {},
-	};
-
-	// Create content repository for checking existing items
-	const contentRepo = new ContentRepository(dineway.db);
-	const bylineRepo = new BylineRepository(dineway.db);
-	const bylineCache = new Map<string, string>();
-
-	for (const post of posts) {
-		const postType = post.postType || "post";
-		const mapping = config.postTypeMappings[postType];
-
-		// Skip if not mapped or disabled
-		if (!mapping || !mapping.enabled) {
-			result.skipped++;
-			continue;
-		}
-
-		// Defensive: mapping.collection is already sanitized by prepare, but the user
-		// could manually edit the import config between prepare and execute.
-		const collection = sanitizeWordPressImportSlug(mapping.collection);
-
-		// Check if collection exists in manifest
-		if (!manifest?.collections[collection]) {
-			result.errors.push({
-				title: post.title || "Untitled",
-				error: `Collection "${collection}" does not exist`,
-			});
-			continue;
-		}
-
-		try {
-			// Convert content to Portable Text
-			const content = post.content ? gutenbergToPortableText(post.content) : [];
-
-			// Generate slug from post name or title
-			const slug = post.postName || slugify(post.title || `post-${post.id || Date.now()}`);
-
-			// Check if already exists (idempotency)
-			if (config.skipExisting) {
-				const existing = await contentRepo.findBySlug(collection, slug);
-				if (existing) {
-					result.skipped++;
-					continue;
-				}
-			}
-
-			// Map WordPress status to Dineway status
-			const status = mapStatus(post.status);
-
-			// Build data object with required fields
-			const data: Record<string, unknown> = {
-				title: post.title || "Untitled",
-				content,
-				excerpt: post.excerpt || undefined,
-			};
-
-			// Only add featured_image if the collection has this field and we have a value
-			const collectionSchema = manifest.collections[collection];
-			const hasFeaturedImageField = collectionSchema?.fields
-				? "featured_image" in collectionSchema.fields
-				: false;
-			if (hasFeaturedImageField) {
-				const thumbnailId = post.meta.get("_thumbnail_id");
-				const featuredImage = thumbnailId ? attachmentMap.get(String(thumbnailId)) : undefined;
-				if (featuredImage) {
-					data.featured_image = featuredImage;
-				}
-			}
-
-			// Resolve author ID from mappings
-			let authorId: string | undefined;
-			if (config.authorMappings && post.creator) {
-				const mappedUserId = config.authorMappings[post.creator];
-				if (mappedUserId !== undefined && mappedUserId !== null) {
-					authorId = mappedUserId;
-				}
-			}
-
-			const bylineId = await resolveImportByline(
-				post.creator,
-				authorDisplayNames?.get(post.creator ?? "") ?? post.creator,
-				authorId,
-				bylineRepo,
-				bylineCache,
-			);
-
-			// Preserve original WordPress dates using the shared WXR date parser.
-			// Fallback chain: postDateGmt (UTC) → pubDate (RFC 2822) → postDate (site-local).
-			const parsedDate = parseWxrDate(post.postDateGmt, post.pubDate, post.postDate);
-			const createdAt = parsedDate ? parsedDate.toISOString() : undefined;
-			const publishedAt = status === "published" && createdAt ? createdAt : undefined;
-
-			// Create the content item
-			const createResult = await dineway.handleContentCreate(collection, {
-				data,
-				slug,
-				status,
-				authorId,
-				bylines: bylineId ? [{ bylineId }] : undefined,
-				locale,
-				createdAt,
-				publishedAt,
-			});
-
-			if (createResult.success) {
-				result.imported++;
-				result.byCollection[collection] = (result.byCollection[collection] || 0) + 1;
-			} else {
-				result.errors.push({
-					title: post.title || "Untitled",
-					error:
-						typeof createResult.error === "object" && createResult.error !== null
-							? (createResult.error as { message?: string }).message || "Unknown error"
-							: String(createResult.error),
-				});
-			}
-		} catch (error) {
-			console.error(`Import error for "${post.title || "Untitled"}":`, error);
-			result.errors.push({
-				title: post.title || "Untitled",
-				error: error instanceof Error && error.message ? error.message : "Failed to import item",
-			});
-		}
-	}
-
-	result.success = result.errors.length === 0;
-	return result;
-}
-
-function mapStatus(wpStatus: string | undefined): string {
-	switch (wpStatus) {
-		case "publish":
-			return "published";
-		case "draft":
-			return "draft";
-		case "pending":
-			return "draft";
-		case "private":
-			return "draft";
-		default:
-			return "draft";
-	}
-}

package/src/astro/routes/api/import/wordpress/media.ts

@@ -1,338 +0,0 @@
-/**
- * WordPress media import endpoint
- *
- * POST /_dineway/api/import/wordpress/media
- *
- * Downloads media attachments from WordPress URLs and uploads to Dineway storage.
- * Streams progress updates as newline-delimited JSON (NDJSON).
- * Each line is either a progress update or the final result.
- */
-
-import * as path from "node:path";
-
-import type { APIRoute } from "astro";
-import { MediaRepository, computeContentHash } from "dineway";
-import mime from "mime/lite";
-import { ulid } from "ulidx";
-
-import { requirePerm } from "#api/authorize.js";
-import { apiError, apiSuccess, handleError } from "#api/error.js";
-import { isParseError, parseBody } from "#api/parse.js";
-import { wpMediaImportBody } from "#api/schemas.js";
-import { validateExternalUrl, ssrfSafeFetch, SsrfError } from "#import/ssrf.js";
-import type { DinewayHandlers } from "#types";
-
-import type { AttachmentInfo } from "./analyze.js";
-
-export const prerender = false;
-
-/** Progress update sent during streaming */
-export interface MediaImportProgress {
-	type: "progress";
-	current: number;
-	total: number;
-	filename?: string;
-	status: "downloading" | "uploading" | "done" | "skipped" | "failed";
-	error?: string;
-}
-
-/** Final result sent at end of stream */
-export interface MediaImportResult {
-	type?: "result";
-	/** Successfully imported items */
-	imported: Array<{
-		wpId?: number;
-		originalUrl: string;
-		newUrl: string;
-		mediaId: string;
-	}>;
-	/** Failed items */
-	failed: Array<{
-		wpId?: number;
-		originalUrl: string;
-		error: string;
-	}>;
-	/** Map of old URLs to new URLs (for content rewriting) */
-	urlMap: Record<string, string>;
-}
-
-export const POST: APIRoute = async ({ request, locals }) => {
-	const { dineway, user } = locals;
-
-	const denied = requirePerm(user, "import:execute");
-	if (denied) return denied;
-
-	if (!dineway?.storage) {
-		return apiError("NO_STORAGE", "Storage not configured. Media import requires storage.", 501);
-	}
-
-	if (!dineway?.db) {
-		return apiError("NO_DB", "Database not initialized", 500);
-	}
-
-	try {
-		const body = await parseBody(request, wpMediaImportBody);
-		if (isParseError(body)) return body;
-
-		const attachments = body.attachments as AttachmentInfo[];
-
-		// Check if streaming is requested (default: true)
-		const shouldStream = body.stream !== false;
-
-		if (shouldStream) {
-			// Stream progress updates as NDJSON
-			const stream = new ReadableStream({
-				async start(controller) {
-					const encoder = new TextEncoder();
-					const sendProgress = (progress: MediaImportProgress) => {
-						controller.enqueue(encoder.encode(JSON.stringify(progress) + "\n"));
-					};
-
-					const result = await importMediaWithProgress(
-						attachments,
-						dineway.db,
-						dineway.storage,
-						request.url,
-						sendProgress,
-					);
-
-					// Send final result
-					controller.enqueue(encoder.encode(JSON.stringify({ ...result, type: "result" }) + "\n"));
-					controller.close();
-				},
-			});
-
-			return new Response(stream, {
-				status: 200,
-				headers: {
-					"Content-Type": "application/x-ndjson",
-					"Cache-Control": "private, no-store",
-					"Transfer-Encoding": "chunked",
-				},
-			});
-		}
-
-		// Non-streaming mode
-		const result = await importMediaWithProgress(
-			attachments,
-			dineway.db,
-			dineway.storage,
-			request.url,
-			() => {}, // No-op progress callback
-		);
-
-		return apiSuccess(result);
-	} catch (error) {
-		return handleError(error, "Failed to import media", "IMPORT_ERROR");
-	}
-};
-
-async function importMediaWithProgress(
-	attachments: AttachmentInfo[],
-	db: NonNullable<DinewayHandlers["db"]>,
-	storage: NonNullable<DinewayHandlers["storage"]>,
-	requestUrl: string,
-	onProgress: (progress: MediaImportProgress) => void,
-): Promise<MediaImportResult> {
-	const repo = new MediaRepository(db);
-	const url = new URL(requestUrl);
-	const baseUrl = `${url.protocol}//${url.host}`;
-	const total = attachments.length;
-
-	const result: MediaImportResult = {
-		imported: [],
-		failed: [],
-		urlMap: {},
-	};
-
-	for (let i = 0; i < attachments.length; i++) {
-		const attachment = attachments[i];
-		const current = i + 1;
-		const filename = attachment.filename || `file-${attachment.id}`;
-
-		if (!attachment.url) {
-			result.failed.push({
-				wpId: attachment.id,
-				originalUrl: "",
-				error: "No URL provided",
-			});
-			onProgress({
-				type: "progress",
-				current,
-				total,
-				filename,
-				status: "failed",
-				error: "No URL provided",
-			});
-			continue;
-		}
-
-		try {
-			// SSRF: validate URL before fetching
-			try {
-				validateExternalUrl(attachment.url);
-			} catch (e) {
-				const msg = e instanceof SsrfError ? e.message : "Invalid URL";
-				result.failed.push({
-					wpId: attachment.id,
-					originalUrl: attachment.url,
-					error: `Blocked: ${msg}`,
-				});
-				onProgress({
-					type: "progress",
-					current,
-					total,
-					filename,
-					status: "failed",
-					error: `Blocked: ${msg}`,
-				});
-				continue;
-			}
-
-			// Report downloading
-			onProgress({
-				type: "progress",
-				current,
-				total,
-				filename,
-				status: "downloading",
-			});
-
-			// Download from WordPress (ssrfSafeFetch re-validates redirect targets)
-			const response = await ssrfSafeFetch(attachment.url, {
-				headers: {
-					"User-Agent": "Dineway-Importer/1.0",
-				},
-			});
-
-			if (!response.ok) {
-				result.failed.push({
-					wpId: attachment.id,
-					originalUrl: attachment.url,
-					error: `HTTP ${response.status}: ${response.statusText}`,
-				});
-				onProgress({
-					type: "progress",
-					current,
-					total,
-					filename,
-					status: "failed",
-					error: `HTTP ${response.status}`,
-				});
-				continue;
-			}
-
-			// Get content type from response or guess from filename
-			const contentType =
-				response.headers.get("content-type") || attachment.mimeType || "application/octet-stream";
-
-			// Get the file data
-			const buffer = await response.arrayBuffer();
-			const size = buffer.byteLength;
-
-			// Compute content hash for deduplication
-			const contentHash = await computeContentHash(buffer);
-
-			// Check if we already have this exact content
-			const existing = await repo.findByContentHash(contentHash);
-			if (existing) {
-				// Same content already exists - reuse it
-				const existingUrl = `${baseUrl}/_dineway/api/media/file/${existing.storageKey}`;
-				result.urlMap[attachment.url] = existingUrl;
-				result.imported.push({
-					wpId: attachment.id,
-					originalUrl: attachment.url,
-					newUrl: existingUrl,
-					mediaId: existing.id,
-				});
-				onProgress({
-					type: "progress",
-					current,
-					total,
-					filename,
-					status: "skipped",
-				});
-				continue;
-			}
-
-			// Report uploading
-			onProgress({
-				type: "progress",
-				current,
-				total,
-				filename,
-				status: "uploading",
-			});
-
-			// Generate storage key
-			const id = ulid();
-			const ext = attachment.filename
-				? path.extname(attachment.filename)
-				: getExtensionFromMimeType(contentType);
-			const storageKey = `${id}${ext}`;
-
-			// Upload to storage
-			await storage.upload({
-				key: storageKey,
-				body: new Uint8Array(buffer),
-				contentType,
-			});
-
-			// Create media record with content hash
-			const mediaItem = await repo.create({
-				filename: attachment.filename || `media-${attachment.id}${ext}`,
-				mimeType: contentType,
-				size,
-				storageKey,
-				contentHash,
-				width: undefined,
-				height: undefined,
-			});
-
-			// Build the new URL
-			const newUrl = `${baseUrl}/_dineway/api/media/file/${storageKey}`;
-
-			result.imported.push({
-				wpId: attachment.id,
-				originalUrl: attachment.url,
-				newUrl,
-				mediaId: mediaItem.id,
-			});
-
-			// Add to URL map
-			result.urlMap[attachment.url] = newUrl;
-
-			// Report done
-			onProgress({
-				type: "progress",
-				current,
-				total,
-				filename,
-				status: "done",
-			});
-		} catch (error) {
-			console.error(`Media import error for "${filename}":`, error);
-			const errorMsg = "Failed to import media";
-			result.failed.push({
-				wpId: attachment.id,
-				originalUrl: attachment.url,
-				error: errorMsg,
-			});
-			onProgress({
-				type: "progress",
-				current,
-				total,
-				filename,
-				status: "failed",
-				error: errorMsg,
-			});
-		}
-	}
-
-	return result;
-}
-
-function getExtensionFromMimeType(mimeType: string): string {
-	const ext = mime.getExtension(mimeType);
-	return ext ? `.${ext}` : "";
-}