domma-cms 1.7.1 → 1.7.2

package/server/services/wpImport/index.js

@@ -0,0 +1,832 @@
+/**
+ * Import from WordPress - a built-in Tool (Tools › Import from WordPress,
+ * #/wordpress-import; permission `import.wordpress`, held by level 0 and
+ * granted once to `admin`).
+ *
+ *   upload  - a WXR export (.xml, or an .xml in a .zip) is staged and read
+ *   preview - planImport() (./plan.js): every item's action and address, the
+ *             media source, redirects; nothing written
+ *   run     - a background job (startRun) working through the plan in small
+ *             steps, so the site keeps answering; progress is polled
+ *   report  - kept in a history list (./store.js)
+ *
+ * Re-runs: map.json remembers what each WordPress id became, with a
+ * fingerprint of what was written, so a second run updates rather than
+ * duplicates, and leaves anything edited here since alone unless told.
+ *
+ * Posts go to the Blog when a Blog Tool owning the `blog-posts`,
+ * `blog-categories` and `blog-comments` collections is switched on
+ * (feature-detected from the collections, see blogTarget()); otherwise they
+ * become pages under /blog, or are skipped - the preview says which.
+ *
+ * The one request hook here answers WordPress's query addresses (/?p=12,
+ * /?page_id=7, /?attachment_id=3) from the mapping; path redirects go into
+ * SEO's redirect store.
+ *
+ * @module services/wpImport
+ */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import fp from 'fastify-plugin';
+import matter from 'gray-matter';
+import {readWxr} from './wxr.js';
+import {openZip, ZipError} from './zip.js';
+import {freeName, isoDate, mediaName, menuItems, originalUploadsPath, planImport, seoFields, uploadsPath} from './plan.js';
+import {convertContent} from './convert.js';
+import {fetchMedia, testHostsAllowed} from './fetch.js';
+import {
+    addHistory, clearRunning, createStaging, dropStaging, fingerprint, markRunning, recordFailedRun, recoverInterrupted, followRename, forgetMapCache, mappingFor, newId, queryTarget, saveMapping,
+    stagingMeta, stagingPath, updateStagingMeta
+} from './store.js';
+import {createPage, getPage, listMedia, listPages, resolveExistingFilePath, saveMedia, updatePage} from '../content.js';
+import {createEntry, getCollection, listEntries, updateEntry} from '../collections.js';
+import {getPluginStates} from '../plugins.js';
+import {createMenu, getMenu, listMenus, updateMenu} from '../menus.js';
+import {listUsers} from '../users.js';
+import {seoEngine} from '../seo/index.js';
+import {hooks as coreHooks} from '../hooks.js';
+import {allowInGitignore} from '../siteGitignore.js';
+import fsSync from 'node:fs';
+
+export const WXR_MAX_BYTES = 50 * 1024 * 1024;
+export const UPLOADS_ZIP_MAX_BYTES = 1024 * 1024 * 1024;
+const DOWNLOAD_CONCURRENCY = 4;
+
+const BLOG = {posts: 'blog-posts', categories: 'blog-categories', comments: 'blog-comments'};
+
+const tick = () => new Promise(r => setImmediate(r));
+
+// ---- The Blog, if there is one --------------------------------------------
+
+/** Fields a collection must have for the importer to write to it as a Blog does. */
+const BLOG_FIELDS = {
+    posts: ['title', 'slug', 'content', 'excerpt', 'featuredImage', 'categories', 'tags', 'authorId', 'status', 'publishedAt'],
+    categories: ['name', 'slug'],
+    comments: ['postId', 'authorName', 'authorEmail', 'body', 'status']
+};
+
+/** Switched-on plugins that ship a collection with this slug (their collections/ folder). */
+async function ownersOf(slug) {
+    const states = getPluginStates() || {};
+    const out = [];
+    for (const [name, st] of Object.entries(states)) {
+        if (st?.enabled !== true || !/^[a-z0-9][a-z0-9_-]*$/.test(name)) continue;
+        const dir = path.resolve('plugins', name, 'collections');
+        let entries = [];
+        try { entries = await fs.readdir(dir, {withFileTypes: true}); } catch { continue; }
+        for (const e of entries) {
+            const file = e.isDirectory() ? path.join(dir, e.name, 'schema.json') : (e.name.endsWith('.json') ? path.join(dir, e.name) : '');
+            if (!file) continue;
+            try {
+                if (JSON.parse(await fs.readFile(file, 'utf8'))?.slug === slug) { out.push(name); break; }
+            } catch { /* not a schema */ }
+        }
+    }
+    return out;
+}
+
+const hasFields = (schema, names) => {
+    const have = new Set((schema?.fields || []).map(f => f?.name));
+    return names.every(n => have.has(n));
+};
+
+/**
+ * Whether posts can go to a Blog: a switched-on Tool ships the `blog-posts`
+ * collection (feature-detected from plugins' collections, never by name), and
+ * the site's `blog-posts` and `blog-categories` have the fields a Blog post
+ * needs. Comments need `blog-comments` in the same shape. The base address is
+ * the owning Tool's `basePath` setting (default /blog).
+ */
+export async function blogTarget() {
+    const out = {available: false, basePath: '/blog', owner: '', comments: false, reason: ''};
+    try {
+        const owners = await ownersOf(BLOG.posts);
+        if (!owners.length) { out.reason = 'No Blog is switched on, so posts can become pages or be left out.'; return out; }
+        const [posts, categories, comments] = await Promise.all([BLOG.posts, BLOG.categories, BLOG.comments].map(s => getCollection(s).catch(() => null)));
+        if (!hasFields(posts, BLOG_FIELDS.posts) || !hasFields(categories, BLOG_FIELDS.categories)) {
+            out.reason = 'The Blog\'s collections on this site are not in the shape the importer writes, so posts can become pages or be left out.';
+            return out;
+        }
+        out.available = true;
+        out.owner = owners[0];
+        out.comments = hasFields(comments, BLOG_FIELDS.comments);
+        const base = getPluginStates()?.[out.owner]?.settings?.basePath;
+        if (typeof base === 'string' && /^\/[a-z0-9/-]*$/i.test(base.trim())) out.basePath = base.trim().replace(/\/+$/, '') || '/blog';
+    } catch { /* no Blog */ }
+    return out;
+}
+
+const entriesOf = (r) => (Array.isArray(r) ? r : Array.isArray(r?.entries) ? r.entries : []);
+
+// ---- What the site has now ----------------------------------------------
+
+async function pageFingerprint(urlPath) {
+    try {
+        const file = await resolveExistingFilePath(urlPath);
+        const raw = await fs.readFile(file, 'utf8');
+        const parsed = matter(raw);
+        return fingerprint(parsed.data?.title, parsed.content);
+    } catch { return ''; }
+}
+
+const postFingerprint = (d) => fingerprint(d?.title, d?.content, d?.excerpt);
+const menuFingerprint = (items) => fingerprint(JSON.stringify(items || []));
+
+/** The site's state as the planner needs it. */
+export async function siteState(mapping = {}) {
+    const pages = await listPages();
+    const pagePaths = new Set(pages.map(p => String(p.urlPath || '/').toLowerCase()));
+    const pageHashes = {};
+    for (const m of Object.values(mapping.items || {})) {
+        if (m.path && pagePaths.has(m.path)) pageHashes[m.path] = await pageFingerprint(m.path);
+    }
+    const media = await listMedia().catch(() => []);
+    const menus = await listMenus().catch(() => []);
+    const menuHashes = {};
+    for (const m of Object.values(mapping.items || {})) {
+        if (m?.kind !== 'menu' || !m.slug) continue;
+        const menu = await getMenu(m.slug).catch(() => null);
+        if (menu) menuHashes[m.slug] = menuFingerprint(menu.items);
+    }
+    const blog = await blogTarget();
+    blog.posts = new Map();
+    blog.postSlugs = new Set();
+    blog.categories = new Map();
+    if (blog.available) {
+        for (const e of entriesOf(await listEntries(BLOG.posts, {limit: 100000}).catch(() => []))) {
+            const d = e.data || {};
+            blog.posts.set(e.id, {slug: d.slug, hash: postFingerprint(d)});
+            if (d.slug) blog.postSlugs.add(d.slug);
+        }
+        for (const e of entriesOf(await listEntries(BLOG.categories, {limit: 10000}).catch(() => []))) {
+            if (e.data?.slug) blog.categories.set(e.data.slug, e.id);
+        }
+    }
+    const seo = seoEngine();
+    return {
+        pagePaths, pageHashes,
+        mediaNames: new Set(media.map(m => String(m.name).toLowerCase())),
+        menuSlugs: new Set(menus.map(m => m.slug)),
+        menuHashes,
+        blog,
+        redirectsAvailable: Boolean(seo),
+        redirectFroms: new Set(seo ? seo.store.rules().map(r => r.from) : [])
+    };
+}
+
+// ---- Staging an upload ---------------------------------------------------
+
+const parsed = new Map(); // staging id -> {wxr, at} (the last few, so preview after preview is quick)
+
+/**
+ * Read a staged upload: an .xml as it is, or the (largest) .xml inside a .zip.
+ * Records what it is in the staging meta.
+ */
+export async function inspectStaged(id) {
+    const meta = await stagingMeta(id);
+    if (!meta) throw Object.assign(new Error('That upload has gone - upload the export again.'), {statusCode: 404});
+    if (meta.kind === 'zip') {
+        const zip = await openZip(stagingPath(id, 'upload.zip'));
+        try {
+            const xml = zip.entries.filter(e => /\.xml$/i.test(e.name) && !/(^|\/)__MACOSX\//.test(e.name)).sort((a, b) => b.size - a.size)[0];
+            if (!xml) throw new ZipError('There is no .xml file in that zip - export from WordPress (Tools › Export) and upload the file it gives you.');
+            if (xml.size > WXR_MAX_BYTES) throw new ZipError(`The export is ${Math.round(xml.size / 1048576)} MB - over the ${WXR_MAX_BYTES / 1048576} MB limit.`);
+            await fs.writeFile(stagingPath(id, 'export.xml'), await zip.read(xml));
+        } finally {
+            await zip.close();
+        }
+        await fs.rm(stagingPath(id, 'upload.zip'), {force: true});
+        await updateStagingMeta(id, {kind: 'xml'});
+    }
+    const wxr = await loadExport(id);
+    const counts = {};
+    for (const it of wxr.items) counts[it.type] = (counts[it.type] || 0) + 1;
+    return updateStagingMeta(id, {siteTitle: wxr.site.title, siteUrl: wxr.site.baseSiteUrl || wxr.site.link, wxrVersion: wxr.site.wxrVersion, counts});
+}
+
+export async function loadExport(id) {
+    const hit = parsed.get(id);
+    if (hit) return hit;
+    const text = await fs.readFile(stagingPath(id, 'export.xml'), 'utf8');
+    const wxr = await readWxr(text);
+    parsed.set(id, wxr);
+    while (parsed.size > 2) parsed.delete(parsed.keys().next().value);
+    return wxr;
+}
+
+export async function discardStaged(id) {
+    parsed.delete(id);
+    await dropStaging(id);
+}
+
+export {createStaging, stagingMeta, stagingPath, updateStagingMeta};
+
+// ---- The uploads zip -------------------------------------------------------
+
+/** uploads-relative path (lower case) for a zip entry: "site/wp-content/uploads/2019/05/a.jpg" -> "2019/05/a.jpg". */
+export function zipRel(name) {
+    const s = String(name);
+    const at = s.toLowerCase().lastIndexOf('uploads/');
+    if (at !== -1) return s.slice(at + 8).toLowerCase();
+    const m = /(\d{4}\/\d{2}\/[^/]+)$/.exec(s);
+    return (m ? m[1] : s).toLowerCase();
+}
+
+/** The uploads zip's files by uploads-relative path, or null when there is none. */
+export async function uploadsIndex(id) {
+    const meta = await stagingMeta(id);
+    if (!meta?.uploads) return null;
+    const zip = await openZip(stagingPath(id, 'uploads.zip'));
+    try {
+        return new Set(zip.entries.map(e => zipRel(e.name)));
+    } finally {
+        await zip.close();
+    }
+}
+
+// ---- Preview ---------------------------------------------------------------
+
+/** The plan for a staged export with these options - nothing written. */
+export async function previewImport(id, options = {}) {
+    const meta = await stagingMeta(id);
+    if (!meta) throw Object.assign(new Error('That upload has gone - upload the export again.'), {statusCode: 404});
+    const wxr = await loadExport(id);
+    const {sourceKey} = await import('./plan.js');
+    const mapping = await mappingFor(sourceKey(wxr.site));
+    const state = await siteState(mapping);
+    const zipPaths = options.media === 'zip' ? await uploadsIndex(id) : null;
+    const plan = planImport(wxr, state, mapping, {...cleanOptions(options), zipPaths, anyPort: testHostsAllowed()});
+    return {plan, state, wxr, mapping, meta};
+}
+
+/** Options from a request - whitelisted. */
+export function cleanOptions(o = {}) {
+    const choices = {};
+    for (const [k, v] of Object.entries(o.choices || {}).slice(0, 20000)) {
+        if (/^[a-z_]+:[\w.-]{1,120}$/i.test(k) && ['skip', 'overwrite', 'import'].includes(v)) choices[k] = v;
+    }
+    return {
+        ...(['blog', 'pages', 'skip'].includes(o.posts) && {posts: o.posts}),
+        ...(['download', 'zip', 'skip'].includes(o.media) && {media: o.media}),
+        comments: o.comments !== false,
+        menus: o.menus !== false,
+        redirects: o.redirects !== false,
+        drafts: o.drafts !== false,
+        choices
+    };
+}
+
+// ---- The run ---------------------------------------------------------------
+
+const jobs = new Map();
+let running = null;
+
+export const getJob = (id) => jobs.get(id) || null;
+export const runningJob = () => running;
+
+/**
+ * Start an import in the background. One at a time per site.
+ *
+ * @returns {Promise<object>} the job (poll getJob(id) for progress)
+ */
+export async function startRun(id, options, user, log = console) {
+    if (running) throw Object.assign(new Error('An import is already running - wait for it to finish.'), {statusCode: 409});
+    // Claimed before the first await, so two quick Run requests cannot both start.
+    const job = {id: newId(), staged: id, status: 'running', phase: 'Starting', done: 0, total: 0, startedAt: new Date().toISOString(), reportId: null, error: ''};
+    running = job;
+    let pre;
+    try {
+        pre = await previewImport(id, options);
+        if (pre.plan.options.media === 'zip' && !pre.meta.uploads) {
+            throw Object.assign(new Error('Upload the wp-content/uploads zip first, or choose another media source.'), {statusCode: 400});
+        }
+    } catch (err) {
+        running = null;
+        throw err;
+    }
+    jobs.set(job.id, job);
+    for (const [k, j] of jobs) if (j !== job && j.status !== 'running' && jobs.size > 10) jobs.delete(k);
+    const record = {id: job.id, by: user?.name || user?.email || '', source: pre.plan.source, siteTitle: pre.plan.site.title,
+        siteUrl: pre.plan.site.url, file: pre.meta.fileName || '', staged: id, startedAt: job.startedAt};
+    await markRunning(record).catch(() => {});
+    runJob(job, pre, user, log)
+        .catch(async (err) => {
+            job.status = 'failed';
+            job.error = err.message || String(err);
+            log.error?.(`[wordpress-import] the run failed: ${err.stack || err}`);
+            await recordFailedRun(record, `The import stopped: ${job.error} What it had already made is kept; run it again to finish - nothing is made twice.`).catch(() => {});
+            job.reportId = record.id;
+        })
+        .finally(async () => { await clearRunning().catch(() => {}); running = null; job.finishedAt = new Date().toISOString(); });
+    return job;
+}
+
+async function runJob(job, {plan, wxr, mapping: oldMapping, meta}, user, log) {
+    const started = Date.now();
+    const mapping = {
+        items: {...(oldMapping.items || {})},
+        redirects: {...(oldMapping.redirects || {})},
+        query: {p: {...oldMapping.query?.p}, page_id: {...oldMapping.query?.page_id}, attachment_id: {...oldMapping.query?.attachment_id}}
+    };
+    try {
+        await runPhases(job, {plan, wxr, oldMapping, meta}, user, log, mapping, started);
+    } catch (err) {
+        // What was written before the failure is recorded, so a retry updates it rather than duplicating it.
+        await saveMapping(plan.source, mapping).catch(() => {});
+        forgetMapCache();
+        throw err;
+    }
+}
+
+async function runPhases(job, {plan, wxr, oldMapping, meta}, user, log, mapping, started) {
+    const byId = new Map(wxr.items.map(i => [i.id, i]));
+    const rows = new Map(plan.items.map(r => [r.key, r]));
+    const results = new Map(); // key -> {result, error?, target?, notes?}
+    const set = (key, r) => results.set(key, r);
+    const works = (r) => ['create', 'rename', 'update'].includes(r.action);
+    job.total = plan.items.filter(works).length + plan.redirects.filter(r => r.action !== 'skip').length;
+    // What has been made is saved as the run goes (every few items, and at each phase), so a
+    // restart in the middle loses nothing a second run would then make twice.
+    let unsaved = 0;
+    let savedAt = Date.now();
+    const checkpoint = async (force = false) => {
+        if (!force && unsaved < 5 && Date.now() - savedAt < 2000) return;
+        unsaved = 0;
+        savedAt = Date.now();
+        await saveMapping(plan.source, mapping);
+    };
+    const step = async (label) => { job.done++; unsaved++; if (label) job.current = label; await checkpoint(); await tick(); };
+
+    // ---- 1. Media ----
+    job.phase = 'Media';
+    const mediaRows = plan.items.filter(r => r.kind === 'media' && works(r));
+    const mediaTarget = new Map();   // uploads rel (lower) -> /media/name, original and variants
+    const mediaById = new Map();     // attachment id -> {url, alt, title}
+    const remember = (it, url) => {
+        const rel = uploadsPath(it.attachmentUrl).toLowerCase();
+        if (rel) { mediaTarget.set(rel, url); mediaTarget.set(originalUploadsPath(rel), url); }
+        mediaById.set(it.id, {url, alt: it.meta._wp_attachment_image_alt || '', title: it.title});
+    };
+    for (const r of plan.items.filter(x => x.kind === 'media' && x.action === 'keep')) {
+        remember(byId.get(r.wpId), r.target);
+        set(r.key, {result: 'kept', target: r.target});
+        mapping.items[r.key] = oldMapping.items[r.key];
+    }
+    const zip = plan.options.media === 'zip' ? await openZip(stagingPath(job.staged, 'uploads.zip')) : null;
+    const zipEntries = zip ? new Map(zip.entries.map(e => [zipRel(e.name), e])) : null;
+    const takenNow = new Set((await listMedia().catch(() => [])).map(m => m.name.toLowerCase()));
+    const fetchOpts = {
+        hosts: wxr.site.hosts,
+        allowPrivate: testHostsAllowed()
+    };
+    try {
+        const queue = [...mediaRows];
+        const worker = async () => {
+            while (queue.length) {
+                const r = queue.shift();
+                const it = byId.get(r.wpId);
+                try {
+                    let buffer;
+                    let contentType = '';
+                    if (zip) {
+                        const entry = zipEntries.get(r.rel.toLowerCase());
+                        if (!entry) throw new Error('It is not in the uploaded zip.');
+                        buffer = await zip.read(entry);
+                    } else {
+                        ({buffer, contentType} = await fetchMedia(it.attachmentUrl, fetchOpts));
+                    }
+                    const ext = r.name.split('.').pop().toLowerCase();
+                    const bad = badContent(ext, buffer, contentType);
+                    if (bad) throw new Error(bad);
+                    const name = await placeMedia(r.name, buffer, takenNow, oldMapping.items[r.key], mediaName(r.rel || it.attachmentUrl, it.id));
+                    const url = `/media/${name}`;
+                    remember(it, url);
+                    mapping.items[r.key] = {kind: 'media', name, path: null, sha: fingerprint(buffer.toString('base64')), at: new Date().toISOString()};
+                    set(r.key, {result: r.action === 'update' ? 'updated' : 'created', target: url});
+                } catch (err) {
+                    set(r.key, {result: 'failed', error: err.message || String(err)});
+                }
+                await step(r.title);
+            }
+        };
+        await Promise.all(Array.from({length: zip ? 1 : DOWNLOAD_CONCURRENCY}, worker));
+    } finally {
+        await zip?.close();
+    }
+    for (const [aid, m] of mediaById) mapping.query.attachment_id[aid] = m.url;
+
+    // Address lookups for content and menus.
+    const targetOf = new Map(plan.items.filter(r => (r.kind === 'page' || r.kind === 'post') && r.target && r.action !== 'skip').map(r => [r.wpId, r.target]));
+    const oldPathTarget = new Map(plan.items.filter(r => r.oldPath && r.target && r.action !== 'skip').map(r => [r.oldPath, r.target]));
+    const {oldPathOf, basePathOf} = await import('./plan.js');
+    const mediaFor = (url) => {
+        const rel = uploadsPath(url).toLowerCase();
+        if (!rel) return null;
+        let host = '';
+        try { host = new URL(url, 'https://relative.invalid').hostname.toLowerCase(); } catch { return null; }
+        if (host !== 'relative.invalid' && !wxr.site.hosts.includes(host)) return null;
+        return mediaTarget.get(rel) || mediaTarget.get(originalUploadsPath(rel)) || null;
+    };
+    const linkFor = (url) => {
+        const raw = String(url || '').trim();
+        if (!raw || raw.startsWith('#') || /^(mailto|tel):/i.test(raw)) return null;
+        let u;
+        try { u = new URL(raw, wxr.site.baseSiteUrl || wxr.site.link); } catch { return null; }
+        if (!wxr.site.hosts.includes(u.hostname.toLowerCase())) return null;
+        const q = u.searchParams;
+        for (const k of ['p', 'page_id']) {
+            const id = Number.parseInt(q.get(k), 10);
+            if (id && targetOf.has(id)) return targetOf.get(id) + u.hash;
+        }
+        const p = oldPathOf(u.href, wxr.site);
+        // The old site's home is this site's home, whatever became of WordPress's front page.
+        if (p === '/' || p === basePathOf(wxr.site)) return `/${u.hash}`;
+        if (p && oldPathTarget.has(p)) return oldPathTarget.get(p) + u.hash;
+        // Elsewhere on the old site: keep it on this one (a redirect or a page may answer).
+        return p ? `${p}${u.hash}` : null;
+    };
+    const attachmentsOf = (postId) => wxr.items.filter(i => i.type === 'attachment' && i.parent === postId).sort((a, b) => a.order - b.order || a.id - b.id).map(i => i.id);
+    const convert = (it) => convertContent(it.content, {mediaFor, mediaById: (aid) => mediaById.get(aid) || null, linkFor, attachmentsOf, postId: it.id});
+
+    await checkpoint(true);
+    // ---- 2. Blog categories ----
+    const blogMode = plan.options.posts === 'blog';
+    const categoryIds = new Map();
+    if (blogMode) {
+        job.phase = 'Categories';
+        const existing = new Map();
+        for (const e of entriesOf(await listEntries(BLOG.categories, {limit: 10000}))) if (e.data?.slug) existing.set(e.data.slug, e.id);
+        for (const r of plan.items.filter(x => x.kind === 'category')) {
+            if (existing.has(r.slug)) categoryIds.set(r.slug, existing.get(r.slug));
+            if (r.action === 'skip' || existing.has(r.slug)) { set(r.key, {result: r.action === 'skip' ? 'skipped' : 'kept'}); continue; }
+            const c = wxr.categories.find(x => x.slug === r.key.slice(9));
+            try {
+                const e = await createEntry(BLOG.categories, {name: clip(c?.name || r.slug, 120), slug: r.slug, description: clip(plainText(c?.description), 500)}, {createdBy: user?.id || null, source: 'import'});
+                categoryIds.set(r.slug, e.id);
+                mapping.items[r.key] = {kind: 'category', entryId: e.id, at: new Date().toISOString()};
+                set(r.key, {result: 'created', target: r.target});
+            } catch (err) {
+                set(r.key, {result: 'failed', error: err.message});
+            }
+            await step(r.title);
+        }
+    }
+
+    await checkpoint(true);
+    // ---- 3. Pages (and posts as pages) ----
+    job.phase = 'Pages';
+    const users = await listUsers().catch(() => []);
+    const userByEmail = new Map(users.filter(u => u.email).map(u => [u.email.toLowerCase(), u.id]));
+    const authorOf = (it) => {
+        const a = wxr.authors.find(x => x.login === it.creator);
+        return (a?.email && userByEmail.get(a.email.toLowerCase())) || user?.id || users[0]?.id || 'import';
+    };
+    const docRows = plan.items.filter(r => (r.kind === 'page' || (r.kind === 'post' && !blogMode)));
+    for (const r of docRows) {
+        if (!works(r)) {
+            if (r.action === 'keep') { mapping.items[r.key] = oldMapping.items[r.key]; recordQuery(mapping, r); }
+            set(r.key, {result: r.action === 'keep' ? 'kept' : 'skipped', target: r.target});
+            continue;
+        }
+        const it = byId.get(r.wpId);
+        try {
+            const conv = convert(it);
+            const fm = pageFrontmatter(it, r, plan, wxr);
+            const exists = await getPage(r.target).catch(() => null);
+            if (exists && (r.action === 'update')) await updatePage(r.target, fm, conv.body, {author: user?.email || 'WordPress import'});
+            else if (exists) throw new Error(`${r.target} was taken while the import ran.`);
+            else await createPage(r.target, fm, conv.body);
+            mapping.items[r.key] = {kind: r.kind, path: r.target, hash: await pageFingerprint(r.target), at: new Date().toISOString()};
+            recordQuery(mapping, r);
+            set(r.key, {result: r.action === 'update' ? 'updated' : 'created', target: r.target, notes: conv.notes, format: conv.format});
+        } catch (err) {
+            set(r.key, {result: 'failed', error: err.message});
+        }
+        await step(r.title);
+    }
+
+    await checkpoint(true);
+    // ---- 4. Blog posts and comments ----
+    let commentsMade = 0;
+    commentErrors.length = 0;
+    if (blogMode) {
+        job.phase = 'Posts';
+        const commentsOn = plan.options.comments && (await blogTarget()).comments;
+        for (const r of plan.items.filter(x => x.kind === 'post')) {
+            if (!works(r)) {
+                if (r.action === 'keep') { mapping.items[r.key] = oldMapping.items[r.key]; recordQuery(mapping, r); }
+                set(r.key, {result: r.action === 'keep' ? 'kept' : 'skipped', target: r.target});
+                continue;
+            }
+            const it = byId.get(r.wpId);
+            try {
+                const conv = convert(it);
+                const data = blogPostData(it, r, conv.body, {authorId: authorOf(it), categoryIds, mediaById, wxr});
+                let entry;
+                if (r.action === 'update' && r.entryId) {
+                    const {getEntry} = await import('../collections.js');
+                    const now = await getEntry(BLOG.posts, r.entryId);
+                    entry = await updateEntry(BLOG.posts, r.entryId, {...(now?.data || {}), ...data, seo: {...(now?.data?.seo || {}), ...data.seo}});
+                } else {
+                    entry = await createEntry(BLOG.posts, data, {createdBy: user?.id || null, source: 'import'});
+                }
+                const id = entry?.id || r.entryId;
+                mapping.items[r.key] = {kind: 'post', entryId: id, slug: data.slug, hash: postFingerprint(data), at: new Date().toISOString()};
+                recordQuery(mapping, r);
+                set(r.key, {result: r.action === 'update' ? 'updated' : 'created', target: r.target, notes: conv.notes, format: conv.format});
+                if (commentsOn) commentsMade += await importComments(it, id, mapping, oldMapping, user);
+            } catch (err) {
+                set(r.key, {result: 'failed', error: err.message});
+            }
+            await step(r.title);
+        }
+    }
+
+    await checkpoint(true);
+    // ---- 5. Menus ----
+    job.phase = 'Menus';
+    const categoryAddress = new Map(plan.items.filter(r => r.kind === 'category' && r.action !== 'skip').map(r => [r.slug, r.target]));
+    for (const r of plan.items.filter(x => x.kind === 'menu')) {
+        if (!works(r)) {
+            if (r.action === 'keep') mapping.items[r.key] = oldMapping.items[r.key];
+            set(r.key, {result: r.action === 'keep' ? 'kept' : 'skipped', target: r.target});
+            continue;
+        }
+        try {
+            const wpSlug = r.key.slice(5);
+            const items = menuItems(wxr, wpSlug, (wpId) => (wpId === 'archive' ? plan.blogBase : targetOf.get(wpId) || null), linkFor, categoryAddress);
+            const name = clip(r.title, 100) || r.slug;
+            const existing = await getMenu(r.slug);
+            if (existing && r.action === 'update') await updateMenu(r.slug, {...existing, items});
+            else if (existing) throw new Error(`A menu called ${r.slug} was made while the import ran.`);
+            else await createMenu({slug: r.slug, name, description: 'Imported from WordPress', items});
+            mapping.items[r.key] = {kind: 'menu', slug: r.slug, hash: menuFingerprint(items), at: new Date().toISOString()};
+            set(r.key, {result: r.action === 'update' ? 'updated' : 'created', target: r.slug});
+        } catch (err) {
+            set(r.key, {result: 'failed', error: err.message});
+        }
+        await step(r.title);
+    }
+
+    await checkpoint(true);
+    // ---- 6. Redirects ----
+    job.phase = 'Redirects';
+    const seo = seoEngine();
+    const redirectResults = [];
+    const failedTargets = new Set([...results.entries()].filter(([, v]) => v.result === 'failed').map(([k]) => rows.get(k)?.target).filter(Boolean));
+    for (const r of plan.redirects) {
+        if (r.action === 'skip' || !seo) { redirectResults.push({...r, result: 'skipped'}); continue; }
+        if (failedTargets.has(r.to)) { redirectResults.push({...r, result: 'skipped', reason: 'What it would lead to was not imported.'}); await step(r.from); continue; }
+        try {
+            const own = mapping.redirects[r.key];
+            const rule = own && seo.store.rules().find(x => x.id === own.id);
+            let out;
+            if (rule) out = seo.updateRule(rule.id, {from: r.from, to: r.to, code: 301});
+            else out = seo.addRule({from: r.from, to: r.to, code: 301, note: 'WordPress import'});
+            if (out.errors) throw new Error(Object.values(out.errors).join(' '));
+            mapping.redirects[r.key] = {id: out.rule.id, from: out.rule.from, to: out.rule.to};
+            redirectResults.push({...r, result: rule ? 'updated' : 'created'});
+        } catch (err) {
+            redirectResults.push({...r, result: 'failed', error: err.message});
+        }
+        await step(r.from);
+    }
+
+    // ---- 7. Save the mapping, the report, the history ----
+    job.phase = 'Finishing';
+    await saveMapping(plan.source, mapping);
+    forgetMapCache();
+    const report = buildReport({job, plan, results, redirectResults, meta, user, started, commentsMade});
+    await addHistory(report.run, report);
+    await discardStaged(job.staged);
+    job.reportId = report.run.id;
+    job.summary = report.run.summary;
+    job.status = 'done';
+    job.phase = 'Done';
+    log.info?.(`[wordpress-import] ${plan.site.title || plan.source}: ${JSON.stringify(report.run.summary)}`);
+}
+
+function recordQuery(mapping, r) {
+    if (!r.target) return;
+    if (r.kind === 'page') { mapping.query.page_id[r.wpId] = r.target; mapping.query.p[r.wpId] = r.target; }
+    if (r.kind === 'post') mapping.query.p[r.wpId] = r.target;
+}
+
+/** A free media name for this file: the planned one, or the same file already here, or the next free. */
+async function placeMedia(planned, buffer, taken, was, original) {
+    const dir = path.resolve('content', 'media');
+    const sameFile = async (name) => {
+        try { return (await fs.readFile(path.join(dir, name))).equals(buffer); } catch { return false; }
+    };
+    if (was?.name && await sameFile(was.name)) return was.name;
+    // The very same file is here already under its own name (another import, an upload): use it.
+    if (original && original !== planned && await sameFile(original)) return original;
+    let name = planned;
+    if (taken.has(name.toLowerCase()) && !(await sameFile(name))) name = freeName(name, taken);
+    taken.add(name.toLowerCase());
+    if (!(await sameFile(name))) await saveMedia(name, buffer);
+    return name;
+}
+
+/** Why these bytes are not the file they claim to be ('' when they are). */
+export function badContent(ext, buf, contentType = '') {
+    if (!buf?.length) return 'The file is empty.';
+    if (/^text\/html/i.test(contentType) && ext !== 'txt') return 'The old site answered with a web page, not the file.';
+    const starts = (...bytes) => bytes.every((b, i) => buf[i] === b);
+    const ascii = (s, at = 0) => buf.slice(at, at + s.length).toString('latin1') === s;
+    const checks = {
+        jpg: () => starts(0xff, 0xd8), jpeg: () => starts(0xff, 0xd8),
+        png: () => starts(0x89, 0x50, 0x4e, 0x47),
+        gif: () => ascii('GIF8'),
+        webp: () => ascii('RIFF') && ascii('WEBP', 8),
+        pdf: () => ascii('%PDF'),
+        bmp: () => ascii('BM')
+    };
+    if (checks[ext] && !checks[ext]()) return `It is not really a .${ext} file.`;
+    // Anything else lands in /media and is served from this site: never a web page or a script, whatever it is called.
+    if (!checks[ext]) {
+        if (/^\s*(text\/html|application\/xhtml|text\/javascript|application\/(x-)?javascript|image\/svg)/i.test(contentType)) return 'The old site sent a web page or a script, not the file.';
+        if (looksLikeMarkup(buf)) return 'It contains a web page or a script - refused.';
+    }
+    return '';
+}
+
+
+const clip = (s, n) => String(s ?? '').replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, '').trim().slice(0, n);
+
+/** HTML (an excerpt, a comment) as plain text. */
+export function plainText(html) {
+    return String(html || '').replace(/<br\s*\/?>/gi, '\n').replace(/<\/p>\s*<p[^>]*>/gi, '\n\n').replace(/<[^>]*>/g, '')
+        .replace(/&nbsp;/g, ' ').replace(/&#0*39;|&apos;/g, "'").replace(/&quot;/g, '"').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
+        .replace(/&#8217;/g, '’').replace(/&#8216;/g, '‘').replace(/&#8220;/g, '“').replace(/&#8221;/g, '”')
+        .replace(/&#8211;/g, '–').replace(/&#8212;/g, '—').replace(/&#8230;/g, '…').replace(/&amp;/g, '&')
+        .replace(/[ \t]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
+}
+
+/** A title as text: the few tags WordPress allows in one removed, entities decoded - "Hello <World>" kept. */
+export function titleText(s) {
+    return plainText(String(s ?? '').replace(/<\/?(b|i|em|strong|span|br|a|small|sup|sub|code|mark|u|s|del|ins)\b[^>]*>/gi, '').replace(/</g, '&lt;').replace(/>/g, '&gt;'));
+}
+
+/** Does this look like HTML, SVG or script a browser would run? (First 1 KB, as browsers sniff.) */
+export function looksLikeMarkup(buf) {
+    const head = buf.subarray(0, 1024).toString('latin1').replace(/^\uFEFF/, '').replace(/\0/g, '').toLowerCase();
+    return /^\s*<(!doctype|html|head|body|script|svg|iframe|\?xml|!--|meta|title|div|a\s|img|object|embed|style)/.test(head) || /<script\b|javascript:/.test(head);
+}
+
+/** A page's frontmatter - every value plain, clipped and free of control characters (gray-matter quotes it). */
+export function pageFrontmatter(it, r, plan, wxr) {
+    const seo = seoFields(it);
+    const description = seo.description || clip(plainText(it.excerpt).replace(/\s+/g, ' '), 300);
+    const fm = {
+        title: clip(titleText(it.title), 200) || `Untitled ${r.kind} ${it.id}`,
+        description,
+        status: r.status === 'published' ? 'published' : 'draft',
+        sortOrder: Number.isInteger(it.order) && it.order > 0 ? it.order : 99,
+        seo: Object.fromEntries(Object.entries({...seo, description: seo.description || ''}).filter(([, v]) => v !== '')),
+        importedFrom: {source: 'wordpress', id: it.id, url: clip(it.link, 500)}
+    };
+    if (r.kind === 'post') {
+        const tags = it.terms.filter(t => t.domain === 'post_tag').map(t => clip(t.name || t.slug, 60)).filter(Boolean);
+        if (tags.length) fm.tags = tags;
+        const date = isoDate(it.dateGmt, it.date);
+        if (date) fm.date = date;
+    }
+    void plan; void wxr;
+    return fm;
+}
+
+/** A Blog post's fields. */
+export function blogPostData(it, r, body, {authorId, categoryIds, mediaById}) {
+    const seo = seoFields(it);
+    const thumb = Number.parseInt(it.meta._thumbnail_id, 10);
+    const date = isoDate(it.dateGmt, it.date);
+    return {
+        title: clip(titleText(it.title), 200) || `Untitled post ${it.id}`,
+        slug: r.slug,
+        excerpt: clip(plainText(it.excerpt), 1000),
+        content: body,
+        featuredImage: (thumb && mediaById.get(thumb)?.url) || '',
+        categories: it.terms.filter(t => t.domain === 'category' && categoryIds.has(t.slug)).map(t => categoryIds.get(t.slug)),
+        tags: [...new Set(it.terms.filter(t => t.domain === 'post_tag').map(t => clip(t.name || t.slug, 60)).filter(Boolean))],
+        authorId,
+        status: r.status,
+        publishedAt: date || new Date().toISOString(),
+        template: '',
+        seo: {...(seo.title && {title: seo.title}), ...(seo.description && {description: seo.description}), ...(seo.noindex && {noindex: true}), ...(seo.focus && {focus: seo.focus})}
+    };
+}
+
+const commentErrors = [];
+
+async function importComments(it, postEntryId, mapping, oldMapping, user) {
+    const approved = it.comments.filter(c => c.approved === '1' && (!c.type || c.type === 'comment')).sort((a, b) => a.id - b.id);
+    let made = 0;
+    const idFor = (wpId) => mapping.items[`comment:${wpId}`]?.entryId || '';
+    for (const c of approved) {
+        const key = `comment:${c.id}`;
+        if (oldMapping.items[key]) { mapping.items[key] = oldMapping.items[key]; continue; }
+        try {
+            const e = await createEntry(BLOG.comments, {
+                postId: postEntryId,
+                parentId: c.parent ? idFor(c.parent) : '',
+                authorName: clip(plainText(c.author), 100) || 'Anonymous',
+                authorEmail: clip(c.email, 200) || 'unknown@wordpress-import.invalid',
+                authorUrl: /^https?:\/\//i.test(c.url) ? clip(c.url, 300) : '',
+                body: clip(plainText(c.content), 10000) || '…',
+                status: 'approved',
+                createdAt: isoDate(c.dateGmt, c.date) || new Date().toISOString()
+            }, {createdBy: user?.id || null, source: 'import'});
+            mapping.items[key] = {kind: 'comment', entryId: e.id, post: postEntryId};
+            made++;
+        } catch (err) {
+            // One bad comment does not stop the rest; the report says what went wrong.
+            commentErrors.push(`A comment by ${clip(plainText(c.author), 60) || 'someone'} on "${clip(titleText(it.title), 80)}": ${err.message}`);
+        }
+        await tick();
+    }
+    return made;
+}
+
+function buildReport({job, plan, results, redirectResults, meta, user, started, commentsMade}) {
+    const items = plan.items.map(r => {
+        const res = results.get(r.key) || {result: r.action === 'skip' ? 'skipped' : r.action === 'keep' ? 'kept' : 'skipped'};
+        return {
+            key: r.key, kind: r.kind, title: r.title, wpId: r.wpId, oldUrl: r.oldUrl || '', target: res.target || r.target || '',
+            planned: r.action, result: res.result, reason: res.error || r.reason || '',
+            shortcodes: r.shortcodes || [], builder: r.builder || '', notes: res.notes || [], format: res.format || ''
+        };
+    });
+    const tally = {};
+    for (const i of items) {
+        tally[i.kind] ||= {created: 0, updated: 0, kept: 0, skipped: 0, failed: 0};
+        tally[i.kind][i.result] = (tally[i.kind][i.result] || 0) + 1;
+    }
+    const redirects = {created: 0, updated: 0, skipped: 0, failed: 0};
+    for (const r of redirectResults) redirects[r.result] = (redirects[r.result] || 0) + 1;
+    const failed = items.filter(i => i.result === 'failed').length + redirects.failed + commentErrors.length;
+    const summary = {
+        kinds: tally,
+        comments: commentsMade,
+        redirects,
+        failed,
+        rebuild: items.filter(i => i.shortcodes.length || i.builder).length
+    };
+    const run = {
+        id: job.id,
+        at: new Date().toISOString(),
+        by: user?.name || user?.email || '',
+        source: plan.source,
+        siteTitle: plan.site.title,
+        siteUrl: plan.site.url,
+        file: meta.fileName || '',
+        status: failed ? 'partial' : 'done',
+        durationMs: Date.now() - started,
+        summary
+    };
+    return {run, options: plan.options, items, redirects: redirectResults, notes: [...plan.notes, ...commentErrors.slice(0, 50)], blogBase: plan.blogBase};
+}
+
+// ---- The request hook: WordPress's query addresses ------------------------
+
+const QUERY_PARAMS = ['p', 'page_id', 'attachment_id'];
+
+/**
+ * /?p=12 -> the post's new address (301), from the mapping. Only on the site
+ * root or /index.php, only GET and HEAD, only a whole-number id that an import
+ * actually made - anything else goes on to the site as usual.
+ */
+async function wordpressQueryRedirects(fastify) {
+    // Everything the importer keeps (uploads, mapping, history with names) stays out of the site's repo.
+    try { if (fsSync.existsSync(path.resolve('content', 'imports', 'wordpress'))) allowInGitignore(process.cwd(), ['/content/imports/wordpress/']); } catch { /* not a managed repo */ }
+    // A run the last process stopped in the middle of goes into the history as failed.
+    try {
+        const run = await recoverInterrupted();
+        if (run) fastify.log.warn?.(`[wordpress-import] the import of ${run.siteTitle || run.source} was interrupted by a restart - recorded as failed.`);
+    } catch { /* nothing to recover */ }
+    // The redirects below belong to the migrated site, not to the Tool: they answer
+    // whether or not the importer is switched on. Cheap: only "/" or "/index.php" with a query is looked at.
+    const onRenamed = ({from, to} = {}) => { followRename(from, to).catch(() => {}); };
+    coreHooks.on?.('content:pageRenamed', onRenamed);
+    fastify.addHook('onClose', (_i, done) => { coreHooks.off?.('content:pageRenamed', onRenamed); done(); });
+    fastify.addHook('onRequest', async (request, reply) => {
+        if (request.method !== 'GET' && request.method !== 'HEAD') return;
+        const url = String(request.url || '');
+        const q = url.indexOf('?');
+        if (q === -1) return;
+        const p = url.slice(0, q);
+        if (p !== '/' && p !== '/index.php') return;
+        let params;
+        try { params = new URLSearchParams(url.slice(q + 1)); } catch { return; }
+        for (const name of QUERY_PARAMS) {
+            const v = params.get(name);
+            if (!v || !/^\d{1,12}$/.test(v)) continue;
+            const to = queryTarget(name, v);
+            if (!to) continue;
+            reply.code(301).header('location', to).header('cache-control', 'public, max-age=3600').send();
+            return reply;
+        }
+    });
+}
+
+export const wordpressImportPlugin = fp(wordpressQueryRedirects, {name: 'core-wordpress-import'});