domma-cms 1.8.8 → 1.8.10

package/server/services/wpImport/index.js

@@ -35,6 +35,13 @@
  * left out; what they wrote is credited to whoever they became. A user here is
  * only ever credited - never changed.
  *
+ * Widgets (./widgets.js) come from a .wie file - the JSON the "Widget Importer
+ * & Exporter" WordPress plugin writes - uploaded beside the export (or inside
+ * its zip). Each widget area becomes a block (wp-<area>); a footer area's block
+ * is added to the site's footer (site.json footer.blocks, drawn on every page by
+ * services/renderer.js) - added once, never taken out, and not put back once
+ * someone removed it here.
+ *
  * The one request hook here answers WordPress's query addresses (/?p=12,
  * /?page_id=7, /?attachment_id=3) from the mapping; path redirects go into
  * SEO's redirect store.
@@ -67,12 +74,19 @@ import {adminRightsGiven} from '../notification-sources.js';
 import {getRoleLevel} from '../roles.js';
 import {creditMap} from './people.js';
 import {seoEngine} from '../seo/index.js';
-import {hooks as coreHooks} from '../hooks.js';
+import {getShortcodeProcessors, hooks as coreHooks} from '../hooks.js';
+import {getBlock, listBlocks, saveBlock} from '../blocks.js';
+import {getConfig, saveConfig} from '../../config.js';
+import * as cache from '../cache/index.js';
+import {getSearchSettings} from '../search.js';
+import {readWie, widgetAreaHtml, WIE_MAX_BYTES} from './widgets.js';
+import {FOOTER_BLOCKS_MAX} from '../renderer.js';
 import {allowInGitignore} from '../siteGitignore.js';
 import fsSync from 'node:fs';
 
 export const WXR_MAX_BYTES = 50 * 1024 * 1024;
 export const UPLOADS_ZIP_MAX_BYTES = 1024 * 1024 * 1024;
+export {WIE_MAX_BYTES};
 const DOWNLOAD_CONCURRENCY = 4;
 
 const BLOG = {posts: 'blog-posts', categories: 'blog-categories', comments: 'blog-comments'};
@@ -202,6 +216,16 @@ const postFingerprint = (d) => fingerprint(d?.title, d?.content, d?.excerpt);
 /** A custom type's entry: its non-empty values (a re-save that adds blank fields is not an edit). */
 export const customFingerprint = (d) => fingerprint(entryFingerprintText(d));
 const menuFingerprint = (items) => fingerprint(JSON.stringify(items || []));
+/** A widget area's block: its template as written. */
+const blockFingerprint = (content) => fingerprint(String(content ?? ''));
+
+/** The blocks in the footer now (site.json footer.blocks), names only. */
+export function footerBlockList(site = null) {
+    let s = site;
+    if (!s) { try { s = getConfig('site'); } catch { s = {}; } }
+    const list = s?.footer?.blocks;
+    return Array.isArray(list) ? list.filter(n => typeof n === 'string' && /^[a-z0-9][a-z0-9-]*$/.test(n)) : [];
+}
 
 /** The site's state as the planner needs it. */
 export async function siteState(mapping = {}) {
@@ -252,11 +276,23 @@ export async function siteState(mapping = {}) {
         for (const e of entriesOf(await listEntries(m.slug, {limit: 100000}).catch(() => []))) hashes.set(e.id, customFingerprint(e.data || {}));
         customEntries.set(m.slug, hashes);
     }
+    // Blocks: their names (a new one must not take one), and the templates of those an earlier run made.
+    const blockNames = new Set((await listBlocks().catch(() => [])).map(b => b.name));
+    const blockHashes = {};
+    for (const m of Object.values(mapping.items || {})) {
+        if (m?.kind !== 'widgets' || !m.block || !blockNames.has(m.block)) continue;
+        const b = await getBlock(m.block).catch(() => null);
+        if (b) blockHashes[m.block] = blockFingerprint(b.content);
+    }
+    let searchOn = false;
+    try { searchOn = getSearchSettings().enabled !== false; } catch { /* no search settings */ }
     const seo = seoEngine();
     // The users here - only what matching an author needs (never a password or a token).
     const users = (await listUsers().catch(() => [])).map(u => ({id: u.id, email: String(u.email || '').toLowerCase(), name: u.name || ''}));
     return {
         pagePaths, pageHashes, shop, collections, customEntries, users,
+        blockNames, blockHashes, footerBlocks: footerBlockList(), searchOn,
+        shortcodes: new Set(getShortcodeProcessors().map(p => p.name)),
         mediaNames: new Set(media.map(m => String(m.name).toLowerCase())),
         menuSlugs: new Set(menus.map(m => m.slug)),
         menuHashes,
@@ -284,6 +320,16 @@ export async function inspectStaged(id) {
             if (!xml) throw new ZipError('There is no .xml file in that zip - export from WordPress (Tools › Export) and upload the file it gives you.');
             if (xml.size > WXR_MAX_BYTES) throw new ZipError(`The export is ${Math.round(xml.size / 1048576)} MB - over the ${WXR_MAX_BYTES / 1048576} MB limit.`);
             await fs.writeFile(stagingPath(id, 'export.xml'), await zip.read(xml));
+            // A widgets file packed with the export comes along, if it reads as one.
+            const wie = zip.entries.filter(e => /\.wie$/i.test(e.name) && !/(^|\/)__MACOSX\//.test(e.name) && e.size <= WIE_MAX_BYTES).sort((a, b) => b.size - a.size)[0];
+            if (wie) {
+                try {
+                    const buf = await zip.read(wie);
+                    const read = readWie(buf);
+                    await fs.writeFile(stagingPath(id, 'widgets.wie'), buf, {mode: 0o600});
+                    await updateStagingMeta(id, {widgets: widgetsMeta(String(wie.name).split('/').pop(), buf.length, read)});
+                } catch { /* not a widgets file - the export imports without it */ }
+            }
         } finally {
             await zip.close();
         }
@@ -306,6 +352,22 @@ export async function loadExport(id) {
     return wxr;
 }
 
+/** What the staging meta says about a widgets file. */
+export function widgetsMeta(fileName, size, read) {
+    return {
+        fileName: String(fileName || 'widgets.wie').slice(0, 200).replace(/[^\w.() -]/g, '_'), size,
+        areas: read.areas.filter(a => a.widgets.length).length,
+        widgets: read.areas.reduce((n, a) => n + a.widgets.length, 0)
+    };
+}
+
+/** The staged widgets file, read, or null when there is none. */
+export async function loadWidgets(id) {
+    const meta = await stagingMeta(id);
+    if (!meta?.widgets) return null;
+    return readWie(await fs.readFile(stagingPath(id, 'widgets.wie')));
+}
+
 export async function discardStaged(id) {
     parsed.delete(id);
     await dropStaging(id);
@@ -347,9 +409,10 @@ export async function previewImport(id, options = {}) {
     const mapping = await mappingFor(sourceKey(wxr.site));
     const state = await siteState(mapping);
     const zipPaths = options.media === 'zip' ? await uploadsIndex(id) : null;
+    const wie = await loadWidgets(id);
     // `people` (what the person may do with users) comes from the route, never from the request body.
-    const plan = planImport(wxr, state, mapping, {...cleanOptions(options), people: options.people || null, zipPaths, anyPort: testHostsAllowed()});
-    return {plan, state, wxr, mapping, meta};
+    const plan = planImport(wxr, state, mapping, {...cleanOptions(options), people: options.people || null, zipPaths, wie, anyPort: testHostsAllowed()});
+    return {plan, state, wxr, mapping, meta, wie};
 }
 
 /** Options from a request - whitelisted. */
@@ -368,6 +431,7 @@ export function cleanOptions(o = {}) {
         products: o.products !== false,
         types: cleanTypeChoices(o.types, ['collection', 'pages', 'skip']),
         listPages: cleanTypeChoices(o.listPages, [false]),
+        footer: cleanAreaChoices(o.footer),
         choices,
         users: cleanUserChoices(o.users),
         userRole: typeof o.userRole === 'string' && /^[a-z0-9][a-z0-9_-]{0,59}$/i.test(o.userRole) ? o.userRole : '',
@@ -383,6 +447,14 @@ function cleanUserChoices(map) {
     return out;
 }
 
+/** {widgetAreaId: false} - a footer widget area NOT placed in the footer (placing is the default). */
+function cleanAreaChoices(map) {
+    const out = {};
+    if (!map || typeof map !== 'object') return out;
+    for (const [k, v] of Object.entries(map).slice(0, 100)) if (/^[A-Za-z0-9_-]{1,100}$/.test(k) && v === false) out[k] = false;
+    return out;
+}
+
 /** {postType: choice} - only real post type names, only the allowed values. */
 function cleanTypeChoices(map, allowed) {
     const out = {};
@@ -438,7 +510,7 @@ export async function startRun(id, options, user, log = console) {
     return job;
 }
 
-async function runJob(job, {plan, wxr, mapping: oldMapping, meta, inviteOrigin}, user, log) {
+async function runJob(job, {plan, wxr, mapping: oldMapping, meta, inviteOrigin, wie}, user, log) {
     const started = Date.now();
     const mapping = {
         items: {...(oldMapping.items || {})},
@@ -446,7 +518,7 @@ async function runJob(job, {plan, wxr, mapping: oldMapping, meta, inviteOrigin},
         query: {p: {...oldMapping.query?.p}, page_id: {...oldMapping.query?.page_id}, attachment_id: {...oldMapping.query?.attachment_id}}
     };
     try {
-        await runPhases(job, {plan, wxr, oldMapping, meta, inviteOrigin}, user, log, mapping, started);
+        await runPhases(job, {plan, wxr, oldMapping, meta, inviteOrigin, wie}, user, log, mapping, started);
     } catch (err) {
         // What was written before the failure is recorded, so a retry updates it rather than duplicating it.
         await saveMapping(plan.source, mapping).catch(() => {});
@@ -455,7 +527,7 @@ async function runJob(job, {plan, wxr, mapping: oldMapping, meta, inviteOrigin},
     }
 }
 
-async function runPhases(job, {plan, wxr, oldMapping, meta, inviteOrigin}, user, log, mapping, started) {
+async function runPhases(job, {plan, wxr, oldMapping, meta, inviteOrigin, wie}, user, log, mapping, started) {
     const byId = new Map(wxr.items.map(i => [i.id, i]));
     const rows = new Map(plan.items.map(r => [r.key, r]));
     const results = new Map(); // key -> {result, error?, target?, notes?}
@@ -833,6 +905,93 @@ async function runPhases(job, {plan, wxr, oldMapping, meta, inviteOrigin}, user,
         await step(r.title);
     }
 
+    await checkpoint(true);
+    // ---- 7b. Widgets: one block per widget area; footer areas join the footer ----
+    const widgetRows = plan.items.filter(x => x.kind === 'widgets');
+    if (widgetRows.length && wie) {
+        job.phase = 'Widgets';
+        const areas = new Map(wie.areas.map(a => [a.id, a]));
+        // The menus as this run left them: a WordPress menu's term id -> the menu made from it.
+        const menuSlugs = new Map();
+        for (const m of wxr.menus) {
+            const made = mapping.items[`menu:${m.slug}`];
+            const ok = ['created', 'updated', 'kept'].includes(results.get(`menu:${m.slug}`)?.result);
+            if (m.id && made?.slug && ok) menuSlugs.set(m.id, made.slug);
+        }
+        const mediaUrl = (aid, url) => (aid && mediaById.get(aid)?.url) || (url && mediaFor(url)) || '';
+        const blog = await blogTarget();
+        const shortcodes = new Set(getShortcodeProcessors().map(p => p.name));
+        let searchOn = false;
+        try { searchOn = getSearchSettings().enabled !== false; } catch { /* no search settings */ }
+        const wctx = {
+            convert: {mediaFor, mediaById: (aid) => mediaById.get(aid) || null, linkFor, attachmentsOf},
+            mediaUrl,
+            hasMedia: (aid, url) => Boolean(mediaUrl(aid, url)),
+            menuSlug: (termId) => menuSlugs.get(termId) || '',
+            blog: {posts: blogMode && blog.available, list: shortcodes.has('blog-list'), categories: shortcodes.has('blog-categories'), basePath: plan.blogBase},
+            search: searchOn
+        };
+        const place = [];
+        for (const r of widgetRows) {
+            const was = oldMapping.items[r.key];
+            if (!works(r)) {
+                if (r.action === 'keep') {
+                    mapping.items[r.key] = {...was};
+                    if (r.place === 'add') place.push(r);
+                    else if (r.place === 'there') mapping.items[r.key].placed = true;
+                }
+                set(r.key, {result: r.action === 'keep' ? 'kept' : 'skipped', target: r.target, place: r.action === 'keep' ? r.place : ''});
+                continue;
+            }
+            try {
+                const area = areas.get(r.area);
+                if (!area) throw new Error('It is not in the widgets file any more.');
+                const {html, notes, rebuild} = widgetAreaHtml(area, wctx);
+                let existing = null;
+                try { existing = await getBlock(r.target); } catch { /* not there */ }
+                if (existing && r.action !== 'update') throw new Error(`A block called ${r.target} was made while the import ran.`);
+                await saveBlock(r.target, html, {meta: {...(existing?.meta || {}), importedFrom: {source: 'wordpress', site: plan.source, widgetArea: r.area}}});
+                mapping.items[r.key] = {kind: 'widgets', block: r.target, area: r.area, hash: blockFingerprint(html), ...(was?.placed && {placed: true}), at: new Date().toISOString()};
+                if (r.place === 'add') place.push(r);
+                else if (r.place === 'there') mapping.items[r.key].placed = true;
+                set(r.key, {result: r.action === 'update' ? 'updated' : 'created', target: r.target, notes, place: r.place,
+                    needs: rebuild.length ? `Rebuild: ${rebuild.join('; ')}` : ''});
+            } catch (err) {
+                set(r.key, {result: 'failed', error: err.message || String(err)});
+            }
+            await step(r.title);
+        }
+        // The footer: added after whatever is there, in the widgets file's order - nothing is ever taken out.
+        if (place.length) {
+            try {
+                const site = getConfig('site');
+                const now = footerBlockList(site);
+                const fresh = place.filter(r => !now.includes(r.target));
+                // The footer draws at most FOOTER_BLOCKS_MAX (and Site Settings refuses more): the rest stay blocks only.
+                const room = Math.max(0, FOOTER_BLOCKS_MAX - now.length);
+                const added = fresh.slice(0, room);
+                for (const r of fresh.slice(room)) {
+                    const res = results.get(r.key);
+                    if (res) { res.place = 'full'; (res.notes ||= []).push(`The footer already has ${FOOTER_BLOCKS_MAX} blocks, so this one was not added - make room in Site Settings › Footer and add ${r.target} there.`); }
+                }
+                if (added.length) {
+                    saveConfig('site', {...site, footer: {...(site.footer || {}), blocks: [...now, ...added.map(r => r.target)]}});
+                    await cache.invalidateTags(['site']);
+                }
+                for (const r of place.filter(x => !fresh.slice(room).includes(x))) {
+                    mapping.items[r.key] = {...mapping.items[r.key], placed: true};
+                    const res = results.get(r.key);
+                    if (res) res.placed = true;
+                }
+            } catch (err) {
+                for (const r of place) {
+                    const res = results.get(r.key);
+                    if (res) (res.notes ||= []).push(`It was not added to the footer: ${err.message || err}`);
+                }
+            }
+        }
+    }
+
     await checkpoint(true);
     // ---- 8. Redirects ----
     job.phase = 'Redirects';
@@ -1195,7 +1354,8 @@ function buildReport({job, plan, results, redirectResults, meta, user, started,
         return {
             key: r.key, kind: r.kind, title: r.title, wpId: r.wpId, oldUrl: r.oldUrl || '', target: res.target || r.target || '',
             planned: r.action, result: res.result, reason: res.error || r.reason || '',
-            shortcodes: r.shortcodes || [], builder: r.builder || '', needs: r.needs || '', notes: res.notes || [], format: res.format || '',
+            shortcodes: r.shortcodes || [], builder: r.builder || '', needs: res.needs ?? r.needs ?? '', notes: res.notes || [], format: res.format || '',
+            ...(r.kind === 'widgets' && {area: r.area, footer: Boolean(r.footer), place: res.place || r.place || '', placed: Boolean(res.placed)}),
             ...(res.entryId && {entryId: res.entryId}),
             ...((res.collection || r.collection) && {collection: res.collection || r.collection}),
             ...(r.postType && {postType: r.postType})

package/server/services/wpImport/plan.js

@@ -21,7 +21,9 @@
  * at their old address (default), pages, or nothing; WordPress's plumbing is
  * listed with the reason it is not imported (summary.customTypes / internalTypes).
  * The authors (./people.js) are matched to users here by email, or made into
- * new accounts, or left out - plan.people.
+ * new accounts, or left out - plan.people. Widgets, from a .wie file uploaded
+ * beside the export (./widgets.js), become one block per widget area, and a
+ * footer area's block joins the footer (summary.widgets).
  *
  * @module services/wpImport/plan
  */
@@ -31,6 +33,7 @@ import {FETCH_PORTS, portOf} from './fetch.js';
 import {readWooProducts, shopProductInput, wooPageRole} from './woo.js';
 import {customFields, customOldPath, customTypesOf, mediaContext, rewriteSlugOf, typeLabel} from './custom.js';
 import {planPeople} from './people.js';
+import {planWidgets} from './widgets.js';
 
 /** Addresses under the Shop's that are its own pages, never a product's. */
 const SHOP_RESERVED = ['cart', 'checkout', 'basket', 'account', 'order'];
@@ -191,6 +194,11 @@ export function isoDate(gmt, local) {
  * @param {{id: string, email: string, name: string}[]} [site.users] - the users here (authors are matched by email)
  * @param {boolean} site.redirectsAvailable  - SEO is running (its redirects take the old addresses)
  * @param {Set<string>} site.redirectFroms   - addresses that already redirect
+ * @param {Set<string>} [site.blockNames]    - the blocks here
+ * @param {Object<string,string>} [site.blockHashes] - block -> fingerprint now (blocks an earlier run made)
+ * @param {string[]} [site.footerBlocks]     - the blocks in the footer now (site.json footer.blocks)
+ * @param {Set<string>} [site.shortcodes]    - shortcodes registered here (the Blog's [blog-list]…)
+ * @param {boolean} [site.searchOn]          - site search is switched on
  * @param {object} [mapping] - what earlier runs of this import made ({items: {key: {...}}})
  * @param {object} [options]
  * @param {'blog'|'pages'|'skip'} [options.posts]
@@ -209,13 +217,15 @@ export function isoDate(gmt, local) {
  * @param {string} [options.userRole] - the role for new accounts
  * @param {boolean} [options.invite] - email new accounts a link to set their password
  * @param {object} [options.people] - what the person running it may do with users (never from a request)
+ * @param {object|null} [options.wie] - the widgets file, from readWie() (./widgets.js)
+ * @param {Object<string,false>} [options.footer] - per widget area: false = not placed in the footer
  * @returns {object}
  */
 export function planImport(wxr, site, mapping = {}, options = {}) {
     const opt = {
         posts: site.blog?.available ? 'blog' : 'pages',
         media: 'download', comments: true, menus: true, redirects: true, drafts: true, products: true, choices: {}, zipPaths: null,
-        types: {}, listPages: {}, users: {}, userRole: '', invite: false,
+        types: {}, listPages: {}, users: {}, userRole: '', invite: false, footer: {}, wie: null,
         ...options
     };
     opt.types ||= {};
@@ -505,6 +515,27 @@ export function planImport(wxr, site, mapping = {}, options = {}) {
         }
     }
 
+    // ---- Widgets: one block per widget area (./widgets.js) ----
+    const menuSlugOf = new Map(menuRows.filter(r => r.action !== 'skip' && r.wpId).map(r => [r.wpId, r.target]));
+    const mediaIds = new Set(items.filter(r => r.kind === 'media' && r.action !== 'skip').map(r => r.wpId));
+    const widgetPlan = planWidgets(opt.wie || null, {
+        mapped, choices, footer: opt.footer || {},
+        blockNames: site.blockNames || new Set(), blockHashes: site.blockHashes || {}, footerBlocks: site.footerBlocks || [],
+        menuSlug: (termId) => menuSlugOf.get(termId) || '',
+        hasMedia: (id, url) => {
+            if (id && mediaIds.has(id)) return true;
+            const rel = uploadsPath(url).toLowerCase();
+            return Boolean(rel && (mediaByRel[rel] || mediaByRel[originalUploadsPath(rel)]));
+        },
+        blog: {posts: opt.posts === 'blog', list: Boolean(site.shortcodes?.has('blog-list')), categories: Boolean(site.shortcodes?.has('blog-categories')), basePath: blogBase},
+        search: site.searchOn === true
+    });
+    for (const r of widgetPlan.rows) push(r);
+    const sidebars = widgetPlan.rows.filter(r => !r.footer && r.action !== 'skip' && r.target);
+    if (sidebars.length) {
+        notes.push(`Widget areas that are not a footer (${sidebars.map(r => r.target).join(', ')}) become blocks only - nothing here shows them by itself. Put one on a page with [block template="${sidebars[0].target}" /]; Content › Blocks lists them.`);
+    }
+
     // ---- Comments (with the Blog) ----
     let comments = 0;
     if (opt.posts === 'blog' && opt.comments && site.blog?.comments !== false) {
@@ -641,6 +672,7 @@ export function planImport(wxr, site, mapping = {}, options = {}) {
         customTypes,
         internalTypes,
         people: people.counts,
+        widgets: widgetPlan.summary,
         products: {
             total: wooProducts.length,
             types: productTypes,
@@ -656,7 +688,7 @@ export function planImport(wxr, site, mapping = {}, options = {}) {
     return {
         source: sourceKey(wxr.site),
         site: {title: wxr.site.title, url: wxr.site.baseSiteUrl || wxr.site.link, wxrVersion: wxr.site.wxrVersion, hosts: wxr.site.hosts},
-        options: {...opt, zipPaths: undefined, anyPort: undefined, people: undefined, inviteOrigin: undefined, invite: people.invite, choices},
+        options: {...opt, zipPaths: undefined, anyPort: undefined, people: undefined, inviteOrigin: undefined, wie: undefined, invite: people.invite, choices},
         blogBase,
         shopBase,
         items,