domma-cms 1.8.4 → 1.8.5

package/server/services/wpImport/custom.js

@@ -0,0 +1,376 @@
+/**
+ * Custom post types - a theme's or plugin's own content (events, team
+ * members, portfolio pieces…) - as plain data. PURE: no I/O.
+ *
+ * Every post type the importer does not already handle is one of two things:
+ *   - plumbing (revisions, block templates, form definitions, orders…) - listed
+ *     once as "not imported" with the reason (INTERNAL_TYPES, the one list), or
+ *   - content - offered as a collection (default), as pages, or skipped.
+ *
+ * A collection's fields come from the items: title, slug, body, excerpt, date,
+ * status, featured image, one per taxonomy the type uses, and one per public
+ * postmeta key (not starting with "_"; ACF keeps its values under the field's
+ * own name and its field reference under "_name", so ACF values come across).
+ * Field types are inferred from every value: date, number, url, else text.
+ * PHP-serialised values go through woo.js's safe reader and are stored as text.
+ *
+ * @module services/wpImport/custom
+ */
+import {readSerialised} from './woo.js';
+
+/** Post types the importer handles elsewhere. */
+export const HANDLED_TYPES = new Set(['post', 'page', 'attachment', 'nav_menu_item', 'product', 'product_variation']);
+
+/**
+ * WordPress's and plugins' own plumbing - never offered, listed once with the reason.
+ * Extend here: a string matches the type exactly, a RegExp matches by pattern.
+ * `record` names WooCommerce records (counted in the Products card too).
+ */
+export const INTERNAL_TYPES = [
+    {match: 'revision', reason: 'Earlier versions of posts and pages - WordPress keeps them for undo.'},
+    {match: 'wp_block', reason: 'Reusable blocks - their content is already inside the pages that use them.'},
+    {match: /^wp_template(_part)?$/, reason: 'Block theme templates - the old theme\'s layout, not content.'},
+    {match: 'wp_global_styles', reason: 'The block theme\'s style settings - set this site\'s look in Content › Theme.'},
+    {match: 'wp_navigation', reason: 'Block-editor navigation - the export\'s menus come across as menus.'},
+    {match: /^wp_font_(family|face)$/, reason: 'Fonts installed in WordPress - add fonts here in Content › Theme.'},
+    {match: 'custom_css', reason: 'Extra CSS from the WordPress Customizer - copy what you need into this site\'s custom CSS.'},
+    {match: 'customize_changeset', reason: 'Unsaved Customizer drafts.'},
+    {match: 'oembed_cache', reason: 'WordPress\'s cache of embedded videos and posts.'},
+    {match: 'user_request', reason: 'Privacy requests - they hold people\'s personal details.'},
+    {match: /^acf-(field-group|field|post-type|taxonomy|ui-options-page)$/, reason: 'Advanced Custom Fields settings - the field values come across with the content that uses them.'},
+    {match: 'wpcf7_contact_form', reason: 'Contact Form 7 forms - rebuild them with Forms.'},
+    {match: /^(wpforms|wpforms_log|nf_sub|frm_form_actions|frm_styles|mc4wp-form|flamingo_\w+|feedback)$/, reason: 'A forms plugin\'s forms or submissions - submissions hold people\'s details; rebuild forms with Forms.'},
+    {match: /^(elementor_library|elementor_\w+|e-landing-page|et_pb_layout|fl-builder-template|vc_grid_item|templatera|ct_template|oxy_user_library|brizy_template)$/, reason: 'A page builder\'s saved templates - layout, not content.'},
+    {match: /^(scheduled-action|action_scheduler|wp_log|postman_sent_mail|wpcode|jp_pay_order|jp_pay_product|amp_validated_url|wp_privacy_\w+)$/, reason: 'A plugin\'s own records.'},
+    {match: /^shop_order(_placehold)?$/, reason: 'WooCommerce orders - they hold customers\' personal details.', record: 'orders'},
+    {match: 'shop_order_refund', reason: 'WooCommerce refunds - part of the orders.', record: 'refunds'},
+    {match: 'shop_coupon', reason: 'WooCommerce coupons - set discounts up in the Shop.', record: 'coupons'},
+    {match: 'shop_webhook', reason: 'WooCommerce webhooks.', record: 'webhooks'},
+    {match: 'shop_subscription', reason: 'WooCommerce subscriptions - they hold customers\' details.', record: 'subscriptions'},
+    {match: 'wc_user_membership', reason: 'WooCommerce memberships - they hold customers\' details.', record: 'memberships'},
+    {match: /^wc_(membership_plan|admin_note|order_\w+)$/, reason: 'WooCommerce\'s own records.'}
+];
+
+/** The INTERNAL_TYPES entry for a type, or null when it is content. */
+export function internalType(type) {
+    const t = String(type || '');
+    return INTERNAL_TYPES.find(x => (typeof x.match === 'string' ? x.match === t : x.match.test(t))) || null;
+}
+
+/** "tribe_events" -> "Tribe events". */
+export function typeLabel(type) {
+    const s = String(type || '').replace(/[-_]+/g, ' ').trim();
+    return s ? s[0].toUpperCase() + s.slice(1) : 'Items';
+}
+
+/** "Über uns" -> "uber-uns" (as plan.js's slugify; kept here so this module stands alone). */
+const slugify = (s) => String(s ?? '').normalize('NFKD').replace(/[̀-ͯ]/g, '').toLowerCase()
+    .replace(/['’]/g, '').replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 80);
+
+/** Statuses that are not an item at all. */
+const NOT_ITEMS = new Set(['trash', 'auto-draft', 'inherit']);
+
+/**
+ * The export's post types the importer does not handle: content types with
+ * their items, and plumbing with its reason.
+ *
+ * @param {object} wxr
+ * @returns {{content: {type: string, items: object[]}[], internal: {type: string, count: number, reason: string, record?: string}[]}}
+ */
+export function customTypesOf(wxr) {
+    const byType = new Map();
+    for (const it of wxr.items) {
+        if (HANDLED_TYPES.has(it.type)) continue;
+        if (!byType.has(it.type)) byType.set(it.type, []);
+        byType.get(it.type).push(it);
+    }
+    const content = [];
+    const internal = [];
+    for (const [type, items] of byType) {
+        const known = internalType(type);
+        if (known) { internal.push({type, count: items.length, reason: known.reason, ...(known.record && {record: known.record})}); continue; }
+        if (!/^[a-z0-9_-]{1,40}$/.test(type)) { internal.push({type: type.slice(0, 60), count: items.length, reason: 'Its name is not one WordPress allows - not imported.'}); continue; }
+        if (!items.some(i => !NOT_ITEMS.has(i.status))) { internal.push({type, count: items.length, reason: 'Only automatic copies and items in the bin - nothing to import.'}); continue; }
+        content.push({type, items});
+    }
+    content.sort((a, b) => b.items.length - a.items.length || a.type.localeCompare(b.type));
+    return {content, internal};
+}
+
+/** The path of an item's link relative to the old site, or null when it is not a pretty address. */
+function prettyPath(link, site, base) {
+    const raw = String(link || '').trim();
+    if (!raw) return null;
+    let u;
+    try { u = new URL(raw, site.baseSiteUrl || site.link || 'https://wordpress.invalid'); } catch { return null; }
+    if (!(site.hosts || []).includes(u.hostname.toLowerCase())) return null;
+    if (u.search) return null;   // ?event=x, ?post_type=event&p=12 - plain permalinks, drafts
+    let p = u.pathname;
+    try { p = decodeURI(p); } catch { /* keep */ }
+    p = `/${p}`.replace(/\/{2,}/g, '/').replace(/\/+$/, '').toLowerCase() || '/';
+    if (base && (p === base || p.startsWith(`${base}/`))) p = p.slice(base.length) || '/';
+    return p;
+}
+
+/**
+ * The type's address on the old site ("events" for /events/summer-fair/), read
+ * from the items' own links - never guessed. '' when no link shows it (plain
+ * permalinks, or only drafts). The most common prefix wins.
+ *
+ * @param {object[]} items
+ * @param {object} site - wxr.site
+ * @param {string} [base] - the WordPress folder ("/wp"), from basePathOf
+ */
+export function rewriteSlugOf(items, site, base = '') {
+    const votes = new Map();
+    for (const it of items) {
+        const p = prettyPath(it.link, site, base);
+        if (!p) continue;
+        const segs = p.split('/').filter(Boolean);
+        if (segs.length < 2) continue;
+        const last = segs[segs.length - 1];
+        const slug = String(it.slug || '').toLowerCase();
+        if (slug && last !== slug && last !== slugify(slug)) continue;
+        // A child item's address carries its parent's slug too: the type's part is the first segment(s)
+        // before the item's own chain, so a hierarchical type still votes for its prefix.
+        const prefix = segs.slice(0, it.parent ? 1 : -1).map(slugify).filter(Boolean).join('/');
+        if (prefix) votes.set(prefix, (votes.get(prefix) || 0) + 1);
+    }
+    let best = '';
+    let n = 0;
+    for (const [k, v] of votes) if (v > n || (v === n && k.length < best.length)) { best = k; n = v; }
+    return best;
+}
+
+/** An item's old address, when it is a pretty one; null for query addresses. */
+export function customOldPath(it, site, base = '') {
+    const p = prettyPath(it.link, site, base);
+    return p && p !== '/' ? `${base}${p}` : null;
+}
+
+// ---- Fields --------------------------------------------------------------
+
+/** The fields every imported type's collection starts with. */
+export const CORE_FIELDS = [
+    {name: 'title', label: 'Title', type: 'string'},
+    {name: 'slug', label: 'Slug', type: 'string'},
+    {name: 'body', label: 'Body', type: 'textarea'},
+    {name: 'excerpt', label: 'Excerpt', type: 'textarea'},
+    {name: 'date', label: 'Date', type: 'date'},
+    {name: 'status', label: 'Status', type: 'select', options: ['published', 'draft']},
+    {name: 'featuredImage', label: 'Featured image', type: 'url'}
+];
+const CORE_NAMES = new Set(CORE_FIELDS.map(f => f.name));
+/** Meta keys that are a plugin's housekeeping, not a field (SEO plugins write public keys too). */
+const META_SKIP = /^(rank_math_|wpseo_|yoast_|_|ekit_|elementor_|fusion_|avada_|slide_template$|pyre_|sbg_|ocean_|astra-|site-|theme-|wp_|jetpack_|amp_|spay_|footnotes$|classic-editor-remember$|inline_featured_image$|enclosure$|pingback|session_tokens$)/i;
+/** Taxonomies that are not a field: menus, and post formats (a theme's display hint). */
+const TAX_SKIP = new Set(['nav_menu', 'post_format', 'link_category']);
+const MAX_META_FIELDS = 60;
+const MAX_VALUE = 20000;
+
+/** A meta key or taxonomy as a field name the collections accept (letters, digits, _ and -). */
+export function fieldName(key, taken) {
+    let n = String(key || '').normalize('NFKD').replace(/[̀-ͯ]/g, '').replace(/[^A-Za-z0-9_-]+/g, '_').replace(/^[^A-Za-z_]+/, '').replace(/_+$/, '').slice(0, 60);
+    if (!n) n = 'field';
+    if (CORE_NAMES.has(n) || taken.has(n.toLowerCase())) {
+        const stem = `meta_${n}`.slice(0, 60);
+        n = stem;
+        for (let i = 2; taken.has(n.toLowerCase()) || CORE_NAMES.has(n); i++) n = `${stem}_${i}`;
+    }
+    taken.add(n.toLowerCase());
+    return n;
+}
+
+// Objects, enums and references count too - so the safe reader refuses them, rather than their text passing as a value.
+const looksSerialised = (s) => /^(a:\d+:\{|s:\d+:"|i:-?\d+;|d:[-\d.eE+]+;|b:[01];|N;$|[OCE]:\d+:"|[rR]:\d+;)/.test(s);
+const ISO_DAY = /^(\d{4})-(\d{2})-(\d{2})(?:[ T]\d{2}:\d{2}(?::\d{2})?)?$/;
+const YMD = /^(\d{4})(\d{2})(\d{2})$/;   // ACF's date picker
+const NUMBER = /^-?(\d{1,15}(\.\d{1,8})?|\.\d{1,8})$/;
+const URL_RE = /^https?:\/\/[^\s<>"']+$/i;
+
+/** "20261031" or "2026-10-31 10:00" -> "2026-10-31"; '' when not a real date. */
+export function dayOf(s) {
+    const m = ISO_DAY.exec(s) || YMD.exec(s);
+    if (!m) return '';
+    const [y, mo, d] = [Number(m[1]), Number(m[2]), Number(m[3])];
+    if (y < 1000 || y > 2999 || mo < 1 || mo > 12 || d < 1 || d > 31) return '';
+    const t = new Date(Date.UTC(y, mo - 1, d));
+    return t.getUTCMonth() === mo - 1 ? `${m[1]}-${m[2]}-${m[3]}` : '';
+}
+
+/**
+ * What the export says about its files, for telling an image field from a number:
+ * every attachment's parent, and the ACF field keys whose settings say image or file.
+ *
+ * @param {object} wxr
+ * @returns {{attachments: Map<number, number>, acfMedia: Set<string>}}
+ */
+export function mediaContext(wxr) {
+    const attachments = new Map();
+    const acfMedia = new Set();
+    for (const it of wxr.items || []) {
+        if (it.type === 'attachment') attachments.set(it.id, it.parent || 0);
+        if (it.type === 'acf-field' && it.slug) {
+            const s = readSerialised(it.content);
+            if (s.value && typeof s.value === 'object' && ['image', 'file'].includes(s.value.type)) acfMedia.add(it.slug);
+        }
+    }
+    return {attachments, acfMedia};
+}
+
+/**
+ * One field's type from all its values (non-empty ones): date, number, url,
+ * textarea or string. `media` - the export shows the key holds files (see
+ * customFields) - makes attachment ids a url: the file's address here. A
+ * number that merely matches some attachment's id stays a number.
+ */
+export function inferType(values, {media = false} = {}) {
+    const vals = values.map(v => String(v ?? '').trim()).filter(Boolean);
+    if (!vals.length) return 'string';
+    if (vals.some(looksSerialised)) return 'textarea';
+    if (media && vals.every(v => /^\d{1,12}$/.test(v))) return 'url';
+    if (vals.every(v => dayOf(v))) return 'date';
+    if (vals.every(v => NUMBER.test(v))) return 'number';
+    if (vals.every(v => URL_RE.test(v))) return 'url';
+    if (vals.some(v => v.length > 200 || /\n/.test(v))) return 'textarea';
+    return 'string';
+}
+
+/**
+ * A custom type's collection fields.
+ *
+ * @param {object[]} items - the type's items
+ * @param {{attachments: Map<number, number>, acfMedia: Set<string>}} [media] - from mediaContext()
+ * @returns {{name: string, label: string, type: string, options?: string[], from: {kind: 'core'|'tax'|'meta', key: string}}[]}
+ */
+export function customFields(items, media = {attachments: new Map(), acfMedia: new Set()}) {
+    const fields = CORE_FIELDS.map(f => ({...f, from: {kind: 'core', key: f.name}}));
+    const taken = new Set(CORE_FIELDS.map(f => f.name.toLowerCase()));
+    const live = items.filter(i => !NOT_ITEMS.has(i.status));
+    const taxonomies = [];
+    for (const it of live) for (const t of it.terms) if (!TAX_SKIP.has(t.domain) && !taxonomies.includes(t.domain)) taxonomies.push(t.domain);
+    for (const tax of taxonomies.slice(0, 20)) {
+        fields.push({name: fieldName(tax === 'category' ? 'categories' : tax === 'post_tag' ? 'tags' : tax, taken), label: tax === 'category' ? 'Categories' : tax === 'post_tag' ? 'Tags' : typeLabel(tax), type: 'string', from: {kind: 'tax', key: tax}});
+    }
+    const keys = [];
+    const values = new Map();
+    const pairs = new Map();
+    for (const it of live) {
+        for (const [k, v] of Object.entries(it.meta || {})) {
+            if (!k || META_SKIP.test(k)) continue;
+            if (!values.has(k)) { values.set(k, []); pairs.set(k, []); keys.push(k); }
+            values.get(k).push(v);
+            pairs.get(k).push([it, String(v ?? '').trim()]);
+        }
+    }
+    // A key holds files only on evidence: ACF says its field is an image or file, or every file it names
+    // was uploaded to the item itself. An id that merely matches some attachment is a number.
+    const holdsFiles = (k) => {
+        const set = pairs.get(k).filter(([, v]) => v);
+        return set.length > 0 && set.every(([it, v]) => /^\d{1,12}$/.test(v) && media.attachments.has(Number(v))
+            && (media.acfMedia.has(String(it.meta?.[`_${k}`] || '')) || media.attachments.get(Number(v)) === it.id));
+    };
+    for (const k of keys.slice(0, MAX_META_FIELDS)) {
+        // An ACF field's label is not in the export; its name, spelled out, is the best there is.
+        fields.push({name: fieldName(k, taken), label: typeLabel(k).slice(0, 80), type: inferType(values.get(k), {media: holdsFiles(k)}), from: {kind: 'meta', key: k}});
+    }
+    return fields;
+}
+
+/** The fields as a schema stores them (without where each came from). */
+export const schemaFields = (fields) => fields.map(({from: _from, ...f}) => f);
+
+/**
+ * A schema that only ever gains fields: the existing ones kept as they are
+ * (the person may have changed a type or a label), fields the export has that
+ * were never offered before added at the end. One offered before and missing
+ * now was removed by the person - it stays removed.
+ *
+ * @param {object[]} existing - the schema's fields now
+ * @param {object[]} wanted - this export's fields
+ * @param {string[]} [offered] - field names earlier runs offered
+ * @returns {{fields: object[], added: string[]}}
+ */
+export function mergeFields(existing = [], wanted = [], offered = []) {
+    const have = new Set([...existing.map(f => String(f?.name || '')), ...offered].map(n => n.toLowerCase()));
+    const added = wanted.filter(f => !have.has(f.name.toLowerCase()));
+    return {fields: [...existing, ...added], added: added.map(f => f.name)};
+}
+
+/** A serialised value as text: a list of plain values joined, anything else as JSON. */
+function serialText(v) {
+    if (v == null) return '';
+    if (Array.isArray(v) && v.every(x => x == null || typeof x !== 'object')) return v.filter(x => x != null && x !== '').join(', ');
+    if (typeof v !== 'object') return String(v);
+    return JSON.stringify(v);
+}
+
+const clip = (s, n) => String(s ?? '').replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, '').trim().slice(0, n);
+
+/**
+ * One item's entry data.
+ *
+ * @param {object} it - the WordPress item
+ * @param {object[]} fields - from customFields()
+ * @param {object} ctx
+ * @param {string} ctx.title - the title as text
+ * @param {string} ctx.slug
+ * @param {string} ctx.body - the converted content
+ * @param {string} ctx.excerpt - as text
+ * @param {string} ctx.date - ISO, or ''
+ * @param {string} ctx.status - 'published' | 'draft'
+ * @param {(id: number) => string} ctx.mediaUrl - an attachment's address here (or its old one), '' when unknown
+ * @param {(url: string) => string|null} [ctx.mediaFor] - an old uploads address -> its address here
+ * @returns {{data: object, notes: string[]}}
+ */
+export function customEntryData(it, fields, ctx) {
+    const data = {};
+    const notes = [];
+    const thumb = Number.parseInt(it.meta?._thumbnail_id, 10);
+    const core = {
+        title: ctx.title, slug: ctx.slug, body: ctx.body, excerpt: ctx.excerpt, date: ctx.date ? ctx.date.slice(0, 10) : '',
+        status: ctx.status, featuredImage: (thumb && ctx.mediaUrl(thumb)) || ''
+    };
+    for (const f of fields) {
+        if (f.from.kind === 'core') { data[f.name] = core[f.from.key] ?? ''; continue; }
+        if (f.from.kind === 'tax') {
+            data[f.name] = [...new Set(it.terms.filter(t => t.domain === f.from.key).map(t => clip(t.name || t.slug, 80)).filter(Boolean))].join(', ');
+            continue;
+        }
+        const raw = String(it.meta?.[f.from.key] ?? '');
+        let v = raw.trim();
+        if (v && looksSerialised(v)) {
+            const read = readSerialised(v);
+            if (read.error) { notes.push(`"${f.label}" could not be read safely and was left empty (${read.error}).`); v = ''; }
+            else v = serialText(read.value);
+        }
+        if (f.type === 'number') data[f.name] = v === '' ? '' : (NUMBER.test(v) ? Number(v) : clip(v, 100));
+        else if (f.type === 'date') data[f.name] = dayOf(v) || clip(v, 40);
+        else if (f.type === 'url') {
+            if (/^\d{1,12}$/.test(v)) data[f.name] = ctx.mediaUrl(Number(v)) || '';
+            else data[f.name] = URL_RE.test(v) ? (ctx.mediaFor?.(v) || clip(v, 2000)) : clip(v, 2000);
+        } else data[f.name] = clip(v, MAX_VALUE);
+    }
+    return {data, notes};
+}
+
+/** A fingerprint's parts for an entry: its non-empty values, keys in order - so a re-save that adds blanks is not an edit. */
+export function entryFingerprintText(data = {}) {
+    const keep = Object.keys(data).filter(k => !k.startsWith('_') && data[k] !== '' && data[k] != null && !(Array.isArray(data[k]) && !data[k].length)).sort();
+    return JSON.stringify(keep.map(k => [k, data[k]]));
+}
+
+/**
+ * Shortcode tags taken out of a text value ("[button url=x]Go[/button]" -> "Go"), as WordPress does to an
+ * excerpt. The list page's [collection] display hands its rows to the later shortcode stages, so a
+ * WordPress shortcode left in would run as Domma's own ([button], [embed], [form]...).
+ */
+export function stripShortcodes(s) {
+    return String(s ?? '').replace(/\[\[|\]\]/g, '').replace(/\[\/?[a-z][a-z0-9_-]*(?:\s[^\[\]]*)?\/?\]/g, '').replace(/[ \t]{2,}/g, ' ').trim();
+}
+
+/** The list page's body: the collection, newest first, published only. */
+export function listPageBody(slug, fields) {
+    const show = ['title', 'date', 'excerpt', ...fields.filter(f => f.from?.kind === 'tax').map(f => f.name)].slice(0, 6);
+    return `[collection slug="${slug}" display="cards" title-field="title" fields="${show.join(',')}" where_status="published" sort="date" order="desc" empty="Nothing here yet." /]\n`;
+}