@materializecms/server 0.0.0-stage → 0.1.0

package/src/wpimport.js

@@ -0,0 +1,352 @@
+import { slugify } from '@materializecms/core';
+import { htmlToPage, htmlToRichText, imagesIn, mainContent, originalOf, plain, } from './wordpress.js';
+/** Which sources fit a field of this kind. */
+export function sourcesFor(f) {
+    switch (f.kind) {
+        case 'text':
+            return f.options.multiline
+                ? ['excerpt', 'title', 'categories', 'tags', 'author']
+                : ['title', 'categories', 'tags', 'author', 'excerpt'];
+        case 'slug':
+            return ['slug'];
+        case 'richText':
+        case 'page':
+            return ['content'];
+        case 'date':
+            return ['date'];
+        case 'media':
+            return ['featured'];
+        case 'select':
+            return ['categories', 'tags'];
+        case 'relation':
+            return ['author'];
+        case 'seo':
+            return ['seo'];
+        default:
+            return [];
+    }
+}
+/** A first guess at where an item kind goes and which field takes what. */
+export function suggest(collections, kind) {
+    const has = (c, k) => Object.values(c.fields).some((f) => f.kind === k);
+    const pool = collections.filter((c) => !c.singleton);
+    const target = kind === 'page'
+        ? (pool.find((c) => has(c, 'page')) ?? pool.find((c) => /page|pagin/.test(c.name)))
+        : (pool.find((c) => /post|blog|news|artic|diario|journal/.test(c.name) && has(c, 'richText')) ??
+            pool.find((c) => has(c, 'richText') && has(c, 'date')));
+    if (!target)
+        return null;
+    const fields = {};
+    const taken = new Set();
+    const pick = (name, source) => {
+        if (taken.has(source) && source !== 'categories')
+            return;
+        fields[name] = source;
+        taken.add(source);
+    };
+    for (const [name, f] of Object.entries(target.fields)) {
+        if (name === target.titleField)
+            pick(name, 'title');
+        else if (f.kind === 'slug')
+            pick(name, 'slug');
+        else if (f.kind === 'page' || (f.kind === 'richText' && !taken.has('content')))
+            pick(name, 'content');
+        else if (f.kind === 'text' && f.options.multiline && /excerpt|summary|somm|riass|intro|desc/i.test(name))
+            pick(name, 'excerpt');
+        else if (f.kind === 'date')
+            pick(name, 'date');
+        else if (f.kind === 'media' && !taken.has('featured'))
+            pick(name, 'featured');
+        else if ((f.kind === 'select' || f.kind === 'text') && /categ/i.test(name))
+            pick(name, 'categories');
+        else if (f.kind === 'text' && /tag|etichett/i.test(name))
+            pick(name, 'tags');
+        else if ((f.kind === 'relation' || f.kind === 'text') && /author|autor/i.test(name))
+            pick(name, 'author');
+        else if (f.kind === 'seo')
+            pick(name, 'seo');
+    }
+    // A multi-line text not taken yet gets the excerpt.
+    if (!taken.has('excerpt'))
+        for (const [name, f] of Object.entries(target.fields))
+            if (!fields[name] && f.kind === 'text' && f.options.multiline) {
+                pick(name, 'excerpt');
+                break;
+            }
+    return { collection: target.name, fields };
+}
+const fold = (s) => s.normalize('NFKD').replace(/[̀-ͯ]/g, '').toLowerCase().trim();
+/** The options of a choice field, as value and label. */
+export function optionsOf(f) {
+    const raw = (f.options.options ?? []);
+    return raw.map((o) => (typeof o === 'string' ? { value: o, label: o } : o));
+}
+/** A WordPress category to an option of a choice field, by value or label; null if none fits. */
+export function matchOption(name, options) {
+    const n = fold(name);
+    return options.find((o) => fold(o.value) === n || fold(o.label) === n)?.value ?? null;
+}
+/** "Il titolo • Yoast" → "Il titolo": the site's name is added by Materialize itself. */
+function seoTitle(title, site) {
+    if (!title)
+        return undefined;
+    const clean = plain(title);
+    const cut = clean.replace(new RegExp(`\\s*[|•\\-–—·:]\\s*${site.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*$`, 'i'), '');
+    return cut || clean;
+}
+const pathOf = (link, base) => {
+    try {
+        const u = new URL(link, base);
+        return decodeURI(u.pathname).replace(/\/+$/, '') || '/';
+    }
+    catch {
+        return null;
+    }
+};
+export async function runImport(site, plan, deps) {
+    const progress = {
+        phase: 'media',
+        done: 0,
+        total: 0,
+        created: [],
+        skipped: [],
+        problems: [],
+        media: { downloaded: 0, failed: 0, reasons: {} },
+        redirects: 0,
+    };
+    const report = () => deps.onProgress({ ...progress });
+    const targetOf = (item) => (item.kind === 'post' ? plan.posts : plan.pages);
+    const defOf = (t) => deps.collections.find((c) => c.name === t.collection);
+    const items = site.items.filter((i) => {
+        const t = targetOf(i);
+        return t && defOf(t);
+    });
+    // 0. Items the API gave empty (some themes and page builders keep the page elsewhere): their
+    // published page is read instead, between its header and its footer.
+    for (const item of items)
+        if (plain(item.html).length < 40 && item.status === 'publish' && item.link) {
+            const page = await deps.fetchPage(item.link);
+            const main = page ? mainContent(page) : '';
+            if (plain(main).length > plain(item.html).length)
+                item.html = main;
+        }
+    // 1. Images: each address once, the original size when WordPress has it.
+    const wanted = new Map();
+    for (const item of items) {
+        const t = targetOf(item);
+        const fields = Object.values(t.fields);
+        if (fields.includes('content'))
+            for (const src of imagesIn(item.html))
+                wanted.set(new URL(src, site.url).href, '');
+        if (fields.includes('featured') && item.featured?.url)
+            wanted.set(new URL(item.featured.url, site.url).href, item.featured.alt);
+    }
+    const images = new Map();
+    progress.total = wanted.size;
+    report();
+    const queue = [...wanted];
+    const worker = async () => {
+        for (let next = queue.shift(); next; next = queue.shift()) {
+            const [url, alt] = next;
+            const original = originalOf(url);
+            let saved = original !== url ? await deps.saveImage(original, alt) : null;
+            if (!saved || 'error' in saved)
+                saved = await deps.saveImage(url, alt);
+            if ('error' in saved) {
+                progress.media.failed++;
+                progress.media.reasons[saved.error] = (progress.media.reasons[saved.error] ?? 0) + 1;
+            }
+            else {
+                images.set(url, saved);
+                progress.media.downloaded++;
+            }
+            progress.done++;
+            report();
+        }
+    };
+    await Promise.all([worker(), worker(), worker()]);
+    // 2. Where every item will live, so links between them point to the new pages.
+    const plans = items.map((item) => {
+        const t = targetOf(item);
+        const def = defOf(t);
+        const slug = slugify(decodeURIComponent(item.slug)) || slugify(item.title) || item.sourceId;
+        const sample = {};
+        for (const [name, source] of Object.entries(t.fields))
+            if (source === 'slug')
+                sample[name] = slug;
+        return { item, t, def, slug, url: deps.urlOf(def, sample) };
+    });
+    const moved = new Map();
+    for (const p of plans) {
+        const from = pathOf(p.item.link, site.url);
+        if (from && p.url)
+            moved.set(from, p.url);
+    }
+    const host = (() => {
+        try {
+            return new URL(site.url).host;
+        }
+        catch {
+            return '';
+        }
+    })();
+    const ctx = {
+        image: (src) => images.get(new URL(src, site.url).href),
+        link: (href) => {
+            try {
+                const u = new URL(href, site.url);
+                if (u.host !== host)
+                    return href;
+                const image = images.get(u.href) ?? images.get(originalOf(u.href));
+                if (image)
+                    return image.src;
+                const to = moved.get(decodeURI(u.pathname).replace(/\/+$/, '') || '/');
+                return to ? to + u.hash : href;
+            }
+            catch {
+                return href;
+            }
+        },
+    };
+    // 3. The documents.
+    progress.phase = 'content';
+    progress.done = 0;
+    progress.total = plans.length;
+    report();
+    const relations = new Map();
+    const moves = [];
+    // Addresses taken by this import: WordPress allows the same slug under different parent pages.
+    const taken = new Map();
+    for (const entry of plans) {
+        const { item, t, def } = entry;
+        let { slug, url } = entry;
+        try {
+            const mine = taken.get(def.name) ?? new Set();
+            taken.set(def.name, mine);
+            if (mine.has(slug)) {
+                let n = 2;
+                while (mine.has(`${slug}-${n}`) || (await deps.exists(def.name, `${slug}-${n}`)))
+                    n++;
+                const before = slug;
+                slug = `${slug}-${n}`;
+                const sample = {};
+                for (const [name, source] of Object.entries(t.fields))
+                    if (source === 'slug')
+                        sample[name] = slug;
+                url = deps.urlOf(def, sample);
+                progress.problems.push({
+                    title: item.title,
+                    reason: `indirizzo già usato da un’altra pagina importata: ora è ${url ?? slug}, era ${before}`,
+                });
+            }
+            else if (await deps.exists(def.name, slug)) {
+                // Already there from before (a second import): left as it is.
+                progress.skipped.push({ title: item.title, reason: 'c’è già un contenuto con lo stesso indirizzo' });
+                continue;
+            }
+            mine.add(slug);
+            const data = {};
+            for (const [name, source] of Object.entries(t.fields)) {
+                const f = def.fields[name];
+                if (!f)
+                    continue;
+                const value = await valueFor(f, source, item, ctx, plan, site, relations, deps);
+                if (value !== undefined)
+                    data[name] = value;
+            }
+            for (const [name, source] of Object.entries(t.fields))
+                if (source === 'slug')
+                    data[name] = slug;
+            const doc = await deps.create(def, data);
+            let published = false;
+            if (plan.publish && item.status === 'publish')
+                try {
+                    await deps.publish(doc.id);
+                    published = true;
+                }
+                catch {
+                    progress.problems.push({ title: doc.title, reason: 'importato come bozza: ha campi da completare' });
+                }
+            progress.created.push({ id: doc.id, collection: def.name, title: doc.title, published });
+            const from = pathOf(item.link, site.url);
+            if (published && from && url && from !== url)
+                moves.push({ from, to: url });
+        }
+        catch (error) {
+            progress.problems.push({ title: item.title, reason: error.message });
+        }
+        finally {
+            progress.done++;
+            report();
+        }
+    }
+    if (plan.redirects && moves.length)
+        progress.redirects = await deps.addRedirects(moves);
+    progress.phase = 'done';
+    report();
+    return progress;
+}
+async function valueFor(f, source, item, ctx, plan, site, relations, deps) {
+    const max = typeof f.options.max === 'number' ? f.options.max : undefined;
+    const clip = (s) => (max && s.length > max ? `${s.slice(0, max - 1).trimEnd()}…` : s);
+    switch (source) {
+        case 'title':
+            return clip(item.title || 'Senza titolo');
+        case 'slug':
+            return undefined;
+        case 'excerpt':
+            return item.excerpt ? clip(item.excerpt.replace(/\s*\[(…|&hellip;|\.\.\.)\]\s*$/, '…')) : undefined;
+        case 'date':
+            return item.date.slice(0, 10);
+        case 'content':
+            if (f.kind === 'page')
+                return htmlToPage(item.html, item.title, ctx);
+            if (f.kind === 'richText')
+                return htmlToRichText(item.html, ctx);
+            return clip(plain(item.html));
+        case 'featured':
+            return item.featured ? ctx.image(item.featured.url) : undefined;
+        case 'categories':
+        case 'tags': {
+            const names = source === 'categories' ? item.categories : item.tags;
+            if (f.kind === 'select') {
+                const options = optionsOf(f);
+                for (const name of names) {
+                    const chosen = plan.categories?.[name] ?? matchOption(name, options);
+                    if (chosen)
+                        return chosen;
+                }
+                return plan.fallbackCategory || (f.required ? options[0]?.value : undefined);
+            }
+            return names.length ? clip(names.join(', ')) : undefined;
+        }
+        case 'author': {
+            if (!item.author)
+                return undefined;
+            if (f.kind === 'relation') {
+                const to = String(f.options.to);
+                if (!relations.has(to))
+                    relations.set(to, await deps.titles(to));
+                return relations.get(to)?.get(fold(item.author));
+            }
+            return clip(item.author);
+        }
+        case 'seo': {
+            if (!item.seo)
+                return undefined;
+            // Search engines show about 60 and 160 characters: longer ones are cut on a word.
+            const cut = (s, n) => {
+                if (!s)
+                    return undefined;
+                if (s.length <= n)
+                    return s;
+                const head = s.slice(0, n - 1);
+                return `${head.slice(0, Math.max(head.lastIndexOf(' '), n - 20)).trimEnd()}…`;
+            };
+            const title = cut(seoTitle(item.seo.title, site.name), 70);
+            const description = cut(item.seo.description ? plain(item.seo.description) : undefined, 170);
+            return { ...(title ? { title } : {}), ...(description ? { description } : {}) };
+        }
+    }
+}
+export { fold };