domma-cms 1.38.0 → 1.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/CLAUDE.md +17 -0
- package/admin/js/api.js +1 -1
- package/admin/js/app.js +1 -1
- package/admin/js/lib/plugins-arrange.js +2 -2
- package/admin/js/lib/wordpress-import-arrange.js +1 -1
- package/admin/js/templates/docs/usage-tools.html +18 -3
- package/admin/js/templates/plugins.html +2 -0
- package/admin/js/templates/wordpress-import.html +23 -5
- package/admin/js/views/plugins.js +24 -12
- package/admin/js/views/wordpress-import.js +10 -10
- package/bin/update.js +2 -0
- package/docs/DommaFeatures.md +7 -0
- package/docs/configuration.md +1 -0
- package/docs/plugin-development.md +11 -0
- package/package.json +1 -1
- package/server/routes/api/plugin-marketplace.js +249 -19
- package/server/routes/api/wordpress-import.js +62 -3
- package/server/server.js +1 -1
- package/server/services/siteGitignore.js +2 -0
- package/server/services/storeClient.js +333 -0
- package/server/services/tools.js +1 -1
- package/server/services/wpImport/crawl.js +619 -0
- package/server/services/wpImport/index.js +53 -8
- package/server/services/wpImport/plan.js +5 -3
- package/server/services/wpImport/platforms.js +186 -0
- package/server/services/wpImport/wxr.js +6 -2
|
@@ -357,7 +357,7 @@ export async function inspectStaged(id) {
|
|
|
357
357
|
const wxr = await loadExport(id);
|
|
358
358
|
const counts = {};
|
|
359
359
|
for (const it of wxr.items) counts[it.type] = (counts[it.type] || 0) + 1;
|
|
360
|
-
return updateStagingMeta(id, {siteTitle: wxr.site.title, siteUrl: wxr.site.baseSiteUrl || wxr.site.link, wxrVersion: wxr.site.wxrVersion, counts});
|
|
360
|
+
return updateStagingMeta(id, {siteTitle: wxr.site.title, siteUrl: wxr.site.baseSiteUrl || wxr.site.link, wxrVersion: wxr.site.wxrVersion, platform: wxr.site.platform, platformName: wxr.site.platformName, counts});
|
|
361
361
|
}
|
|
362
362
|
|
|
363
363
|
export async function loadExport(id) {
|
|
@@ -416,6 +416,12 @@ export async function uploadsIndex(id) {
|
|
|
416
416
|
}
|
|
417
417
|
}
|
|
418
418
|
|
|
419
|
+
/** "WordPress", "Squarespace", "Wix" - or "the old site" for any other site read live. */
|
|
420
|
+
const sourceName = (wxr, capital = false) => {
|
|
421
|
+
const n = wxr?.site?.platformName || 'WordPress';
|
|
422
|
+
return n === 'website' ? (capital ? 'Site' : 'the old site') : n;
|
|
423
|
+
};
|
|
424
|
+
|
|
419
425
|
// ---- Preview ---------------------------------------------------------------
|
|
420
426
|
|
|
421
427
|
/** The plan for a staged export with these options - nothing written. */
|
|
@@ -430,9 +436,28 @@ export async function previewImport(id, options = {}) {
|
|
|
430
436
|
const wie = await loadWidgets(id);
|
|
431
437
|
// `people` (what the person may do with users) comes from the route, never from the request body.
|
|
432
438
|
const plan = planImport(wxr, state, mapping, {...cleanOptions(options), people: options.people || null, zipPaths, wie, anyPort: testHostsAllowed()});
|
|
439
|
+
plan.notes.unshift(...platformNotes(wxr, meta));
|
|
433
440
|
return {plan, state, wxr, mapping, meta, wie};
|
|
434
441
|
}
|
|
435
442
|
|
|
443
|
+
/** What the preview says first about an export that is not from WordPress (./platforms.js, ./crawl.js). */
|
|
444
|
+
export function platformNotes(wxr, meta = {}) {
|
|
445
|
+
const notes = [];
|
|
446
|
+
const list = (urls, n) => urls.slice(0, 5).map(u => { try { return new URL(u).pathname; } catch { return u; } }).join(', ') + (n > 5 ? ` and ${n - 5} more` : '');
|
|
447
|
+
if (meta.source === 'crawl' && meta.crawl) {
|
|
448
|
+
const c = meta.crawl;
|
|
449
|
+
const day = String(meta.at || '').slice(0, 10);
|
|
450
|
+
notes.push(`Read from the live ${c.builder === 'wix' ? 'Wix ' : c.builder === 'squarespace' ? 'Squarespace ' : ''}site${day ? ` on ${day}` : ''}: ${c.pages} page${c.pages === 1 ? '' : 's'}, ${c.posts} blog post${c.posts === 1 ? '' : 's'}, ${c.files} file${c.files === 1 ? '' : 's'}${c.menu ? ' and its menu' : ''}. The words, headings, images and links come across; layouts, forms, galleries that load as you scroll, shops and bookings do not - rebuild those with Tools. The old site is not changed.`);
|
|
451
|
+
if (!c.sitemap) notes.push('The site has no sitemap, so its pages were found by following links from the home page - a page nothing links to is not here.');
|
|
452
|
+
if (c.capped) notes.push('The reading stopped at its limit (500 pages, or 15 minutes) - import these, then read the site again for the rest.');
|
|
453
|
+
if (c.failedCount) notes.push(`${c.failedCount} address${c.failedCount === 1 ? '' : 'es'} could not be read: ${list(c.failed.map(f => f.url), c.failedCount)}.`);
|
|
454
|
+
if (c.thin?.length) notes.push(`${c.thin.length} page${c.thin.length === 1 ? ' has' : 's have'} little or no text (often a page drawn by an app) - check ${c.thin.length === 1 ? 'it' : 'them'} after the import: ${list(c.thin, c.thin.length)}.`);
|
|
455
|
+
} else if (wxr.site.platform === 'squarespace') {
|
|
456
|
+
notes.push('This is a Squarespace export. It holds your pages, one blog and your gallery images; Squarespace leaves out shop products, events, audio and video blocks, your styling and any other blogs. The images are fetched from Squarespace, so import before you close the Squarespace account.');
|
|
457
|
+
}
|
|
458
|
+
return notes;
|
|
459
|
+
}
|
|
460
|
+
|
|
436
461
|
/** Options from a request - whitelisted. */
|
|
437
462
|
export function cleanOptions(o = {}) {
|
|
438
463
|
const choices = {};
|
|
@@ -593,6 +618,8 @@ async function runPhases(job, {plan, wxr, oldMapping, meta, inviteOrigin, wie},
|
|
|
593
618
|
const remember = (it, url) => {
|
|
594
619
|
const rel = uploadsPath(it.attachmentUrl).toLowerCase();
|
|
595
620
|
if (rel) { mediaTarget.set(rel, url); mediaTarget.set(originalUploadsPath(rel), url); }
|
|
621
|
+
// By its full address too: a file of a site read live (./crawl.js) has no uploads path.
|
|
622
|
+
if (it.attachmentUrl) mediaTarget.set(`url:${it.attachmentUrl.toLowerCase()}`, url);
|
|
596
623
|
mediaById.set(it.id, {url, alt: it.meta._wp_attachment_image_alt || '', title: it.title});
|
|
597
624
|
};
|
|
598
625
|
for (const r of plan.items.filter(x => x.kind === 'media' && x.action === 'keep')) {
|
|
@@ -604,7 +631,7 @@ async function runPhases(job, {plan, wxr, oldMapping, meta, inviteOrigin, wie},
|
|
|
604
631
|
const zipEntries = zip ? new Map(zip.entries.map(e => [zipRel(e.name), e])) : null;
|
|
605
632
|
const takenNow = new Set((await listMedia().catch(() => [])).map(m => m.name.toLowerCase()));
|
|
606
633
|
const fetchOpts = {
|
|
607
|
-
hosts: wxr.site.hosts,
|
|
634
|
+
hosts: wxr.site.mediaHosts || wxr.site.hosts,
|
|
608
635
|
allowPrivate: testHostsAllowed()
|
|
609
636
|
};
|
|
610
637
|
try {
|
|
@@ -623,10 +650,14 @@ async function runPhases(job, {plan, wxr, oldMapping, meta, inviteOrigin, wie},
|
|
|
623
650
|
} else {
|
|
624
651
|
({buffer, contentType} = await fetchMedia(it.attachmentUrl, fetchOpts));
|
|
625
652
|
}
|
|
626
|
-
|
|
653
|
+
let planned = r.name;
|
|
654
|
+
// Squarespace's CDN serves many images as WebP whatever their name says: keep the real type.
|
|
655
|
+
const real = imageTypeOf(buffer);
|
|
656
|
+
if (real && String(r.rel || '').startsWith('cdn/') && !badContent(real, buffer, contentType)) planned = planned.replace(/\.[a-z0-9]+$/i, '') + `.${real}`;
|
|
657
|
+
const ext = planned.split('.').pop().toLowerCase();
|
|
627
658
|
const bad = badContent(ext, buffer, contentType);
|
|
628
659
|
if (bad) throw new Error(bad);
|
|
629
|
-
const name = await placeMedia(
|
|
660
|
+
const name = await placeMedia(planned, buffer, takenNow, oldMapping.items[r.key], mediaName(r.rel || it.attachmentUrl, it.id));
|
|
630
661
|
const url = `/media/${name}`;
|
|
631
662
|
remember(it, url);
|
|
632
663
|
mapping.items[r.key] = {kind: 'media', name, path: null, sha: fingerprint(buffer.toString('base64')), at: new Date().toISOString()};
|
|
@@ -654,10 +685,14 @@ async function runPhases(job, {plan, wxr, oldMapping, meta, inviteOrigin, wie},
|
|
|
654
685
|
const {oldPathOf, basePathOf} = await import('./plan.js');
|
|
655
686
|
const mediaFor = (url) => {
|
|
656
687
|
const rel = uploadsPath(url).toLowerCase();
|
|
657
|
-
if (!rel)
|
|
688
|
+
if (!rel) {
|
|
689
|
+
let abs;
|
|
690
|
+
try { abs = new URL(String(url || '').replace(/&/g, '&'), wxr.site.link || wxr.site.baseSiteUrl).href; } catch { return null; }
|
|
691
|
+
return mediaTarget.get(`url:${abs.toLowerCase()}`) || null;
|
|
692
|
+
}
|
|
658
693
|
let host = '';
|
|
659
694
|
try { host = new URL(url, 'https://relative.invalid').hostname.toLowerCase(); } catch { return null; }
|
|
660
|
-
if (host !== 'relative.invalid' && !wxr.site.hosts.includes(host)) return null;
|
|
695
|
+
if (host !== 'relative.invalid' && !(wxr.site.mediaHosts || wxr.site.hosts).includes(host)) return null;
|
|
661
696
|
return mediaTarget.get(rel) || mediaTarget.get(originalUploadsPath(rel)) || null;
|
|
662
697
|
};
|
|
663
698
|
const linkFor = (url) => {
|
|
@@ -966,7 +1001,7 @@ async function runPhases(job, {plan, wxr, oldMapping, meta, inviteOrigin, wie},
|
|
|
966
1001
|
const existing = await getMenu(r.slug);
|
|
967
1002
|
if (existing && r.action === 'update') await updateMenu(r.slug, {...existing, items});
|
|
968
1003
|
else if (existing) throw new Error(`A menu called ${r.slug} was made while the import ran.`);
|
|
969
|
-
else await createMenu({slug: r.slug, name, description:
|
|
1004
|
+
else await createMenu({slug: r.slug, name, description: `Imported from ${sourceName(wxr)}`, items});
|
|
970
1005
|
mapping.items[r.key] = {kind: 'menu', slug: r.slug, hash: menuFingerprint(items), at: new Date().toISOString()};
|
|
971
1006
|
set(r.key, {result: r.action === 'update' ? 'updated' : 'created', target: r.slug});
|
|
972
1007
|
} catch (err) {
|
|
@@ -1091,7 +1126,7 @@ async function runPhases(job, {plan, wxr, oldMapping, meta, inviteOrigin, wie},
|
|
|
1091
1126
|
const rule = own && seo.store.rules().find(x => x.id === own.id);
|
|
1092
1127
|
let out;
|
|
1093
1128
|
if (rule) out = seo.updateRule(rule.id, {from: r.from, to: r.to, code: 301});
|
|
1094
|
-
else out = seo.addRule({from: r.from, to: r.to, code: 301, note:
|
|
1129
|
+
else out = seo.addRule({from: r.from, to: r.to, code: 301, note: `${sourceName(wxr, true)} import`});
|
|
1095
1130
|
if (out.errors) throw new Error(Object.values(out.errors).join(' '));
|
|
1096
1131
|
mapping.redirects[r.key] = {id: out.rule.id, from: out.rule.from, to: out.rule.to};
|
|
1097
1132
|
redirectResults.push({...r, result: rule ? 'updated' : 'created'});
|
|
@@ -1395,6 +1430,16 @@ async function placeMedia(planned, buffer, taken, was, original) {
|
|
|
1395
1430
|
}
|
|
1396
1431
|
|
|
1397
1432
|
/** Why these bytes are not the file they claim to be ('' when they are). */
|
|
1433
|
+
/** The image type the bytes are ('jpg', 'png', 'gif', 'webp'), or ''. */
|
|
1434
|
+
export function imageTypeOf(buf) {
|
|
1435
|
+
if (!buf?.length) return '';
|
|
1436
|
+
if (buf[0] === 0xff && buf[1] === 0xd8) return 'jpg';
|
|
1437
|
+
if (buf[0] === 0x89 && buf.slice(1, 4).toString('latin1') === 'PNG') return 'png';
|
|
1438
|
+
if (buf.slice(0, 4).toString('latin1') === 'GIF8') return 'gif';
|
|
1439
|
+
if (buf.slice(0, 4).toString('latin1') === 'RIFF' && buf.slice(8, 12).toString('latin1') === 'WEBP') return 'webp';
|
|
1440
|
+
return '';
|
|
1441
|
+
}
|
|
1442
|
+
|
|
1398
1443
|
export function badContent(ext, buf, contentType = '') {
|
|
1399
1444
|
if (!buf?.length) return 'The file is empty.';
|
|
1400
1445
|
if (/^text\/html/i.test(contentType) && ext !== 'txt') return 'The old site answered with a web page, not the file.';
|
|
@@ -46,6 +46,7 @@ import {readWooProducts, shopProductInput, wooPageRole} from './woo.js';
|
|
|
46
46
|
import {customFields, customOldPath, customTypesOf, mediaContext, rewriteSlugOf, typeLabel} from './custom.js';
|
|
47
47
|
import {planPeople} from './people.js';
|
|
48
48
|
import {planWidgets} from './widgets.js';
|
|
49
|
+
import {platformFilePath} from './platforms.js';
|
|
49
50
|
import {detectFrontPage, suggestPostsPage} from './front.js';
|
|
50
51
|
|
|
51
52
|
/** Addresses under the Shop's that are its own pages, never a product's. */
|
|
@@ -112,7 +113,8 @@ function cleanPath(path) {
|
|
|
112
113
|
/** The uploads-relative path of a media address: ".../wp-content/uploads/2019/05/a.jpg" -> "2019/05/a.jpg". */
|
|
113
114
|
export function uploadsPath(url) {
|
|
114
115
|
const m = /\/wp-content\/uploads\/(.+)$/i.exec(String(url || '').split(/[?#]/)[0]);
|
|
115
|
-
|
|
116
|
+
// A Squarespace or Wix file (./platforms.js): "cdn/<host>/<path>".
|
|
117
|
+
if (!m) return platformFilePath(url);
|
|
116
118
|
try { return decodeURIComponent(m[1]); } catch { return m[1]; }
|
|
117
119
|
}
|
|
118
120
|
|
|
@@ -298,7 +300,7 @@ export function planImport(wxr, site, mapping = {}, options = {}) {
|
|
|
298
300
|
else if (ext === 'svg' || ext === 'svgz') push({...row, action: 'skip', reason: 'SVG files can carry scripts - upload it by hand if you trust it.'});
|
|
299
301
|
else if (!MEDIA_EXTENSIONS.has(ext)) push({...row, action: 'skip', reason: `.${ext || '?'} files are not accepted in Media.`});
|
|
300
302
|
else if (!/^https?:\/\//i.test(it.attachmentUrl)) push({...row, action: 'skip', reason: 'It has no file address.'});
|
|
301
|
-
else if (opt.media === 'download' && !wxr.site.hosts.includes(hostOf(it.attachmentUrl))) push({...row, action: 'skip', reason: `It is on another site (${hostOf(it.attachmentUrl) || '?'}) - only the
|
|
303
|
+
else if (opt.media === 'download' && !(wxr.site.mediaHosts || wxr.site.hosts).includes(hostOf(it.attachmentUrl))) push({...row, action: 'skip', reason: `It is on another site (${hostOf(it.attachmentUrl) || '?'}) - only the old site's own files are downloaded.`});
|
|
302
304
|
else if (opt.media === 'download' && !opt.anyPort && !FETCH_PORTS.includes(portOf(it.attachmentUrl))) push({...row, action: 'skip', reason: `It is on port ${portOf(it.attachmentUrl)} - files are only downloaded over the web ports (80 and 443). Upload the uploads folder as a zip instead.`});
|
|
303
305
|
else if (opt.media === 'zip' && opt.zipPaths && !opt.zipPaths.has(rel.toLowerCase())) push({...row, action: 'skip', reason: 'It is not in the uploaded zip.'});
|
|
304
306
|
else {
|
|
@@ -809,7 +811,7 @@ export function planImport(wxr, site, mapping = {}, options = {}) {
|
|
|
809
811
|
};
|
|
810
812
|
return {
|
|
811
813
|
source: sourceKey(wxr.site),
|
|
812
|
-
site: {title: wxr.site.title, url: wxr.site.baseSiteUrl || wxr.site.link, wxrVersion: wxr.site.wxrVersion, hosts: wxr.site.hosts},
|
|
814
|
+
site: {title: wxr.site.title, url: wxr.site.baseSiteUrl || wxr.site.link, wxrVersion: wxr.site.wxrVersion, hosts: wxr.site.hosts, platform: wxr.site.platform || 'wordpress', platformName: wxr.site.platformName || 'WordPress'},
|
|
813
815
|
options: {...opt, zipPaths: undefined, anyPort: undefined, people: undefined, inviteOrigin: undefined, wie: undefined, invite: people.invite, choices},
|
|
814
816
|
blogBase,
|
|
815
817
|
shopBase,
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Exports that are not from WordPress but use its format.
|
|
3
|
+
*
|
|
4
|
+
* - Squarespace exports a WordPress-format file (Settings › Import & Export ›
|
|
5
|
+
* Export). Its images are not in the file and not on the site's own host:
|
|
6
|
+
* they live on Squarespace's CDN (images.squarespace-cdn.com,
|
|
7
|
+
* static1.squarespace.com), often only inside the page HTML, often with a
|
|
8
|
+
* ?format=750w size and the address in data-src rather than src.
|
|
9
|
+
* - A Wix site has no export at all; ./crawl.js reads the live site and writes
|
|
10
|
+
* a WordPress-format file, whose files live on static.wixstatic.com (with
|
|
11
|
+
* /v1/fill/... resizing paths) and *.filesusr.com.
|
|
12
|
+
*
|
|
13
|
+
* adaptPlatform() runs on every export read (./wxr.js): for a WordPress export
|
|
14
|
+
* it only sets `platform: 'wordpress'` and `mediaHosts` = the site's hosts.
|
|
15
|
+
* For the others it also
|
|
16
|
+
* - allows the platform's file hosts for downloads (`site.mediaHosts`) - the
|
|
17
|
+
* exact host names the export names, never a wildcard; links are still
|
|
18
|
+
* only rewritten for the site's own hosts (`site.hosts`);
|
|
19
|
+
* - puts every image's full-size address in its src (no size query, no
|
|
20
|
+
* resizing path, src filled from data-src);
|
|
21
|
+
* - adds an attachment item for each file the content refers to that the
|
|
22
|
+
* export does not list, so it is downloaded, named and mapped like any
|
|
23
|
+
* other (ids from SYNTHETIC_ID_BASE up, stable for a given address so a
|
|
24
|
+
* second run updates rather than duplicates).
|
|
25
|
+
*
|
|
26
|
+
* @module services/wpImport/platforms
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
export const PLATFORM_NAMES = Object.freeze({wordpress: 'WordPress', squarespace: 'Squarespace', wix: 'Wix', website: 'website'});
|
|
30
|
+
|
|
31
|
+
/** A file host of a platform: exact names, plus *.filesusr.com for Wix documents. */
|
|
32
|
+
const SQUARESPACE_FILE_HOST = /^(images\.squarespace-cdn\.com|static\d*\.squarespace\.com)$/i;
|
|
33
|
+
const WIX_FILE_HOST = /^(static\.wixstatic\.com|video\.wixstatic\.com|music\.wixstatic\.com|[a-z0-9-]+\.filesusr\.com)$/i;
|
|
34
|
+
|
|
35
|
+
/** Ids given to the attachments added here - far above any real WordPress post id. */
|
|
36
|
+
export const SYNTHETIC_ID_BASE = 1_900_000_000;
|
|
37
|
+
|
|
38
|
+
/** Which platform's file host is this ('' when none)? */
|
|
39
|
+
export function fileHostPlatform(host) {
|
|
40
|
+
const h = String(host || '').toLowerCase();
|
|
41
|
+
if (SQUARESPACE_FILE_HOST.test(h)) return 'squarespace';
|
|
42
|
+
if (WIX_FILE_HOST.test(h)) return 'wix';
|
|
43
|
+
return '';
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const hostOf = (url) => {
|
|
47
|
+
try {
|
|
48
|
+
const u = new URL(String(url || '').trim());
|
|
49
|
+
return (u.protocol === 'http:' || u.protocol === 'https:') ? u.hostname.toLowerCase() : '';
|
|
50
|
+
} catch { return ''; }
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* The full-size address of a platform file, or the address unchanged.
|
|
55
|
+
* Squarespace: drop the query (?format=750w) - the bare path is the original.
|
|
56
|
+
* Wix: https://static.wixstatic.com/media/a_b~mv2.jpg/v1/fill/w_980,h_600,.../a_b~mv2.jpg
|
|
57
|
+
* -> https://static.wixstatic.com/media/a_b~mv2.jpg
|
|
58
|
+
*/
|
|
59
|
+
export function fullSizeUrl(url) {
|
|
60
|
+
const raw = String(url || '').trim().replace(/&/g, '&');
|
|
61
|
+
let u;
|
|
62
|
+
try { u = new URL(raw.startsWith('//') ? `https:${raw}` : raw); } catch { return raw; }
|
|
63
|
+
const p = fileHostPlatform(u.hostname);
|
|
64
|
+
if (!p) return raw;
|
|
65
|
+
u.protocol = 'https:';
|
|
66
|
+
u.hash = '';
|
|
67
|
+
if (p === 'squarespace') u.search = '';
|
|
68
|
+
if (p === 'wix' && u.hostname.toLowerCase() === 'static.wixstatic.com') {
|
|
69
|
+
u.search = '';
|
|
70
|
+
const m = /^(\/media\/[^/]+)\/v1\//i.exec(u.pathname);
|
|
71
|
+
if (m) u.pathname = m[1];
|
|
72
|
+
}
|
|
73
|
+
return u.href;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The key a platform file is known by - like uploadsPath() for WordPress:
|
|
78
|
+
* "cdn/<host>/<path>" (decoded), or '' when it is not a platform file.
|
|
79
|
+
*/
|
|
80
|
+
export function platformFilePath(url) {
|
|
81
|
+
const full = fullSizeUrl(url);
|
|
82
|
+
let u;
|
|
83
|
+
try { u = new URL(full); } catch { return ''; }
|
|
84
|
+
if (!fileHostPlatform(u.hostname)) return '';
|
|
85
|
+
let path = u.pathname.replace(/^\/+/, '');
|
|
86
|
+
try { path = decodeURIComponent(path); } catch { /* keep it encoded */ }
|
|
87
|
+
return path ? `cdn/${u.hostname.toLowerCase()}/${path}` : '';
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Which platform made this export? */
|
|
91
|
+
export function detectPlatform(wxr) {
|
|
92
|
+
const g = String(wxr.site.generator || '').toLowerCase();
|
|
93
|
+
if (g.includes('squarespace')) return 'squarespace';
|
|
94
|
+
if (g.includes('wix')) return 'wix';
|
|
95
|
+
if (g.includes('domma cms site reader')) return 'website';
|
|
96
|
+
if ((wxr.site.hosts || []).some(h => /\.squarespace\.com$/.test(h))) return 'squarespace';
|
|
97
|
+
if ((wxr.site.hosts || []).some(h => /\.wixsite\.com$/.test(h))) return 'wix';
|
|
98
|
+
// A custom domain: the files say where it came from.
|
|
99
|
+
const seen = {squarespace: 0, wix: 0};
|
|
100
|
+
for (const it of wxr.items) {
|
|
101
|
+
const p = fileHostPlatform(hostOf(it.attachmentUrl));
|
|
102
|
+
if (p) seen[p]++;
|
|
103
|
+
if (seen.squarespace + seen.wix > 2) break;
|
|
104
|
+
}
|
|
105
|
+
if (seen.squarespace || seen.wix) return seen.squarespace >= seen.wix ? 'squarespace' : 'wix';
|
|
106
|
+
for (const it of wxr.items.slice(0, 200)) {
|
|
107
|
+
if (/squarespace-cdn\.com|static1\.squarespace\.com/i.test(it.content || '')) return 'squarespace';
|
|
108
|
+
if (/static\.wixstatic\.com/i.test(it.content || '')) return 'wix';
|
|
109
|
+
}
|
|
110
|
+
return 'wordpress';
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// <img ...> and <a href> (files linked from a page) carrying a platform address.
|
|
114
|
+
const TAG_RE = /<(img|a|source|video|audio)\b[^>]*>/gi;
|
|
115
|
+
const ATTR_RE = /\s(src|data-src|data-image|href|poster)\s*=\s*("([^"]*)"|'([^']*)')/gi;
|
|
116
|
+
|
|
117
|
+
/** One tag with every platform address made full-size, and an <img> given a src from data-src when it has none. */
|
|
118
|
+
function fixTag(tag, found) {
|
|
119
|
+
let srcFrom = '';
|
|
120
|
+
let out = tag.replace(ATTR_RE, (m, name, _q, dq, sq) => {
|
|
121
|
+
const value = dq ?? sq ?? '';
|
|
122
|
+
const host = hostOf(value.replace(/&/g, '&').replace(/^\/\//, 'https://'));
|
|
123
|
+
if (!fileHostPlatform(host)) return m;
|
|
124
|
+
const full = fullSizeUrl(value);
|
|
125
|
+
found.add(full);
|
|
126
|
+
if (name.toLowerCase() === 'data-src' || name.toLowerCase() === 'data-image') srcFrom ||= full;
|
|
127
|
+
return ` ${name}="${full.replace(/&/g, '&').replace(/"/g, '"')}"`;
|
|
128
|
+
});
|
|
129
|
+
if (/^<img\b/i.test(out) && srcFrom && !/\ssrc\s*=/i.test(out)) out = out.replace(/^<img\b/i, `<img src="${srcFrom.replace(/&/g, '&').replace(/"/g, '"')}"`);
|
|
130
|
+
return out;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** A stable id for an address (FNV-1a, folded into 0..99,999,999). */
|
|
134
|
+
export function syntheticId(url) {
|
|
135
|
+
let h = 0x811c9dc5;
|
|
136
|
+
for (let i = 0; i < url.length; i++) { h ^= url.charCodeAt(i); h = Math.imul(h, 0x01000193) >>> 0; }
|
|
137
|
+
return SYNTHETIC_ID_BASE + (h % 100_000_000);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Fill in what the planner needs for a non-WordPress export. Changes `wxr` in place and returns it.
|
|
142
|
+
*
|
|
143
|
+
* @param {object} wxr - from wxrFromTree()
|
|
144
|
+
*/
|
|
145
|
+
export function adaptPlatform(wxr) {
|
|
146
|
+
const platform = detectPlatform(wxr);
|
|
147
|
+
wxr.site.platform = platform;
|
|
148
|
+
wxr.site.platformName = PLATFORM_NAMES[platform];
|
|
149
|
+
const mediaHosts = new Set(wxr.site.hosts);
|
|
150
|
+
wxr.site.mediaHosts = [...mediaHosts];
|
|
151
|
+
if (platform === 'wordpress') return wxr;
|
|
152
|
+
|
|
153
|
+
const listed = new Set();
|
|
154
|
+
for (const it of wxr.items) {
|
|
155
|
+
if (it.type !== 'attachment' || !it.attachmentUrl) continue;
|
|
156
|
+
it.attachmentUrl = fullSizeUrl(it.attachmentUrl);
|
|
157
|
+
listed.add(it.attachmentUrl);
|
|
158
|
+
const h = hostOf(it.attachmentUrl);
|
|
159
|
+
if (fileHostPlatform(h)) mediaHosts.add(h);
|
|
160
|
+
}
|
|
161
|
+
const extra = [];
|
|
162
|
+
const usedIds = new Set(wxr.items.map(i => i.id));
|
|
163
|
+
for (const it of wxr.items) {
|
|
164
|
+
if (it.type === 'attachment' || !it.content) continue;
|
|
165
|
+
const found = new Set();
|
|
166
|
+
it.content = it.content.replace(TAG_RE, (tag) => fixTag(tag, found));
|
|
167
|
+
for (const url of found) {
|
|
168
|
+
const h = hostOf(url);
|
|
169
|
+
mediaHosts.add(h);
|
|
170
|
+
if (listed.has(url)) continue;
|
|
171
|
+
listed.add(url);
|
|
172
|
+
let id = syntheticId(url);
|
|
173
|
+
while (usedIds.has(id)) id++;
|
|
174
|
+
usedIds.add(id);
|
|
175
|
+
const name = decodeURIComponent(new URL(url).pathname.split('/').pop() || '') || `file-${id}`;
|
|
176
|
+
extra.push({
|
|
177
|
+
id, type: 'attachment', title: name.replace(/\.[a-z0-9]+$/i, ''), slug: '', status: 'inherit',
|
|
178
|
+
parent: it.id, order: 0, date: it.date, dateGmt: it.dateGmt, link: '', guid: url, creator: it.creator,
|
|
179
|
+
content: '', excerpt: '', password: '', attachmentUrl: url, meta: {}, terms: [], comments: [], synthetic: true
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
wxr.items.push(...extra);
|
|
184
|
+
wxr.site.mediaHosts = [...mediaHosts];
|
|
185
|
+
return wxr;
|
|
186
|
+
}
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
* A WordPress export (WXR 1.0-1.2) as plain data - what the planner and the
|
|
3
3
|
* runner work from. Reads the tree from ./xml.js; does no I/O.
|
|
4
4
|
*
|
|
5
|
-
* {site: {title, link, baseSiteUrl, baseBlogUrl, wxrVersion, hosts[]
|
|
5
|
+
* {site: {title, link, baseSiteUrl, baseBlogUrl, wxrVersion, generator, hosts[],
|
|
6
|
+
* platform ('wordpress'|'squarespace'|'wix'), platformName, mediaHosts[] - see ./platforms.js},
|
|
6
7
|
* authors: [{id, login, email, displayName, firstName, lastName}] (personal data - see ./people.js),
|
|
7
8
|
* categories[], tags[], menus[],
|
|
8
9
|
* terms: [{taxonomy, id, slug, name, parent, description}] - every other taxonomy's terms
|
|
@@ -12,6 +13,7 @@
|
|
|
12
13
|
*
|
|
13
14
|
* @module services/wpImport/wxr
|
|
14
15
|
*/
|
|
16
|
+
import {adaptPlatform} from './platforms.js';
|
|
15
17
|
import {child, children, parseXml, textOf, XmlError} from './xml.js';
|
|
16
18
|
|
|
17
19
|
/** Item types the importer acts on; everything else is counted and listed as skipped. */
|
|
@@ -58,6 +60,7 @@ export function wxrFromTree(root) {
|
|
|
58
60
|
link: textOf(channel, 'link'),
|
|
59
61
|
description: textOf(channel, 'description'),
|
|
60
62
|
language: textOf(channel, 'language'),
|
|
63
|
+
generator: textOf(channel, 'generator'),
|
|
61
64
|
baseSiteUrl: textOf(channel, 'wp:base_site_url'),
|
|
62
65
|
baseBlogUrl: textOf(channel, 'wp:base_blog_url'),
|
|
63
66
|
wxrVersion
|
|
@@ -134,7 +137,8 @@ export function wxrFromTree(root) {
|
|
|
134
137
|
if (h) { hostSet.add(h); hostSet.add(bareHost(h)); hostSet.add(`www.${bareHost(h)}`); }
|
|
135
138
|
}
|
|
136
139
|
site.hosts = [...hostSet];
|
|
137
|
-
|
|
140
|
+
// Squarespace's export and the Wix reader's (./crawl.js): their files live on the platform's own hosts.
|
|
141
|
+
return adaptPlatform({site, authors, categories, tags, menus, terms, items});
|
|
138
142
|
}
|
|
139
143
|
|
|
140
144
|
function readItem(node) {
|