domma-cms 1.38.0 → 1.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,619 @@
1
+ /**
2
+ * Read a live website - for platforms with no export (Wix), or any site - and
3
+ * write what it finds as a WordPress-format export, so the rest of the
4
+ * importer (preview, choices, run, re-runs, redirects, report) works on it
5
+ * unchanged.
6
+ *
7
+ * 1. The pages: the site's sitemap (/sitemap.xml, a sitemap index followed
8
+ * one level), or - with none - the links from the home page outwards.
9
+ * Only the site's own host (with or without www.) is read.
10
+ * 2. Each page: its title, description, date and author (meta tags and
11
+ * JSON-LD), and its main content - Wix's page container, an <article>,
12
+ * <main>, or the body - with the header, footer, navigation, scripts,
13
+ * forms and buttons left out and the builder's <div>/<span> layers
14
+ * unwrapped down to headings, paragraphs, lists, links, images, quotes
15
+ * and tables. ./convert.js cleans it again on import.
16
+ * 3. A page is a blog post when it says so (og:type article, JSON-LD
17
+ * BlogPosting/Article/NewsArticle) or comes from a blog sitemap.
18
+ * 4. Images and linked documents become attachments; ./platforms.js turns
19
+ * Wix and Squarespace image addresses into their full-size originals.
20
+ * 5. The home page's navigation becomes a menu.
21
+ *
22
+ * Every request goes through services/outbound.js (public addresses only,
23
+ * ports 80 and 443, the address that was checked is the one connected to).
24
+ * Bounded: MAX_PAGES pages, PAGE_MAX_BYTES each, CRAWL_TIME_MS overall.
25
+ *
26
+ * Ids: each page's is syntheticId() of its address, so reading the site again
27
+ * gives the same ids and a second import updates rather than duplicates.
28
+ *
29
+ * @module services/wpImport/crawl
30
+ */
31
+ import {parseDocument} from 'htmlparser2';
32
+ import {selectAll, selectOne} from 'css-select';
33
+ import {safeFetch, FetchRefused} from '../outbound.js';
34
+ import {children, parseXml, textOf, XmlError} from './xml.js';
35
+ import {fileHostPlatform, fullSizeUrl, syntheticId} from './platforms.js';
36
+
37
+ export const CRAWL_LIMITS = Object.freeze({maxPages: 500, pageMaxBytes: 5 * 1024 * 1024, sitemapMaxBytes: 10 * 1024 * 1024, timeMs: 15 * 60_000, concurrency: 3, maxSitemaps: 25});
38
+
39
+ const USER_AGENT = 'DommaCMS-SiteImport/1 (+https://domma-cms.com)';
40
+
41
+ /** Addresses that are never content: Wix and Squarespace system pages, feeds, files. */
42
+ const NOT_A_PAGE = /\/(_api|_partials|_serverless|wp-json|feed|cart|checkout|account|members-area|login|search|tag|tags|categories?|hashtags?|blog-feed\.xml|robots\.txt|sitemap[^/]*\.xml)(\/|$)|\.(xml|json|rss|pdf|jpe?g|png|gif|webp|svg|zip|docx?|xlsx?|mp[34]|css|js)$/i;
43
+
44
+ /** File types a link may point at to be taken as a document. */
45
+ const DOC_EXT = /\.(pdf|docx?|xlsx?|pptx?|txt|csv|zip|mp3|mp4)$/i;
46
+
47
+ /** Tags kept in content (with only the attributes listed); anything else is unwrapped or dropped. */
48
+ const KEEP = {
49
+ h1: [], h2: [], h3: [], h4: [], h5: [], h6: [], p: [], br: [], hr: [], ul: [], ol: [], li: [], blockquote: [],
50
+ strong: [], b: [], em: [], i: [], u: [], s: [], sub: [], sup: [], code: [], pre: [], a: ['href', 'title'], img: ['src', 'alt', 'title'],
51
+ figure: [], figcaption: [], table: [], thead: [], tbody: [], tr: [], th: ['colspan', 'rowspan'], td: ['colspan', 'rowspan'],
52
+ iframe: ['src']
53
+ };
54
+ /** Tags dropped with everything inside them. */
55
+ const DROP = new Set(['script', 'style', 'noscript', 'template', 'svg', 'canvas', 'form', 'button', 'input', 'select', 'textarea', 'nav', 'header', 'footer', 'aside', 'dialog', 'object', 'embed', 'link', 'meta', 'head']);
56
+ /** Kept tags that sit inside a line of text; the others are blocks. */
57
+ const INLINE_KEEP = new Set(['a', 'strong', 'b', 'em', 'i', 'u', 's', 'sub', 'sup', 'code']);
58
+ /** Blocks that hold text directly (a loose paragraph is never opened inside one). */
59
+ const TEXT_BLOCKS = new Set(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'blockquote', 'figcaption', 'td', 'th', 'pre', 'figure']);
60
+ /** Unwrapped tags that are inline (a <span>): their text runs on. Every other unwrapped tag (a <div>) is a break. */
61
+ const INLINE_UNWRAP = new Set(['span', 'font', 'small', 'big', 'mark', 'abbr', 'time', 'label', 'wow-image', 'picture', 'cite', 'q', 'var', 'samp', 'kbd', 'bdi', 'bdo', 'wbr', 'ins', 'del']);
62
+
63
+ /** Where a builder keeps the page's own content - tried in order. */
64
+ const CONTENT_SELECTORS = [
65
+ '[data-hook="post-description"]', '[data-id="content-viewer"]', '[data-hook="post-page"] article', // Wix blog posts
66
+ '#PAGES_CONTAINER', '#SITE_PAGES', 'main#PAGES_CONTAINER', // Wix pages
67
+ '.blog-item-content', 'article .sqs-layout', '#page .sqs-layout', 'main#page', '#sections', // Squarespace
68
+ 'article', 'main', '[role="main"]', '#content', '.content', 'body'
69
+ ];
70
+ /** Removed from the content root before it is read: site chrome a builder puts inside it. */
71
+ const CHROME_SELECTORS = ['#SITE_HEADER', '#SITE_FOOTER', '[data-hook="post-footer"]', '[data-hook="related-posts"]', '[data-hook="comments"]', '[data-hook="post-title"]',
72
+ '.sqs-block-newsletter', '.sqs-block-form', '[hidden]'];
73
+
74
+ export class CrawlError extends Error {
75
+ constructor(message) { super(message); this.code = 'ECRAWL'; }
76
+ }
77
+
78
+ /** The site's home address from what the person typed: "example.com" -> https://example.com/. Throws CrawlError. */
79
+ export function startUrl(raw) {
80
+ let s = String(raw || '').trim();
81
+ if (!s) throw new CrawlError('Type the address of the site to copy.');
82
+ if (!/^[a-z][a-z0-9+.-]*:\/\//i.test(s)) s = `https://${s}`;
83
+ let u;
84
+ try { u = new URL(s); } catch { throw new CrawlError('That is not a web address.'); }
85
+ if (u.protocol !== 'http:' && u.protocol !== 'https:') throw new CrawlError('Only http and https sites can be read.');
86
+ if (u.username || u.password) throw new CrawlError('Leave the user name and password out of the address.');
87
+ if (!u.hostname.includes('.')) throw new CrawlError('Type the full address, such as www.example.com.');
88
+ u.hash = '';
89
+ u.search = '';
90
+ // A Wix free site lives in a folder (user.wixsite.com/mysite): keep the path; otherwise the home page.
91
+ if (!/(^|\.)(wixsite\.com|wix\.com)$/i.test(u.hostname)) u.pathname = '/';
92
+ return u.href;
93
+ }
94
+
95
+ const bare = (h) => String(h || '').toLowerCase().replace(/^www\./, '');
96
+
97
+ /** An address on the site, cleaned (no hash, no tracking query, no trailing slash), or null when it is elsewhere or not a page. */
98
+ export function pageAddress(href, base, site) {
99
+ let u;
100
+ try { u = new URL(String(href || '').trim(), base); } catch { return null; }
101
+ if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;
102
+ if (bare(u.hostname) !== site.bareHost) return null;
103
+ if (site.basePath && !(`${u.pathname}/`).startsWith(`${site.basePath}/`)) return null;
104
+ u.hash = '';
105
+ u.search = '';
106
+ u.protocol = site.protocol;
107
+ u.hostname = site.host;
108
+ let p = u.pathname.replace(/\/{2,}/g, '/');
109
+ if (p.length > 1) p = p.replace(/\/+$/, '');
110
+ u.pathname = p;
111
+ if (NOT_A_PAGE.test(p)) return null;
112
+ return u.href;
113
+ }
114
+
115
+ // ---- Fetching -------------------------------------------------------------
116
+
117
+ async function get(url, ctx, maxBytes) {
118
+ if (ctx.signal?.aborted) throw new CrawlError('Stopped.');
119
+ const res = await safeFetch(url, {
120
+ headers: {'user-agent': USER_AGENT, accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'accept-language': 'en-GB,en;q=0.8'},
121
+ maxBytes, timeoutMs: 30_000, maxRedirects: 5, signal: ctx.signal, ...ctx.fetchOpts
122
+ });
123
+ return res;
124
+ }
125
+
126
+ /** Every page address the sitemap lists, with whether it came from a blog sitemap. */
127
+ export async function readSitemap(site, ctx) {
128
+ const found = new Map();
129
+ const queue = [`${site.origin}${site.basePath}/sitemap.xml`];
130
+ const seen = new Set();
131
+ while (queue.length && seen.size < CRAWL_LIMITS.maxSitemaps) {
132
+ const url = queue.shift();
133
+ if (seen.has(url)) continue;
134
+ seen.add(url);
135
+ let root;
136
+ try {
137
+ const res = await get(url, ctx, CRAWL_LIMITS.sitemapMaxBytes);
138
+ if (!res.ok) continue;
139
+ root = await parseXml(res.text().replace(/^/, ''));
140
+ } catch (err) {
141
+ if (err instanceof XmlError || err instanceof FetchRefused || err.code) continue;
142
+ throw err;
143
+ }
144
+ const blogMap = /blog|post|news|article/i.test(url.split('/').pop() || '');
145
+ if (root?.name === 'sitemapindex') {
146
+ for (const s of children(root, 'sitemap')) {
147
+ const loc = textOf(s, 'loc');
148
+ // Only the site's own sitemaps; Wix stores' and events' are read like any other.
149
+ try { if (bare(new URL(loc).hostname) === site.bareHost) queue.push(loc); } catch { /* not an address */ }
150
+ }
151
+ } else if (root?.name === 'urlset') {
152
+ for (const u of children(root, 'url')) {
153
+ const addr = pageAddress(textOf(u, 'loc'), site.origin, site);
154
+ if (addr && !found.has(addr)) found.set(addr, {fromBlog: blogMap, lastmod: textOf(u, 'lastmod')});
155
+ }
156
+ }
157
+ }
158
+ return found;
159
+ }
160
+
161
+ // ---- Reading one page -----------------------------------------------------
162
+
163
+ const attr = (n, name) => (n?.attribs?.[name] ?? '').trim();
164
+ const metaContent = (doc, sel) => attr(selectOne(sel, doc), 'content');
165
+
166
+ function textContent(n) {
167
+ if (!n) return '';
168
+ if (n.type === 'text') return n.data;
169
+ return (n.children || []).map(textContent).join('');
170
+ }
171
+
172
+ const decode = (s) => String(s || '').replace(/&#(\d+);/g, (_m, d) => String.fromCodePoint(Number(d))).replace(/&#x([0-9a-f]+);/gi, (_m, h) => String.fromCodePoint(parseInt(h, 16)))
173
+ .replace(/&quot;/g, '"').replace(/&#39;|&apos;/g, "'").replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&');
174
+
175
+ /** JSON-LD objects on the page, flattened (@graph included). */
176
+ function jsonLd(doc) {
177
+ const out = [];
178
+ for (const s of selectAll('script[type="application/ld+json"]', doc)) {
179
+ try {
180
+ const v = JSON.parse(textContent(s));
181
+ const walk = (x) => {
182
+ if (Array.isArray(x)) x.forEach(walk);
183
+ else if (x && typeof x === 'object') { out.push(x); if (x['@graph']) walk(x['@graph']); }
184
+ };
185
+ walk(v);
186
+ } catch { /* not JSON */ }
187
+ }
188
+ return out;
189
+ }
190
+
191
+ const typesOf = (o) => [].concat(o?.['@type'] || []).map(String);
192
+ const POST_TYPES = ['BlogPosting', 'Article', 'NewsArticle', 'SocialMediaPosting'];
193
+
194
+ /** Which builder made this page ('wix', 'squarespace' or ''). */
195
+ export function builderOf(html) {
196
+ if (/<meta[^>]+name=["']generator["'][^>]+content=["'][^"']*Wix/i.test(html) || /static\.wixstatic\.com|wix-thunderbolt|_wixCIDX/i.test(html)) return 'wix';
197
+ if (/<!-- This is Squarespace\. -->|static1\.squarespace\.com|squarespace-cdn\.com/i.test(html)) return 'squarespace';
198
+ return '';
199
+ }
200
+
201
+ /** The best image address of an <img> (data-src or the largest srcset entry beats a placeholder src). */
202
+ function imageSrc(n, base) {
203
+ let src = attr(n, 'data-src') || attr(n, 'data-image') || attr(n, 'data-pin-media') || '';
204
+ const srcset = attr(n, 'srcset') || attr(n, 'data-srcset');
205
+ if (!src && srcset) {
206
+ const best = srcset.split(',').map(s => s.trim().split(/\s+/)).map(([u, w]) => ({u, w: Number.parseFloat(w) || 0})).sort((a, b) => b.w - a.w)[0];
207
+ src = best?.u || '';
208
+ }
209
+ if (!src || /^data:/i.test(src)) src = attr(n, 'src');
210
+ if (!src || /^data:/i.test(src)) return '';
211
+ try { return fullSizeUrl(new URL(src.replace(/^\/\//, 'https://'), base).href); } catch { return ''; }
212
+ }
213
+
214
+ /** A Wix image component's picture: {src (full size), alt}, or null. */
215
+ export function wixImage(info) {
216
+ let d;
217
+ try { d = JSON.parse(decode(info)); } catch { return null; }
218
+ const im = d?.imageData;
219
+ const uri = String(im?.uri || '').trim();
220
+ if (!uri || /^data:/i.test(uri)) return null;
221
+ const src = fullSizeUrl(/^https?:\/\//i.test(uri) ? uri : `https://static.wixstatic.com/media/${uri.replace(/^\/+/, '')}`);
222
+ // Its name is often only the uploaded file's name - not a description.
223
+ const name = String(im.alt || im.name || '').trim();
224
+ const alt = /\.(jpe?g|png|gif|webp|avif)$/i.test(name) || /^[0-9a-f-]{20,}$/i.test(name) ? '' : name;
225
+ return {src, alt};
226
+ }
227
+
228
+ /**
229
+ * The page's content as plain HTML, and the files it uses.
230
+ *
231
+ * @returns {{html: string, files: string[], text: number}}
232
+ */
233
+ export function cleanContent(root, base, site) {
234
+ const files = new Set();
235
+ const alts = {};
236
+ const out = [];
237
+ let textLen = 0;
238
+ const esc = (s) => String(s).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
239
+ const escA = (s) => esc(s).replace(/"/g, '&quot;');
240
+ const abs = (v) => { try { return new URL(v, base).href; } catch { return ''; } };
241
+
242
+ // Text and inline tags outside any block are gathered into a paragraph of their own; the builder's
243
+ // <div>s end that paragraph (outside a block) or are only a space (inside one).
244
+ let inBlock = 0; // open blocks text may sit in (p, h1-6, li, td...)
245
+ let inInline = 0; // open inline tags (a, strong...) - a <div> inside one never splits it
246
+ let loose = false; // a paragraph opened for loose text
247
+ const openLoose = () => { if (!inBlock && !loose) { out.push('<p>'); loose = true; } };
248
+ const closeLoose = () => { if (loose && !inInline) { out.push('</p>\n'); loose = false; } };
249
+
250
+ // An image on its own (outside any text) is a paragraph of its own.
251
+ const pushImg = (src, alt) => {
252
+ files.add(src);
253
+ if (alt && !alts[src]) alts[src] = alt.slice(0, 200);
254
+ const tag = `<img src="${escA(src)}" alt="${escA(alt)}">`;
255
+ if (inBlock || inInline || loose) out.push(tag);
256
+ else out.push(`\n<p>${tag}</p>\n`);
257
+ };
258
+
259
+ const walk = (n) => {
260
+ if (n.type === 'text') {
261
+ const t = n.data.replace(/\s+/g, ' ');
262
+ if (!t.trim()) { if (inBlock || loose) out.push(' '); return; }
263
+ textLen += t.trim().length;
264
+ openLoose();
265
+ out.push(esc(decode(t)));
266
+ return;
267
+ }
268
+ if (n.type !== 'tag' && n.type !== 'script' && n.type !== 'style') return;
269
+ const name = n.name.toLowerCase();
270
+ if (DROP.has(name) || n.type === 'script' || n.type === 'style') return;
271
+ // Wix's image component: the picture is named in data-image-info and its <img> is filled in by script.
272
+ if (name === 'wow-image' && attr(n, 'data-image-info')) {
273
+ const img = wixImage(attr(n, 'data-image-info'));
274
+ if (img) {
275
+ pushImg(img.src, img.alt);
276
+ return;
277
+ }
278
+ }
279
+ if (name === 'img') {
280
+ const src = imageSrc(n, base);
281
+ if (!src) return;
282
+ pushImg(src, decode(attr(n, 'alt')));
283
+ return;
284
+ }
285
+ if (name === 'iframe') {
286
+ const src = abs(attr(n, 'src'));
287
+ if (/youtube\.com|youtu\.be|vimeo\.com/i.test(src)) { closeLoose(); out.push(`\n<iframe src="${escA(src)}"></iframe>\n`); }
288
+ return;
289
+ }
290
+ const keep = KEEP[name];
291
+ if (!keep) {
292
+ const blockish = !INLINE_UNWRAP.has(name);
293
+ if (blockish && !inBlock) closeLoose();
294
+ (n.children || []).forEach(walk);
295
+ if (blockish) { if (inBlock || inInline) out.push(' '); else closeLoose(); }
296
+ return;
297
+ }
298
+ let attrs = '';
299
+ for (const a of keep) {
300
+ let v = attr(n, a);
301
+ if (!v) continue;
302
+ if (a === 'href') {
303
+ if (/^(javascript|data):/i.test(v)) continue;
304
+ if (!/^(mailto|tel):|^#/i.test(v)) v = abs(v);
305
+ const host = (() => { try { return new URL(v).hostname; } catch { return ''; } })();
306
+ if (DOC_EXT.test(v.split(/[?#]/)[0]) && (bare(host) === site.bareHost || fileHostPlatform(host))) files.add(fullSizeUrl(v));
307
+ else if (pageAddress(v, base, site)) v = pageAddress(v, base, site);
308
+ }
309
+ attrs += ` ${a}="${escA(decode(v))}"`;
310
+ }
311
+ if (name === 'br') { if (inBlock || loose) out.push('<br>'); return; }
312
+ if (name === 'hr') { closeLoose(); out.push('\n<hr>\n'); return; }
313
+ const inline = INLINE_KEEP.has(name);
314
+ if (inline) openLoose();
315
+ else if (!inInline) closeLoose();
316
+ const holds = TEXT_BLOCKS.has(name);
317
+ // A paragraph inside a paragraph (Wix nests them in list items) is only its text.
318
+ if (name === 'p' && inBlock) { (n.children || []).forEach(walk); out.push(' '); return; }
319
+ out.push(`${inline ? '' : '\n'}<${name}${attrs}>`);
320
+ if (inline) inInline++;
321
+ if (holds) inBlock++;
322
+ (n.children || []).forEach(walk);
323
+ if (holds) inBlock--;
324
+ if (inline) inInline--;
325
+ out.push(`</${name}>${inline ? '' : '\n'}`);
326
+ };
327
+ walk(root);
328
+ closeLoose();
329
+ let html = out.join('');
330
+ // Empty tags (Wix spaces its text with <p><br></p>), repeated until none are left.
331
+ for (let i = 0; i < 4; i++) html = html.replace(/<(p|h[1-6]|li|strong|em|b|i|u|s|a|blockquote|figcaption)\b[^>]*>(\s|&nbsp;|<br>)*<\/\1>/gi, '');
332
+ html = html.replace(/<(p|h[1-6]|li|td|th|figcaption)>\s+/gi, '<$1>').replace(/\s+<\/(p|h[1-6]|li|td|th|figcaption)>/gi, '</$1>')
333
+ .replace(/(<br>\s*)+<\/(p|li|h[1-6])>/gi, '</$2>').replace(/[ \t]+\n/g, '\n').replace(/\n{2,}/g, '\n').trim()
334
+ .replace(/\n/g, '\n\n');
335
+ return {html, files: [...files], alts, text: textLen};
336
+ }
337
+
338
+ /**
339
+ * Read one page.
340
+ *
341
+ * @returns {{url, title, description, seoTitle, kind: 'post'|'page', date, author, html, files, links[], builder, siteName, nav: {text, url, children}[]|null}}
342
+ */
343
+ export function readPage(html, url, site, {fromBlog = false, wantNav = false} = {}) {
344
+ const doc = parseDocument(html, {decodeEntities: true});
345
+ const ld = jsonLd(doc);
346
+ const ldPost = ld.find(o => typesOf(o).some(t => POST_TYPES.includes(t)));
347
+ const ogType = metaContent(doc, 'meta[property="og:type"]').toLowerCase();
348
+ const kind = (ldPost || ogType === 'article' || (fromBlog && /\/(post|blog|news)\//i.test(new URL(url).pathname))) ? 'post' : 'page';
349
+
350
+ const siteName = decode(metaContent(doc, 'meta[property="og:site_name"]'));
351
+ const rawTitle = decode(ldPost?.headline || metaContent(doc, 'meta[property="og:title"]') || textContent(selectOne('title', doc)) || '').replace(/\s+/g, ' ').trim();
352
+ const docTitle = decode(textContent(selectOne('title', doc))).replace(/\s+/g, ' ').trim();
353
+ // "About | My Site" -> "About"
354
+ const title = (siteName ? rawTitle.replace(new RegExp(`\\s*[|\\-–—·]\\s*${siteName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*$`, 'i'), '') : rawTitle.replace(/\s+[|–—·]\s+[^|–—·]+$/, '')) || rawTitle;
355
+ const description = decode(metaContent(doc, 'meta[name="description"]') || metaContent(doc, 'meta[property="og:description"]')).trim();
356
+ const date = String(ldPost?.datePublished || metaContent(doc, 'meta[property="article:published_time"]') || '').trim();
357
+ const author = (() => {
358
+ const a = [].concat(ldPost?.author || [])[0];
359
+ return decode(typeof a === 'string' ? a : (a?.name || metaContent(doc, 'meta[name="author"]') || '')).trim();
360
+ })();
361
+
362
+ const links = selectAll('a[href]', doc).map(a => pageAddress(attr(a, 'href'), url, site)).filter(Boolean);
363
+ const nav = wantNav ? readNav(doc, url, site) : null;
364
+
365
+ let root = null;
366
+ for (const sel of CONTENT_SELECTORS) {
367
+ const n = selectOne(sel, doc);
368
+ if (n && textContent(n).trim().length > 40) { root = n; break; }
369
+ }
370
+ root ||= selectOne('body', doc) || doc;
371
+ for (const sel of CHROME_SELECTORS) for (const n of selectAll(sel, root)) n.children = [];
372
+ const {html: body, files, alts} = cleanContent(root, url, site);
373
+ // The title is the page's own heading: a first <h1> that repeats it goes.
374
+ const content = body.replace(/^<h1>([\s\S]*?)<\/h1>\s*/i, (m, h) => (h.replace(/<[^>]+>/g, '').trim().toLowerCase() === title.toLowerCase() ? '' : m));
375
+ return {url, title: title || docTitle || new URL(url).pathname, description, seoTitle: docTitle, kind, date, author, html: content, files, alts, links, builder: builderOf(html), siteName, nav};
376
+ }
377
+
378
+ /** The site's main navigation: the first <nav> (or Wix's menu) with links on this site. */
379
+ export function readNav(doc, url, site) {
380
+ const candidates = [...selectAll('#SITE_HEADER nav, header nav, nav', doc), ...selectAll('[data-testid="linkBar"], .header-nav', doc)];
381
+ for (const nav of candidates) {
382
+ const list = selectOne('ul', nav);
383
+ const fromList = (ul) => (ul?.children || []).filter(c => c.name === 'li').map(li => {
384
+ const a = selectOne('a[href]', li);
385
+ const sub = (li.children || []).map(c => (c.name === 'ul' ? c : selectOne('ul', c))).find(Boolean);
386
+ const text = decode(textContent(a || li)).replace(/\s+/g, ' ').trim();
387
+ const href = a ? attr(a, 'href') : '';
388
+ const target = href ? (pageAddress(href, url, site) || (/^https?:/i.test(href) ? href : '')) : '';
389
+ return {text, url: target, children: sub && sub !== ul ? fromList(sub) : []};
390
+ }).filter(i => i.text && (i.url || i.children.length));
391
+ let items = list ? fromList(list) : [];
392
+ if (!items.length) {
393
+ items = selectAll('a[href]', nav).map(a => ({text: decode(textContent(a)).replace(/\s+/g, ' ').trim(), url: pageAddress(attr(a, 'href'), url, site) || '', children: []})).filter(i => i.text && i.url);
394
+ }
395
+ // De-duplicate (Wix draws the menu twice: the bar and the "More" overflow).
396
+ const seen = new Set();
397
+ items = items.filter(i => { const k = `${i.text}\0${i.url}`; if (seen.has(k)) return false; seen.add(k); return true; });
398
+ if (items.length >= 2) return items.slice(0, 40);
399
+ }
400
+ return null;
401
+ }
402
+
403
+ // ---- The crawl ------------------------------------------------------------
404
+
405
+ /**
406
+ * Read a site.
407
+ *
408
+ * @param {string} raw - the address the person typed
409
+ * @param {object} [opts]
410
+ * @param {(p: {phase: string, found: number, read: number, failed: number}) => void} [opts.onProgress]
411
+ * @param {AbortSignal} [opts.signal]
412
+ * @param {number} [opts.maxPages]
413
+ * @param {object} [opts.fetchOpts] - passed to safeFetch (tests: policy, lookup)
414
+ * @returns {Promise<{xml: string, summary: object}>}
415
+ */
416
+ export async function crawlSite(raw, opts = {}) {
417
+ const start = startUrl(raw);
418
+ const s = new URL(start);
419
+ const site = {origin: s.origin, protocol: s.protocol, host: s.hostname.toLowerCase(), bareHost: bare(s.hostname), basePath: s.pathname.replace(/\/+$/, '')};
420
+ const maxPages = Math.min(opts.maxPages || CRAWL_LIMITS.maxPages, CRAWL_LIMITS.maxPages);
421
+ const deadline = Date.now() + (opts.timeMs || CRAWL_LIMITS.timeMs);
422
+ const ctx = {signal: opts.signal, fetchOpts: opts.fetchOpts || {}};
423
+ const progress = {phase: 'Reading the home page', found: 0, read: 0, failed: 0};
424
+ const tell = () => opts.onProgress?.({...progress});
425
+
426
+ // The home page first: it says which builder, which host (www or not) and holds the menu.
427
+ let home;
428
+ try {
429
+ home = await get(start, ctx, CRAWL_LIMITS.pageMaxBytes);
430
+ } catch (err) {
431
+ throw new CrawlError(`The site could not be reached: ${err.message || err}`);
432
+ }
433
+ if (!home.ok) throw new CrawlError(`The site answered ${home.status} for its home page.`);
434
+ if (!/html/i.test(String(home.headers['content-type'] || 'text/html'))) throw new CrawlError('That address is not a web page.');
435
+ // Follow a redirect to www. (or away from it), or to https.
436
+ const landed = new URL(home.url);
437
+ if (bare(landed.hostname) !== site.bareHost) throw new CrawlError(`The site sent us to another address (${landed.hostname}). Type that address instead.`);
438
+ Object.assign(site, {origin: landed.origin, protocol: landed.protocol, host: landed.hostname.toLowerCase()});
439
+ const homeUrl = `${site.origin}${site.basePath || '/'}`;
440
+ const homeHtml = home.text();
441
+ const homePage = readPage(homeHtml, homeUrl, site, {wantNav: true});
442
+ const builder = homePage.builder;
443
+
444
+ // Which pages.
445
+ progress.phase = 'Reading the sitemap';
446
+ tell();
447
+ const pages = new Map([[pageAddress(homeUrl, homeUrl, site) || homeUrl, {fromBlog: false}]]);
448
+ const sitemap = await readSitemap(site, ctx);
449
+ for (const [u, v] of sitemap) if (!pages.has(u)) pages.set(u, v);
450
+ const usedSitemap = sitemap.size > 0;
451
+ if (!usedSitemap) for (const u of homePage.links) if (!pages.has(u)) pages.set(u, {fromBlog: false});
452
+ progress.found = pages.size;
453
+
454
+ const read = new Map();
455
+ read.set([...pages.keys()][0], homePage);
456
+ const failed = [];
457
+ const queue = [...pages.keys()].slice(1);
458
+ let capped = false;
459
+ progress.phase = 'Reading the pages';
460
+ progress.read = 1;
461
+ tell();
462
+
463
+ const worker = async () => {
464
+ while (queue.length) {
465
+ if (Date.now() > deadline) { capped = true; return; }
466
+ if (read.size + failed.length >= maxPages) { capped = true; queue.length = 0; return; }
467
+ const url = queue.shift();
468
+ try {
469
+ const res = await get(url, ctx, CRAWL_LIMITS.pageMaxBytes);
470
+ if (!res.ok) throw new Error(`answered ${res.status}`);
471
+ if (!/html/i.test(String(res.headers['content-type'] || 'text/html'))) throw new Error('not a web page');
472
+ const finalUrl = pageAddress(res.url, res.url, site) || url;
473
+ if (read.has(finalUrl)) continue;
474
+ const page = readPage(res.text(), finalUrl, site, pages.get(url) || {});
475
+ read.set(finalUrl, page);
476
+ // With no sitemap, follow the links outwards.
477
+ if (!usedSitemap) for (const l of page.links) if (!pages.has(l)) { pages.set(l, {fromBlog: false}); queue.push(l); }
478
+ progress.read++;
479
+ } catch (err) {
480
+ if (err instanceof CrawlError) throw err;
481
+ failed.push({url, error: err.message || String(err)});
482
+ progress.failed++;
483
+ }
484
+ progress.found = pages.size;
485
+ tell();
486
+ }
487
+ };
488
+ await Promise.all(Array.from({length: CRAWL_LIMITS.concurrency}, worker));
489
+
490
+ progress.phase = 'Writing the export';
491
+ tell();
492
+ const xml = toWxr({site, homeUrl, homePage, pages: [...read.values()], builder});
493
+ const all = [...read.values()];
494
+ return {
495
+ xml,
496
+ summary: {
497
+ url: homeUrl, builder: builder || 'website', siteName: homePage.siteName || homePage.title,
498
+ pages: all.filter(p => p.kind === 'page').length, posts: all.filter(p => p.kind === 'post').length,
499
+ files: new Set(all.flatMap(p => p.files)).size, menu: homePage.nav?.length || 0,
500
+ sitemap: usedSitemap, capped, failed: failed.slice(0, 100), failedCount: failed.length,
501
+ thin: all.filter(p => p.html.replace(/<[^>]+>/g, '').trim().length < 40).map(p => p.url).slice(0, 100)
502
+ }
503
+ };
504
+ }
505
+
506
+ // ---- Writing the export ---------------------------------------------------
507
+
508
+ const cdata = (s) => `<![CDATA[${String(s ?? '').replace(/]]>/g, ']]]]><![CDATA[>')}]]>`;
509
+ const xmlText = (s) => String(s ?? '').replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
510
+
511
+ /** "2024-03-01T10:00:00Z" -> "2024-03-01 10:00:00" (WordPress's format), '' when not a date. */
512
+ export function wpDate(v) {
513
+ const t = Date.parse(v);
514
+ if (!Number.isFinite(t)) return '';
515
+ return new Date(t).toISOString().slice(0, 19).replace('T', ' ');
516
+ }
517
+
518
+ /** The slug a page is known by: its address's last part. */
519
+ const slugOf = (url, site) => {
520
+ const p = new URL(url).pathname.slice(site.basePath.length).replace(/^\/+|\/+$/g, '');
521
+ return decodeURIComponent(p.split('/').pop() || '');
522
+ };
523
+
524
+ /** The pages read, as a WXR 1.2 document ./wxr.js reads like any other. */
525
+ export function toWxr({site, homeUrl, homePage, pages, builder}) {
526
+ const now = wpDate(new Date().toISOString());
527
+ const items = [];
528
+ const ids = new Set();
529
+ const idFor = (key) => { let id = syntheticId(key); while (ids.has(id)) id++; ids.add(id); return id; };
530
+ const authors = new Map();
531
+ const pageIds = new Map();
532
+ // Parents: /services/cleaning sits under /services when that page was read.
533
+ for (const p of pages) pageIds.set(p.url, idFor(`page:${new URL(p.url).pathname}`));
534
+ const files = new Map();
535
+ for (const p of pages) {
536
+ const id = pageIds.get(p.url);
537
+ const path = new URL(p.url).pathname;
538
+ const up = path.replace(/\/[^/]+$/, '');
539
+ const parentUrl = up && up !== site.basePath ? `${site.origin}${up}` : '';
540
+ const parent = p.kind === 'page' && parentUrl && pageIds.has(parentUrl) && pages.find(x => x.url === parentUrl)?.kind === 'page' ? pageIds.get(parentUrl) : 0;
541
+ const date = wpDate(p.date) || now;
542
+ // No named author: nobody is credited but whoever runs the import.
543
+ const creator = p.author ? p.author.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').slice(0, 60) || 'author' : '';
544
+ if (p.author && !authors.has(creator)) authors.set(creator, p.author);
545
+ const isHome = p.url === homePage.url || p.url === homeUrl;
546
+ const metas = {};
547
+ if (p.description) metas._yoast_wpseo_metadesc = p.description;
548
+ if (p.seoTitle && p.seoTitle !== p.title) metas._yoast_wpseo_title = p.seoTitle;
549
+ items.push(itemXml({
550
+ id, type: p.kind, title: p.title, slug: isHome ? 'home' : slugOf(p.url, site), link: isHome ? `${site.origin}${site.basePath}/` : p.url,
551
+ date, creator, content: p.html, parent, metas
552
+ }));
553
+ for (const f of p.files) if (!files.has(f)) files.set(f, {parent: id, date, alt: p.alts?.[f] || ''});
554
+ }
555
+ for (const [url, f] of files) {
556
+ const name = decodeURIComponent(new URL(url).pathname.split('/').pop() || 'file');
557
+ items.push(itemXml({id: idFor(`file:${url}`), type: 'attachment', title: f.alt || name.replace(/\.[a-z0-9]+$/i, ''), metas: f.alt ? {_wp_attachment_image_alt: f.alt} : {}, slug: '', link: url, date: f.date, creator: '', parent: f.parent, attachmentUrl: url, status: 'inherit'}));
558
+ }
559
+ // The menu.
560
+ let menuXml = '';
561
+ if (homePage.nav?.length) {
562
+ menuXml = `<wp:term><wp:term_id>1</wp:term_id><wp:term_taxonomy>nav_menu</wp:term_taxonomy><wp:term_slug>main-menu</wp:term_slug><wp:term_name>${cdata('Main menu')}</wp:term_name></wp:term>`;
563
+ let order = 0;
564
+ const add = (entries, parentId) => {
565
+ for (const e of entries) {
566
+ const id = idFor(`menu:${parentId}:${e.text}:${e.url}`);
567
+ order++;
568
+ items.push(itemXml({
569
+ id, type: 'nav_menu_item', title: e.text, slug: String(id), link: '', date: now, creator: '', order,
570
+ terms: [['nav_menu', 'main-menu', 'Main menu']],
571
+ metas: {_menu_item_type: 'custom', _menu_item_url: e.url || '#', _menu_item_menu_item_parent: String(parentId || 0), _menu_item_object: 'custom', _menu_item_object_id: String(id)}
572
+ }));
573
+ if (e.children?.length) add(e.children, id);
574
+ }
575
+ };
576
+ add(homePage.nav, 0);
577
+ }
578
+ const generator = `Domma CMS site reader (${builder || 'website'})`;
579
+ const title = homePage.siteName || homePage.title || site.host;
580
+ return `<?xml version="1.0" encoding="UTF-8"?>
581
+ <rss version="2.0" xmlns:excerpt="http://wordpress.org/export/1.2/excerpt/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:wfw="http://wellformedweb.org/CommentAPI/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:wp="http://wordpress.org/export/1.2/">
582
+ <channel>
583
+ <title>${xmlText(title)}</title>
584
+ <link>${xmlText(`${site.origin}${site.basePath}`)}</link>
585
+ <description>${xmlText(homePage.description)}</description>
586
+ <language>en</language>
587
+ <wp:wxr_version>1.2</wp:wxr_version>
588
+ <wp:base_site_url>${xmlText(`${site.origin}${site.basePath}`)}</wp:base_site_url>
589
+ <wp:base_blog_url>${xmlText(`${site.origin}${site.basePath}`)}</wp:base_blog_url>
590
+ <generator>${xmlText(generator)}</generator>
591
+ ${[...authors].map(([login, name], i) => ` <wp:author><wp:author_id>${i + 1}</wp:author_id><wp:author_login>${cdata(login)}</wp:author_login><wp:author_email></wp:author_email><wp:author_display_name>${cdata(name)}</wp:author_display_name></wp:author>`).join('\n')}
592
+ ${menuXml}
593
+ ${items.join('\n')}
594
+ </channel>
595
+ </rss>
596
+ `;
597
+ }
598
+
599
+ function itemXml({id, type, title, slug, link, date, creator, content = '', parent = 0, order = 0, attachmentUrl = '', status = 'publish', metas = {}, terms = []}) {
600
+ return ` <item>
601
+ <title>${cdata(title)}</title>
602
+ <link>${xmlText(link)}</link>
603
+ <dc:creator>${cdata(creator)}</dc:creator>
604
+ <guid isPermaLink="false">${xmlText(link || `site-import:${id}`)}</guid>
605
+ <content:encoded>${cdata(content)}</content:encoded>
606
+ <excerpt:encoded>${cdata('')}</excerpt:encoded>
607
+ <wp:post_id>${id}</wp:post_id>
608
+ <wp:post_date>${cdata(date)}</wp:post_date>
609
+ <wp:post_date_gmt>${cdata(date)}</wp:post_date_gmt>
610
+ <wp:post_name>${cdata(slug)}</wp:post_name>
611
+ <wp:status>${cdata(status)}</wp:status>
612
+ <wp:post_parent>${parent}</wp:post_parent>
613
+ <wp:menu_order>${order}</wp:menu_order>
614
+ <wp:post_type>${cdata(type)}</wp:post_type>
615
+ ${attachmentUrl ? `<wp:attachment_url>${cdata(attachmentUrl)}</wp:attachment_url>` : ''}
616
+ ${terms.map(t => `<category domain="${t[0]}" nicename="${t[1]}">${cdata(t[2])}</category>`).join('')}
617
+ ${Object.entries(metas).map(([k, v]) => `<wp:postmeta><wp:meta_key>${cdata(k)}</wp:meta_key><wp:meta_value>${cdata(v)}</wp:meta_value></wp:postmeta>`).join('')}
618
+ </item>`;
619
+ }